diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index ec18a650b..42c61d51f 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -82,9 +82,17 @@ body: label: 复现链接(可选) description: | 请提供能复现此问题的链接。 - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI生成内容(可选) - description: | - 如果此问题是由AI辅助您发现的,请提供全部聊天记录,包括使用的模型信息。 + label: AI生成内容 + description: 必须且只能勾选一项,请勿删除或修改声明文字。 + options: + - label: 我使用了AI工具生成此内容 + - label: 我没有使用AI工具生成此内容 + - type: input + id: ai-model + attributes: + label: AI模型是 + description: 如果使用了AI工具,请填写模型名称;未使用则留空。 + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 0085d4de5..3e9001915 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -82,9 +82,17 @@ body: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI Generated Content (optional) - description: | - If this issue was identified with the assistance of AI, please provide the complete chat log, including information about the model used. + label: AI Generated Content + description: Select exactly one option. Do not delete or modify the disclosure text. + options: + - label: I used AI tools to generate this content + - label: I did not use AI tools to generate this content + - type: input + id: ai-model + attributes: + label: AI model used + description: If you used AI tools, enter the model name; otherwise leave this blank. + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 907240261..def688e2f 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -48,9 +48,17 @@ body: label: 附加信息 description: | 相关的任何其他上下文或截图,或者你觉得有帮助的信息 - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI生成内容(可选) - description: | - 如果此请求是由AI辅助您提交的,请提供全部聊天记录,包括使用的模型信息。 + label: AI生成内容 + description: 必须且只能勾选一项,请勿删除或修改声明文字。 + options: + - label: 我使用了AI工具生成此内容 + - label: 我没有使用AI工具生成此内容 + - type: input + id: ai-model + attributes: + label: AI模型是 + description: 如果使用了AI工具,请填写模型名称;未使用则留空。 + placeholder: xxx diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 393118592..796f0e6c1 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -48,9 +48,17 @@ body: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. - - type: textarea + - type: checkboxes id: aigenerated attributes: - label: AI Generated Content (optional) - description: | - If this request was submitted with the assistance of an AI, please provide the complete chat log, including information about the model used. + label: AI Generated Content + description: Select exactly one option. Do not delete or modify the disclosure text. + options: + - label: I used AI tools to generate this content + - label: I did not use AI tools to generate this content + - type: input + id: ai-model + attributes: + label: AI model used + description: If you used AI tools, enter the model name; otherwise leave this blank. + placeholder: xxx diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f443d6896..21e6f1d62 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: benjlevesque/short-sha@dbe07338b37c456ce06d23409b35a56a7815eef4 # v4.0 id: short-sha @@ -28,7 +28,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.26.4" + go-version: "1.27.1" - name: Setup web run: bash build.sh dev web @@ -37,7 +37,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.5 + uses: OpenListTeam/cgo-actions@d760a8ec1a6be1f8ec181229e11cb2671797f9e0 # v1.3.0 with: targets: ${{ matrix.target }} flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} @@ -47,7 +47,7 @@ jobs: out-dir: build x-flags: | github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$built_at - github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors + github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling @@ -64,7 +64,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8fa3117a0..423bd1069 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 diff --git a/.github/workflows/issue_comment.yml b/.github/workflows/issue_comment.yml new file mode 100644 index 000000000..f8b209035 --- /dev/null +++ b/.github/workflows/issue_comment.yml @@ -0,0 +1,75 @@ +name: Issue Auto Reply + +on: + issues: + types: [opened] + +permissions: + issues: write + +jobs: + auto-reply: + runs-on: ubuntu-latest + steps: + - name: Check issue for unchecked tasks and reply + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + if (context.payload.issue.title.startsWith('[Announcements]')) return; + const titleNotEdited = /(请修改标题|Please modify the title)/i.test(context.payload.issue.title); + const issueBody = context.payload.issue.body || ""; + const aiSection = issueBody.match(/^### (AI生成内容|AI Generated Content)\r?\n([\s\S]*?)(?=^### |$(?![\s\S]))/m); + const aiOptionsPattern = aiSection?.[1] === 'AI生成内容' + ? /^\s*- \[([ xX])\] 我使用了AI工具生成此内容\r?\n- \[([ xX])\] 我没有使用AI工具生成此内容\s*$/ + : /^\s*- \[([ xX])\] I used AI tools to generate this content\r?\n- \[([ xX])\] I did not use AI tools to generate this content\s*$/; + const aiOptions = (aiSection?.[2] || '').match(aiOptionsPattern); + const validAiDisclosure = aiOptions !== null && (aiOptions[1] !== ' ') !== (aiOptions[2] !== ' '); + const aiModelSection = issueBody.match(/^### (AI模型是|AI model used)\r?\n([\s\S]*?)(?=^### |$(?![\s\S]))/m); + const aiModel = (aiModelSection?.[2] || '').trim(); + const missingAiModel = validAiDisclosure && aiOptions[1] !== ' ' && (aiModel === '' || aiModel === '_No response_'); + const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + const closeIssue = titleNotEdited || confirmNotRead || !validAiDisclosure || missingAiModel; + let comment; + if (titleNotEdited) { + comment = `⚠️ 请修改标题以更好地描述您的问题或需求,并删除示例提示。当前 Issue 将被自动关闭并锁定。如需继续提交,请创建新的 Issue。 + ⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed and locked. If you wish to proceed, please create a new issue. + `; + } else if (confirmNotRead || !validAiDisclosure || missingAiModel) { + comment = `⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭并锁定。如需继续提交,请确认已了解规则后创建新的 Issue。 + ⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed and locked. If you wish to proceed, please confirm that you have reviewed the rules before creating a new issue. + `; + } else if (/- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody.replace(aiSection[0], ''))) { + comment = `感谢您联系OpenList。我们会尽快回复您。 + Thanks for contacting OpenList. We will reply to you as soon as possible. + + 由于您提出的 Issue 中包含部分未确认的项目,为了更好地管理项目,在人工审核后可能会直接关闭此问题。 + 如果您能确认并补充相关未确认项目的信息,欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持! + Since your issue contains some unchecked tasks, it may be closed after manual review. + If you can confirm and provide information for the unchecked tasks, feel free to resubmit. + We will pay attention and handle it in a timely manner. + + 感谢您的理解与支持! + Thank you for your understanding and support! + `; + } else { + return; + } + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + if (closeIssue) { + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + await github.rest.issues.lock({ + ...context.repo, + issue_number: context.issue.number + }); + } diff --git a/.github/workflows/pr_title_check.yml b/.github/workflows/pr_title_check.yml new file mode 100644 index 000000000..45707af7c --- /dev/null +++ b/.github/workflows/pr_title_check.yml @@ -0,0 +1,34 @@ +name: PR Title Check + +on: + pull_request_target: + types: [opened] + +permissions: + pull-requests: write + +jobs: + pr-title-check: + runs-on: ubuntu-latest + steps: + - name: Check PR title for required prefix and comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const title = context.payload.pull_request.title; + if (/^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title)) return; + const comment = `⚠️ PR 标题需以 \`feat(): \`, \`docs(): \`, \`fix(): \`, \`style(): \`, \`refactor(): \`, \`chore(): \` 其中之一开头,例如:\`feat(component): 新增功能\`。 + ⚠️ The PR title must start with \`feat(): \`, \`docs(): \`, \`fix(): \`, \`style(): \`, or \`refactor(): \`, \`chore(): \`. For example: \`feat(component): add new feature\`. + + 如果跨多个组件,请使用主要组件作为前缀,并在标题中枚举、描述中说明。 + If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + + 如果是破坏性变更,请在类型后添加 \`!\`,例如 \`feat(component)!: 破坏性变更\`。 + For breaking changes, add \`!\` after the type, e.g., \`feat(component)!: breaking change\`. + + `; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 783a9abcb..99b721bc4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Prerelease - uses: irongut/EditRelease@v1.2.0 + uses: irongut/EditRelease@ccf529ad26dddf9996e7dd0f24ca5da4ea507cc2 # v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} id: ${{ github.event.release.id }} @@ -33,7 +33,7 @@ jobs: - name: Free Disk Space (Ubuntu) if: matrix.target-platform == '' - uses: jlumbroso/free-disk-space@main + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main with: tool-cache: false android: true @@ -46,10 +46,10 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.26.4' + go-version: '1.27.1' - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 1 fetch-tags: true diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index adba09ea0..b8f9d7b59 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -43,7 +43,7 @@ jobs: - name: Cache Musl id: cache-musl - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -61,7 +61,7 @@ jobs: run: bash build.sh release docker-multiplatform - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true diff --git a/.github/workflows/trigger-makefile-update.yml b/.github/workflows/trigger-makefile-update.yml index b8c4491cb..e35343dd7 100644 --- a/.github/workflows/trigger-makefile-update.yml +++ b/.github/workflows/trigger-makefile-update.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Trigger Makefile hash update - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} script: | diff --git a/build.sh b/build.sh index f1079f756..f5587951a 100644 --- a/build.sh +++ b/build.sh @@ -1,7 +1,9 @@ set -e appName="alist" builtAt="$(date +'%F %T %z')" + gitAuthor="power721" + gitCommit=$(git log --pretty=format:"%h" -1) # Set frontend repository, default to OpenListTeam/OpenList-Frontend @@ -98,7 +100,7 @@ AssertStaticBinary() { return 0 } FetchWebRolling() { - pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/edge\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') # There is no lite for rolling @@ -536,8 +538,8 @@ BuildReleaseFreeBSD() { sed 's/\.0$//') if [ -z "$freebsd_version" ]; then - echo "Failed to get FreeBSD version, falling back to 14.3" - freebsd_version="14.3" + echo "Failed to get FreeBSD version, falling back to 14.4" + freebsd_version="14.4" fi echo "Using FreeBSD version: $freebsd_version" diff --git a/cmd/start.go b/cmd/start.go index 57edff38c..a1838eadd 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -31,14 +32,23 @@ func start() { return } } - args := os.Args - args[1] = "server" - args = append(args, "--force-bin-dir") - cmd := &exec.Cmd{ - Path: args[0], - Args: args, - Env: os.Environ(), + exe, err := os.Executable() + if err != nil { + log.Fatal("failed to resolve executable path: ", err) + } + childArgs := append([]string{"server"}, os.Args[2:]...) + hasForceBinDir := false + for _, arg := range childArgs { + if arg == "--force-bin-dir" || strings.HasPrefix(arg, "--force-bin-dir=") { + hasForceBinDir = true + break + } + } + if !hasForceBinDir { + childArgs = append(childArgs, "--force-bin-dir") } + cmd := exec.Command(exe, childArgs...) + cmd.Env = os.Environ() stdout, err := os.OpenFile(filepath.Join(filepath.Dir(pidFile), "start.log"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) if err != nil { log.Fatal(os.Getpid(), ": failed to open start log file:", err) diff --git a/drivers/115/util.go b/drivers/115/util.go index f5b27c763..96e22971f 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -393,9 +393,11 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试,共尝试3次 for retry := 0; retry < 3; retry++ { - select { - case <-ctx.Done(): + if utils.IsCanceled(ctx) { + err = ctx.Err() break + } + select { case <-ticker.C: if ossToken, err = d.client.GetOSSToken(); err != nil { // 到时重新获取ossToken errCh <- errors.Wrap(err, "刷新token时出现错误") diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2c..ac0aad38b 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "encoding/base64" + "errors" "io" "time" @@ -70,6 +71,19 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } `json:"data"` // } +// retryExpiredToken retries only the rejected OSS operation, preserving the upload ID. +func retryExpiredToken(refresh func() error, operation func() error) error { + err := operation() + var serviceErr oss.ServiceError + if !errors.As(err, &serviceErr) || serviceErr.Code != "SecurityTokenExpired" { + return err + } + if err := refresh(); err != nil { + return err + } + return operation() +} + func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { @@ -80,7 +94,32 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, return err } - imur, err := bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential()) + refresh := func() error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + token, err := d.client.UploadGetToken(ctx) + if err != nil { + return err + } + client, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return err + } + newBucket, err := client.Bucket(initResp.Bucket) + if err != nil { + return err + } + bucket = newBucket + return nil + } + + var imur oss.InitiateMultipartUploadResult + err = retryExpiredToken(refresh, func() error { + var err error + imur, err = bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential(), oss.WithContext(ctx)) + return err + }) if err != nil { return err } @@ -109,13 +148,17 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, return err } err = retry.Do(func() error { - rd.Seek(0, io.SeekStart) - part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) - if err != nil { - return err - } - parts[i-1] = part - return nil + return retryExpiredToken(refresh, func() error { + if _, err := rd.Seek(0, io.SeekStart); err != nil { + return err + } + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i), oss.WithContext(ctx)) + if err != nil { + return err + } + parts[i-1] = part + return nil + }) }, retry.Context(ctx), retry.Attempts(3), @@ -134,14 +177,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up(float64(offset) * 100 / float64(fileSize)) } - // callbackRespBytes := make([]byte, 1024) - _, err = bucket.CompleteMultipartUpload( - imur, - parts, - oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), - oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), - // oss.CallbackResult(&callbackRespBytes), - ) + err = retryExpiredToken(refresh, func() error { + _, err := bucket.CompleteMultipartUpload( + imur, + parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + oss.WithContext(ctx), + ) + return err + }) if err != nil { return err } diff --git a/drivers/139/driver.go b/drivers/139/driver.go index c0d5ee50a..c05ff8fb7 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -42,17 +42,8 @@ func (d *Yun139) GetAddition() driver.Additional { func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { - if len(d.Authorization) == 0 { - if d.Username != "" && d.Password != "" { - log.Infof("139yun: authorization is empty, trying to login with password.") - newAuth, err := d.loginWithPassword() - log.Debugf("newAuth: Ok: %s", newAuth) - if err != nil { - return fmt.Errorf("login with password failed: %w", err) - } - } else { - return fmt.Errorf("authorization is empty and username/password is not provided") - } + if err := d.validateAndInitCredentials(); err != nil { + return err } err := d.refreshToken() if err != nil { diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 1714ea67f..73bd1b18e 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { //Account string `json:"account" required:"true"` - Authorization string `json:"authorization" type:"text" required:"true"` - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true" secret:"true"` - MailCookies string `json:"mail_cookies" required:"true" type:"text" help:"Cookies from mail.139.com used for login authentication."` + Authorization string `json:"authorization" type:"text" help:"Authorization can be used alone. If empty, use username + password; mail_cookies is optional and will be established/updated automatically. Existing mail_cookies can also be used alone for fast login."` + Username string `json:"username" help:"Use together with password when Authorization is empty. mail_cookies may be left empty on the first login."` + Password string `json:"password" secret:"true" help:"Use together with username when Authorization is empty. mail_cookies may be left empty on the first login."` + SmsCode string `json:"sms_code" secret:"true" help:"Fill this only after OpenList reports that a 139 Mail SMS verification code was sent, then save the storage again."` + MailCookies string `json:"mail_cookies" type:"text" help:"Optional cookies from mail.10086.cn. Leave empty for a first username/password login; cookies created or updated by password/SMS login are persisted and reused as device context. Existing cookies may also be used alone for fast login."` driver.RootID Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` diff --git a/drivers/139/util.go b/drivers/139/util.go index 5488b0083..c50d14eca 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -6,9 +6,12 @@ import ( "crypto/aes" "crypto/cipher" crypto_rand "crypto/rand" + "crypto/rsa" "crypto/sha1" + "crypto/x509" "encoding/base64" "encoding/hex" + "encoding/xml" "errors" "fmt" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -27,6 +30,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + cookiepkg "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" @@ -35,8 +39,22 @@ import ( ) const ( - KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 - KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 + KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 + KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 + mailPublicKey = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnsOHTFtwW5rq/8gGhPlM5Z3RPdeN/d+FYIHb5JmcfBOCozXuT8c+0anvxtkjzghixwNlnmBuhN8OYfS789YuH/qReQbHC7OdlisLildNWPHRNUYcPa0W3lXSG3+81CXK7FDXPvXo5ubw2GqVbIsccMarI1dyfXdi4ITiCXvmM9wYBdUs9yXtoorhlpyYUI2GV8HNsQjWK9P5QZHT3ox5Qy+mjRmvv6RUFJLPOMkOS/pGZ0DwC1ypFZBxstW0/ftVupdOmGWvW7J2/e3dq3A/UvIkC4YUY/diL1wighJx1G9MiRROISjNMvNyUDSTqPJy516+l3sgHEbc067QIJx2NQIDAQAB" +) + +var ( + mailPasswordURL = "https://mail.10086.cn/Login/Login.ashx" + mailSMSURL = "https://mail.10086.cn/s" +) + +type credentialState int + +const ( + credentialStateAuthorization credentialState = iota + credentialStateFullLogin + credentialStateCookiesOnly ) // do others that not defined in Driver interface @@ -77,29 +95,28 @@ func (d *Yun139) refreshToken() error { } decode, err := base64.StdEncoding.DecodeString(d.Authorization) if err != nil { - return fmt.Errorf("authorization decode failed: %s", err) + return d.loginAfterAuthorizationFailure(fmt.Errorf("authorization decode failed: %w", err)) } decodeStr := string(decode) splits := strings.Split(decodeStr, ":") if len(splits) < 3 { - return fmt.Errorf("authorization is invalid, splits < 3") + return d.loginAfterAuthorizationFailure(errors.New("authorization is invalid, splits < 3")) } d.Account = splits[1] strs := strings.Split(splits[2], "|") if len(strs) < 4 { - return fmt.Errorf("authorization is invalid, strs < 4") + return d.loginAfterAuthorizationFailure(errors.New("authorization is invalid, strs < 4")) } expiration, err := strconv.ParseInt(strs[3], 10, 64) if err != nil { - return fmt.Errorf("authorization is invalid") + return d.loginAfterAuthorizationFailure(errors.New("authorization expiration is invalid")) } expiration -= time.Now().UnixMilli() if expiration > 1000*60*60*24*15 { - // Authorization有效期大于15天无需刷新 return nil } if expiration < 0 { - return fmt.Errorf("authorization has expired") + return d.loginAfterAuthorizationFailure(errors.New("authorization has expired")) } url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" @@ -111,13 +128,7 @@ func (d *Yun139) refreshToken() error { SetResult(&resp). Post(url) if err != nil || resp.Return != "0" { - log.Warnf("139yun: failed to refresh token with old token: %v, desc: %s. trying to login with password.", err, resp.Desc) - newAuth, loginErr := d.loginWithPassword() - log.Debugf("newAuth: Ok: %s", newAuth) - if loginErr != nil { - return fmt.Errorf("failed to login with password after refresh failed: %w", loginErr) - } - return nil + return d.loginAfterAuthorizationFailure(fmt.Errorf("token refresh failed: %v, desc: %s", err, resp.Desc)) } d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) @@ -126,6 +137,18 @@ func (d *Yun139) refreshToken() error { return nil } +// loginAfterAuthorizationFailure deliberately skips cookie fast login. Mail +// cookies are only reused as device context for the password login request. +func (d *Yun139) loginAfterAuthorizationFailure(cause error) error { + log.Warnf("139yun: %v; trying password login.", cause) + newAuth, err := d.loginWithPassword() + log.Debugf("139yun: password fallback generated authorization: %t", newAuth != "") + if err != nil { + return fmt.Errorf("%v; password login failed: %w", cause, err) + } + return nil +} + func (d *Yun139) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() randStr := random.String(16) @@ -530,7 +553,7 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R "X-Yun-Api-Version": "v1", "X-Yun-App-Channel": "10000034", "X-Yun-Channel-Source": "10000034", - "X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + "X-Yun-Client-Info": "||13|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", "X-Yun-Module-Type": "100", "X-Yun-Svc-Type": "1", }) @@ -743,15 +766,212 @@ func (d *Yun139) getDiskQuotaDetail(ctx context.Context) (*DiskQuotaDetail, erro return &resp, nil } +type mailLoginResponse struct { + Code string `json:"code"` + Summary string `json:"summary"` + Var struct { + LoginSuccessURL string `json:"loginSuccessUrl"` + } `json:"var"` +} + +func parseMailLoginResponse(body []byte) mailLoginResponse { + var response mailLoginResponse + if utils.Json.Unmarshal(body, &response) == nil && (response.Code != "" || response.Summary != "" || response.Var.LoginSuccessURL != "") { + return response + } + if match := regexp.MustCompile(`['"]?code['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Code = string(match[1]) + } + if match := regexp.MustCompile(`['"]?summary['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Summary = string(match[1]) + } + if match := regexp.MustCompile(`['"]?loginSuccessUrl['"]?\s*:\s*['"]([^'"]+)`).FindSubmatch(body); len(match) == 2 { + response.Var.LoginSuccessURL = string(match[1]) + } + return response +} + +func mergeMailCookieHeader(existing string, responseCookies []*http.Cookie) string { + cookies := cookiepkg.Parse(existing) + for _, responseCookie := range responseCookies { + if responseCookie != nil && responseCookie.Name != "" { + cookies = cookiepkg.SetCookie(cookies, responseCookie.Name, responseCookie.Value) + } + } + return cookiepkg.ToString(cookies) +} + +func mailRiskCode(location string) string { + parsed, err := url.Parse(location) + if err != nil { + return "" + } + return parsed.Query().Get("ec") +} + +func smsSceneForRisk(riskCode string) (int, bool) { + switch riskCode { + case "PML401010062": + return 2, true + case "MW0016": + return 4, true + case "S025", "S035": + return 1, true + default: + return 0, false + } +} + +func encryptMailLoginName(account string) (string, error) { + der, err := base64.StdEncoding.DecodeString(mailPublicKey) + if err != nil { + return "", fmt.Errorf("decode mail public key: %w", err) + } + parsed, err := x509.ParsePKIXPublicKey(der) + if err != nil { + return "", fmt.Errorf("parse mail public key: %w", err) + } + publicKey, ok := parsed.(*rsa.PublicKey) + if !ok { + return "", errors.New("mail public key is not RSA") + } + encrypted, err := rsa.EncryptPKCS1v15(crypto_rand.Reader, publicKey, []byte(account)) + if err != nil { + return "", fmt.Errorf("encrypt mail login name: %w", err) + } + return base64.StdEncoding.EncodeToString(encrypted), nil +} + +func mailXMLHeaders(cookie string) map[string]string { + return map[string]string{ + "Cookie": cookie, + "Content-Type": "application/xml; charset=utf-8", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/4.12.0", + } +} + +func new139RestyClient() *resty.Client { + if base.RestyClient != nil { + return base.RestyClient.Clone() + } + return base.NewRestyClient() +} + +func (d *Yun139) sendSMSVerificationCode(riskCode string) error { + scene, ok := smsSceneForRisk(riskCode) + if !ok { + return fmt.Errorf("139 Mail risk control does not support SMS verification: %s", riskCode) + } + loginName, err := encryptMailLoginName(d.Username) + if err != nil { + return err + } + body := strings.Join([]string{ + "", + mailXMLField("loginName", loginName), + mailXMLField("fv", "4"), + mailXMLField("clientId", "1003"), + mailXMLField("eMode", "1"), + mailXMLField("loginFailureUrl", ""), + mailXMLField("loginSuccessUrl", ""), + mailXMLField("verifyCode", ""), + mailXMLField("version", "1.0"), + mailXMLField("scene", strconv.Itoa(scene)), + "", + }, "") + res, err := new139RestyClient().SetRetryCount(0).R(). + SetHeaders(mailXMLHeaders(d.MailCookies)). + SetBody(body). + Post(mailSMSURL + "?func=" + url.QueryEscape("login:sendSmsCodeByScene") + "&cguid=" + strconv.FormatInt(time.Now().UnixMilli(), 10)) + if err != nil { + return fmt.Errorf("send 139 Mail SMS verification code: %w", err) + } + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) + response := parseMailLoginResponse(res.Body()) + if response.Code == "S_OK" { + return nil + } + switch response.Code { + case "PML401010021", "PML401010022": + return fmt.Errorf("139 Mail requires picture verification before SMS can be sent: %s", response.Code) + case "PML404010001": + return errors.New("139 Mail SMS verification code was requested too frequently") + case "PML401010002": + return errors.New("139 Mail rejected the SMS verification parameters") + default: + return fmt.Errorf("send 139 Mail SMS verification code failed: code=%s summary=%s", response.Code, response.Summary) + } +} + +func (d *Yun139) verifySMSCode(riskCode string) (string, error) { + if _, ok := smsSceneForRisk(riskCode); !ok { + return "", fmt.Errorf("139 Mail risk control does not support SMS verification: %s", riskCode) + } + loginName, err := encryptMailLoginName(d.Username) + if err != nil { + return "", err + } + pwdType := "" + if riskCode == "MW0016" { + pwdType = mailXMLField("pwdType", "1") + } + body := strings.Join([]string{ + "", + mailXMLField("clientId", "1003"), + mailXMLField("version", "4"), + mailXMLField("loginType", "0"), + mailXMLField("authType", "2"), + mailXMLField("loginName", loginName), + mailXMLField("eMode", "1"), + mailXMLField("loginPassword", sha1Hash("fetion.com.cn:"+d.SmsCode)), + mailXMLField("createAutoLoginSecretKey", "1"), + mailXMLField("verifyCode", ""), + mailXMLField("verifyAgentId", ""), + mailXMLField("reqFrom", "3"), + mailXMLField("needWCookie", "1"), + pwdType, + "", + }, "") + res, err := new139RestyClient().SetRetryCount(0).R(). + SetHeaders(mailXMLHeaders(d.MailCookies)). + SetBody(body). + Post(mailSMSURL + "?func=" + url.QueryEscape("/login/inlogin.action") + "&cguid=" + strconv.FormatInt(time.Now().UnixMilli(), 10)) + if err != nil { + return "", fmt.Errorf("verify 139 Mail SMS code: %w", err) + } + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) + response := parseMailLoginResponse(res.Body()) + if response.Code != "S_OK" { + return "", fmt.Errorf("verify 139 Mail SMS code failed: code=%s summary=%s", response.Code, response.Summary) + } + sid := "" + if response.Var.LoginSuccessURL != "" { + if successURL, parseErr := url.Parse(response.Var.LoginSuccessURL); parseErr == nil { + sid = successURL.Query().Get("sid") + } + } + if sid == "" { + for _, cookie := range cookiepkg.Parse(d.MailCookies) { + if cookie.Name == "Os_SSo_Sid" || cookie.Name == "sid" { + sid = cookie.Value + break + } + } + } + if sid == "" { + return "", errors.New("139 Mail SMS verification succeeded but did not return sid") + } + d.SmsCode = "" + return sid, nil +} + func (d *Yun139) step1_password_login() (string, error) { log.Debugf("--- 执行步骤 1: 登录 API ---") - loginURL := "https://mail.10086.cn/Login/Login.ashx" + loginURL := mailPasswordURL // 密码 SHA1 哈希 hashedPassword := sha1Hash(fmt.Sprintf("fetion.com.cn:%s", d.Password)) - log.Debugf("DEBUG: 原始密码: %s", d.Password) - log.Debugf("DEBUG: SHA1 输入: fetion.com.cn:%s", d.Password) - log.Debugf("DEBUG: 生成的 Password 哈希: %s", hashedPassword) cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) // 随机生成 cguid @@ -787,79 +1007,56 @@ func (d *Yun139) step1_password_login() (string, error) { loginData.Set("authType", "2") log.Debugf("DEBUG: 登录请求 URL: %s", loginURL) - log.Debugf("DEBUG: 登录请求 Headers: %+v", loginHeaders) - log.Debugf("DEBUG: 登录请求 Body: %s", loginData.Encode()) + log.Debugf("DEBUG: 登录请求已准备") - // 设置客户端不跟随重定向 - client := base.RestyClient.SetRedirectPolicy(resty.NoRedirectPolicy()) - res, err := client.R(). + res, err := new139RestyClient(). + SetRetryCount(0). + SetRedirectPolicy(resty.RedirectPolicyFunc(func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + })).R(). SetHeaders(loginHeaders). SetFormDataFromValues(loginData). Post(loginURL) - if err != nil { - // 如果是重定向错误,则不作为失败处理,因为我们禁止了自动重定向 - if res != nil && res.StatusCode() >= 300 && res.StatusCode() < 400 { - log.Debugf("DEBUG: 登录响应 Status Code: %d (Redirect)", res.StatusCode()) - } else { - return "", fmt.Errorf("step1 login request failed: %w", err) - } - } else { - log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + return "", fmt.Errorf("step1 login request failed: %w", err) } - // 恢复客户端的默认重定向策略,以免影响后续请求 - base.RestyClient.SetRedirectPolicy(resty.FlexibleRedirectPolicy(10)) - log.Debugf("DEBUG: 登录响应 Headers: %+v", res.Header()) + log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + log.Debugf("DEBUG: 登录响应 Location present: %t", res.Header().Get("Location") != "") - var sid, extractedCguid string + d.MailCookies = mergeMailCookieHeader(d.MailCookies, res.Cookies()) - // 从 Location 头部提取 sid 和 cguid + sid := "" locationHeader := res.Header().Get("Location") if locationHeader != "" { - sidMatch := regexp.MustCompile(`sid=([^&]+)`).FindStringSubmatch(locationHeader) - cguidMatch := regexp.MustCompile(`cguid=([^&]+)`).FindStringSubmatch(locationHeader) - if len(sidMatch) > 1 { - sid = sidMatch[1] - log.Debugf("DEBUG: 从 Location 提取到 sid: %s", sid) + if riskCode := mailRiskCode(locationHeader); riskCode != "" { + if _, ok := smsSceneForRisk(riskCode); !ok { + return "", fmt.Errorf("139 Mail risk control triggered: %s", riskCode) + } + if strings.TrimSpace(d.SmsCode) == "" { + if sendErr := d.sendSMSVerificationCode(riskCode); sendErr != nil { + return "", sendErr + } + op.MustSaveDriverStorage(d) + return "", errors.New("139 Mail SMS verification code sent; fill sms_code and save the storage again") + } + return d.verifySMSCode(riskCode) } - if len(cguidMatch) > 1 { - extractedCguid = cguidMatch[1] - log.Debugf("DEBUG: 从 Location 提取到 cguid: %s", extractedCguid) + if redirectURL, parseErr := url.Parse(locationHeader); parseErr == nil { + sid = redirectURL.Query().Get("sid") } } - // 如果 Location 中没有,尝试从 Set-Cookie 中提取 - if sid == "" || extractedCguid == "" { - setCookieHeaders := res.Header().Values("Set-Cookie") - for _, cookieStr := range setCookieHeaders { - ssoSidMatch := regexp.MustCompile(`Os_SSo_Sid=([^;]+)`).FindStringSubmatch(cookieStr) - cookieCguidMatch := regexp.MustCompile(`cguid=([^;]+)`).FindStringSubmatch(cookieStr) - if len(ssoSidMatch) > 1 && sid == "" { - sid = ssoSidMatch[1] - log.Debugf("DEBUG: 从 Set-Cookie 提取到 sid: %s", sid) - } - if len(cookieCguidMatch) > 1 && extractedCguid == "" { - extractedCguid = cookieCguidMatch[1] - log.Debugf("DEBUG: 从 Set-Cookie 提取到 cguid: %s", extractedCguid) + if sid == "" { + for _, cookie := range res.Cookies() { + if cookie.Name == "Os_SSo_Sid" || cookie.Name == "sid" { + sid = cookie.Value + break } } } - - if sid == "" || extractedCguid == "" { - return "", errors.New("failed to extract sid or cguid from login response") + if sid == "" { + return "", errors.New("failed to extract sid from login response") } - - // 提取并记录 cookies - loginUrlObj, _ := url.Parse(loginURL) - cookies := base.RestyClient.GetClient().Jar.Cookies(loginUrlObj) - var cookieStrings []string - for _, cookie := range cookies { - cookieStrings = append(cookieStrings, cookie.Name+"="+cookie.Value) - } - cookieStr := strings.Join(cookieStrings, "; ") - log.Debugf("DEBUG: 提取到的 Cookies: %s", cookieStr) - d.MailCookies = cookieStr - return sid, nil } @@ -915,6 +1112,16 @@ func (d *Yun139) step2_get_single_token(sid string) (string, error) { return dycpwd, nil } +func mailXMLField(name, value string) string { + return `` + escapeXML(value) + `` +} + +func escapeXML(value string) string { + var escaped bytes.Buffer + _ = xml.EscapeText(&escaped, []byte(value)) + return escaped.String() +} + // --- 辅助函数:加密/解密 --- // sha1Hash 计算 SHA1 哈希值,返回十六进制字符串。 @@ -1211,9 +1418,126 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { return newAuthorization, nil } +func extractFastLoginCookies(mailCookies string) (sid string, rmkey string) { + for _, c := range cookiepkg.Parse(mailCookies) { + switch c.Name { + case "Os_SSo_Sid": + sid = c.Value + case "RMKEY": + rmkey = c.Value + } + if sid != "" && rmkey != "" { + return sid, rmkey + } + } + return sid, rmkey +} + +func hasCookiePair(raw string) bool { + for _, part := range strings.Split(raw, ";") { + name, value, ok := strings.Cut(strings.TrimSpace(part), "=") + if ok && strings.TrimSpace(name) != "" && value != "" { + return true + } + } + return false +} + +func (d *Yun139) tryFastLoginWithCookies() bool { + sid, rmkey := extractFastLoginCookies(d.MailCookies) + if sid == "" || rmkey == "" { + log.Warnf("139yun: fast login skipped, required cookies missing: Os_SSo_Sid=%t RMKEY=%t", sid != "", rmkey != "") + return false + } + + log.Infof("139yun: attempting fast login using existing SID/Cookies (Step 2).") + token, err := d.step2_get_single_token(sid) + if err != nil || token == "" { + log.Warnf("139yun: fast login Step 2 failed: %v", err) + return false + } + + log.Infof("139yun: Step 2 success. Proceeding to Step 3.") + auth, err := d.step3_third_party_login(token) + if err != nil { + log.Warnf("139yun: fast login Step 3 failed: %v", err) + return false + } + + d.Authorization = auth + op.MustSaveDriverStorage(d) + log.Infof("139yun: fast login success (Step 2 -> Step 3).") + return true +} + +func (d *Yun139) validateAndInitCredentials() error { + state, err := d.credentialState() + if err != nil { + return err + } + + switch state { + case credentialStateAuthorization: + // Authorization is refreshed by Init immediately after this helper returns. + log.Debugf("139yun: Authorization exists, skipping initialization login.") + return nil + case credentialStateFullLogin, credentialStateCookiesOnly: + log.Infof("139yun: Authorization missing, attempting login...") + if d.MailCookies != "" && d.tryFastLoginWithCookies() { + return nil + } + + if state == credentialStateCookiesOnly { + return fmt.Errorf("fast login with cookies failed, and cannot fallback to password login (missing username/password)") + } + + log.Infof("139yun: fast login failed or not possible, performing full password login (Step 1).") + _, err := d.loginWithPassword() + if err != nil { + return fmt.Errorf("login with password failed: %w", err) + } + return nil + default: + return fmt.Errorf("unsupported credential state: %d", state) + } +} + +func (d *Yun139) credentialState() (credentialState, error) { + d.Authorization = strings.TrimSpace(d.Authorization) + d.Username = strings.TrimSpace(d.Username) + d.MailCookies = strings.TrimSpace(d.MailCookies) + + if d.Authorization != "" { + if strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + return 0, fmt.Errorf("authorization should not include Basic prefix") + } + return credentialStateAuthorization, nil + } + + if d.MailCookies != "" && !hasCookiePair(d.MailCookies) { + return 0, fmt.Errorf("MailCookies format is invalid, please check your configuration") + } + + hasUsername := d.Username != "" + hasPassword := strings.TrimSpace(d.Password) != "" + + if hasUsername != hasPassword { + return 0, fmt.Errorf("username and password must be provided together") + } + if hasUsername { + return credentialStateFullLogin, nil + } + + if d.MailCookies != "" { + return credentialStateCookiesOnly, nil + } + + return 0, fmt.Errorf("authorization is empty and credentials are not provided") +} + func (d *Yun139) loginWithPassword() (string, error) { - if d.Username == "" || d.Password == "" || d.MailCookies == "" { - return "", errors.New("username, password or mail_cookies is empty") + if d.Username == "" || d.Password == "" { + return "", errors.New("username or password is empty") } passId, err := d.step1_password_login() diff --git a/drivers/139/util_test.go b/drivers/139/util_test.go new file mode 100644 index 000000000..d4abdc4d9 --- /dev/null +++ b/drivers/139/util_test.go @@ -0,0 +1,474 @@ +package _139 + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/go-resty/resty/v2" +) + +func useRetryingTestClient(t *testing.T) { + t.Helper() + oldClient := base.RestyClient + base.RestyClient = resty.New().SetRetryCount(3) + t.Cleanup(func() { + base.RestyClient = oldClient + }) +} + +func TestMergeMailCookiesPreservesExistingOrderAndAppendsNewNames(t *testing.T) { + got := mergeMailCookieHeader("z=zv; behaviorid=b; Os_SSo_Sid=old", []*http.Cookie{ + {Name: "RMKEY", Value: "rm"}, + {Name: "Os_SSo_Sid", Value: "sid"}, + {Name: "a", Value: "av"}, + }) + want := "z=zv;behaviorid=b;Os_SSo_Sid=sid;RMKEY=rm;a=av" + if got != want { + t.Fatalf("mergeMailCookieHeader() = %q, want %q", got, want) + } +} + +func TestExtractFastLoginCookies(t *testing.T) { + sid, rmkey := extractFastLoginCookies("RMKEY=rm; Os_SSo_Sid=sid") + if sid != "sid" || rmkey != "rm" { + t.Fatalf("extractFastLoginCookies() = %q, %q; want sid, rm", sid, rmkey) + } +} + +func TestCredentialState(t *testing.T) { + tests := []struct { + name string + d Yun139 + want credentialState + err bool + }{ + { + name: "authorization", + d: Yun139{Addition: Addition{Authorization: " auth "}}, + want: credentialStateAuthorization, + }, + { + name: "full login", + d: Yun139{Addition: Addition{ + MailCookies: "RMKEY=rm; Os_SSo_Sid=sid", + Username: "user", + Password: "password", + }}, + want: credentialStateFullLogin, + }, + { + name: "full login without initial cookies", + d: Yun139{Addition: Addition{ + Username: "user", + Password: "password", + }}, + want: credentialStateFullLogin, + }, + { + name: "cookies only", + d: Yun139{Addition: Addition{MailCookies: "RMKEY=rm; Os_SSo_Sid=sid"}}, + want: credentialStateCookiesOnly, + }, + { + name: "partial password login", + d: Yun139{Addition: Addition{Username: "user"}}, + err: true, + }, + { + name: "missing credentials", + d: Yun139{}, + err: true, + }, + { + name: "invalid cookie", + d: Yun139{Addition: Addition{MailCookies: "invalid-cookie"}}, + err: true, + }, + { + name: "authorization with basic prefix", + d: Yun139{Addition: Addition{Authorization: "Basic abc"}}, + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.d.credentialState() + if tt.err { + if err == nil { + t.Fatal("credentialState() expected error") + } + return + } + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("credentialState() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIntegrationLoginObtainsAuthorization(t *testing.T) { + if os.Getenv("OPENLIST_139_INTEGRATION") != "1" { + t.Skip("set OPENLIST_139_INTEGRATION=1 to run live 139Yun login checks") + } + base.RestyClient = resty.New(). + SetHeader("user-agent", base.UserAgent). + SetRetryCount(3). + SetRetryResetReaders(true). + SetTimeout(30 * time.Second) + + username := os.Getenv("OPENLIST_139_USERNAME") + password := os.Getenv("OPENLIST_139_PASSWORD") + mailCookies := os.Getenv("OPENLIST_139_MAIL_COOKIES") + authorization := strings.TrimSpace(os.Getenv("OPENLIST_139_AUTHORIZATION")) + + if authorization != "" { + t.Run("authorization", func(t *testing.T) { + d := Yun139{Addition: Addition{Authorization: authorization}} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateAuthorization { + t.Fatalf("credentialState() = %v, want authorization", state) + } + if d.Authorization == "" || strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + t.Fatal("authorization should be present without Basic prefix") + } + }) + } + + if mailCookies == "" { + t.Fatal("OPENLIST_139_MAIL_COOKIES is required") + } + + runFastLogin := func(mailCookies string) error { + d := Yun139{Addition: Addition{MailCookies: mailCookies}} + state, err := d.credentialState() + if err != nil { + return fmt.Errorf("credentialState() error: %w", err) + } + if state != credentialStateCookiesOnly { + return fmt.Errorf("credentialState() = %v, want cookies only", state) + } + sid, rmkey := extractFastLoginCookies(d.MailCookies) + if sid == "" || rmkey == "" { + return fmt.Errorf("mail cookies are missing Os_SSo_Sid or RMKEY") + } + token, err := d.step2_get_single_token(sid) + if err != nil { + return fmt.Errorf("step2_get_single_token() error: %w", err) + } + auth, err := d.step3_third_party_login(token) + if err != nil { + return fmt.Errorf("step3_third_party_login() error: %w", err) + } + if auth == "" { + return fmt.Errorf("authorization is empty after fast login") + } + return nil + } + + if username == "" || password == "" { + t.Fatal("OPENLIST_139_USERNAME and OPENLIST_139_PASSWORD are required for password fallback") + } + + var refreshedMailCookies string + var generatedAuthorization string + t.Run("password login fallback", func(t *testing.T) { + d := Yun139{Addition: Addition{ + MailCookies: mailCookies, + Username: username, + Password: password, + }} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateFullLogin { + t.Fatalf("credentialState() = %v, want full login", state) + } + passId, err := d.step1_password_login() + if err != nil { + t.Fatalf("step1_password_login() error: %v", err) + } + token, err := d.step2_get_single_token(passId) + if err != nil { + t.Fatalf("step2_get_single_token() error: %v", err) + } + auth, err := d.step3_third_party_login(token) + if err != nil { + t.Fatalf("step3_third_party_login() error: %v", err) + } + d.Authorization = auth + if auth == "" || d.Authorization == "" { + t.Fatal("authorization is empty after password login") + } + generatedAuthorization = auth + refreshedMailCookies = d.MailCookies + }) + + t.Run("authorization generated by password login", func(t *testing.T) { + if generatedAuthorization == "" { + t.Fatal("password login did not generate authorization") + } + d := Yun139{Addition: Addition{Authorization: generatedAuthorization}} + state, err := d.credentialState() + if err != nil { + t.Fatalf("credentialState() unexpected error: %v", err) + } + if state != credentialStateAuthorization { + t.Fatalf("credentialState() = %v, want authorization", state) + } + if strings.HasPrefix(strings.ToLower(d.Authorization), "basic ") { + t.Fatal("authorization should not include Basic prefix") + } + }) + + t.Run("mail cookies fast login from input", func(t *testing.T) { + sid, rmkey := extractFastLoginCookies(mailCookies) + if sid == "" || rmkey == "" { + t.Skip("input mail cookies are missing Os_SSo_Sid or RMKEY") + } + if err := runFastLogin(mailCookies); err != nil { + t.Skipf("input mail cookies are stale or unusable: %v", err) + } + }) + + t.Run("mail cookies fast login after password login", func(t *testing.T) { + if refreshedMailCookies == "" { + t.Fatal("password login did not refresh mail cookies") + } + if err := runFastLogin(refreshedMailCookies); err != nil { + t.Fatalf("refreshed mail cookies fast login failed: %v", err) + } + }) +} + +func TestInvalidAuthorizationDoesNotUseCookieFastLogin(t *testing.T) { + d := Yun139{Addition: Addition{ + Authorization: "not-base64", + MailCookies: "Os_SSo_Sid=sid; RMKEY=rmkey", + }} + + err := d.refreshToken() + if err == nil || !strings.Contains(err.Error(), "password login failed") { + t.Fatalf("refreshToken() error = %v, want password login fallback error", err) + } + if d.Authorization != "not-base64" { + t.Fatalf("Authorization = %q, want original invalid value retained", d.Authorization) + } +} + +func TestSMSSceneForRisk(t *testing.T) { + tests := map[string]int{ + "S025": 1, + "S035": 1, + "PML401010062": 2, + "MW0016": 4, + } + for riskCode, want := range tests { + got, ok := smsSceneForRisk(riskCode) + if !ok || got != want { + t.Fatalf("smsSceneForRisk(%q) = %d, %t; want %d, true", riskCode, got, ok, want) + } + } + if _, ok := smsSceneForRisk("PICTURE_ONLY"); ok { + t.Fatal("smsSceneForRisk() accepted unsupported risk code") + } +} + +func TestSendSMSVerificationCode(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read request body: %v", err) + } + if r.URL.Query().Get("func") != "login:sendSmsCodeByScene" { + t.Errorf("func = %q", r.URL.Query().Get("func")) + } + if !strings.Contains(string(body), `1`) { + t.Errorf("request body = %s", body) + } + if !strings.Contains(r.Header.Get("Cookie"), "device=fingerprint") { + t.Errorf("Cookie = %q", r.Header.Get("Cookie")) + } + http.SetCookie(w, &http.Cookie{Name: "challenge", Value: "sms"}) + _, _ = io.WriteString(w, `{"code":"S_OK"}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", MailCookies: "device=fingerprint"}} + if err := d.sendSMSVerificationCode("S025"); err != nil { + t.Fatalf("sendSMSVerificationCode() error = %v", err) + } + if !strings.Contains(d.MailCookies, "challenge=sms") { + t.Fatalf("MailCookies = %q", d.MailCookies) + } +} + +func TestSendSMSVerificationCodeStopsAtPictureChallenge(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"code":"PML401010021"}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000"}} + err := d.sendSMSVerificationCode("S025") + if err == nil || !strings.Contains(err.Error(), "requires picture verification") { + t.Fatalf("sendSMSVerificationCode() error = %v", err) + } +} + +func TestVerifySMSCode(t *testing.T) { + useRetryingTestClient(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read request body: %v", err) + } + if r.URL.Query().Get("func") != "/login/inlogin.action" { + t.Errorf("func = %q", r.URL.Query().Get("func")) + } + wantHash := sha1Hash("fetion.com.cn:123456") + if !strings.Contains(string(body), ``+wantHash+``) { + t.Errorf("request body does not contain SMS code hash: %s", body) + } + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "new-rmkey"}) + _, _ = io.WriteString(w, `{"code":"S_OK","var":{"loginSuccessUrl":"https://mail.10086.cn/?sid=sms-sid"}}`) + })) + defer server.Close() + + oldURL := mailSMSURL + mailSMSURL = server.URL + defer func() { mailSMSURL = oldURL }() + + d := Yun139{Addition: Addition{ + Username: "18800000000", + SmsCode: "123456", + MailCookies: "challenge=sms", + }} + sid, err := d.verifySMSCode("S025") + if err != nil { + t.Fatalf("verifySMSCode() error = %v", err) + } + if sid != "sms-sid" { + t.Fatalf("sid = %q", sid) + } + if d.SmsCode != "" { + t.Fatalf("SmsCode = %q, want cleared", d.SmsCode) + } + if !strings.Contains(d.MailCookies, "RMKEY=new-rmkey") { + t.Fatalf("MailCookies = %q", d.MailCookies) + } +} + +func TestPasswordLoginWithoutInitialCookiesStopsAtRedirectAndPersistsCookies(t *testing.T) { + useRetryingTestClient(t) + + var loginRequests, redirectRequests int + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/Login/Login.ashx": + loginRequests++ + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "new-rm"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "new-sid"}) + w.Header().Set("Location", server.URL+"/appmail?sid=single-sid") + w.WriteHeader(http.StatusFound) + case "/appmail": + redirectRequests++ + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + oldURL := mailPasswordURL + mailPasswordURL = server.URL + "/Login/Login.ashx" + defer func() { mailPasswordURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", Password: "password"}} + sid, err := d.step1_password_login() + if err != nil || sid != "single-sid" { + t.Fatalf("sid=%q err=%v", sid, err) + } + if loginRequests != 1 || redirectRequests != 0 { + t.Fatalf("login/redirect requests=%d/%d, want 1/0", loginRequests, redirectRequests) + } + if !strings.Contains(d.MailCookies, "RMKEY=new-rm") || !strings.Contains(d.MailCookies, "Os_SSo_Sid=new-sid") { + t.Fatalf("response cookies were not persisted: %q", d.MailCookies) + } +} + +func TestPasswordLoginReusesRawRefreshedCookies(t *testing.T) { + useRetryingTestClient(t) + + var n int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n++ + cookie := r.Header.Get("Cookie") + if !strings.Contains(cookie, "JSESSIONID=stale") || !strings.Contains(cookie, "behaviorid=device") { + t.Errorf("login %d lost raw cookie context: %q", n, cookie) + } + if n == 1 { + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "rm-1"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "sid-1"}) + w.Header().Set("Location", "https://mail.10086.cn/?sid=first") + } else { + if !strings.Contains(cookie, "RMKEY=rm-1") || !strings.Contains(cookie, "Os_SSo_Sid=sid-1") { + t.Errorf("second login did not reuse first refreshed cookies: %q", cookie) + } + http.SetCookie(w, &http.Cookie{Name: "RMKEY", Value: "rm-2"}) + http.SetCookie(w, &http.Cookie{Name: "Os_SSo_Sid", Value: "sid-2"}) + w.Header().Set("Location", "https://mail.10086.cn/?sid=second") + } + w.WriteHeader(http.StatusFound) + })) + defer server.Close() + + oldURL := mailPasswordURL + mailPasswordURL = server.URL + defer func() { mailPasswordURL = oldURL }() + + d := Yun139{Addition: Addition{Username: "18800000000", Password: "password", MailCookies: "Os_SSo_Sid=old; RMKEY=old; JSESSIONID=stale; behaviorid=device"}} + if sid, err := d.step1_password_login(); err != nil || sid != "first" { + t.Fatalf("first login sid=%q err=%v", sid, err) + } + refreshed := d.MailCookies + if !strings.Contains(refreshed, "RMKEY=rm-1") || !strings.Contains(refreshed, "Os_SSo_Sid=sid-1") { + t.Fatalf("first login did not persist refreshed cookies: %q", refreshed) + } + d.Authorization = "" + d.MailCookies = refreshed + if sid, err := d.step1_password_login(); err != nil || sid != "second" { + t.Fatalf("second login sid=%q err=%v", sid, err) + } + if n != 2 { + t.Fatalf("login count=%d, want 2", n) + } +} diff --git a/drivers/189_tv/help.go b/drivers/189_tv/help.go index cfd9e18ab..1f130c560 100644 --- a/drivers/189_tv/help.go +++ b/drivers/189_tv/help.go @@ -5,12 +5,15 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/hex" + "encoding/json" "encoding/xml" "fmt" "net/http" "regexp" "strings" "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func clientSuffix() map[string]string { @@ -57,7 +60,13 @@ func timestamp() int64 { type Time time.Time -func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return t.Unmarshal([]byte(s)) +} func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { @@ -72,10 +81,19 @@ func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") + // 189 时间串里 AM/PM 前可能使用 U+202F 窄不换行空格或 U+00A0 不换行空格,统一替换为普通空格 + bs = strings.ReplaceAll(bs, "\u202f", " ") + bs = strings.ReplaceAll(bs, "\u00a0", " ") var v time.Time var err error - for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { - v, err = time.ParseInLocation(f, bs+" +08", time.Local) + // 189 返回的时间可能自带时区(如 "Aug 11, 2026, 10:37:18 PM +08"),也可能不带,分别尝试 + for _, s := range []string{bs, bs + " +08"} { + for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 3:04:05 PM -07", "Jan 2, 2006, 3:04:05 PM -07"} { + v, err = time.ParseInLocation(f, s, utils.CNLoc) + if err == nil { + break + } + } if err == nil { break } diff --git a/drivers/189_tv/help_test.go b/drivers/189_tv/help_test.go new file mode 100644 index 000000000..6643a5aa9 --- /dev/null +++ b/drivers/189_tv/help_test.go @@ -0,0 +1,65 @@ +package _189_tv + +import ( + "encoding/json" + "encoding/xml" + "testing" + "time" +) + +func TestTimeUnmarshal(t *testing.T) { + tests := []struct { + name string + input string + want time.Time + }{ + {"numeric date", `"2026-08-11 10:37:18"`, time.Date(2026, 8, 11, 10, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"legacy month date", `"Aug 11, 2026 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format with tz", `"Aug 11, 2026, 10:37:18 PM +08"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format no tz", `"Aug 11, 2026, 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"narrow no-break space (U+202F)", "\"Aug 12, 2026, 12:35:41\u202fAM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"no-break space (U+00A0)", "\"Aug 12, 2026, 12:35:41\u00a0AM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped narrow space", `"Sep 4, 2026, 10:59:33\u202fPM +08"`, time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped space without timezone", `"Sep 4, 2026, 12:59:33\u00a0AM"`, time.Date(2026, 9, 4, 0, 59, 33, 0, time.FixedZone("", 8*3600))}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tm Time + if err := json.Unmarshal([]byte(tt.input), &tm); err != nil { + t.Fatalf("Unmarshal(%s) error: %v", tt.input, err) + } + if !tt.want.Equal(time.Time(tm)) { + t.Fatalf("Unmarshal(%s) = %v, want %v", tt.input, time.Time(tm), tt.want) + } + }) + } +} + +func TestTimeUnmarshalRejectsInvalid(t *testing.T) { + var tm Time + if err := tm.Unmarshal([]byte("Aug 12, 2026, 25:32:44 AM")); err == nil { + t.Fatal("Unmarshal accepted an invalid time") + } +} + +func TestTimeUnmarshalJSONRejectsInvalid(t *testing.T) { + for _, input := range []string{`"invalid"`, `123`, `null`, `"Sep 4, 2026, 10:59:33\uZZZZPM +08"`} { + t.Run(input, func(t *testing.T) { + var tm Time + if err := tm.UnmarshalJSON([]byte(input)); err == nil { + t.Fatalf("UnmarshalJSON(%s) accepted invalid input", input) + } + }) + } +} + +func TestTimeUnmarshalXML(t *testing.T) { + var tm Time + if err := xml.Unmarshal([]byte(``), &tm); err != nil { + t.Fatal(err) + } + want := time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600)) + if !want.Equal(time.Time(tm)) { + t.Fatalf("UnmarshalXML = %v, want %v", time.Time(tm), want) + } +} diff --git a/drivers/189_tv/utils.go b/drivers/189_tv/utils.go index cea9411ec..313f1dc5e 100644 --- a/drivers/189_tv/utils.go +++ b/drivers/189_tv/utils.go @@ -184,6 +184,7 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) } fullUrl += "/listFiles.action" + pageSize := 130 // 每一页返回的文件数量 res := make([]model.Obj, 0, 130) for pageNum := 1; ; pageNum++ { var resp Cloud189FilesResp @@ -195,7 +196,7 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) "mediaAttr": "0", "iconOption": "5", "pageNum": fmt.Sprint(pageNum), - "pageSize": "130", + "pageSize": fmt.Sprint(pageSize), }) if isFamily { r.SetQueryParams(map[string]string{ @@ -218,13 +219,22 @@ func (y *Cloud189TV) getFiles(ctx context.Context, fileId string, isFamily bool) if resp.FileListAO.Count == 0 { break } + + FolderCount := len(resp.FileListAO.FolderList) // 当前文件夹总数 + FileCount := len(resp.FileListAO.FileList) // 当前文件总数 + PageCount := FolderCount + FileCount // 当前页数总数 - for i := 0; i < len(resp.FileListAO.FolderList); i++ { + for i := 0; i < FolderCount; i++ { res = append(res, &resp.FileListAO.FolderList[i]) } - for i := 0; i < len(resp.FileListAO.FileList); i++ { + for i := 0; i < FileCount; i++ { res = append(res, &resp.FileListAO.FileList[i]) } + + // 文件数量小于设定数量时跳出 + if PageCount < pageSize { + break + } } return res, nil } diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index c4ce35e30..555891d63 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -5,6 +5,7 @@ import ( "fmt" log "github.com/sirupsen/logrus" "net/http" + "path" "strconv" "strings" "sync" @@ -410,6 +411,14 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } +func familyTransferTempName(srcName string) string { + ext := path.Ext(srcName) + if ext == "" { + ext = ".transfer" + } + return fmt.Sprintf("0%s%s", uuid.NewString(), ext) +} + func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { if y.shouldRestoreSourceFromCAS(stream.GetName()) { obj, err := y.restoreSourceFromCAS(ctx, dstDir, stream) @@ -479,7 +488,7 @@ func (y *Cloud189PC) uploadFile(ctx context.Context, dstDir model.Obj, stream mo srcName := stream.GetName() stream = &WrapFileStreamer{ FileStreamer: stream, - Name: fmt.Sprintf("0%s.transfer", uuid.NewString()), + Name: familyTransferTempName(srcName), } // 使用家庭云上传 diff --git a/drivers/189pc/driver_test.go b/drivers/189pc/driver_test.go new file mode 100644 index 000000000..03611f6e1 --- /dev/null +++ b/drivers/189pc/driver_test.go @@ -0,0 +1,45 @@ +package _189pc + +import ( + "path" + "slices" + "strings" + "testing" +) + +func TestFamilyTransferTempNamePreservesExtension(t *testing.T) { + tests := []struct { + name string + want string + }{ + {name: "photo.jpg", want: ".jpg"}, + {name: "PHOTO.PNG", want: ".PNG"}, + {name: "archive.tar.gz", want: ".gz"}, + {name: "no-extension", want: ".transfer"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := familyTransferTempName(tt.name) + if !strings.HasPrefix(got, "0") { + t.Fatalf("temporary name %q must start with 0", got) + } + if ext := path.Ext(got); ext != tt.want { + t.Fatalf("temporary name extension = %q, want %q", ext, tt.want) + } + }) + } +} + +func TestUploadProgressKeysIncludeExtension(t *testing.T) { + jpgKeys := uploadProgressKeys("session", "same-md5", "photo.jpg") + txtKeys := uploadProgressKeys("session", "same-md5", "photo.txt") + retryKeys := uploadProgressKeys("session", "same-md5", "different-name.jpg") + + if slices.Equal(jpgKeys, txtKeys) { + t.Fatal("files with different extensions must not share upload progress") + } + if !slices.Equal(jpgKeys, retryKeys) { + t.Fatal("files with the same session, MD5, and extension should share upload progress") + } +} diff --git a/drivers/189pc/help.go b/drivers/189pc/help.go index 6f6c59f30..5d6959402 100644 --- a/drivers/189pc/help.go +++ b/drivers/189pc/help.go @@ -9,6 +9,7 @@ import ( "crypto/sha1" "crypto/x509" "encoding/hex" + "encoding/json" "encoding/pem" "encoding/xml" "fmt" @@ -19,6 +20,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) @@ -101,7 +103,13 @@ func MustParseTime(str string) *time.Time { type Time time.Time -func (t *Time) UnmarshalJSON(b []byte) error { return t.Unmarshal(b) } +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return t.Unmarshal([]byte(s)) +} func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { b, err := e.Token() if err != nil { @@ -116,10 +124,19 @@ func (t *Time) UnmarshalXML(e *xml.Decoder, ee xml.StartElement) error { } func (t *Time) Unmarshal(b []byte) error { bs := strings.Trim(string(b), "\"") + // 189 时间串里 AM/PM 前可能使用 U+202F 窄不换行空格或 U+00A0 不换行空格,统一替换为普通空格 + bs = strings.ReplaceAll(bs, "\u202f", " ") + bs = strings.ReplaceAll(bs, "\u00a0", " ") var v time.Time var err error - for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 15:04:05 PM -07"} { - v, err = time.ParseInLocation(f, bs+" +08", time.Local) + // 189 返回的时间可能自带时区(如 "Aug 11, 2026, 10:37:18 PM +08"),也可能不带,分别尝试 + for _, s := range []string{bs, bs + " +08"} { + for _, f := range []string{"2006-01-02 15:04:05 -07", "Jan 2, 2006 3:04:05 PM -07", "Jan 2, 2006, 3:04:05 PM -07"} { + v, err = time.ParseInLocation(f, s, utils.CNLoc) + if err == nil { + break + } + } if err == nil { break } diff --git a/drivers/189pc/help_test.go b/drivers/189pc/help_test.go new file mode 100644 index 000000000..6c91df126 --- /dev/null +++ b/drivers/189pc/help_test.go @@ -0,0 +1,65 @@ +package _189pc + +import ( + "encoding/json" + "encoding/xml" + "testing" + "time" +) + +func TestTimeUnmarshal(t *testing.T) { + tests := []struct { + name string + input string + want time.Time + }{ + {"numeric date", `"2026-08-11 10:37:18"`, time.Date(2026, 8, 11, 10, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"legacy month date", `"Aug 11, 2026 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format with tz", `"Aug 11, 2026, 10:37:18 PM +08"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"new format no tz", `"Aug 11, 2026, 10:37:18 PM"`, time.Date(2026, 8, 11, 22, 37, 18, 0, time.FixedZone("", 8*3600))}, + {"narrow no-break space (U+202F)", "\"Aug 12, 2026, 12:35:41\u202fAM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"no-break space (U+00A0)", "\"Aug 12, 2026, 12:35:41\u00a0AM +08\"", time.Date(2026, 8, 12, 0, 35, 41, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped narrow space", `"Sep 4, 2026, 10:59:33\u202fPM +08"`, time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600))}, + {"JSON escaped space without timezone", `"Sep 4, 2026, 12:59:33\u00a0AM"`, time.Date(2026, 9, 4, 0, 59, 33, 0, time.FixedZone("", 8*3600))}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tm Time + if err := json.Unmarshal([]byte(tt.input), &tm); err != nil { + t.Fatalf("Unmarshal(%s) error: %v", tt.input, err) + } + if !tt.want.Equal(time.Time(tm)) { + t.Fatalf("Unmarshal(%s) = %v, want %v", tt.input, time.Time(tm), tt.want) + } + }) + } +} + +func TestTimeUnmarshalRejectsInvalid(t *testing.T) { + var tm Time + if err := tm.Unmarshal([]byte("Aug 12, 2026, 25:32:44 AM")); err == nil { + t.Fatal("Unmarshal accepted an invalid time") + } +} + +func TestTimeUnmarshalJSONRejectsInvalid(t *testing.T) { + for _, input := range []string{`"invalid"`, `123`, `null`, `"Sep 4, 2026, 10:59:33\uZZZZPM +08"`} { + t.Run(input, func(t *testing.T) { + var tm Time + if err := tm.UnmarshalJSON([]byte(input)); err == nil { + t.Fatalf("UnmarshalJSON(%s) accepted invalid input", input) + } + }) + } +} + +func TestTimeUnmarshalXML(t *testing.T) { + var tm Time + if err := xml.Unmarshal([]byte(``), &tm); err != nil { + t.Fatal(err) + } + want := time.Date(2026, 9, 4, 22, 59, 33, 0, time.FixedZone("", 8*3600)) + if !want.Equal(time.Time(tm)) { + t.Fatalf("UnmarshalXML = %v, want %v", time.Time(tm), want) + } +} diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 9ba20b142..da60bf266 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -7,10 +7,16 @@ import ( type Addition struct { LoginType string `json:"login_type" type:"select" options:"password,qrcode" default:"password" required:"true"` - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" help:"Not needed when an access token or refresh token is provided"` + Password string `json:"password" help:"Not needed when an access token or refresh token is provided"` VCode string `json:"validate_code"` + SmsCode string `json:"sms_code" help:"SMS code for the second device verification, fill it in and save again when login asks for it"` + AccessToken string `json:"access_token" required:"false"` RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"` + DeviceID string `json:"device_id" help:"DEVICEID cookie issued after the second device verification, keep it to avoid verifying again"` + ClientSn string `json:"client_sn" help:"Device serial number captured from the official client, leave it empty if you do not have one"` + JgOpenId string `json:"jg_open_id" help:"Optional push id reported by the official client"` + UserFinger string `json:"user_finger" help:"Device fingerprint sent with login requests, generated and kept automatically when empty"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index f6a9e678f..0f2873379 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -72,6 +72,8 @@ type BaseLoginParam struct { // 请求头参数 Lt string ReqId string + // logbox页面地址,作为后续请求的Referer,缺失会被判定为陌生设备 + Referer string // 表单参数 ParamId string @@ -97,10 +99,20 @@ type LoginParam struct { // rsa密钥 jRsaKey string + // 加密字段的前缀,服务端下发(如 {NRP}) + rsaPrefix string + + // 设备二次校验时服务端返回的加密手机号 + SecondAuthMobile string BaseLoginParam } +// encryptSecret 用登陆时拿到的公钥加密敏感值,格式与userName/epd一致 +func (p *LoginParam) encryptSecret(value string) string { + return p.rsaPrefix + RsaEncrypt(p.jRsaKey, value) +} + // 登陆加密相关 type EncryptConfResp struct { Result int `json:"result"` @@ -116,6 +128,35 @@ type LoginResp struct { Msg string `json:"msg"` Result int `json:"result"` ToUrl string `json:"toUrl"` + // 设备二次校验时返回的加密手机号 + Mobile string `json:"mobile"` +} + +// 登陆页配置,新版登陆页的paramId由该接口下发 +// 该接口的result可能是数字也可能是字符串 +type AppConfResp struct { + Result any `json:"result"` + Msg string `json:"msg"` + Data struct { + ParamId string `json:"paramId"` + AccountType string `json:"accountType"` + ReturnUrl string `json:"returnUrl"` + MailSuffix string `json:"mailSuffix"` + } `json:"data"` +} + +func (r *AppConfResp) Succeeded() bool { + switch v := r.Result.(type) { + case nil: + return true + case string: + return v == "0" || v == "" + case float64: + return v == 0 + case int: + return v == 0 + } + return false } // 刷新session返回 @@ -149,6 +190,27 @@ type AppSessionResp struct { RefreshToken string `json:"refreshToken"` } +// 刷新token返回,失败时以HTTP 200返回result/msg,需要单独判断 +type RefreshTokenResp struct { + AccessToken string `json:"accessToken"` + RefreshToken string `json:"refreshToken"` + ExpiresIn int `json:"expiresIn"` + + Result int `json:"result"` + Msg string `json:"msg"` +} + +func (r *RefreshTokenResp) HasError() bool { + return r.Result != 0 || r.AccessToken == "" +} + +func (r *RefreshTokenResp) Error() string { + if r.Msg != "" { + return fmt.Sprintf("refresh token failed, result: %d, msg: %s", r.Result, r.Msg) + } + return fmt.Sprintf("refresh token failed, result: %d", r.Result) +} + // 家庭云账户 type FamilyInfoListResp struct { FamilyInfoResp []FamilyInfoResp `json:"familyInfoResp"` @@ -191,6 +253,10 @@ type Cloud189File struct { // StarLabel int64 `json:"starLabel"` } +func normalizeCloud189Name(name string) string { + return strings.ReplaceAll(name, "\\'", "'") +} + func (c *Cloud189File) CreateTime() time.Time { return time.Time(c.CreateDate) } @@ -208,6 +274,8 @@ func (c *Cloud189File) GetID() string { return string(c.ID) } func (c *Cloud189File) GetPath() string { return "" } func (c *Cloud189File) Thumb() string { return c.Icon.SmallUrl } +func (c *Cloud189File) GetDisplayName() string { return normalizeCloud189Name(c.Name) } + // 文件夹 type Cloud189Folder struct { ID String `json:"id"` @@ -240,6 +308,8 @@ func (c *Cloud189Folder) IsDir() bool { return true } func (c *Cloud189Folder) GetID() string { return string(c.ID) } func (c *Cloud189Folder) GetPath() string { return "" } +func (c *Cloud189Folder) GetDisplayName() string { return normalizeCloud189Name(c.Name) } + type Cloud189FilesResp struct { //ResCode int `json:"res_code"` //ResMessage string `json:"res_message"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index b0026b841..36854485a 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -14,6 +14,7 @@ import ( "net/http/cookiejar" "net/url" "os" + "path" "regexp" "sort" "strconv" @@ -30,6 +31,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/skip2/go-qrcode" "github.com/avast/retry-go" @@ -41,9 +43,13 @@ import ( const ( ACCOUNT_TYPE = "02" - APP_ID = "8025431004" - CLIENT_TYPE = "10020" - VERSION = "6.2" + // 官方 PC 端(cloud.189.cn 网页/客户端)使用的 appId, + // 登录、生成二维码、换取 session 必须全程使用同一个 appId + APP_ID = "9317140619" + CLIENT_TYPE = "10020" + // 扫码状态轮询使用的 clientType,与密码登录的 10020 不同 + QR_CLIENT_TYPE = "1" + VERSION = "7.2.4.0" WEB_URL = "https://cloud.189.cn" AUTH_URL = "https://open.e.189.cn" @@ -57,8 +63,18 @@ const ( CHANNEL_ID = "web_cloud.189.cn" + // 服务端通过短信二次校验后下发的设备标识,复用它可以避免再次触发校验 + DEVICE_ID_COOKIE = "DEVICEID" + + // 扫码登录本地轮询参数,超时后把二维码交回前端,避免请求被反向代理掐断 + QRCODE_POLL_INTERVAL = 2 * time.Second + QRCODE_POLL_TIMEOUT = 20 * time.Second + // Error codes UserInvalidOpenTokenError = "UserInvalidOpenToken" + + // 密码登录返回该结果表示需要设备二次校验 + SecondDeviceAuthResult = -133 ) func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { @@ -195,9 +211,10 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { + pageSize := 1000 // 每一页返回的文件数量 res := make([]model.Obj, 0, 100) for pageNum := 1; ; pageNum++ { - resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, 1000, y.OrderBy, y.OrderDirection) + resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, pageSize, y.OrderBy, y.OrderDirection) if err != nil { return nil, err } @@ -206,13 +223,22 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) break } - for i := 0; i < len(resp.FileListAO.FolderList); i++ { + FolderCount := len(resp.FileListAO.FolderList) // 当前文件夹总数 + FileCount := len(resp.FileListAO.FileList) // 当前文件总数 + PageCount := FolderCount + FileCount // 当前页数总数 + + for i := 0; i < FolderCount; i++ { res = append(res, &resp.FileListAO.FolderList[i]) } - for i := 0; i < len(resp.FileListAO.FileList); i++ { + for i := 0; i < FileCount; i++ { resp.FileListAO.FileList[i].ParentID = fileId res = append(res, &resp.FileListAO.FileList[i]) } + + // 当前文件数量小于设定数量则跳出 + if PageCount < pageSize { + break + } } return res, nil } @@ -278,9 +304,72 @@ func (y *Cloud189PC) login() error { if y.LoginType == "qrcode" { return y.loginByQRCode() } + if y.Username == "" || y.Password == "" { + return errors.New("please fill in the username and password, or provide an access token / refresh token") + } return y.loginByPassword() } +// 设备指纹,为空时生成并保存,服务端以此识别是否为同一台设备 +func (y *Cloud189PC) getUserFinger() string { + if y.Addition.UserFinger == "" { + y.Addition.UserFinger = fmt.Sprint(random.Rand.Int63n(9e9) + 1e9) + op.MustSaveDriverStorage(y) + } + return y.Addition.UserFinger +} + +// 换取会话时携带的设备参数,与官方PC客户端保持一致 +// clientSn/jgOpenId 只在用户从官方客户端抓到并填写后才发送,避免上报一个服务端不认识的设备号 +func (y *Cloud189PC) deviceParams() map[string]string { + params := map[string]string{"returnType": "JSON"} + if y.Addition.ClientSn != "" { + params["clientSn"] = y.Addition.ClientSn + } + if y.Addition.JgOpenId != "" { + params["jgOpenId"] = y.Addition.JgOpenId + } + return params +} + +// logbox接口的公共请求头,缺少user-finger和Referer会被判定为陌生设备 +func (y *Cloud189PC) loginHeaders(param BaseLoginParam) map[string]string { + return map[string]string{ + "REQID": param.ReqId, + "lt": param.Lt, + "user-finger": y.getUserFinger(), + "Referer": IF(param.Referer != "", param.Referer, AUTH_URL), + } +} + +// 把已保存的设备标识写入cookie,避免重复触发设备二次校验 +func (y *Cloud189PC) applyDeviceID(jar http.CookieJar) { + if y.Addition.DeviceID == "" { + return + } + authUrl, err := url.Parse(AUTH_URL) + if err != nil { + return + } + jar.SetCookies(authUrl, []*http.Cookie{{ + Name: DEVICE_ID_COOKIE, + Value: y.Addition.DeviceID, + Domain: "e.189.cn", + Path: "/", + }}) +} + +// 保存服务端下发的设备标识,下次登陆复用即可跳过设备二次校验 +func (y *Cloud189PC) saveDeviceID(res *resty.Response) { + for _, cookie := range res.Cookies() { + if cookie.Name == DEVICE_ID_COOKIE && cookie.Value != "" && cookie.Value != y.Addition.DeviceID { + y.Addition.DeviceID = cookie.Value + op.MustSaveDriverStorage(y) + return + } + } +} + func (y *Cloud189PC) loginByPassword() (err error) { // 初始化登陆所需参数 if y.loginParam == nil { @@ -289,9 +378,16 @@ func (y *Cloud189PC) loginByPassword() (err error) { return err } } + // 设备二次校验必须复用同一套登陆参数,此时不能销毁也不能重新初始化 + keepLoginParam := false defer func() { // 销毁验证码 y.VCode = "" + if keepLoginParam { + y.Status = err.Error() + op.MustSaveDriverStorage(y) + return + } // 销毁登陆参数 y.loginParam = nil // 遇到错误,重新加载登陆参数(刷新验证码) @@ -309,17 +405,18 @@ func (y *Cloud189PC) loginByPassword() (err error) { param := y.loginParam var loginresp LoginResp - _, err = y.client.R(). + res, err := y.client.R(). ForceContentType("application/json;charset=UTF-8").SetResult(&loginresp). - SetHeaders(map[string]string{ - "REQID": param.ReqId, - "lt": param.Lt, - }). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). SetFormData(map[string]string{ + "version": "v2.0", + "apToken": "", "appKey": APP_ID, + "pageKey": "normal", "accountType": ACCOUNT_TYPE, "userName": param.RsaUsername, "password": param.RsaPassword, + "epd": param.RsaPassword, "validateCode": y.VCode, "captchaToken": param.CaptchaToken, "returnUrl": RETURN_URL, @@ -335,17 +432,106 @@ func (y *Cloud189PC) loginByPassword() (err error) { if err != nil { return err } + y.saveDeviceID(res) + + // 设备二次校验:服务端要求短信验证,保留登陆参数并引导填写短信验证码 + if loginresp.Result == SecondDeviceAuthResult { + err = y.secondDeviceAuth(loginresp.Mobile) + // 校验未完成时保留登陆参数,等待用户回填短信验证码 + keepLoginParam = err != nil && y.loginParam != nil + return err + } + if loginresp.ToUrl == "" { return fmt.Errorf("login failed,No toUrl obtained, msg: %s", loginresp.Msg) } - // 获取Session + return y.getSessionByRedirectURL(loginresp.ToUrl) +} + +// 设备二次校验:先发短信,用户回填验证码后再提交 +func (y *Cloud189PC) secondDeviceAuth(mobile string) error { + param := y.loginParam + if mobile != "" { + param.SecondAuthMobile = mobile + } + if param.SecondAuthMobile == "" { + return errors.New("second device verification is required, but no mobile was returned") + } + + // 已填写短信验证码,直接提交校验 + if y.SmsCode != "" { + smsCode := y.SmsCode + y.SmsCode = "" + op.MustSaveDriverStorage(y) + return y.submitSecondDeviceAuth(smsCode) + } + + var smsResp LoginResp + _, err := y.client.R(). + ForceContentType("application/json;charset=UTF-8").SetResult(&smsResp). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). + SetFormData(map[string]string{ + "mobile": param.SecondAuthMobile, + "appKey": APP_ID, + }). + Post(AUTH_URL + "/api/logbox/oauth2/sendSmsCodeForSecondAuth.do") + if err != nil { + return err + } + if smsResp.Result != 0 { + return fmt.Errorf("failed to send the verification SMS: %s", smsResp.Msg) + } + // 保留登陆参数,等待用户回填短信验证码后重新保存 + return errors.New("second device verification is required, an SMS code has been sent, please fill it into `sms_code` and save again") +} + +// 提交短信验证码完成设备二次校验 +// 注意:该接口没有独立的短信码字段,短信码要加密后放在epd里(登陆时epd装的是密码) +func (y *Cloud189PC) submitSecondDeviceAuth(smsCode string) error { + param := y.loginParam + var authResp LoginResp + res, err := y.client.R(). + ForceContentType("application/json;charset=UTF-8").SetResult(&authResp). + SetHeaders(y.loginHeaders(param.BaseLoginParam)). + SetFormData(map[string]string{ + "mobile": param.SecondAuthMobile, + "appKey": APP_ID, + "userName": param.RsaUsername, + "epd": param.encryptSecret(smsCode), + "accountType": ACCOUNT_TYPE, + "returnUrl": RETURN_URL, + "isOauth2": "false", + "cb_SaveName": "1", + "state": "", + "paramId": param.ParamId, + }). + Post(AUTH_URL + "/api/logbox/oauth2/submitForSecondAuth.do") + if err != nil { + return err + } + // 校验通过后服务端会下发DEVICEID,保存下来以后就不会再触发二次校验 + y.saveDeviceID(res) + + if authResp.Result != 0 { + return fmt.Errorf("second device verification failed: %s", authResp.Msg) + } + if authResp.ToUrl == "" { + return fmt.Errorf("second device verification failed, no toUrl obtained, msg: %s", authResp.Msg) + } + return y.getSessionByRedirectURL(authResp.ToUrl) +} + +// 用登陆结果的跳转地址换取会话 +func (y *Cloud189PC) getSessionByRedirectURL(redirectURL string) error { var erron RespErr var tokenInfo AppSessionResp - _, err = y.client.R(). + _, err := y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", loginresp.ToUrl). + SetQueryParams(y.deviceParams()). + SetQueryParam("redirectURL", redirectURL). + SetHeader("X-Request-ID", uuid.NewString()). Post(API_URL + "/getSessionForPC.action") if err != nil { return err @@ -355,13 +541,12 @@ func (y *Cloud189PC) loginByPassword() (err error) { return &erron } if tokenInfo.ResCode != 0 { - err = errors.New(tokenInfo.ResMessage) - return err + return errors.New(tokenInfo.ResMessage) } y.Addition.RefreshToken = tokenInfo.RefreshToken y.tokenInfo = &tokenInfo op.MustSaveDriverStorage(y) - return err + return nil } func (y *Cloud189PC) loginByQRCode() error { @@ -372,65 +557,74 @@ func (y *Cloud189PC) loginByQRCode() error { } } - var state struct { - Status int `json:"status"` - RedirectUrl string `json:"redirectUrl"` - Msg string `json:"msg"` + // 本地轮询,扫码确认后自动继续,不需要用户反复保存 + deadline := time.Now().Add(QRCODE_POLL_TIMEOUT) + lastStatus := -106 + for { + state, err := y.checkQRCodeState() + if err != nil { + return fmt.Errorf("failed to check QR code state: %w", err) + } + lastStatus = state.Status + + switch state.Status { + case 0: // 登录成功 + y.qrcodeParam = nil + return y.getSessionByRedirectURL(state.RedirectUrl) + case -106, -11002: // -106 等待扫描,-11002 已扫描等待确认 + case -11001: // 二维码过期 + y.qrcodeParam = nil + return errors.New("QR code expired, please try again") + default: // 其他错误 + y.qrcodeParam = nil + return fmt.Errorf("QR code login failed with status %d: %s", state.Status, state.Msg) + } + + if time.Now().Add(QRCODE_POLL_INTERVAL).After(deadline) { + break + } + time.Sleep(QRCODE_POLL_INTERVAL) + } + + // 轮询超时,把二维码交回前端等待下一次保存 + if lastStatus == -11002 { + return y.genQRCode("QR code has been scanned, please confirm the login on your phone and save again") } + return y.genQRCode("QR code has not been scanned yet, please scan and save again") +} + +type qrCodeState struct { + Status int `json:"status"` + RedirectUrl string `json:"redirectUrl"` + Msg string `json:"msg"` +} +// 查询扫码状态,参数需与官方PC端一致,否则服务端不会返回授权结果 +func (y *Cloud189PC) checkQRCodeState() (*qrCodeState, error) { now := time.Now() + var state qrCodeState _, err := y.client.R(). - SetHeaders(map[string]string{ - "Referer": AUTH_URL, - "Reqid": y.qrcodeParam.ReqId, - "lt": y.qrcodeParam.Lt, - }). + SetHeaders(y.loginHeaders(y.qrcodeParam.BaseLoginParam)). SetFormData(map[string]string{ - "appId": APP_ID, - "clientType": CLIENT_TYPE, - "returnUrl": RETURN_URL, - "paramId": y.qrcodeParam.ParamId, - "uuid": y.qrcodeParam.UUID, - "encryuuid": y.qrcodeParam.EncryUUID, - "date": formatDate(now), - "timeStamp": fmt.Sprint(now.UTC().UnixNano() / 1e6), + "appId": APP_ID, + "clientType": QR_CLIENT_TYPE, + "returnUrl": RETURN_URL, + "paramId": y.qrcodeParam.ParamId, + "uuid": y.qrcodeParam.UUID, + "encryuuid": y.qrcodeParam.EncryUUID, + "cb_SaveName": "3", + "isOauth2": "false", + "state": "", + "date": formatDate(now), + "timeStamp": fmt.Sprint(now.UTC().UnixNano() / 1e6), }). ForceContentType("application/json;charset=UTF-8"). SetResult(&state). Post(AUTH_URL + "/api/logbox/oauth2/qrcodeLoginState.do") if err != nil { - return fmt.Errorf("failed to check QR code state: %w", err) - } - - switch state.Status { - case 0: // 登录成功 - var tokenInfo AppSessionResp - _, err = y.client.R(). - SetResult(&tokenInfo). - SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", state.RedirectUrl). - Post(API_URL + "/getSessionForPC.action") - if err != nil { - return err - } - if tokenInfo.ResCode != 0 { - return errors.New(tokenInfo.ResMessage) - } - y.Addition.RefreshToken = tokenInfo.RefreshToken - y.tokenInfo = &tokenInfo - op.MustSaveDriverStorage(y) - return nil - case -11001: // 二维码过期 - y.qrcodeParam = nil - return errors.New("QR code expired, please try again") - case -106: // 等待扫描 - return y.genQRCode("QR code has not been scanned yet, please scan and save again") - case -11002: // 等待确认 - return y.genQRCode("QR code has been scanned, please confirm the login on your phone and save again") - default: // 其他错误 - y.qrcodeParam = nil - return fmt.Errorf("QR code login failed with status %d: %s", state.Status, state.Msg) + return nil, err } + return &state, nil } func (y *Cloud189PC) genQRCode(text string) error { @@ -456,8 +650,9 @@ func (y *Cloud189PC) genQRCode(text string) error { } func (y *Cloud189PC) initBaseParams() (*BaseLoginParam, error) { - // 清除cookie + // 清除cookie,并带上已保存的设备标识 jar, _ := cookiejar.New(nil) + y.applyDeviceID(jar) y.client.SetCookieJar(jar) res, err := y.client.R(). @@ -472,14 +667,98 @@ func (y *Cloud189PC) initBaseParams() (*BaseLoginParam, error) { return nil, err } + // 当前登陆页把lt/reqId放在跳转地址上,老页面则写在页内变量里,两种都要支持 + param, err := parseBaseParamFromRedirect(res.RawResponse.Request.URL) + if err != nil { + param, err = parseBaseParamFromPage(res.String()) + if err != nil { + return nil, err + } + return param, nil + } + + // 跳转地址上没有paramId,需要再问一次appConf.do + var appConf AppConfResp + _, err = y.client.R(). + SetHeaders(y.loginHeaders(*param)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&appConf). + SetFormData(map[string]string{ + "version": "2.0", + "appKey": APP_ID, + }). + Post(AUTH_URL + "/api/logbox/oauth2/appConf.do") + if err != nil { + return nil, err + } + if !appConf.Succeeded() || appConf.Data.ParamId == "" { + return nil, fmt.Errorf("failed to get the login paramId: %s", appConf.Msg) + } + param.ParamId = appConf.Data.ParamId + return param, nil +} + +// parseBaseParamFromRedirect 从logbox跳转地址提取登陆参数,并以该地址作为后续请求的Referer +func parseBaseParamFromRedirect(finalUrl *url.URL) (*BaseLoginParam, error) { + if finalUrl == nil { + return nil, errors.New("no login page redirect") + } + query := finalUrl.Query() + lt, reqId := query.Get("lt"), query.Get("reqId") + if lt == "" || reqId == "" { + return nil, errors.New("no lt/reqId in the login page redirect") + } return &BaseLoginParam{ - CaptchaToken: regexp.MustCompile(`'captchaToken' value='(.+?)'`).FindStringSubmatch(res.String())[1], - Lt: regexp.MustCompile(`lt = "(.+?)"`).FindStringSubmatch(res.String())[1], - ParamId: regexp.MustCompile(`paramId = "(.+?)"`).FindStringSubmatch(res.String())[1], - ReqId: regexp.MustCompile(`reqId = "(.+?)"`).FindStringSubmatch(res.String())[1], + Lt: lt, + ReqId: reqId, + Referer: finalUrl.String(), }, nil } +// parseBaseParamFromPage 兼容把参数写在页内变量里的老登陆页 +func parseBaseParamFromPage(body string) (*BaseLoginParam, error) { + lt, err := matchLoginParam(body, `lt = "(.+?)"`, "lt") + if err != nil { + return nil, err + } + reqId, err := matchLoginParam(body, `reqId = "(.+?)"`, "reqId") + if err != nil { + return nil, err + } + paramId, err := matchLoginParam(body, `paramId = "(.+?)"`, "paramId") + if err != nil { + return nil, err + } + // 老页面才有内嵌的图形验证码token + captchaToken, _ := matchLoginParam(body, `'captchaToken' value='(.+?)'`, "captchaToken") + + param := &BaseLoginParam{ + CaptchaToken: captchaToken, + Lt: lt, + ParamId: paramId, + ReqId: reqId, + } + encryptUrl, _ := matchLoginParam(body, `encryptUrl = "(.+?)"`, "encryptUrl") + param.Referer = AUTH_URL + "/api/logbox/separate/web/index.html?" + strings.Join([]string{ + "appId=" + url.QueryEscape(APP_ID), + "lt=" + url.QueryEscape(param.Lt), + "reqId=" + url.QueryEscape(param.ReqId), + }, "&") + if encryptUrl != "" { + param.Referer += "&encryptUrl=" + url.QueryEscape(encryptUrl) + } + return param, nil +} + +// matchLoginParam 从登陆页面提取参数,缺失时返回可读的错误而不是panic +func matchLoginParam(body, pattern, name string) (string, error) { + matches := regexp.MustCompile(pattern).FindStringSubmatch(body) + if len(matches) < 2 { + return "", fmt.Errorf("failed to get %s from the login page", name) + } + return matches[1], nil +} + /* 初始化登陆需要的参数 * 如果遇到验证码返回错误 */ @@ -504,12 +783,13 @@ func (y *Cloud189PC) initLoginParam() error { } y.loginParam.jRsaKey = fmt.Sprintf("-----BEGIN PUBLIC KEY-----\n%s\n-----END PUBLIC KEY-----", encryptConf.Data.PubKey) - y.loginParam.RsaUsername = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Username) - y.loginParam.RsaPassword = encryptConf.Data.Pre + RsaEncrypt(y.loginParam.jRsaKey, y.Password) + y.loginParam.rsaPrefix = encryptConf.Data.Pre + y.loginParam.RsaUsername = y.loginParam.encryptSecret(y.Username) + y.loginParam.RsaPassword = y.loginParam.encryptSecret(y.Password) // 判断是否需要验证码 resp, err := y.client.R(). - SetHeader("REQID", y.loginParam.ReqId). + SetHeaders(y.loginHeaders(y.loginParam.BaseLoginParam)). SetFormData(map[string]string{ "appKey": APP_ID, "accountType": ACCOUNT_TYPE, @@ -564,6 +844,7 @@ func (y *Cloud189PC) initQRCodeParam() (err error) { var qrcodeParam QRLoginParam _, err = y.client.R(). + SetHeaders(y.loginHeaders(*baseParam)). SetFormData(map[string]string{"appId": APP_ID}). ForceContentType("application/json;charset=UTF-8"). SetResult(&qrcodeParam). @@ -571,6 +852,9 @@ func (y *Cloud189PC) initQRCodeParam() (err error) { if err != nil { return err } + if qrcodeParam.UUID == "" { + return errors.New("failed to get the QR code uuid") + } qrcodeParam.BaseLoginParam = *baseParam y.qrcodeParam = &qrcodeParam @@ -591,6 +875,7 @@ func (y *Cloud189PC) refreshSessionWithRetry(retryCount int) (err error) { _, err = y.client.R(). SetResult(&userSessionResp).SetError(&erron). SetQueryParams(clientSuffix()). + SetQueryParams(y.deviceParams()). SetQueryParams(map[string]string{ "appId": APP_ID, "accessToken": y.tokenInfo.AccessToken, @@ -631,12 +916,11 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { return errors.New("refresh token failed after maximum retries") } - var erron RespErr - var tokenInfo AppSessionResp + // 该接口刷新失败时以HTTP 200返回 result/msg,SetError不会触发,必须解析响应体判断 + var tokenInfo RefreshTokenResp _, err = y.client.R(). SetResult(&tokenInfo). ForceContentType("application/json;charset=UTF-8"). - SetError(&erron). SetFormData(map[string]string{ "clientId": APP_ID, "refreshToken": y.tokenInfo.RefreshToken, @@ -649,7 +933,8 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { } // 如果刷新失败,返回错误给上层处理 - if erron.HasError() { + if tokenInfo.HasError() { + refreshErr := tokenInfo.Error() if y.Addition.RefreshToken != "" { y.Addition.RefreshToken = "" op.MustSaveDriverStorage(y) @@ -657,14 +942,19 @@ func (y *Cloud189PC) refreshTokenWithRetry(retryCount int) (err error) { // 根据登录类型决定下一步行为 if y.LoginType == "qrcode" { - return errors.New("QR code session has expired, please re-scan the code to log in") + return fmt.Errorf("QR code session has expired, please re-scan the code to log in: %s", refreshErr) + } + // 没有账号密码时无法回退到完整登录,直接把刷新失败的原因返回 + if y.Username == "" || y.Password == "" { + return errors.New(refreshErr) } // 密码登录模式下,尝试回退到完整登录 return y.login() } y.Addition.RefreshToken = tokenInfo.RefreshToken - y.tokenInfo = &tokenInfo + y.tokenInfo.AccessToken = tokenInfo.AccessToken + y.tokenInfo.RefreshToken = tokenInfo.RefreshToken op.MustSaveDriverStorage(y) return y.refreshSessionWithRetry(retryCount + 1) } @@ -925,6 +1215,10 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode return y.fastUpload(ctx, dstDir, file, up, isFamily, overwrite, generateTorrent) } +func uploadProgressKeys(sessionKey, fileMD5, fileName string) []string { + return []string{sessionKey, fileMD5, path.Ext(fileName)} +} + func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool, generateTorrent bool) (model.Obj, *casUploadInfo, error) { var ( cache = file.GetFile() @@ -1027,7 +1321,8 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode } // 尝试恢复进度 - uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex) + progressKeys := uploadProgressKeys(y.getTokenInfo().SessionKey, fileMd5Hex, file.GetName()) + uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, progressKeys...) if !ok { // step.2 预上传 params := Params{ @@ -1095,7 +1390,7 @@ func (y *Cloud189PC) fastUpload(ctx context.Context, dstDir model.Obj, file mode if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" }) - base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex) + base.SaveUploadProgress(y, uploadProgress, progressKeys...) } return nil, nil, err } diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 7008495ce..d69d6cf50 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -328,7 +328,6 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } resultLink := link.Clone() // 复制一份,避免修改到原始link - resultLink.Expiration = nil if args.Redirect { return resultLink, nil } diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 0b869a9a9..df4aa1c62 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -98,6 +98,7 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: path.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, diff --git a/drivers/alist_v3/driver_test.go b/drivers/alist_v3/driver_test.go new file mode 100644 index 000000000..bb8e009ea --- /dev/null +++ b/drivers/alist_v3/driver_test.go @@ -0,0 +1,65 @@ +package alist_v3 + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +// TestListSetsChildPaths descends two levels the way op.Get does, feeding an +// object from one listing back into List as dir. Without a Path on that object +// the driver asks upstream for "", which a real server answers with its own +// root -- hence the fake upstream's fallback, and the endless self-similar tree. +func TestListSetsChildPaths(t *testing.T) { + tree := map[string][]ObjResp{ + "/": {{Name: "root-marker", IsDir: true}}, + "/drive": {{Name: "concerts", IsDir: true}}, + "/drive/concerts": {{Name: "show.mkv"}}, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req ListReq + _ = json.NewDecoder(r.Body).Decode(&req) + content, ok := tree[req.Path] + if !ok { + content = tree["/"] + } + w.Header().Set("Content-Type", "application/json") // resty only unmarshals JSON + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 200, "message": "success", + "data": map[string]any{"content": content, "total": len(content)}, + }) + })) + t.Cleanup(srv.Close) + + // conf.Conf is nil outside a booted server, so base.InitClient() is unusable. + prev := base.RestyClient + base.RestyClient = resty.New().SetTimeout(5 * time.Second) + t.Cleanup(func() { base.RestyClient = prev }) + + d := &AListV3{Addition: Addition{ + RootPath: driver.RootPath{RootFolderPath: "/drive"}, + Address: srv.URL, + }} + dir := model.Obj(&model.Object{Path: "/drive", IsFolder: true}) + for _, want := range []string{"/drive/concerts", "/drive/concerts/show.mkv"} { + objs, err := d.List(context.Background(), dir, model.ListArgs{}) + if err != nil { + t.Fatalf("List(%q): %v", dir.GetPath(), err) + } + if len(objs) != 1 { + t.Fatalf("List(%q) returned %d objects, want 1", dir.GetPath(), len(objs)) + } + if got := objs[0].GetPath(); got != want { + t.Fatalf("child of %q has path %q, want %q", dir.GetPath(), got, want) + } + dir = objs[0] + } +} diff --git a/drivers/aliyundrive_open/callback.go b/drivers/aliyundrive_open/callback.go new file mode 100644 index 000000000..54b0e88b8 --- /dev/null +++ b/drivers/aliyundrive_open/callback.go @@ -0,0 +1,295 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "io" + "math/rand/v2" + "net/http" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + anet "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +const ( + defaultCallbackConcurrency = 1 + callbackAcquireTimeout = time.Second + callbackRequestAttempts = 3 + callbackRetryBaseDelay = 200 * time.Millisecond + callbackErrorBodyLimit = 64 << 10 +) + +var callbackLimiters = struct { + sync.Mutex + byUser map[string]*callbackLimiter +}{byUser: make(map[string]*callbackLimiter)} + +type callbackLimiter struct { + userID string + mu sync.Mutex + active int + nextID uint64 + registrations map[uint64]int + changed chan struct{} +} + +type callbackRegistration struct { + limiter *callbackLimiter + id uint64 + once sync.Once +} + +type callbackPermit struct { + limiter *callbackLimiter + once sync.Once +} + +func normalizeCallbackConcurrency(limit int) int { + if limit <= 0 { + return defaultCallbackConcurrency + } + return limit +} + +func registerCallbackLimiter(userID string, limit int) *callbackRegistration { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + + limiter := callbackLimiters.byUser[userID] + if limiter == nil { + limiter = &callbackLimiter{ + userID: userID, + registrations: make(map[uint64]int), + changed: make(chan struct{}), + } + callbackLimiters.byUser[userID] = limiter + } + limiter.mu.Lock() + limiter.nextID++ + id := limiter.nextID + limiter.registrations[id] = normalizeCallbackConcurrency(limit) + limiter.signalLocked() + limiter.mu.Unlock() + return &callbackRegistration{limiter: limiter, id: id} +} + +func (r *callbackRegistration) unregister() { + if r == nil || r.limiter == nil { + return + } + r.once.Do(func() { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + r.limiter.mu.Lock() + delete(r.limiter.registrations, r.id) + r.limiter.signalLocked() + if len(r.limiter.registrations) == 0 && r.limiter.active == 0 { + delete(callbackLimiters.byUser, r.limiter.userID) + } + r.limiter.mu.Unlock() + }) +} + +func (r *callbackRegistration) acquire(ctx context.Context) (*callbackPermit, error) { + if r == nil || r.limiter == nil { + return nil, errs.NewErr(errs.TemporaryCapacity, "callback limiter is unavailable") + } + if err := ctx.Err(); err != nil { + return nil, err + } + waitCtx, cancel := context.WithTimeout(ctx, callbackAcquireTimeout) + defer cancel() + for { + r.limiter.mu.Lock() + if r.limiter.active < r.limiter.limitLocked() { + r.limiter.active++ + r.limiter.mu.Unlock() + return &callbackPermit{limiter: r.limiter}, nil + } + changed := r.limiter.changed + r.limiter.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-waitCtx.Done(): + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, errs.NewErr(errs.TemporaryCapacity, "timed out waiting for callback admission") + case <-changed: + } + } +} + +func (l *callbackLimiter) limitLocked() int { + limit := 0 + for _, registered := range l.registrations { + if limit == 0 || registered < limit { + limit = registered + } + } + return limit +} + +func (l *callbackLimiter) signalLocked() { + close(l.changed) + l.changed = make(chan struct{}) +} + +func (p *callbackPermit) release() { + if p == nil || p.limiter == nil { + return + } + p.once.Do(func() { + callbackLimiters.Lock() + defer callbackLimiters.Unlock() + p.limiter.mu.Lock() + p.limiter.active-- + p.limiter.signalLocked() + if len(p.limiter.registrations) == 0 && p.limiter.active == 0 { + delete(callbackLimiters.byUser, p.limiter.userID) + } + p.limiter.mu.Unlock() + }) +} + +func (d *AliyundriveOpen) callbackRegistration() *callbackRegistration { + if d.callback != nil { + return d.callback + } + if d.ref != nil { + return d.ref.callbackRegistration() + } + return nil +} + +func (d *AliyundriveOpen) callbackRangeReader(url string, size int64) stream.RangeReaderFunc { + return func(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + if requested.Length < 0 || requested.Start+requested.Length > size { + requested.Length = size - requested.Start + } + for attempt := 0; attempt < callbackRequestAttempts; attempt++ { + permit, err := d.callbackRegistration().acquire(ctx) + if err != nil { + return nil, err + } + body, retry, err := openCallbackRange(ctx, url, size, requested) + if !retry && err == nil { + return newCallbackBody(ctx, body, permit.release), nil + } + permit.release() + if !retry { + return nil, err + } + if attempt+1 == callbackRequestAttempts { + return nil, errs.NewErr(errs.TemporaryCapacity, "Aliyun callback concurrency limit rejected %d attempts", callbackRequestAttempts) + } + delay := callbackRetryBaseDelay << attempt + delay += time.Duration(rand.Int64N(int64(delay / 2))) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + return nil, errs.NewErr(errs.TemporaryCapacity, "callback attempts exhausted") + } +} + +func openCallbackRange(ctx context.Context, url string, size int64, requested http_range.Range) (io.ReadCloser, bool, error) { + requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) + header := anet.ProcessHeader(requestHeader, nil) + header = http_range.ApplyRangeToHttpHeader(requested, header) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, false, fmt.Errorf("create Aliyun callback request: %w", err) + } + req.Header = header + response, err := anet.HttpClient().Do(req) + if err != nil { + return nil, false, fmt.Errorf("Aliyun callback request failed: %w", err) + } + if response.StatusCode >= http.StatusBadRequest { + defer response.Body.Close() + body, readErr := io.ReadAll(io.LimitReader(response.Body, callbackErrorBodyLimit)) + if readErr != nil { + return nil, false, fmt.Errorf("read Aliyun callback error response: %w", readErr) + } + if isCallbackCapacityRejection(response.StatusCode, body) { + return nil, true, nil + } + message := strings.ReplaceAll(strings.TrimSpace(string(body)), url, "") + return nil, false, fmt.Errorf("Aliyun callback request failed: %w; response: %s", anet.HttpStatusCodeError(response.StatusCode), message) + } + if requested.Start == 0 && requested.Length == size || response.StatusCode == http.StatusPartialContent || callbackContentRangeStartsAt(response.Header, requested.Start) { + return response.Body, false, nil + } + if response.StatusCode == http.StatusOK { + body, rangeErr := anet.GetRangedHttpReader(response.Body, requested.Start, requested.Length) + if rangeErr != nil { + response.Body.Close() + return nil, false, rangeErr + } + return body, false, nil + } + return response.Body, false, nil +} + +func isCallbackCapacityRejection(status int, body []byte) bool { + return status == http.StatusForbidden && + strings.Contains(string(body), "RequestDeniedByCallback") && + strings.Contains(string(body), "ExceedMaxConcurrency") +} + +func callbackContentRangeStartsAt(header http.Header, offset int64) bool { + start, _, err := http_range.ParseContentRange(header.Get("Content-Range")) + return err == nil && start == offset +} + +type callbackBody struct { + body io.ReadCloser + release func() + once sync.Once + mu sync.Mutex + stop func() bool +} + +func newCallbackBody(ctx context.Context, body io.ReadCloser, release func()) *callbackBody { + b := &callbackBody{body: body, release: release} + stop := context.AfterFunc(ctx, func() { _ = b.Close() }) + b.mu.Lock() + b.stop = stop + b.mu.Unlock() + return b +} + +func (b *callbackBody) Read(p []byte) (int, error) { + n, err := b.body.Read(p) + if err != nil { + _ = b.Close() + } + return n, err +} + +func (b *callbackBody) Close() error { + var err error + b.once.Do(func() { + b.mu.Lock() + stop := b.stop + b.mu.Unlock() + if stop != nil { + stop() + } + err = b.body.Close() + b.release() + }) + return err +} diff --git a/drivers/aliyundrive_open/callback_test.go b/drivers/aliyundrive_open/callback_test.go new file mode 100644 index 000000000..13e685a1f --- /dev/null +++ b/drivers/aliyundrive_open/callback_test.go @@ -0,0 +1,310 @@ +package aliyundrive_open + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestCallbackRangeHoldsPermitUntilBodyClose(t *testing.T) { + oldConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = oldConf }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1") + w.Header().Set("Content-Range", "bytes 0-0/1") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "x") + })) + defer server.Close() + + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + d := &AliyundriveOpen{callback: registration} + body, err := d.callbackRangeReader(server.URL, 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if err != nil { + t.Fatal(err) + } + registration.limiter.mu.Lock() + active := registration.limiter.active + registration.limiter.mu.Unlock() + if active != 1 { + t.Fatalf("active callback bodies = %d, want 1", active) + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + registration.limiter.mu.Lock() + active = registration.limiter.active + registration.limiter.mu.Unlock() + if active != 0 { + t.Fatalf("active callback bodies after Close = %d, want 0", active) + } +} + +func TestCallbackLimiterUsesMinimumRegisteredLimit(t *testing.T) { + firstRegistration := registerCallbackLimiter(t.Name(), 2) + t.Cleanup(firstRegistration.unregister) + first, err := firstRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + second, err := firstRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer first.release() + defer second.release() + + lowerRegistration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(lowerRegistration.unregister) + acquired := make(chan *callbackPermit, 1) + go func() { + permit, acquireErr := lowerRegistration.acquire(t.Context()) + if acquireErr == nil { + acquired <- permit + } + }() + + first.release() + select { + case permit := <-acquired: + permit.release() + t.Fatal("lowering the shared limit must wait for all excess bodies to drain") + case <-time.After(100 * time.Millisecond): + } + second.release() + select { + case permit := <-acquired: + permit.release() + case <-time.After(time.Second): + t.Fatal("admission did not resume after active bodies drained below the new limit") + } +} + +func TestCallbackLimiterSeparatesUsers(t *testing.T) { + firstUser := registerCallbackLimiter(t.Name()+"-first", 1) + secondUser := registerCallbackLimiter(t.Name()+"-second", 1) + t.Cleanup(firstUser.unregister) + t.Cleanup(secondUser.unregister) + first, err := firstUser.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer first.release() + second, err := secondUser.acquire(t.Context()) + if err != nil { + t.Fatalf("independent user was blocked: %v", err) + } + second.release() +} + +func TestCallbackLimiterReconfigureWaitsForOldBodies(t *testing.T) { + userID := t.Name() + oldRegistration := registerCallbackLimiter(userID, 2) + first, err := oldRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + second, err := oldRegistration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + oldRegistration.unregister() + + newRegistration := registerCallbackLimiter(userID, 1) + t.Cleanup(newRegistration.unregister) + acquired := make(chan *callbackPermit, 1) + go func() { + permit, acquireErr := newRegistration.acquire(t.Context()) + if acquireErr == nil { + acquired <- permit + } + }() + first.release() + select { + case permit := <-acquired: + permit.release() + t.Fatal("reconfigured limiter admitted while an old body still occupied the new limit") + case <-time.After(100 * time.Millisecond): + } + second.release() + select { + case permit := <-acquired: + permit.release() + case <-time.After(time.Second): + t.Fatal("reconfigured limiter did not admit after old bodies drained") + } +} + +func TestCallbackLimiterDistinguishesTimeoutAndCancellation(t *testing.T) { + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + permit, err := registration.acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer permit.release() + + started := time.Now() + _, err = registration.acquire(t.Context()) + if !errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("admission timeout error = %v, want TemporaryCapacity", err) + } + if time.Since(started) < callbackAcquireTimeout { + t.Fatal("admission timed out before the configured wait elapsed") + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = registration.acquire(ctx) + if !errors.Is(err, context.Canceled) || errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("canceled admission error = %v, want only context.Canceled", err) + } +} + +func TestCallbackCapacityRejectionRequiresBothExactMarkers(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + {name: "both", body: `{"code":"RequestDeniedByCallback","message":"ExceedMaxConcurrency"}`, want: true}, + {name: "code only", body: `{"code":"RequestDeniedByCallback"}`}, + {name: "message only", body: `{"message":"ExceedMaxConcurrency"}`}, + {name: "case differs", body: `{"code":"requestdeniedbycallback","message":"ExceedMaxConcurrency"}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := isCallbackCapacityRejection(http.StatusForbidden, []byte(test.body)); got != test.want { + t.Fatalf("classification = %v, want %v", got, test.want) + } + }) + } + if isCallbackCapacityRejection(http.StatusTooManyRequests, []byte(`RequestDeniedByCallback ExceedMaxConcurrency`)) { + t.Fatal("non-403 response must not be classified as callback capacity") + } +} + +func TestCallbackRangeRetriesOnlyVerifiedCapacityRejections(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"code":"RequestDeniedByCallback","message":"ExceedMaxConcurrency"}`) + })) + defer server.Close() + + registration := registerCallbackLimiter(t.Name(), 1) + t.Cleanup(registration.unregister) + d := &AliyundriveOpen{callback: registration} + _, err := d.callbackRangeReader(server.URL+"?token=secret", 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if !errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("verified rejection error = %v, want TemporaryCapacity", err) + } + if requests.Load() != callbackRequestAttempts { + t.Fatalf("requests = %d, want %d", requests.Load(), callbackRequestAttempts) + } + if strings.Contains(err.Error(), "secret") { + t.Fatal("capacity error leaked the signed callback URL") + } + permit, acquireErr := registration.acquire(t.Context()) + if acquireErr != nil { + t.Fatalf("capacity retries leaked admission: %v", acquireErr) + } + permit.release() + + requests.Store(0) + permanent := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"code":"RequestDeniedByCallback","message":"denied"}`) + })) + defer permanent.Close() + _, err = d.callbackRangeReader(permanent.URL+"?token=secret", 1).RangeRead(t.Context(), http_range.Range{Length: 1}) + if errors.Is(err, errs.TemporaryCapacity) { + t.Fatalf("permanent 403 error = %v, must not be TemporaryCapacity", err) + } + if requests.Load() != 1 { + t.Fatalf("permanent 403 requests = %d, want 1", requests.Load()) + } + if strings.Contains(err.Error(), "secret") { + t.Fatal("permanent error leaked the signed callback URL") + } +} + +type countingReadCloser struct { + reader io.Reader + closed atomic.Int32 +} + +func (r *countingReadCloser) Read(p []byte) (int, error) { return r.reader.Read(p) } +func (r *countingReadCloser) Close() error { + r.closed.Add(1) + return nil +} + +func TestCallbackBodyReleasesExactlyOnce(t *testing.T) { + underlying := &countingReadCloser{reader: strings.NewReader("x")} + var released atomic.Int32 + body := newCallbackBody(t.Context(), underlying, func() { released.Add(1) }) + _, _ = io.ReadAll(body) + if err := body.Close(); err != nil { + t.Fatal(err) + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + if underlying.closed.Load() != 1 || released.Load() != 1 { + t.Fatalf("close count = %d, release count = %d; want 1, 1", underlying.closed.Load(), released.Load()) + } +} + +type failingReadCloser struct { + closed atomic.Int32 +} + +func (*failingReadCloser) Read([]byte) (int, error) { return 0, errors.New("read failed") } +func (r *failingReadCloser) Close() error { + r.closed.Add(1) + return nil +} + +func TestCallbackBodyReadFailureReleasesPermit(t *testing.T) { + underlying := &failingReadCloser{} + var released atomic.Int32 + body := newCallbackBody(t.Context(), underlying, func() { released.Add(1) }) + if _, err := body.Read(make([]byte, 1)); err == nil { + t.Fatal("read unexpectedly succeeded") + } + if underlying.closed.Load() != 1 || released.Load() != 1 { + t.Fatalf("close count = %d, release count = %d; want 1, 1", underlying.closed.Load(), released.Load()) + } +} + +func TestCallbackBodyCancellationReleasesPermit(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + underlying := &countingReadCloser{reader: strings.NewReader("x")} + released := make(chan struct{}, 1) + _ = newCallbackBody(ctx, underlying, func() { released <- struct{}{} }) + cancel() + select { + case <-released: + case <-time.After(time.Second): + t.Fatal("context cancellation did not release callback admission") + } + if underlying.closed.Load() != 1 { + t.Fatalf("underlying close count = %d, want 1", underlying.closed.Load()) + } +} diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index ae8a9ceab..b78b56c98 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -24,8 +24,9 @@ type AliyundriveOpen struct { DriveId string - limiter *limiter - ref *AliyundriveOpen + limiter *limiter + ref *AliyundriveOpen + callback *callbackRegistration TempDirId string IsVip bool @@ -40,6 +41,7 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.CallbackConcurrency = normalizeCallbackConcurrency(d.CallbackConcurrency) d.limiter = getLimiterForUser(globalLimiterUserID) // First create a globally shared limiter to limit the initial requests. if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" @@ -93,6 +95,10 @@ func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { } func (d *AliyundriveOpen) Drop(ctx context.Context) error { + if d.callback != nil { + d.callback.unregister() + d.callback = nil + } d.limiter.free() d.limiter = nil d.ref = nil diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 33d10f137..01d9d32f4 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -22,11 +22,12 @@ type Addition struct { LIVPDownloadFormat string `json:"livp_download_format" type:"select" options:"jpeg,mov" default:"jpeg"` AccessToken string `json:"access_token"` - AccountId int `json:"account_id" required:"true"` - RefreshToken2 string `json:"refresh_token2" required:"true"` - AccessToken2 string - Concurrency int `json:"concurrency" type:"number" default:"4"` - ChunkSize int `json:"chunk_size" type:"number" default:"256"` + AccountId int `json:"account_id" required:"true"` + RefreshToken2 string `json:"refresh_token2" required:"true"` + AccessToken2 string + Concurrency int `json:"concurrency" type:"number" default:"4"` + ChunkSize int `json:"chunk_size" type:"number" default:"256"` + CallbackConcurrency int `json:"callback_concurrency" type:"number" default:"1" help:"Maximum active proxied downloads shared by Aliyun user"` } var config = driver.Config{ diff --git a/drivers/emby/driver.go b/drivers/emby/driver.go index b2406aa6e..39122d43b 100644 --- a/drivers/emby/driver.go +++ b/drivers/emby/driver.go @@ -56,8 +56,8 @@ func (d *Emby) Init(ctx context.Context) error { return nil } - if strings.TrimSpace(d.Username) == "" || strings.TrimSpace(d.Password) == "" { - return fmt.Errorf("please provide api_key+user_id or username+password") + if strings.TrimSpace(d.Username) == "" { + return fmt.Errorf("please provide api_key+user_id or username(+password)") } if err := d.login(ctx); err != nil { diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 7c35aee1b..adf4a0023 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -154,15 +154,17 @@ func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } } return ret, nil - } else { - ret := make([]model.Obj, 0, len(obj.Entries)) - for _, entry := range obj.Entries { - if entry.Name != ".gitkeep" { - ret = append(ret, entry.toModelObj()) - } + } + + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name == ".gitkeep" { + continue } - return ret, nil + ret = append(ret, entry.toModelObj()) } + d.fetchAccurateModifiedTimes(ctx, dir.GetPath(), ret) + return ret, nil } func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/github/meta.go b/drivers/github/meta.go index ae313dcc2..366d3850f 100644 --- a/drivers/github/meta.go +++ b/drivers/github/meta.go @@ -7,23 +7,24 @@ import ( type Addition struct { driver.RootPath - Token string `json:"token" type:"string" required:"true"` - Owner string `json:"owner" type:"string" required:"true"` - Repo string `json:"repo" type:"string" required:"true"` - Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` - GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` - GPGPrivateKey string `json:"gpg_private_key" type:"text"` - GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` - CommitterName string `json:"committer_name" type:"string"` - CommitterEmail string `json:"committer_email" type:"string"` - AuthorName string `json:"author_name" type:"string"` - AuthorEmail string `json:"author_email" type:"string"` - MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` - DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` - PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` - RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` - CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` - MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + AccurateModifiedTime bool `json:"accurate_modified_time" type:"bool" default:"false" help:"Best-effort accurate modified time for small directory listings. Default disabled. Adds one GitHub GraphQL request and falls back to legacy zero-time values on failure."` + GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` + GPGPrivateKey string `json:"gpg_private_key" type:"text"` + GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` } var config = driver.Config{ diff --git a/drivers/github/mtime.go b/drivers/github/mtime.go new file mode 100644 index 000000000..f2ec39755 --- /dev/null +++ b/drivers/github/mtime.go @@ -0,0 +1,106 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" +) + +const ( + mtimeMaxEntries = 200 + githubGraphQLEndpoint = "https://api.github.com/graphql" +) + +var githubZeroTime = time.Unix(0, 0) + +type graphQLHistory struct { + Nodes []struct { + CommittedDate time.Time `json:"committedDate"` + } `json:"nodes"` +} + +type graphQLMtimeResponse struct { + Data struct { + Repository struct { + Commit map[string]graphQLHistory `json:"commit"` + } `json:"repository"` + } `json:"data"` + Errors []struct{} `json:"errors"` +} + +func quoteGraphQLString(value string) string { + quoted, _ := json.Marshal(value) + return string(quoted) +} + +func buildMtimeQuery(owner, repo, ref string, objs []model.Obj) string { + histories := make([]string, 0, len(objs)) + for i, obj := range objs { + path := strings.TrimPrefix(obj.GetPath(), "/") + histories = append(histories, fmt.Sprintf(`p%d: history(first: 1, path: %s) { nodes { committedDate } }`, i, quoteGraphQLString(path))) + } + + return fmt.Sprintf(`query { + repository(owner: %s, name: %s) { + commit: object(expression: %s) { + ... on Commit { + %s + } + } + } +}`, + quoteGraphQLString(owner), + quoteGraphQLString(repo), + quoteGraphQLString(ref+"^{commit}"), + strings.Join(histories, "\n\t\t\t\t"), + ) +} + +func (d *Github) fetchAccurateModifiedTimes(ctx context.Context, dirPath string, objs []model.Obj) { + token := strings.TrimSpace(d.Token) + if !d.AccurateModifiedTime || token == "" || len(objs) == 0 || len(objs) > mtimeMaxEntries { + return + } + + res, err := d.client.R(). + SetContext(ctx). + SetHeader("Accept", "application/vnd.github+json"). + SetHeader("Authorization", "Bearer "+token). + SetBody(map[string]string{"query": buildMtimeQuery(d.Owner, d.Repo, d.Ref, objs)}). + Post(githubGraphQLEndpoint) + if err != nil { + log.WithError(err).Warnf("github accurate mtime failed for %s: transport", dirPath) + return + } + if res.StatusCode() != http.StatusOK { + log.Warnf("github accurate mtime failed for %s: http_%d", dirPath, res.StatusCode()) + return + } + + var response graphQLMtimeResponse + if err := utils.Json.Unmarshal(res.Body(), &response); err != nil { + log.WithError(err).Warnf("github accurate mtime failed for %s: graphql", dirPath) + return + } + if len(response.Errors) > 0 || response.Data.Repository.Commit == nil { + log.Warnf("github accurate mtime failed for %s: graphql", dirPath) + return + } + + for i, obj := range objs { + history := response.Data.Repository.Commit[fmt.Sprintf("p%d", i)] + if len(history.Nodes) == 0 { + continue + } + if raw, ok := obj.(*model.Object); ok { + raw.Modified = history.Nodes[0].CommittedDate + } + } +} diff --git a/drivers/github/mtime_test.go b/drivers/github/mtime_test.go new file mode 100644 index 000000000..a6b2bdc68 --- /dev/null +++ b/drivers/github/mtime_test.go @@ -0,0 +1,378 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + odriver "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/go-resty/resty/v2" +) + +func TestDriverInfoIncludesAccurateModifiedTimeDefault(t *testing.T) { + info := op.GetDriverInfoMap()["GitHub API"] + for _, item := range info.Additional { + if item.Name != "accurate_modified_time" { + continue + } + if item.Default != "false" { + t.Fatalf("unexpected default: %q", item.Default) + } + return + } + t.Fatal("accurate_modified_time item not registered") +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +func newGithubTestDriver(rt roundTripFunc, token string, enabled bool) *Github { + return &Github{ + Storage: model.Storage{MountPath: "/github-test", CacheExpiration: 10}, + Addition: Addition{ + RootPath: odriver.RootPath{RootFolderPath: "/"}, + Token: token, + Owner: "owner", + Repo: "repo", + Ref: "main", + AccurateModifiedTime: enabled, + }, + client: resty.New().SetTransport(rt), + } +} + +func newJSONResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + } +} + +func mustJSON(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal json: %v", err) + } + return string(data) +} + +func graphQLQueryFromRequest(t *testing.T, r *http.Request) string { + t.Helper() + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read graphql request body: %v", err) + } + var payload map[string]string + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode graphql request body: %v", err) + } + query := payload["query"] + if query == "" { + t.Fatalf("graphql request missing query: %s", string(body)) + } + return query +} + +func newContentsPayload(t *testing.T, entries []Object) string { + t.Helper() + return mustJSON(t, map[string]any{ + "type": "dir", + "sha": "tree-sha", + "entries": entries, + }) +} + +func newTreePayload(t *testing.T, sha string, trees []TreeObjResp) string { + t.Helper() + return mustJSON(t, map[string]any{ + "sha": sha, + "truncated": false, + "tree": trees, + }) +} + +func newCommitGraphQLPayload(t *testing.T, histories map[string][]string) string { + t.Helper() + commit := make(map[string]any, len(histories)) + for alias, dates := range histories { + nodes := make([]map[string]string, 0, len(dates)) + for _, date := range dates { + nodes = append(nodes, map[string]string{"committedDate": date}) + } + commit[alias] = map[string]any{"nodes": nodes} + } + return mustJSON(t, map[string]any{ + "data": map[string]any{ + "repository": map[string]any{"commit": commit}, + }, + }) +} + +func newSequentialEntries(count int) []Object { + entries := make([]Object, 0, count) + for i := range count { + name := fmt.Sprintf("%03d.md", i) + entries = append(entries, Object{Name: name, Path: "docs/" + name, Type: "file", Size: 1}) + } + return entries +} + +func mustObject(t *testing.T, obj model.Obj) *model.Object { + t.Helper() + raw, ok := model.UnwrapObjName(obj).(*model.Object) + if !ok { + t.Fatalf("unexpected obj type %T", obj) + } + return raw +} + +func TestListAppliesAccurateModifiedTimeInOneRequest(t *testing.T) { + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + entries := []Object{ + {Name: "a.md", Path: "docs/a.md", Type: "file", Size: 1}, + {Name: ".gitkeep", Path: "docs/.gitkeep", Type: "file"}, + {Name: `quote " 文.md`, Path: `docs/quote " 文.md`, Type: "file", Size: 1}, + {Name: "control.md", Path: "docs/control\x01.md", Type: "file", Size: 1}, + } + graphqlCalls := 0 + var query string + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, entries)), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + query = graphQLQueryFromRequest(t, r) + if got := r.Header.Get("Authorization"); got != "Bearer token" { + t.Fatalf("unexpected authorization header: %q", got) + } + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{ + "p0": {stamp.Format(time.RFC3339)}, + "p1": {}, + "p2": {}, + })), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL request, got %d", graphqlCalls) + } + if strings.Count(query, "history(first: 1") != 3 || + !strings.Contains(query, `object(expression: "main^{commit}")`) || + !strings.Contains(query, `p0: history(first: 1, path: "docs/a.md")`) || + !strings.Contains(query, `p1: history(first: 1, path: "docs/quote \" 文.md")`) || + !strings.Contains(query, `p2: history(first: 1, path: "docs/control\u0001.md")`) { + t.Fatalf("query should peel the ref and contain all listed paths once:\n%s", query) + } + if len(objs) != 3 { + t.Fatalf("expected three objects after .gitkeep filtering, got %d", len(objs)) + } + if first := mustObject(t, objs[0]); !first.ModTime().Equal(stamp) || !first.CreateTime().Equal(githubZeroTime) { + t.Fatalf("unexpected first timestamps: mod=%v create=%v", first.ModTime(), first.CreateTime()) + } + if second := mustObject(t, objs[1]); !second.ModTime().Equal(githubZeroTime) || !second.CreateTime().Equal(githubZeroTime) { + t.Fatalf("unmatched entry should retain legacy timestamps: mod=%v create=%v", second.ModTime(), second.CreateTime()) + } +} + +func TestListSkipsAccurateModifiedTime(t *testing.T) { + tests := []struct { + name string + enabled bool + token string + entries int + }{ + {name: "disabled", enabled: false, token: "token", entries: 1}, + {name: "missing token", enabled: true, token: "", entries: 1}, + {name: "over entry limit", enabled: true, token: "token", entries: 201}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + graphqlCalls := 0 + payload := newContentsPayload(t, newSequentialEntries(tc.entries)) + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, payload), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, nil)), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), tc.token, tc.enabled) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 0 { + t.Fatalf("expected zero GraphQL requests, got %d", graphqlCalls) + } + for _, obj := range objs { + raw := mustObject(t, obj) + if !raw.ModTime().Equal(githubZeroTime) || !raw.CreateTime().Equal(githubZeroTime) { + t.Fatalf("legacy timestamps should be preserved: mod=%v create=%v", raw.ModTime(), raw.CreateTime()) + } + } + }) + } +} + +func TestListFallsBackWhenGraphQLFails(t *testing.T) { + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, newSequentialEntries(1))), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, `{"errors":[{"message":"rate limited"}]}`), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("GraphQL failure should be best-effort: %v", err) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL request, got %d", graphqlCalls) + } + obj := mustObject(t, objs[0]) + if !obj.ModTime().Equal(githubZeroTime) || !obj.CreateTime().Equal(githubZeroTime) { + t.Fatalf("failed GraphQL request should retain legacy timestamps: mod=%v create=%v", obj.ModTime(), obj.CreateTime()) + } +} + +func TestListUsesOneGraphQLRequestAtEntryLimit(t *testing.T) { + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + graphqlCalls := 0 + var query string + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, newSequentialEntries(200))), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + query = graphQLQueryFromRequest(t, r) + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{ + "p199": {stamp.Format(time.RFC3339)}, + })), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if graphqlCalls != 1 || strings.Count(query, "history(first: 1") != 200 { + t.Fatalf("200 entries should share one request: calls=%d histories=%d", graphqlCalls, strings.Count(query, "history(first: 1")) + } + if !strings.Contains(query, `p199: history(first: 1, path: "docs/199.md")`) { + t.Fatalf("query missing final entry:\n%s", query) + } + if len(objs) != 200 || !mustObject(t, objs[199]).ModTime().Equal(stamp) { + t.Fatalf("unexpected final object timestamp") + } +} + +func TestListKeepsTreeFallbackOnLegacyPath(t *testing.T) { + entries := make([]Object, 0, 1000) + for i := range 1000 { + name := fmt.Sprintf("dir-%d", i) + entries = append(entries, Object{Name: name, Path: "docs/" + name, Type: "dir"}) + } + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, entries)), nil + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/git/trees/"): + return newJSONResponse(http.StatusOK, newTreePayload(t, "tree-sha", []TreeObjResp{{TreeObjReq: TreeObjReq{Path: "child.md", Mode: "100644", Type: "blob", Sha: "blob-sha"}, Size: 1, URL: "https://example.invalid/blob"}})), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, nil)), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + objs, err := drv.List(context.Background(), &model.Object{Path: "/docs", Name: "docs", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(objs) != 1 { + t.Fatalf("unexpected tree fallback result length: %d", len(objs)) + } + first := mustObject(t, objs[0]) + if first.GetPath() != "/child.md" { + t.Fatalf("unexpected tree fallback path: %s", first.GetPath()) + } + if !first.ModTime().Equal(githubZeroTime) || !first.CreateTime().Equal(githubZeroTime) { + t.Fatalf("tree fallback should preserve legacy timestamps: mod=%v create=%v", first.ModTime(), first.CreateTime()) + } + if graphqlCalls != 0 { + t.Fatalf("tree fallback should skip GraphQL, got %d calls", graphqlCalls) + } +} + +func TestOpListCacheHitDoesNotRepeatGraphQL(t *testing.T) { + op.Cache.ClearAll() + defer op.Cache.ClearAll() + stamp := time.Date(2025, 12, 22, 4, 52, 41, 0, time.UTC) + graphqlCalls := 0 + drv := newGithubTestDriver(roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/contents/"): + return newJSONResponse(http.StatusOK, newContentsPayload(t, []Object{{Name: "a.md", Path: "a.md", Type: "file", Size: 1}})), nil + case r.Method == http.MethodPost && r.URL.String() == githubGraphQLEndpoint: + graphqlCalls++ + return newJSONResponse(http.StatusOK, newCommitGraphQLPayload(t, map[string][]string{"p0": {stamp.Format(time.RFC3339)}})), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.String()) + } + }), "token", true) + + first, err := op.List(context.Background(), drv, "/", model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected first list error: %v", err) + } + second, err := op.List(context.Background(), drv, "/", model.ListArgs{}) + if err != nil { + t.Fatalf("unexpected second list error: %v", err) + } + if len(first) != 1 || len(second) != 1 { + t.Fatalf("unexpected cached results: first=%d second=%d", len(first), len(second)) + } + if graphqlCalls != 1 { + t.Fatalf("expected one GraphQL call across cached lists, got %d", graphqlCalls) + } + if !mustObject(t, first[0]).ModTime().Equal(stamp) { + t.Fatalf("expected first list to include backfilled modified time, got %v", mustObject(t, first[0]).ModTime()) + } + if !mustObject(t, second[0]).ModTime().Equal(stamp) { + t.Fatalf("expected cached list to retain modified time, got %v", mustObject(t, second[0]).ModTime()) + } +} diff --git a/drivers/github/types.go b/drivers/github/types.go index d0d24839c..d6d30642b 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -1,8 +1,6 @@ package github import ( - "time" - "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -35,7 +33,8 @@ func (o *Object) toModelObj() *model.Object { return &model.Object{ Name: o.Name, Size: o.Size, - Modified: time.Unix(0, 0), + Modified: githubZeroTime, + Ctime: githubZeroTime, IsFolder: o.Type == "dir", Path: utils.FixAndCleanPath(o.Path), } @@ -69,7 +68,8 @@ func (o *TreeObjResp) toModelObj() *model.Object { return &model.Object{ Name: o.Path, Size: o.Size, - Modified: time.Unix(0, 0), + Modified: githubZeroTime, + Ctime: githubZeroTime, IsFolder: o.Type == "tree", Path: utils.FixAndCleanPath(o.Path), } diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 73b31014e..a640d8669 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "context" @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/http/cookiejar" "net/url" "strconv" "strings" @@ -27,11 +28,13 @@ type ILanZou struct { model.Storage Addition - userID string - account string - upClient *resty.Client - conf Conf - config driver.Config + userID string + account string + apiClient *resty.Client + linkClient *resty.Client + upClient *resty.Client + conf Conf + config driver.Config } func (d *ILanZou) Config() driver.Config { @@ -43,6 +46,18 @@ func (d *ILanZou) GetAddition() driver.Additional { } func (d *ILanZou) Init(ctx context.Context) error { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + // Keep state isolated per storage. The console and CDN issue cookies that + // must survive a retry but must never leak to another configured account. + d.apiClient = base.NewRestyClient().SetCookieJar(jar) + d.linkClient = base.NewRestyClient().SetCookieJar(jar).SetRedirectPolicy( + resty.RedirectPolicyFunc(func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }), + ) d.upClient = base.NewRestyClient().SetTimeout(time.Minute * 10) if d.UUID == "" { res, err := d.unproved("/getUuid", http.MethodGet, nil) @@ -77,8 +92,7 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) "folderId=" + dir.GetID(), "type=0", } - queryString := strings.Join(params, "&") - req.SetQueryString(queryString).SetResult(&resp) + req.SetQueryString(strings.Join(params, "&")).SetResult(&resp) }) if err != nil { return nil, err @@ -96,14 +110,12 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } obj := model.Object{ - ID: strconv.FormatInt(f.FileId, 10), - // Path: "", + ID: strconv.FormatInt(f.FileId, 10), Name: f.FileName, Size: f.FileSize * 1024, Modified: updTime, Ctime: updTime, IsFolder: false, - // HashInfo: utils.HashInfo{}, } if f.FileType == 2 { obj.IsFolder = true @@ -133,26 +145,23 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", "timestamp=" + tsStr, - "appToken=" + url.QueryEscape(d.Token), + "appToken=" + appTokenQueryValue(d.Token), "enable=1", } - - downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) + downloadID, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { return nil, err } - params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadId))) - + params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadID))) auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), ts)), d.conf.secret) if err != nil { return nil, err } params = append(params, "auth="+url.QueryEscape(hex.EncodeToString(auth))) - u.RawQuery = strings.Join(params, "&") realURL := u.String() - // get the url after redirect - req := base.NoRedirectClient.R().SetContext(ctx) + + req := d.linkClient.R().SetContext(ctx) req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", @@ -162,7 +171,6 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } - res, err := req.Get(realURL) if err != nil { return nil, err @@ -172,49 +180,50 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) realURL = location } else if res.StatusCode() == http.StatusOK && location != "" { realURL = location + } else if res.StatusCode() == http.StatusOK { + // Some file types return a 200 JSON resolver response instead of a 3xx. + realURL = utils.Json.Get(res.Body(), "url").ToString() + if realURL == "" { + realURL = utils.Json.Get(res.Body(), "data", "url").ToString() + } + if realURL == "" { + return nil, fmt.Errorf("download resolver returned no URL: %s", utils.Json.Get(res.Body(), "msg").ToString()) + } } else { return nil, fmt.Errorf("redirect failed, status: %d, location: %s, msg: %s", res.StatusCode(), location, utils.Json.Get(res.Body(), "msg").ToString()) } - link := model.Link{URL: realURL} - return &link, nil + link := &model.Link{URL: realURL} + // Probe the CDN for the actual object size; API metadata can differ from + // the bytes served by the final URL. The timeout bounds Link latency. + headCtx, cancel := context.WithTimeout(ctx, linkHeadTimeout) + defer cancel() + if response, err := d.apiClient.R().SetContext(headCtx).Head(realURL); err == nil && response.StatusCode() >= http.StatusOK && response.StatusCode() < http.StatusMultipleChoices { + if size, parseErr := strconv.ParseInt(response.Header().Get("Content-Length"), 10, 64); parseErr == nil && size > 0 { + link.ContentLength = size + } + } + return link, nil } func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { res, err := d.proved("/file/folder/save", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderDesc": "", - "folderId": parentDir.GetID(), - "folderName": dirName, - }) + req.SetBody(base.Json{"folderDesc": "", "folderId": parentDir.GetID(), "folderName": dirName}) }) if err != nil { return nil, err } - return &model.Object{ - ID: utils.Json.Get(res, "list", 0, "id").ToString(), - // Path: "", - Name: dirName, - Size: 0, - Modified: time.Now(), - Ctime: time.Now(), - IsFolder: true, - // HashInfo: utils.HashInfo{}, - }, nil + return &model.Object{ID: utils.Json.Get(res, "list", 0, "id").ToString(), Name: dirName, Modified: time.Now(), Ctime: time.Now(), IsFolder: true}, nil } func (d *ILanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - var fileIds, folderIds []string + var fileIDs, folderIDs []string if srcObj.IsDir() { - folderIds = []string{srcObj.GetID()} + folderIDs = []string{srcObj.GetID()} } else { - fileIds = []string{srcObj.GetID()} + fileIDs = []string{srcObj.GetID()} } _, err := d.proved("/file/folder/move", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderIds": strings.Join(folderIds, ","), - "fileIds": strings.Join(fileIds, ","), - "targetId": dstDir.GetID(), - }) + req.SetBody(base.Json{"folderIds": strings.Join(folderIDs, ","), "fileIds": strings.Join(fileIDs, ","), "targetId": dstDir.GetID()}) }) if err != nil { return nil, err @@ -226,58 +235,47 @@ func (d *ILanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) var err error if srcObj.IsDir() { _, err = d.proved("/file/folder/edit", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderDesc": "", - "folderId": srcObj.GetID(), - "folderName": newName, - }) + req.SetBody(base.Json{"folderDesc": "", "folderId": srcObj.GetID(), "folderName": newName}) }) } else { _, err = d.proved("/file/edit", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "fileDesc": "", - "fileId": srcObj.GetID(), - "fileName": newName, - }) + req.SetBody(base.Json{"fileDesc": "", "fileId": srcObj.GetID(), "fileName": newName}) }) } if err != nil { return nil, err } - return &model.Object{ - ID: srcObj.GetID(), - // Path: "", - Name: newName, - Size: srcObj.GetSize(), - Modified: time.Now(), - Ctime: srcObj.CreateTime(), - IsFolder: srcObj.IsDir(), - }, nil + return &model.Object{ID: srcObj.GetID(), Name: newName, Size: srcObj.GetSize(), Modified: time.Now(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir()}, nil } +// iLanzou has no server-side copy primitive. Returning NotImplement delegates +// to OpenList's persistent CopyTaskManager instead of blocking the HTTP request. func (d *ILanZou) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - // TODO copy obj, optional return nil, errs.NotImplement } func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { - var fileIds, folderIds []string + var fileIDs, folderIDs []string if obj.IsDir() { - folderIds = []string{obj.GetID()} + folderIDs = []string{obj.GetID()} } else { - fileIds = []string{obj.GetID()} + fileIDs = []string{obj.GetID()} } _, err := d.proved("/file/delete", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "folderIds": strings.Join(folderIds, ","), - "fileIds": strings.Join(fileIds, ","), - "status": 0, - }) + req.SetBody(base.Json{"folderIds": strings.Join(folderIDs, ","), "fileIds": strings.Join(fileIDs, ","), "status": 0}) }) return err } -const DefaultPartSize = 1024 * 1024 * 8 +const ( + DefaultPartSize = 1024 * 1024 * 8 + + // The results endpoint is eventually consistent after the upload commit. + maxUploadCommitRetries = 10 + uploadCommitRetryDelay = time.Second + + linkHeadTimeout = 5 * time.Second +) func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { etag := s.GetHash().GetHash(utils.MD5) @@ -288,55 +286,31 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame return nil, err } } - // get upToken + fileSizeKiB := (s.GetSize() + 1023) / 1024 + if fileSizeKiB < 1 { + fileSizeKiB = 1 + } res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "fileId": "", - "fileName": s.GetName(), - "fileSize": s.GetSize()/1024 + 1, - "folderId": dstDir.GetID(), - "md5": etag, - "type": 1, - }) + req.SetBody(base.Json{"fileId": "", "fileName": s.GetName(), "fileSize": fileSizeKiB, "folderId": dstDir.GetID(), "md5": etag, "type": 1}) }) if err != nil { return nil, err } upToken := utils.Json.Get(res, "upToken").ToString() if upToken == "-1" { - // 支持秒传 var resp UploadTokenRapidResp - err := utils.Json.Unmarshal(res, &resp) - if err != nil { + if err := utils.Json.Unmarshal(res, &resp); err != nil { return nil, err } - return &model.Object{ - ID: strconv.FormatInt(resp.Map.FileID, 10), - Name: resp.Map.FileName, - Size: s.GetSize(), - Modified: s.ModTime(), - Ctime: s.CreateTime(), - IsFolder: false, - HashInfo: utils.NewHashInfo(utils.MD5, etag), - }, nil + return &model.Object{ID: strconv.FormatInt(resp.Map.FileID, 10), Name: resp.Map.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag)}, nil } now := time.Now() - key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) - reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ - Reader: &driver.SimpleReaderWithSize{ - Reader: s, - Size: s.GetSize(), - }, - UpdateProgress: up, - }) + // Match the current console's generated Qiniu object key. + key := fmt.Sprintf("disk/%04d/%02d/%02d/%s/%d.rar", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{Reader: &driver.SimpleReaderWithSize{Reader: s, Size: s.GetSize()}, UpdateProgress: up}) var token string if s.GetSize() <= DefaultPartSize { - res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ - "token": upToken, - "key": key, - "fname": s.GetName(), - }).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader). - Post("https://upload.qiniup.com/") + res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{"token": upToken, "key": key, "fname": s.GetName()}).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader).Post("https://upload.qiniup.com/") if err != nil { return nil, err } @@ -347,40 +321,27 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame if err != nil { return nil, err } - uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() + uploadID := utils.Json.Get(res.Body(), "uploadId").ToString() parts := make([]Part, 0) partNum := (s.GetSize() + DefaultPartSize - 1) / DefaultPartSize for i := 1; i <= int(partNum); i++ { - u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) + u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadID, i) res, err = d.upClient.R().SetContext(ctx).SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(reader, DefaultPartSize)).Put(u) if err != nil { return nil, err } - etag := utils.Json.Get(res.Body(), "etag").ToString() - parts = append(parts, Part{ - PartNumber: i, - ETag: etag, - }) + parts = append(parts, Part{PartNumber: i, ETag: utils.Json.Get(res.Body(), "etag").ToString()}) } - res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ - "fnmae": s.GetName(), - "parts": parts, - }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) + res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{"fnmae": s.GetName(), "parts": parts}).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadID)) if err != nil { return nil, err } token = utils.Json.Get(res.Body(), "token").ToString() } - // commit upload var resp UploadResultResp - for i := 0; i < 10; i++ { + for i := 0; i < maxUploadCommitRetries; i++ { _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { - params := []string{ - "tokenList=" + token, - "tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), - } - queryString := strings.Join(params, "&") - req.SetQueryString(queryString).SetResult(&resp) + req.SetQueryString("tokenList=" + token + "&tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")).SetResult(&resp) }) if err != nil { return nil, err @@ -391,22 +352,17 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame if resp.List[0].Status == 1 { break } - time.Sleep(time.Second * 1) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(uploadCommitRetryDelay): + } } file := resp.List[0] if file.Status != 1 { - return nil, fmt.Errorf("upload failed, status: %d", resp.List[0].Status) + return nil, fmt.Errorf("upload failed, status: %d", file.Status) } - return &model.Object{ - ID: strconv.FormatInt(file.FileId, 10), - // Path: , - Name: file.FileName, - Size: s.GetSize(), - Modified: s.ModTime(), - Ctime: s.CreateTime(), - IsFolder: false, - HashInfo: utils.NewHashInfo(utils.MD5, etag), - }, nil + return &model.Object{ID: strconv.FormatInt(file.FileId, 10), Name: file.FileName, Size: s.GetSize(), Modified: s.ModTime(), Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag)}, nil } func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) { @@ -419,18 +375,8 @@ func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) vipSize := utils.Json.Get(res, "map", "vipSize").ToInt64() * 1024 totalSize := utils.Json.Get(res, "map", "totalSize").ToInt64() * 1024 rewardSize := utils.Json.Get(res, "map", "rewardSize").ToInt64() * 1024 - total := totalSize + rewardSize + vipSize used := utils.Json.Get(res, "map", "usedSize").ToInt64() * 1024 - return &model.StorageDetails{ - DiskUsage: model.DiskUsage{ - TotalSpace: total, - UsedSpace: used, - }, - }, nil + return &model.StorageDetails{DiskUsage: model.DiskUsage{TotalSpace: totalSize + rewardSize + vipSize, UsedSpace: used}}, nil } -//func (d *ILanZou) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} - var _ driver.Driver = (*ILanZou)(nil) diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index 35637aa51..b20146b14 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" diff --git a/drivers/ilanzou/types.go b/drivers/ilanzou/types.go index 04579c36d..70dc8a03a 100644 --- a/drivers/ilanzou/types.go +++ b/drivers/ilanzou/types.go @@ -1,4 +1,4 @@ -package template +package ilanzou type ListResp struct { Msg string `json:"msg"` diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index ea03cfd02..11087ac4a 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -1,4 +1,4 @@ -package template +package ilanzou import ( "encoding/hex" @@ -18,10 +18,7 @@ import ( func (d *ILanZou) login() error { res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "loginName": d.Username, - "loginPwd": d.Password, - }) + req.SetBody(base.Json{"loginName": d.Username, "loginPwd": d.Password}) }) if err != nil { return err @@ -35,20 +32,33 @@ func (d *ILanZou) login() error { func getTimestamp(secret []byte) (int64, string, error) { ts := time.Now().UnixMilli() - tsStr := strconv.FormatInt(ts, 10) - res, err := mopan.AesEncrypt([]byte(tsStr), secret) + res, err := mopan.AesEncrypt([]byte(strconv.FormatInt(ts, 10)), secret) if err != nil { return 0, "", err } return ts, hex.EncodeToString(res), nil } +// isCDNChallenge detects a transient CDN challenge (409 Conflict + HTML 403). +// A retry with the session cookie jar can resolve transient challenges. +func isCDNChallenge(res *resty.Response) bool { + if res == nil || res.StatusCode() != http.StatusConflict { + return false + } + return strings.Contains(res.Header().Get("Content-Type"), "text/html") && strings.Contains(string(res.Body()), "403") +} + +func appTokenQueryValue(token string) string { + // iLanzou requires a literal colon, while other reserved token characters + // must remain query-escaped. + return strings.ReplaceAll(url.QueryEscape(token), "%3A", ":") +} + func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { - _, ts_str, err := getTimestamp(d.conf.secret) + _, timestamp, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } - params := []string{ "uuid=" + url.QueryEscape(d.UUID), "devType=6", @@ -56,48 +66,51 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "devModel=chrome", "devVersion=" + url.QueryEscape(d.conf.devVersion), "appVersion=", - "timestamp=" + ts_str, + "timestamp=" + timestamp, } - if proved { - params = append(params, "appToken="+url.QueryEscape(d.Token)) + params = append(params, "appToken="+appTokenQueryValue(d.Token)) } - params = append(params, "extra=2") - queryString := strings.Join(params, "&") - - req := base.RestyClient.R() + if d.apiClient == nil { + return nil, fmt.Errorf("iLanzou driver is not initialized") + } + req := d.apiClient.R() req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", "Accept-Encoding": "gzip", "Accept-Language": "zh-CN,zh;q=0.9,en-US,en;q=0.8", }) - if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) } - if callback != nil { callback(req) } - - res, err := req.Execute(method, d.conf.base+pathname+"?"+queryString) + res, err := req.Execute(method, d.conf.base+pathname+"?"+strings.Join(params, "&")) if err != nil { if res != nil { - log.Errorf("[iLanZou] request error: %s", res.String()) + log.Errorf("[ilanzou] request error: %s", res.String()) } return nil, err } isRetry := len(retry) > 0 && retry[0] + if isCDNChallenge(res) { + // Resty's cookie jar records the challenge cookie from the first response. + // A second request proves whether it is a transient challenge or hard block. + if !isRetry { + return d.request(pathname, method, callback, proved, true) + } + return nil, fmt.Errorf("iLanzou CDN rejected %s with HTML 403; retry later or delete from the console", pathname) + } body := res.Body() code := utils.Json.Get(body, "code").ToInt() msg := utils.Json.Get(body, "msg").ToString() if code != 200 { if !isRetry && proved && (utils.SliceContains([]int{-1, -2}, code) || d.Token == "") { - err = d.login() - if err != nil { + if err := d.login(); err != nil { return nil, err } return d.request(pathname, method, callback, proved, true) diff --git a/drivers/ilanzou/util_test.go b/drivers/ilanzou/util_test.go new file mode 100644 index 000000000..10d6eb23f --- /dev/null +++ b/drivers/ilanzou/util_test.go @@ -0,0 +1,11 @@ +package ilanzou + +import "testing" + +func TestAppTokenQueryValue(t *testing.T) { + got := appTokenQueryValue("alpha:beta+gamma&delta#percent%") + want := "alpha:beta%2Bgamma%26delta%23percent%25" + if got != want { + t.Fatalf("appTokenQueryValue() = %q, want %q", got, want) + } +} diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index f2599b283..e702ebcac 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -170,23 +170,17 @@ var findKVReg = regexp.MustCompile(`'(.+?)':('?([^' },]*)'?)`) // 拆分kv // 根据key查询js变量 func findJSVarFunc(key, data string) string { - var values []string - if key != "sasign" { - values = regexp.MustCompile(`var ` + key + `\s*=\s*['"]?(.+?)['"]?;`).FindStringSubmatch(data) - } else { - matches := regexp.MustCompile(`var `+key+`\s*=\s*['"]?(.+?)['"]?;`).FindAllStringSubmatch(data, -1) - if len(matches) == 3 { - values = matches[1] - } else { - if len(matches) > 0 { - values = matches[0] - } - } - } - if len(values) == 0 { + re := regexp.MustCompile(`var\s+` + regexp.QuoteMeta(key) + `\s*=\s*['"]?([^'"]*)['"]?\s*;`) + matches := re.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { return "" } - return values[1] + for i := len(matches) - 1; i >= 0; i-- { + if matches[i][1] != "" { + return matches[i][1] + } + } + return matches[len(matches)-1][1] } var findFunction = regexp.MustCompile(`(?ims)^function[^{]+`) diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index f20f5ae0b..c02df953f 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -313,7 +313,37 @@ var findSubFolderReg = regexp.MustCompile(`(?i)(?:folderlink|mbxfolder).+href="/ var findDownPageParamReg = regexp.MustCompile(` 1 { - fileID = fileIDs[1] - } else { + matches := findFileIDReg.FindStringSubmatch(sharePageData) + if len(matches) < 2 { return nil, fmt.Errorf("not find file id") } + ajaxUrl := d.ShareUrl + matches[0][1:len(matches[0])-1] var resp FileShareInfoAndUrlResp[string] - _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(ajaxUrl, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -439,20 +467,23 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( return nil, err } nextPageData := RemoveNotes(string(data)) - param, err = htmlJsonToMap(nextPageData) - if err != nil { - return nil, err - } - fileIDs := findFileIDReg.FindStringSubmatch(nextPageData) - var fileID string - if len(fileIDs) > 1 { - fileID = fileIDs[1] + var resp FileShareInfoAndUrlResp[int] + matches := findFileIDReg.FindStringSubmatch(nextPageData) + if len(matches) >= 2 { + // 旧版结构:相对路径 /ajaxm.php?file=N + param, err = htmlJsonToMap(nextPageData) + if err != nil { + return nil, err + } + ajaxUrl := d.ShareUrl + matches[0][1:len(matches[0])-1] + _, err = d.post(ajaxUrl, func(req *resty.Request) { req.SetFormData(param) }, &resp) + } else if fnUrl, fnForm, ferr := parseFnPage(nextPageData); ferr == nil { + // 2026-10 改版结构:/fn? 内页携带 apifile 绝对地址与签名参数 + _, err = d.post(fnUrl, func(req *resty.Request) { req.SetFormData(fnForm) }, &resp) } else { return nil, fmt.Errorf("not find file id") } - var resp FileShareInfoAndUrlResp[int] - _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } diff --git a/drivers/local/driver.go b/drivers/local/driver.go index b3515f805..c51b1e762 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -154,8 +153,7 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" if d.Thumbnail { - typeName := utils.GetFileType(f.Name()) - if typeName == conf.IMAGE || typeName == conf.VIDEO { + if d.supportsThumbnail(f.Name()) { thumb = common.GetApiUrl(ctx) + stdpath.Join("/d", reqPath, f.Name()) thumb = utils.EncodePath(thumb, true) thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) @@ -241,7 +239,7 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( var thumbPath *string err := d.thumbTokenBucket.Do(ctx, func() error { var err error - buf, thumbPath, err = d.getThumb(file) + buf, thumbPath, err = d.getThumb(ctx, file) return err }) if err != nil { diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 9073e2c4f..7ef3e3a4c 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -9,6 +9,7 @@ type Addition struct { driver.RootPath DirectorySize bool `json:"directory_size" default:"false" help:"This might impact host performance"` Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + PDFThumbnail bool `json:"pdf_thumbnail" default:"false" required:"false" help:"Generate PDF first-page thumbnails with Quick Look on macOS"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` diff --git a/drivers/local/pdf_thumb_darwin.go b/drivers/local/pdf_thumb_darwin.go new file mode 100644 index 000000000..6542cb399 --- /dev/null +++ b/drivers/local/pdf_thumb_darwin.go @@ -0,0 +1,45 @@ +//go:build darwin + +package local + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" +) + +func pdfThumbnailSupported() bool { + return true +} + +func renderPDFThumbnail(ctx context.Context, fullPath string) (*bytes.Buffer, error) { + tempDir, err := os.MkdirTemp("", "openlist-pdf-thumb-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tempDir) + + renderCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + cmd := exec.CommandContext(renderCtx, "/usr/bin/qlmanage", "-t", "-s", "512", "-o", tempDir, fullPath) + if output, err := cmd.CombinedOutput(); err != nil { + if renderCtx.Err() == context.DeadlineExceeded { + return nil, fmt.Errorf("render PDF thumbnail timed out: %w", renderCtx.Err()) + } + if renderCtx.Err() != nil { + return nil, fmt.Errorf("render PDF thumbnail canceled: %w", renderCtx.Err()) + } + return nil, fmt.Errorf("render PDF thumbnail: %w: %s", err, bytes.TrimSpace(output)) + } + + thumbPath := filepath.Join(tempDir, filepath.Base(fullPath)+".png") + data, err := os.ReadFile(thumbPath) + if err != nil { + return nil, fmt.Errorf("read rendered PDF thumbnail: %w", err) + } + return bytes.NewBuffer(data), nil +} diff --git a/drivers/local/pdf_thumb_darwin_test.go b/drivers/local/pdf_thumb_darwin_test.go new file mode 100644 index 000000000..ec00ed633 --- /dev/null +++ b/drivers/local/pdf_thumb_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package local + +import ( + "bytes" + "context" + "errors" + "image/png" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestRenderPDFThumbnailDarwin(t *testing.T) { + tempDir := t.TempDir() + textPath := filepath.Join(tempDir, "source.txt") + pdfPath := filepath.Join(tempDir, "source 文件.pdf") + if err := os.WriteFile(textPath, []byte("OpenList PDF thumbnail integration test\n"), 0o600); err != nil { + t.Fatal(err) + } + + cmd := exec.Command("/usr/sbin/cupsfilter", textPath) + pdfData, err := cmd.Output() + if err != nil { + t.Fatalf("create fixture PDF: %v", err) + } + if err := os.WriteFile(pdfPath, pdfData, 0o600); err != nil { + t.Fatal(err) + } + + thumb, err := renderPDFThumbnail(context.Background(), pdfPath) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(thumb.Bytes(), []byte("\x89PNG\r\n\x1a\n")) { + t.Fatal("rendered thumbnail is not PNG") + } + cfg, err := png.DecodeConfig(bytes.NewReader(thumb.Bytes())) + if err != nil { + t.Fatalf("decode thumbnail: %v", err) + } + if cfg.Width <= 0 || cfg.Height <= 0 { + t.Fatalf("invalid thumbnail dimensions: %dx%d", cfg.Width, cfg.Height) + } + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := renderPDFThumbnail(canceledCtx, pdfPath); !errors.Is(err, context.Canceled) { + t.Fatalf("renderPDFThumbnail with canceled context returned %v, want context.Canceled", err) + } +} diff --git a/drivers/local/pdf_thumb_test.go b/drivers/local/pdf_thumb_test.go new file mode 100644 index 000000000..614232fb7 --- /dev/null +++ b/drivers/local/pdf_thumb_test.go @@ -0,0 +1,40 @@ +package local + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestSupportsThumbnail(t *testing.T) { + oldImages := conf.SlicesMap[conf.ImageTypes] + oldVideos := conf.SlicesMap[conf.VideoTypes] + conf.SlicesMap[conf.ImageTypes] = []string{"jpg"} + conf.SlicesMap[conf.VideoTypes] = []string{"mp4"} + t.Cleanup(func() { + conf.SlicesMap[conf.ImageTypes] = oldImages + conf.SlicesMap[conf.VideoTypes] = oldVideos + }) + + tests := []struct { + name string + fileName string + pdfThumbnail bool + want bool + }{ + {name: "image", fileName: "cover.jpg", want: true}, + {name: "video", fileName: "movie.mp4", want: true}, + {name: "PDF disabled by default", fileName: "document.pdf", want: false}, + {name: "unrelated document", fileName: "document.txt", pdfThumbnail: true, want: false}, + {name: "PDF enabled when renderer is available", fileName: "document.PDF", pdfThumbnail: true, want: pdfThumbnailSupported()}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Local{Addition: Addition{PDFThumbnail: tt.pdfThumbnail}} + if got := d.supportsThumbnail(tt.fileName); got != tt.want { + t.Fatalf("supportsThumbnail(%q) = %v, want %v", tt.fileName, got, tt.want) + } + }) + } +} diff --git a/drivers/local/pdf_thumb_unsupported.go b/drivers/local/pdf_thumb_unsupported.go new file mode 100644 index 000000000..deb8e99f2 --- /dev/null +++ b/drivers/local/pdf_thumb_unsupported.go @@ -0,0 +1,17 @@ +//go:build !darwin + +package local + +import ( + "bytes" + "context" + "errors" +) + +func pdfThumbnailSupported() bool { + return false +} + +func renderPDFThumbnail(context.Context, string) (*bytes.Buffer, error) { + return nil, errors.New("PDF thumbnails are not supported on this platform") +} diff --git a/drivers/local/util.go b/drivers/local/util.go index f9f4f4c45..b3ccc1fd8 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -2,6 +2,7 @@ package local import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -127,7 +128,19 @@ func (d *Local) removeThumbCache(fullPath string) { _ = os.Remove(thumbPath) } -func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { +func (d *Local) supportsThumbnail(name string) bool { + typeName := utils.GetFileType(name) + if typeName == conf.IMAGE || typeName == conf.VIDEO { + return true + } + return d.supportsPDFThumbnail(name) +} + +func (d *Local) supportsPDFThumbnail(name string) bool { + return d.PDFThumbnail && pdfThumbnailSupported() && strings.EqualFold(filepath.Ext(name), ".pdf") +} + +func (d *Local) getThumb(ctx context.Context, file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail @@ -140,12 +153,19 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } } var srcBuf *bytes.Buffer - if utils.GetFileType(file.GetName()) == conf.VIDEO { + typeName := utils.GetFileType(file.GetName()) + if typeName == conf.VIDEO { videoBuf, err := d.GetSnapshot(fullPath) if err != nil { return nil, nil, err } srcBuf = videoBuf + } else if d.supportsPDFThumbnail(file.GetName()) { + pdfBuf, err := renderPDFThumbnail(ctx, fullPath) + if err != nil { + return nil, nil, err + } + srcBuf = pdfBuf } else { imgData, err := os.ReadFile(fullPath) if err != nil { diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index e01c1f7c1..1d82b8c28 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -52,16 +52,20 @@ func (d *Onedrive) InitReference(refStorage driver.Driver) error { } func (d *Onedrive) Drop(ctx context.Context) error { + d.mutex.Lock() + defer d.mutex.Unlock() + // RootFolderPath may change before this driver instance is reinitialized. + d.root = nil d.ref = nil return nil } func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) { + d.mutex.Lock() + defer d.mutex.Unlock() if d.root != nil { return d.root, nil } - d.mutex.Lock() - defer d.mutex.Unlock() root := &Object{ ObjThumb: model.ObjThumb{ Object: model.Object{ diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go index 13785939f..a0f7450d3 100644 --- a/drivers/onedrive_sharelink/util.go +++ b/drivers/onedrive_sharelink/util.go @@ -2,7 +2,6 @@ package onedrive_sharelink import ( "context" - "crypto/tls" "encoding/json" "fmt" "io" @@ -13,30 +12,28 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" + internalNet "github.com/OpenListTeam/OpenList/v4/internal/net" log "github.com/sirupsen/logrus" "golang.org/x/net/html" ) // NewNoRedirectClient creates an HTTP client that doesn't follow redirects func NewNoRedirectCLient() *http.Client { - return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, - // Prevent following redirects - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, + client := internalNet.NewHttpClient() + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse } + return client } // getCookiesWithPassword fetches cookies required for authenticated access using the provided password -func getCookiesWithPassword(link, password string) (string, error) { +func getCookiesWithPassword(ctx context.Context, link, password string) (string, error) { // Send GET request - resp, err := http.Get(link) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil) + if err != nil { + return "", err + } + resp, err := base.HttpClient.Do(req) if err != nil { return "", err } @@ -95,16 +92,18 @@ func getCookiesWithPassword(link, password string) (string, error) { "__VIEWSTATEENCRYPTED": []string{""}, } - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } + client := NewNoRedirectCLient() // Send the POST request, preventing redirects - resp, err = client.PostForm(newURL, data) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, newURL, strings.NewReader(data.Encode())) if err != nil { return "", err } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err = client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() // Extract the desired cookie value cookie := resp.Cookies() @@ -153,6 +152,7 @@ func (d *OnedriveSharelink) getHeaders(ctx context.Context) (http.Header, error) if err != nil { return nil, err } + defer answerNoRedirect.Body.Close() redirectUrl := answerNoRedirect.Header.Get("Location") log.Debugln("redirectUrl:", redirectUrl) if redirectUrl == "" { @@ -169,7 +169,7 @@ func (d *OnedriveSharelink) getHeaders(ctx context.Context) (http.Header, error) header.Set("authority", u.Host) return header, nil } else { - cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword) + cookie, err := getCookiesWithPassword(ctx, d.ShareLinkURL, d.ShareLinkPassword) if err != nil { return nil, err } @@ -197,6 +197,7 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, if err != nil { return nil, err } + _ = answerNoRedirect.Body.Close() redirectUrl = answerNoRedirect.Header.Get("Location") } else { header = d.Headers @@ -205,6 +206,7 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, if err != nil { return nil, err } + _ = answerNoRedirect.Body.Close() redirectUrl = answerNoRedirect.Header.Get("Location") } redirectSplitURL := strings.Split(redirectUrl, "/") @@ -289,9 +291,9 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, } tempHeader["Content-Type"] = []string{"application/json;odata=verbose"} - client := &http.Client{} + client := base.HttpClient postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql" - req, err = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(graphqlVar)) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, postUrl, strings.NewReader(graphqlVar)) if err != nil { return nil, err } @@ -326,7 +328,10 @@ func (d *OnedriveSharelink) getFiles(ctx context.Context, path string) ([]Item, log.Debugln("nextHref:", nextHref) graphqlReqNEW := GraphQLNEWRequest{} postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref - req, _ = http.NewRequest(http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, postUrl, strings.NewReader(renderListDataAsStreamVar)) + if err != nil { + return nil, err + } req.Header = tempHeader resp, err := client.Do(req) diff --git a/drivers/onedrive_sharelink/util_test.go b/drivers/onedrive_sharelink/util_test.go new file mode 100644 index 000000000..e0bdbad23 --- /dev/null +++ b/drivers/onedrive_sharelink/util_test.go @@ -0,0 +1,53 @@ +package onedrive_sharelink + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + internalNet "github.com/OpenListTeam/OpenList/v4/internal/net" +) + +func TestNoRedirectClientUsesSharedSettings(t *testing.T) { + oldConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = oldConf }) + client := NewNoRedirectCLient() + sharedClient := internalNet.NewHttpClient() + if client.Timeout != sharedClient.Timeout { + t.Fatalf("expected shared timeout %s, got %s", sharedClient.Timeout, client.Timeout) + } + if reflect.TypeOf(client.Transport) != reflect.TypeOf(sharedClient.Transport) { + t.Fatalf("expected shared transport type %T, got %T", sharedClient.Transport, client.Transport) + } +} + +func TestNoRedirectClientStopsRedirects(t *testing.T) { + oldConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = oldConf }) + targetRequested := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/target" { + targetRequested = true + w.WriteHeader(http.StatusNoContent) + return + } + http.Redirect(w, r, "/target", http.StatusFound) + })) + defer server.Close() + + resp, err := NewNoRedirectCLient().Get(server.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusFound { + t.Fatalf("expected redirect response, got %d", resp.StatusCode) + } + if targetRequested { + t.Fatal("redirect target was requested") + } +} diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index c602bd2a1..da3da551d 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -10,10 +10,11 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` - RefreshToken string `json:"refresh_token" required:"true" default:""` + RefreshToken string `json:"refresh_token" required:"false" default:""` CaptchaToken string `json:"captcha_token" default:""` DeviceID string `json:"device_id" required:"false" default:""` DisableMediaLink bool `json:"disable_media_link" default:"true"` + SkipVerification bool `json:"skip_verification" default:"false" help:"ignore the human verification URL returned by the captcha API instead of failing; enabling this may trigger PikPak risk control"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 1d091217a..2d5b0729a 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -100,12 +100,13 @@ func (d *PikPak) login() error { return errors.New("username or password is empty") } + // Clear expired access token so captcha requests don't carry a stale bearer + d.AccessToken = "" + url := "https://user.mypikpak.net/v1/auth/signin" - // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) - if d.GetCaptchaToken() == "" { - if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { - return err - } + // Always refresh captcha token before signin (it may be expired) + if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { + return err } var e ErrResp @@ -125,7 +126,12 @@ func (d *PikPak) login() error { data := res.Body() d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() + if d.AccessToken == "" || d.RefreshToken == "" { + return errors.New("login failed: server returned empty tokens") + } d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) return nil } @@ -159,9 +165,14 @@ func (d *PikPak) refreshToken(refreshToken string) error { return errors.New(e.Error()) } data := res.Body() + newAccessToken := jsoniter.Get(data, "access_token").ToString() + newRefreshToken := jsoniter.Get(data, "refresh_token").ToString() + if newAccessToken == "" || newRefreshToken == "" { + return errors.New("refresh failed: server returned empty tokens") + } d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.RefreshToken = newRefreshToken + d.AccessToken = newAccessToken d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) @@ -197,12 +208,18 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r case 0: return res.Body(), nil case 4122, 4121, 16: - // access_token 过期 + if strings.Contains(url, "/v1/auth/") || strings.Contains(url, "/v1/shield/captcha/") { + return nil, errors.New(e.Error()) + } + // access_token expired, refresh and retry if err1 := d.refreshToken(d.RefreshToken); err1 != nil { return nil, err1 } return d.request(url, method, callback, resp) - case 9: // 验证码token过期 + case 9: // captcha token expired + if strings.Contains(url, "/v1/shield/captcha/") { + return nil, errors.New(e.Error()) + } if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { return nil, err } @@ -369,6 +386,9 @@ func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { } else { metas["username"] = username } + metas["client_version"] = d.ClientVersion + metas["package_name"] = d.PackageName + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() return d.refreshCaptchaToken(action, metas) } @@ -407,7 +427,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err return errors.New(e.Error()) } - if resp.Url != "" { + if resp.Url != "" && !d.Addition.SkipVerification { return fmt.Errorf(`need verify: Click Here`, resp.Url) } @@ -503,9 +523,11 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试,共尝试3次 for retry := 0; retry < 3; retry++ { - select { - case <-ctx.Done(): + if utils.IsCanceled(ctx) { + err = ctx.Err() break + } + select { case <-ticker.C: errCh <- errors.Wrap(err, "ossToken 过期") default: diff --git a/drivers/pikpak/util_test.go b/drivers/pikpak/util_test.go new file mode 100644 index 000000000..7c69ea532 --- /dev/null +++ b/drivers/pikpak/util_test.go @@ -0,0 +1,720 @@ +package pikpak + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/glebarez/sqlite" + "github.com/go-resty/resty/v2" + "gorm.io/gorm" +) + +// --- Helper function tests --- + +func TestGetAction(t *testing.T) { + tests := []struct { + method string + url string + want string + }{ + {"GET", "https://api-drive.mypikpak.net/drive/v1/files", "GET:/drive/v1/files"}, + {"POST", "https://user.mypikpak.net/v1/auth/signin", "POST:/v1/auth/signin"}, + {"POST", "https://user.mypikpak.net/v1/shield/captcha/init", "POST:/v1/shield/captcha/init"}, + {"GET", "https://api-drive.mypikpak.net/drive/v1/files?page_token=abc", "GET:/drive/v1/files"}, + {"POST", "https://user.mypikpak.net/v1/auth/token", "POST:/v1/auth/token"}, + } + for _, tt := range tests { + t.Run(tt.method+":"+tt.url, func(t *testing.T) { + got := GetAction(tt.method, tt.url) + if got != tt.want { + t.Errorf("GetAction(%q, %q) = %q, want %q", tt.method, tt.url, got, tt.want) + } + }) + } +} + +func TestGetCaptchaSign(t *testing.T) { + c := &Common{ + ClientID: "YNxT9w7GMdWvEOKa", + ClientVersion: "1.53.2", + PackageName: "com.pikcloud.pikpak", + DeviceID: "test-device-id", + Algorithms: AndroidAlgorithms, + } + + timestamp, sign := c.GetCaptchaSign() + if timestamp == "" { + t.Fatal("timestamp should not be empty") + } + if len(sign) != 34 { + t.Fatalf("sign length should be 34 (\"1.\" + 32 hex), got %d: %q", len(sign), sign) + } + if sign[:2] != "1." { + t.Errorf("sign should start with '1.', got %q", sign[:2]) + } +} + +func TestGenerateDeviceSign(t *testing.T) { + sign := generateDeviceSign("test-device", "com.pikcloud.pikpak") + if len(sign) < 7 { + t.Fatal("device sign too short") + } + if sign[:7] != "div101." { + t.Errorf("device sign should start with 'div101.', got %q", sign[:7]) + } + // Deterministic + if sign != generateDeviceSign("test-device", "com.pikcloud.pikpak") { + t.Error("generateDeviceSign should be deterministic") + } +} + +func TestBuildCustomUserAgent(t *testing.T) { + ua := BuildCustomUserAgent("dev123", AndroidClientID, AndroidPackageName, + AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "user456") + for _, want := range []string{"ANDROID-", "clientid/", "deviceid/dev123", "usrno/user456"} { + if !strings.Contains(ua, want) { + t.Errorf("user agent should contain %q", want) + } + } +} + +// --- Auth recovery behavior tests --- + +func TestErrRespErrorClassification(t *testing.T) { + tests := []struct { + name string + resp ErrResp + wantError bool + wantCode int64 + }{ + {"success", ErrResp{ErrorCode: 0}, false, 0}, + {"access_token_expired_4122", ErrResp{ErrorCode: 4122, ErrorMsg: "access_token expired"}, true, 4122}, + {"access_token_expired_4121", ErrResp{ErrorCode: 4121, ErrorMsg: "access_token expired"}, true, 4121}, + {"unauthenticated_16", ErrResp{ErrorCode: 16, ErrorMsg: "unauthenticated"}, true, 16}, + {"refresh_token_invalid_4126", ErrResp{ErrorCode: 4126, ErrorMsg: "invalid_grant"}, true, 4126}, + {"captcha_expired_9", ErrResp{ErrorCode: 9, ErrorMsg: "captcha_invalid"}, true, 9}, + {"rate_limit_10", ErrResp{ErrorCode: 10, ErrorDescription: "too frequent"}, true, 10}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotError := tt.resp.IsError() + if gotError != tt.wantError { + t.Errorf("IsError() = %v, want %v", gotError, tt.wantError) + } + if tt.resp.ErrorCode != tt.wantCode { + t.Errorf("ErrorCode = %d, want %d", tt.resp.ErrorCode, tt.wantCode) + } + }) + } +} + +// TestGuardClauseOnAuthURLDoesNotRefresh verifies that when the auth endpoint +// itself reports 4122, request() fails fast instead of calling refreshToken() +// (which would recurse). Real behavior, real code path: with the guard +// removed from request(), the token endpoint would be hit a second time. +func TestGuardClauseOnAuthURLDoesNotRefresh(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + + _, err := d.request("https://user.mypikpak.net/v1/auth/token", http.MethodPost, nil, nil) + if err == nil { + t.Fatal("request() to an auth URL must fail on 4122 instead of refreshing") + } + if got := m.count(pathToken); got != 1 { + t.Errorf("guard clause violated: token endpoint hit %d times, want exactly 1 (no refreshToken recursion)", got) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("no re-login expected, got %d signin calls", got) + } +} + +// --- Integration scaffolding: in-memory DB + mock PikPak endpoints --- + +var ( + setupDBOnce sync.Once + setupDBErr error + rowSeq int64 +) + +// setupTestDB mirrors internal/op/storage_test.go: an in-memory SQLite +// database behind internal/db, so op.MustSaveDriverStorage really persists +// and tests can assert on the saved row instead of on comments. +func setupTestDB(t *testing.T) { + t.Helper() + setupDBOnce.Do(func() { + var gormDB *gorm.DB + gormDB, setupDBErr = gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if setupDBErr != nil { + return + } + conf.Conf = conf.DefaultConfig("testdata") + db.Init(gormDB) + }) + if setupDBErr != nil { + t.Fatalf("failed to set up test database: %v", setupDBErr) + } +} + +// createStorageRow inserts a fresh storage row and returns it, so that +// MustSaveDriverStorage during a test performs an UPDATE that can be read +// back afterwards. +func createStorageRow(t *testing.T) *model.Storage { + t.Helper() + rowSeq++ + st := &model.Storage{ + Driver: "PikPak", + MountPath: fmt.Sprintf("/pikpak-test-%d", rowSeq), + Addition: `{"username":"tester@example.com","password":"pw"}`, + } + if err := db.CreateStorage(st); err != nil { + t.Fatalf("failed to create storage row: %v", err) + } + return st +} + +func persistedRefreshToken(t *testing.T, id uint) string { + t.Helper() + st, err := db.GetStorageById(id) + if err != nil { + t.Fatalf("failed to read storage back: %v", err) + } + var a Addition + if err := json.Unmarshal([]byte(st.Addition), &a); err != nil { + t.Fatalf("failed to decode persisted addition %q: %v", st.Addition, err) + } + return a.RefreshToken +} + +// mockCall records one request received by the mock server. +type mockCall struct { + headers http.Header + body map[string]any +} + +func (c mockCall) captchaToken() string { + s, _ := c.body["captcha_token"].(string) + return s +} + +// pikpakMock emulates the captcha/auth endpoints used by login() and +// refreshToken(), plus one drive endpoint that serves as the entry point of +// the recovery chain. The drive endpoint fails exactly once (with the code +// configured in driveFirstStatus) and succeeds afterwards, so request() can +// only complete if recovery actually ran. +type pikpakMock struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + calls map[string][]mockCall + + captchaTokenOut string + captchaURL string + + tokenStatus int + tokenBody map[string]any + + signinStatus int + signinBody map[string]any + + driveFirstStatus int + driveFirstBody map[string]any // body served on the first drive call only + driveBody map[string]any // body served afterwards + driveHits int +} + +func newPikpakMock(t *testing.T) *pikpakMock { + t.Helper() + m := &pikpakMock{ + t: t, + calls: map[string][]mockCall{}, + captchaTokenOut: "cap-fresh", + tokenStatus: http.StatusOK, + tokenBody: map[string]any{"access_token": "at-2", "refresh_token": "rt-2", "sub": "user-1"}, + signinStatus: http.StatusOK, + signinBody: map[string]any{"access_token": "at-new", "refresh_token": "rt-new", "sub": "user-1"}, + driveFirstStatus: http.StatusOK, + driveFirstBody: map[string]any{"files": []any{}, "next_page_token": ""}, + driveBody: map[string]any{"files": []any{}, "next_page_token": ""}, + } + m.srv = httptest.NewServer(http.HandlerFunc(m.serve)) + return m +} + +func (m *pikpakMock) close() { m.srv.Close() } + +func (m *pikpakMock) serve(w http.ResponseWriter, r *http.Request) { + body := map[string]any{} + if raw, err := io.ReadAll(r.Body); err == nil && len(raw) > 0 { + _ = json.Unmarshal(raw, &body) + } + m.mu.Lock() + m.calls[r.URL.Path] = append(m.calls[r.URL.Path], mockCall{headers: r.Header.Clone(), body: body}) + status := http.StatusOK + payload := any(map[string]any{}) + switch { + case strings.HasSuffix(r.URL.Path, "/v1/shield/captcha/init"): + payload = map[string]any{"captcha_token": m.captchaTokenOut, "expires_in": 3600, "url": m.captchaURL} + case strings.HasSuffix(r.URL.Path, "/v1/auth/signin"): + status = m.signinStatus + payload = m.signinBody + case strings.HasSuffix(r.URL.Path, "/v1/auth/token"): + status = m.tokenStatus + payload = m.tokenBody + case strings.HasSuffix(r.URL.Path, "/drive/v1/files"): + m.driveHits++ + if m.driveHits == 1 { + status = m.driveFirstStatus + payload = m.driveFirstBody + } else { + payload = m.driveBody + } + default: + m.mu.Unlock() + m.t.Errorf("unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + m.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func (m *pikpakMock) count(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.calls[path]) +} + +func (m *pikpakMock) reset() { + m.mu.Lock() + defer m.mu.Unlock() + m.calls = map[string][]mockCall{} + m.driveHits = 0 +} + +func (m *pikpakMock) last(path string) mockCall { + m.mu.Lock() + defer m.mu.Unlock() + calls := m.calls[path] + if len(calls) == 0 { + m.t.Fatalf("no recorded call for %s", path) + } + return calls[len(calls)-1] +} + +// installMockClient replaces base.RestyClient with a client whose requests to +// the hard-coded PikPak hosts are rewritten onto the mock server, and returns +// a restore function. The rewrite happens in OnBeforeRequest, which resty +// runs before its internal parseRequestURL/createHTTPRequest middlewares. +func installMockClient(m *pikpakMock) func() { + old := base.RestyClient + client := resty.New() + client.OnBeforeRequest(func(_ *resty.Client, req *resty.Request) error { + for _, host := range []string{"https://user.mypikpak.net", "https://api-drive.mypikpak.net"} { + if strings.HasPrefix(req.URL, host) { + req.URL = strings.Replace(req.URL, host, m.srv.URL, 1) + } + } + return nil + }) + base.RestyClient = client + return func() { base.RestyClient = old } +} + +// newTestDriver builds a PikPak with a fully initialized Common (web platform +// constants) and a fresh storage row in the DB, ready for auth-flow tests. +func newTestDriver(t *testing.T) (*PikPak, uint) { + t.Helper() + setupTestDB(t) + st := createStorageRow(t) + d := &PikPak{} + d.SetStorage(*st) + d.Platform = "web" + d.Username = "tester@example.com" + d.Password = "pw" + d.Common = &Common{ + ClientID: WebClientID, + ClientSecret: WebClientSecret, + ClientVersion: WebClientVersion, + PackageName: WebPackageName, + DeviceID: "test-device", + UserAgent: "test-agent", + Algorithms: WebAlgorithms, + } + d.Common.RefreshCTokenCk = func(token string) { + d.Common.CaptchaToken = token + } + return d, st.ID +} + +const ( + pathCaptchaInit = "/v1/shield/captcha/init" + pathSignin = "/v1/auth/signin" + pathToken = "/v1/auth/token" + pathFiles = "/drive/v1/files" +) + +// --- Main auth recovery path --- + +// TestMainRecoveryPath exercises the full chain the PR is about: a drive +// request fails with 4122, refreshToken fails with 4126, login() runs (fresh +// captcha + password signin), the new refresh token is persisted to the DB, +// and request() retries the original call successfully with the new tokens. +func TestMainRecoveryPath(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, id := newTestDriver(t) + d.RefreshToken = "rt-old" + d.AccessToken = "at-stale" + d.SetCaptchaToken("cap-stale") + d.Addition.RefreshToken = "rt-old" + + // refresh attempt fails with "refresh token invalid" + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4126, "error": "invalid_grant"} + // the first drive call reports an expired access token; the retry succeeds + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() returned error even though recovery should succeed: %v", err) + } + + if got := m.count(pathToken); got != 1 { + t.Errorf("expected exactly 1 refresh request, got %d", got) + } + if got := m.count(pathSignin); got != 1 { + t.Errorf("expected exactly 1 signin (re-login), got %d", got) + } + if got := m.count(pathFiles); got != 2 { + t.Errorf("expected 2 files requests (failed + retried), got %d", got) + } + if got := m.count(pathCaptchaInit); got != 1 { + t.Errorf("expected exactly 1 captcha/init call during re-login, got %d", got) + } + + // The retry must carry the tokens obtained via re-login, not the stale ones. + lastFiles := m.last(pathFiles) + if got := lastFiles.headers.Get("Authorization"); got != "Bearer at-new" { + t.Errorf("retried request Authorization = %q, want %q", got, "Bearer at-new") + } + if got := lastFiles.headers.Get("X-Captcha-Token"); got != "cap-fresh" { + t.Errorf("retried request X-Captcha-Token = %q, want %q", got, "cap-fresh") + } + + // Tokens were rotated in memory... + if d.AccessToken != "at-new" { + t.Errorf("AccessToken = %q, want %q", d.AccessToken, "at-new") + } + if d.RefreshToken != "rt-new" { + t.Errorf("RefreshToken = %q, want %q", d.RefreshToken, "rt-new") + } + // ...and the rotated refresh token was really persisted. + if got := persistedRefreshToken(t, id); got != "rt-new" { + t.Errorf("persisted addition refresh_token = %q, want %q", got, "rt-new") + } +} + +// TestRefreshToken4126WithoutCredentialsDoesNotLogin checks that a 4126 with +// empty username/password yields the "re-provide refresh_token" error instead +// of attempting a password login. +func TestRefreshToken4126WithoutCredentialsDoesNotLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.Username = "" + d.Password = "" + + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 4126, "error": "invalid_grant"} + + err := d.refreshToken("rt-old") + if err == nil { + t.Fatal("refreshToken() with invalid refresh token and no credentials must fail") + } + if !strings.Contains(err.Error(), "re-provide") { + t.Errorf("unexpected error text: %v", err) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("signin must not be attempted without credentials, got %d calls", got) + } +} + +// TestRefreshTokenOtherErrorDoesNotLogin checks that a non-4126 refresh +// failure propagates without triggering a re-login (4126 is the single +// documented trigger). +func TestRefreshTokenOtherErrorDoesNotLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + m.tokenStatus = http.StatusBadRequest + m.tokenBody = map[string]any{"error_code": 10, "error_description": "too frequent"} + + if err := d.refreshToken("rt-old"); err == nil { + t.Fatal("refreshToken() must propagate a non-4126 error") + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("signin must not be attempted for non-4126 errors, got %d calls", got) + } +} + +// --- Token validation (replaces TestTokenValidationRejectsEmpty) --- + +// TestTokenValidationRejectsEmpty drives login() and refreshToken() against +// 200 responses that carry empty tokens and requires both paths to refuse +// them without persisting anything. +func TestTokenValidationRejectsEmpty(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + // login(): signin answers 200 but with an empty access_token. + d, id := newTestDriver(t) + m.signinBody = map[string]any{"access_token": "", "refresh_token": "rt-x", "sub": "user-1"} + if err := d.login(); err == nil { + t.Fatal("login() must reject empty access_token") + } + if got := persistedRefreshToken(t, id); got != "" { + t.Errorf("login() must not persist tokens when validation fails, persisted %q", got) + } + + // login(): symmetric case — empty refresh_token but non-empty access_token. + d3, id3 := newTestDriver(t) + m.reset() + m.signinBody = map[string]any{"access_token": "at-x", "refresh_token": "", "sub": "user-1"} + if err := d3.login(); err == nil { + t.Fatal("login() must reject empty refresh_token") + } + if got := persistedRefreshToken(t, id3); got != "" { + t.Errorf("login() must not persist tokens when validation fails, persisted %q", got) + } + + // refreshToken(): 200 but empty refresh_token. + d2, id2 := newTestDriver(t) + m.tokenStatus = http.StatusOK + m.tokenBody = map[string]any{"access_token": "at-x", "refresh_token": "", "sub": "user-1"} + if err := d2.refreshToken("rt-old"); err == nil { + t.Fatal("refreshToken() must reject empty refresh_token") + } + if got := persistedRefreshToken(t, id2); got != "" { + t.Errorf("refreshToken() must not persist tokens when validation fails, persisted %q", got) + } +} + +// --- Captcha refresh (replaces TestCaptchaAlwaysRefreshedBeforeLogin) --- + +// TestCaptchaAlwaysRefreshedBeforeLogin proves login() fetches a fresh captcha +// even when a (possibly expired) token is already present, and that signin is +// performed with the fresh token rather than the stale one. +func TestCaptchaAlwaysRefreshedBeforeLogin(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.SetCaptchaToken("cap-stale") // non-empty and (conceptually) expired + + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + if got := m.count(pathCaptchaInit); got != 1 { + t.Fatalf("expected exactly 1 captcha/init call despite a non-empty stale token, got %d", got) + } + if got := m.last(pathSignin).captchaToken(); got != "cap-fresh" { + t.Errorf("signin used captcha_token %q, want the fresh %q", got, "cap-fresh") + } + if got := d.GetCaptchaToken(); got != "cap-fresh" { + t.Errorf("driver CaptchaToken = %q after login, want %q", got, "cap-fresh") + } +} + +// --- Stale bearer cleared before login --- + +// TestLoginClearsStaleAccessToken checks that the captcha/init and signin +// requests issued by login() do not carry the expired bearer token. +func TestLoginClearsStaleAccessToken(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.AccessToken = "at-stale" + + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + for _, path := range []string{pathCaptchaInit, pathSignin} { + if got := m.last(path).headers.Get("Authorization"); got != "" { + t.Errorf("%s request carried Authorization %q, want it cleared before login", path, got) + } + } +} + +// --- Captcha meta completeness --- + +// TestCaptchaMetaCompleteness asserts captcha/init on the login path carries +// the same meta fields RefreshCaptchaTokenAtLogin sends on main. +func TestCaptchaMetaCompleteness(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + if err := d.login(); err != nil { + t.Fatalf("login() failed: %v", err) + } + + meta, _ := m.last(pathCaptchaInit).body["meta"].(map[string]any) + for _, key := range []string{"email", "client_version", "package_name", "timestamp", "captcha_sign"} { + if v, ok := meta[key]; !ok || v == "" { + t.Errorf("captcha meta missing or empty %q (got %#v)", key, meta) + } + } +} + +// --- refreshToken success path (highest-frequency production path) --- + +// TestRefreshTokenSuccessRotatesAndPersists covers 4122 -> refreshToken() +// succeeding: rotated tokens land in memory, the retry carries the new bearer, +// and the new refresh token is persisted to the DB. +func TestRefreshTokenSuccessRotatesAndPersists(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, id := newTestDriver(t) + d.RefreshToken = "rt-old" + d.AccessToken = "at-stale" + d.Addition.RefreshToken = "rt-old" + + m.tokenStatus = http.StatusOK + m.tokenBody = map[string]any{"access_token": "at-2", "refresh_token": "rt-2", "sub": "user-1"} + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 4122, "error": "access_token_expired"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() failed even though refresh should succeed: %v", err) + } + + if got := m.count(pathSignin); got != 0 { + t.Errorf("a successful refresh must not fall through to password login, got %d signin calls", got) + } + if d.AccessToken != "at-2" { + t.Errorf("AccessToken = %q, want %q", d.AccessToken, "at-2") + } + if d.RefreshToken != "rt-2" { + t.Errorf("RefreshToken = %q, want %q", d.RefreshToken, "rt-2") + } + if got := m.last(pathFiles).headers.Get("Authorization"); got != "Bearer at-2" { + t.Errorf("retried request Authorization = %q, want %q", got, "Bearer at-2") + } + if got := persistedRefreshToken(t, id); got != "rt-2" { + t.Errorf("persisted addition refresh_token = %q, want %q", got, "rt-2") + } +} + +// --- captcha expired (case 9) --- + +// TestCaptchaExpiredRefreshesAndRetries covers request() case 9: a captcha +// error on a drive call triggers a captcha refresh and one retry. +func TestCaptchaExpiredRefreshesAndRetries(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + d, _ := newTestDriver(t) + d.AccessToken = "at-ok" + d.RefreshToken = "rt-ok" + d.SetCaptchaToken("cap-stale") + + m.driveFirstStatus = http.StatusBadRequest + m.driveFirstBody = map[string]any{"error_code": 9, "error": "captcha_invalid"} + m.driveBody = map[string]any{"files": []any{}, "next_page_token": ""} + + var resp Files + if _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, nil, &resp); err != nil { + t.Fatalf("request() failed even though captcha refresh should recover: %v", err) + } + + if got := m.count(pathCaptchaInit); got == 0 { + t.Fatal("expected a captcha refresh after error code 9") + } + if got := m.count(pathFiles); got != 2 { + t.Errorf("expected 2 files requests (failed + retried), got %d", got) + } + if got := m.count(pathSignin); got != 0 { + t.Errorf("captcha recovery must not re-login, got %d signin calls", got) + } + if got := m.last(pathFiles).headers.Get("X-Captcha-Token"); got != "cap-fresh" { + t.Errorf("retried request X-Captcha-Token = %q, want %q", got, "cap-fresh") + } +} + +// --- SkipVerification (added by this PR) --- + +// TestSkipVerificationControlsVerificationURL covers the new config option: +// a captcha/init response carrying a human-verification url is fatal by +// default and ignored only when skip_verification is enabled. +func TestSkipVerificationControlsVerificationURL(t *testing.T) { + m := newPikpakMock(t) + defer m.close() + restore := installMockClient(m) + defer restore() + + m.captchaURL = "https://user.mypikpak.net/forbidden/test" + + d, _ := newTestDriver(t) + if err := d.login(); err == nil { + t.Fatal("login() must fail on a verification url by default") + } else if !strings.Contains(err.Error(), "need verify") { + t.Errorf("unexpected error: %v", err) + } + + d2, _ := newTestDriver(t) + d2.SkipVerification = true + if err := d2.login(); err != nil { + t.Fatalf("login() with skip_verification must ignore the url, got: %v", err) + } + if d2.AccessToken != "at-new" { + t.Errorf("AccessToken = %q after skipped verification, want %q", d2.AccessToken, "at-new") + } +} diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 4cf2efb14..775e86aea 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -58,20 +58,13 @@ func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCal if err != nil { return nil, err } + // 只认响应 Set-Cookie 里的新 __puus:请求头里的值是构建请求时的 d.Cookie 快照 + // (RestyClient 无 cookie jar),若按它回写,并发请求会把另一请求刚轮换的新值覆盖回旧值。 __puus := cookie.GetCookie(res.Cookies(), "__puus") if __puus != nil { log.Debugf("update __puus: %v", __puus) d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) d.SaveCookie(d.Cookie) - } else { - c := res.Request.Header.Get("Cookie") - v1 := cookie.GetStr(d.Cookie, "__puus") - v2 := cookie.GetStr(c, "__puus") - if v2 != "" && v1 != v2 { - d.Cookie = cookie.SetStr(d.Cookie, "__puus", v2) - log.Debugf("update cookie: %v %v %v", d.Cookie, v1, v2) - d.SaveCookie(d.Cookie) - } } if e.Status >= 400 || e.Code != 0 { diff --git a/drivers/quark_uc_share/driver_test.go b/drivers/quark_uc_share/driver_test.go index 06b67be5b..da2d60b84 100644 --- a/drivers/quark_uc_share/driver_test.go +++ b/drivers/quark_uc_share/driver_test.go @@ -3,6 +3,8 @@ package quark_uc_share import ( "context" "errors" + "net/http" + "net/http/httptest" "testing" "time" @@ -11,6 +13,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/cache" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" ) // stubMultiSourceDisabled 关闭多账号分片下载并还原,避免 Link() 读 setting 死锁(测试里 op 未初始化)。 @@ -395,8 +398,8 @@ func TestSpeedupLinkMarksStorageId(t *testing.T) { getSpeedupToken = func(d *QuarkUCShare, cookie, savedFid, fileName string) (string, error) { return "tok", nil } - speedupDownload = func(d *QuarkUCShare, cookie, savedFid, token string) (string, error) { - return "https://dl-c-x.quark.cn/some/path?auth=1", nil + speedupDownload = func(d *QuarkUCShare, cookie, savedFid, token string) (string, string, error) { + return "https://dl-c-x.quark.cn/some/path?auth=1", "", nil } t.Cleanup(func() { getSpeedupToken = origToken @@ -455,3 +458,56 @@ func TestBindRequestDriverUsesSelectedTVStorageForRequestAndTempDir(t *testing.T t.Fatalf("expected tv binding to reject a different storage instance") } } + +// requestAt 收到响应 Set-Cookie 轮换的 __puus 时,必须同时更新兜底 Cookie 与回调 +// onPuus(供调用方写回实际发请求的账号):否则账号 Cookie 停在登录时刻,约一天后 +// 被夸克 CDN 判失效 → 412(#1082)。 +func TestRequestAtRotatesPuusToFallbackAndSink(t *testing.T) { + var gotCookie, gotReferer string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotCookie = r.Header.Get("Cookie") + gotReferer = r.Header.Get("Referer") + w.Header().Set("Set-Cookie", "__puus=rotated-new; Path=/") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"status":200,"code":0,"data":{}}`)) + })) + defer srv.Close() + + setShareCookie("__puus=old; other=1") + d := &QuarkUCShare{config: driver.Config{Name: "QuarkShare"}, conf: Conf{ua: "ua", referer: "https://pan.quark.cn", api: srv.URL, pr: "ucpro"}} + + sinkCalls := "" + _, err := d.requestAt(srv.URL, "__puus=old; account=1", "/file/download", http.MethodPost, nil, nil, func(puus string) { + sinkCalls = puus + }) + if err != nil { + t.Fatalf("requestAt: %v", err) + } + if sinkCalls != "rotated-new" { + t.Fatalf("expected onPuus sink to receive rotated value, got %q", sinkCalls) + } + if v := cookie.GetStr(getShareCookie(), "__puus"); v != "rotated-new" { + t.Fatalf("expected fallback shareCookie updated, got %q", v) + } + if gotCookie != "__puus=old; account=1" || gotReferer != "https://pan.quark.cn" { + t.Fatalf("expected request to carry account cookie + referer, got cookie=%q referer=%q", gotCookie, gotReferer) + } + + // 无 onPuus 时兜底 Cookie 仍要更新(旧行为保持)。 + _, err = d.requestAt(srv.URL, "", "/x", http.MethodGet, nil, nil, nil) + if err != nil { + t.Fatalf("requestAt(no sink): %v", err) + } + if v := cookie.GetStr(getShareCookie(), "__puus"); v != "rotated-new" { + t.Fatalf("expected fallback shareCookie still updated without sink, got %q", v) + } +} + +// saveQuarkPuus 值未变时不得触发落库(测试桩驱动直接构造,SaveCookie 会触碰未初始化的 op, +// 所以只测跳过分支);新值路径由 requestAt 集成测试与线上日志覆盖。 +func TestSaveQuarkPuusSkipsUnchanged(t *testing.T) { + uc := &quark.QuarkOrUC{} + uc.Cookie = "__puus=same; a=1" + saveQuarkPuus(uc, "same") // 值相同:不落库,不应 panic + saveQuarkPuus(nil, "x") // nil 驱动:静默跳过 +} diff --git a/drivers/quark_uc_share/util.go b/drivers/quark_uc_share/util.go index 4c7d718db..8b6250aea 100755 --- a/drivers/quark_uc_share/util.go +++ b/drivers/quark_uc_share/util.go @@ -60,6 +60,25 @@ func updateShareCookiePuus(value string) { shareCookie = cookie.SetStr(shareCookie, "__puus", value) } +// saveQuarkPuus 把轮换的 __puus 并入网盘账号当前 Cookie 并持久化(SaveCookie 会落库 +// storage 并回写 alist-tvbox);值未变时跳过,避免并发播放重复触发落库与回写。 +func saveQuarkPuus(uc *quark.QuarkOrUC, puus string) { + if uc == nil || puus == "" || cookie.GetStr(uc.Cookie, "__puus") == puus { + return + } + log.Infof("[quark-uc-share] 账号 %d __puus 已轮换,写回账号 Cookie", uc.ID) + uc.SaveCookie(cookie.SetStr(uc.Cookie, "__puus", puus)) +} + +// saveTVPuus 同 saveQuarkPuus,作用于 TV 账号。 +func saveTVPuus(uc *quark_uc_tv.QuarkUCTV, puus string) { + if uc == nil || puus == "" || cookie.GetStr(uc.Cookie, "__puus") == puus { + return + } + log.Infof("[quark-uc-share] TV账号 %d __puus 已轮换,写回账号 Cookie", uc.ID) + uc.SaveCookie(cookie.SetStr(uc.Cookie, "__puus", puus)) +} + // shareFileID 是 fileToObj 拼出的 "{fid}-{share_fid_token}-{pdir_fid}"。 // 旧代码直接 strings.Split(id, "-") 取 s[1]/s[2],遇到不带 token 的 ID 会 panic。 type shareFileID struct { @@ -94,6 +113,10 @@ type shareRequestBinding interface { link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) // cookieValue 返回该绑定账号的 Cookie,供走 drive-pc 端点的分享请求使用。 cookieValue() string + // syncPuus 把响应里轮换的 __puus 写回本绑定账号的 Cookie 并持久化。 + // 免转存/分享请求不走 quark_uc.request(那里会自动滚 __puus),不回写的话 + // 账号 Cookie 停在登录时刻,约一天后被夸克 CDN 判失效 → 412(#1082)。 + syncPuus(puus string) } type requestBinding struct { @@ -115,7 +138,7 @@ func (b requestBinding) doRequest(d *QuarkUCShare, pathname string, method strin if b.requestDriver != nil { return b.requestDriver.Request(pathname, method, callback, resp) } - return d.directRequest(b.cookie, pathname, method, callback, resp) + return d.directRequest(b.cookie, pathname, method, callback, resp, nil) } func (b requestBinding) tempDirID() string { @@ -203,6 +226,14 @@ func (b requestBinding) cookieValue() string { return b.cookie } +func (b requestBinding) syncPuus(puus string) { + if b.requestDriver == nil { + updateShareCookiePuus(puus) + return + } + saveQuarkPuus(b.requestDriver, puus) +} + type requestTVBinding struct { requestDriver *quark_uc_tv.QuarkUCTV cookie string @@ -220,7 +251,7 @@ func bindTVRequestDriver(uc *quark_uc_tv.QuarkUCTV) requestTVBinding { } func (b requestTVBinding) doRequest(d *QuarkUCShare, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return d.directRequest(b.cookie, pathname, method, callback, resp) + return d.directRequest(b.cookie, pathname, method, callback, resp, b.syncPuus) } func (b requestTVBinding) tempDirID() string { @@ -323,6 +354,14 @@ func (b requestTVBinding) cookieValue() string { return b.cookie } +func (b requestTVBinding) syncPuus(puus string) { + if b.requestDriver == nil { + updateShareCookiePuus(puus) + return + } + saveTVPuus(b.requestDriver, puus) +} + func (d *QuarkUCShare) getDriverName() string { name := "Quark" if d.config.Name == "UCShare" { @@ -381,11 +420,11 @@ func (d *QuarkUCShare) request(pathname string, method string, callback base.Req return uc.Request(pathname, method, callback, resp) } - return d.directRequest(getShareCookie(), pathname, method, callback, resp) + return d.directRequest(getShareCookie(), pathname, method, callback, resp, nil) } -func (d *QuarkUCShare) directRequest(cookieStr string, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return d.requestAt(d.conf.api, cookieStr, pathname, method, callback, resp) +func (d *QuarkUCShare) directRequest(cookieStr string, pathname string, method string, callback base.ReqCallback, resp interface{}, onPuus func(string)) ([]byte, error) { + return d.requestAt(d.conf.api, cookieStr, pathname, method, callback, resp, onPuus) } // pcApi 返回夸克/UC 分享操作的 PC 客户端端点。夸克 sharepage/save、sharepage/token、 @@ -402,15 +441,19 @@ func (d *QuarkUCShare) pcApi() string { // requestSharePc 用 PC 端点 + binding(账号)Cookie(无 binding 用兜底 Cookie)发分享请求。 func (d *QuarkUCShare) requestSharePc(binding shareRequestBinding, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { cookieStr := getShareCookie() + var onPuus func(string) if binding != nil { if c := binding.cookieValue(); c != "" { cookieStr = c } + onPuus = binding.syncPuus } - return d.requestAt(d.pcApi(), cookieStr, pathname, method, callback, resp) + return d.requestAt(d.pcApi(), cookieStr, pathname, method, callback, resp, onPuus) } -func (d *QuarkUCShare) requestAt(api, cookieStr, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +// requestAt 用指定端点与 Cookie 发请求。onPuus 非 nil 时,响应里轮换的 __puus 除更新 +// 兜底 Cookie 外,还会回调给它(供把新值写回实际发请求的账号)。 +func (d *QuarkUCShare) requestAt(api, cookieStr, pathname string, method string, callback base.ReqCallback, resp interface{}, onPuus func(string)) ([]byte, error) { u := api + pathname req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -447,6 +490,9 @@ func (d *QuarkUCShare) requestAt(api, cookieStr, pathname string, method string, if __puus != nil { log.Debugf("__puus: %v", __puus) updateShareCookiePuus(__puus.Value) + if onPuus != nil { + onPuus(__puus.Value) + } } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) @@ -995,12 +1041,13 @@ var getSpeedupToken = func(d *QuarkUCShare, cookie, savedFid, fileName string) ( // speedupDownload 走 drive-pc(PC 客户端)端点取带 token 的提速直链。 // 关键:必须用 drive-pc.quark.cn;quark_uc 默认的 drive.quark.cn 对带 speedup token 的请求 // 会返回 "download file size limit",只有 PC 客户端端点接受 speedup token(my.jar 也走 drive-pc)。 +// 第二个返回值是响应里轮换的 __puus(无则空串),调用方须写回账号(#1082)。 // 声明为 var 便于单测替换(离线桩直链)。 -var speedupDownload = func(d *QuarkUCShare, cookie, savedFid, token string) (string, error) { +var speedupDownload = func(d *QuarkUCShare, cookieStr, savedFid, token string) (string, string, error) { u := "https://drive-pc.quark.cn/1/clouddrive/file/download?pr=ucpro&fr=pc" req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "Cookie": cookie, + "Cookie": cookieStr, "User-Agent": d.conf.ua, "Referer": d.conf.referer + "/", "Content-Type": "application/json", @@ -1012,16 +1059,20 @@ var speedupDownload = func(d *QuarkUCShare, cookie, savedFid, token string) (str req.SetError(&e) res, err := req.Execute(http.MethodPost, u) if err != nil { - return "", err + return "", "", err } log.Debugf("[speedup] file/download(drive-pc) http=%d body=%s", res.StatusCode(), truncStr(string(res.Body()), 300)) + newPuus := "" + if c := cookie.GetCookie(res.Cookies(), "__puus"); c != nil { + newPuus = c.Value + } if e.Code != 0 { - return "", errors.New(e.Message) + return "", newPuus, errors.New(e.Message) } if len(resp.Data) == 0 || resp.Data[0].DownloadUrl == "" { - return "", errors.New("empty speedup download url") + return "", newPuus, errors.New("empty speedup download url") } - return resp.Data[0].DownloadUrl, nil + return resp.Data[0].DownloadUrl, newPuus, nil } // speedupLink 带提速 token 取直链;任何一步失败回退 binding.link 的普通直链。 @@ -1038,20 +1089,30 @@ func (d *QuarkUCShare) speedupLink(ctx context.Context, rb requestBinding, saved return rb.link(ctx, savedFile, args) } savedFid := savedFile.GetID() - token, err := getSpeedupToken(d, rb.cookie, savedFid, savedFile.GetName()) + uc := rb.requestDriver + // 用账号的实时 Cookie 而非绑定快照:转存流程(uc.Request)可能刚轮换过 __puus, + // 快照会带旧值被 CDN 拒(412);测试桩驱动 Cookie 为空时回落快照。 + liveCookie := func() string { + if uc != nil && uc.Cookie != "" { + return uc.Cookie + } + return rb.cookie + } + token, err := getSpeedupToken(d, liveCookie(), savedFid, savedFile.GetName()) if err != nil { log.Warnf("[speedup] token 获取失败,回退普通直链: %v", err) return rb.link(ctx, savedFile, args) } - downloadUrl, err := speedupDownload(d, rb.cookie, savedFid, token) + downloadUrl, newPuus, err := speedupDownload(d, liveCookie(), savedFid, token) if err != nil { log.Warnf("[speedup] 带 token 的 file/download 失败,回退普通直链: %v", err) return rb.link(ctx, savedFile, args) } + // 下载域响应轮换的 __puus 写回账号并持久化(#1082),下方播放头重读新值。 + saveQuarkPuus(uc, newPuus) host := urlHost(downloadUrl) // dl-c-* = 提速通道; dl-pc-* = 普通限速通道。 log.Infof("[speedup] 提速直链 host=%s (dl-c=加速/dl-pc=普通) fid=%s", host, savedFid) - uc := rb.requestDriver // URL 带 #storageId 片段标记,对齐 quark_uc.getDownloadLink:alist-tvbox 按它解析取链账号 // 并下发匹配 Cookie。dl-c 提速直链缺此标记时客户端只能回落 master Cookie,多账号并发下 // 首条链常来自非 master 账号 → Cookie 归属错配被 CDN 拒(实测 412)。片段不会发往 CDN; @@ -1059,7 +1120,7 @@ func (d *QuarkUCShare) speedupLink(ctx context.Context, rb requestBinding, saved return &model.Link{ URL: downloadUrl + fmt.Sprintf("#storageId=%d", uc.ID), Header: http.Header{ - "Cookie": []string{rb.cookie}, + "Cookie": []string{liveCookie()}, "Referer": []string{d.conf.referer}, "User-Agent": []string{d.conf.ua}, }, @@ -1402,10 +1463,8 @@ func sourcesFromLinks(links []*model.Link, d *QuarkUCShare) []model.LinkSource { return srcs } -// masterCookie 取当前驱动类型主账号(master)的 Cookie。 -// 夸克 share /file/download 需账号上下文(参考脚本 quarkRequestShareDownload 用 drive.fetch 带账号), -// 匿名请求会失败 → 回退转存。故夸克取链需带主账号 Cookie;UC 匿名即可。无账号返回 ""。 -func (d *QuarkUCShare) masterCookie() string { +// masterDriver 取当前驱动类型主账号(master)的驱动实例,无账号返回 nil。 +func (d *QuarkUCShare) masterDriver() *quark.QuarkOrUC { name := d.getDriverName() prefix := conf.UC if name == "Quark" { @@ -1413,13 +1472,23 @@ func (d *QuarkUCShare) masterCookie() string { } storage := op.GetMasterDriver(name, prefix, 0) if storage == nil { - return "" + return nil } uc, ok := storage.(*quark.QuarkOrUC) if !ok { - return "" + return nil } - return uc.Cookie + return uc +} + +// masterCookie 取当前驱动类型主账号(master)的 Cookie。 +// 夸克 share /file/download 需账号上下文(参考脚本 quarkRequestShareDownload 用 drive.fetch 带账号), +// 匿名请求会失败 → 回退转存。故夸克取链需带主账号 Cookie;UC 匿名即可。无账号返回 ""。 +func (d *QuarkUCShare) masterCookie() string { + if uc := d.masterDriver(); uc != nil { + return uc.Cookie + } + return "" } // resolveShareDirectLink 免转存取链:直接用分享凭据调 /file/download 换直链, @@ -1442,10 +1511,18 @@ var resolveShareDirectLink = func(d *QuarkUCShare, file model.Obj) (*model.Link, } } // 取链请求 Cookie:UC 走匿名(已验证可行);夸克 share /file/download 需账号上下文,走主账号 Cookie。 + // 响应里轮换的 __puus 必须写回主账号并持久化(#1082):此路径不经过 quark_uc.request + // (那里会自动滚 __puus),不回写则账号 Cookie 停在登录时刻,约一天后被 CDN 判失效 → + // API 域仍成功、下载域 412;重扫码即恢复一天,循环复发。写回后下方 masterCookie() + // 重新读取的就是新值,播放头不再带过期 __puus。 isUC := d.getDriverName() == "UC" reqCookie := "" + var onPuus func(string) if !isUC { - reqCookie = d.masterCookie() + if uc := d.masterDriver(); uc != nil { + reqCookie = uc.Cookie + onPuus = func(puus string) { saveQuarkPuus(uc, puus) } + } } body := base.Json{ "fids": []string{fileId}, @@ -1457,14 +1534,14 @@ var resolveShareDirectLink = func(d *QuarkUCShare, file model.Obj) (*model.Link, var resp DownResp _, err := d.requestAt(d.pcApi(), reqCookie, "/file/download", http.MethodPost, func(req *resty.Request) { req.SetBody(body) - }, &resp) + }, &resp, onPuus) // fid_token 失效时,按 pid 重新换取 share_fid_token 后重试一次(复用 saveFile 的回退策略)。 if err != nil && strings.Contains(err.Error(), "token校验异常") && pid != "" { if newToken, e := d.getFileToken(nil, pid, fileId); e == nil && newToken != "" { body["fids_token"] = []string{newToken} _, err = d.requestAt(d.pcApi(), reqCookie, "/file/download", http.MethodPost, func(req *resty.Request) { req.SetBody(body) - }, &resp) + }, &resp, onPuus) } } // stoken 失效(code:50052 "st invalid"):刷新分享 stoken 后重试一次。 @@ -1475,7 +1552,7 @@ var resolveShareDirectLink = func(d *QuarkUCShare, file model.Obj) (*model.Link, log.Infof("[share-direct] stoken 失效(50052),已刷新重试 %s", fileId) _, err = d.requestAt(d.pcApi(), reqCookie, "/file/download", http.MethodPost, func(req *resty.Request) { req.SetBody(body) - }, &resp) + }, &resp, onPuus) } } // 空直链自愈: 强制刷新 stoken + 按 pid 重换 fid_token 后再试一次(对齐不夜 cloud-drive.js:4906-4923)。 @@ -1487,7 +1564,7 @@ var resolveShareDirectLink = func(d *QuarkUCShare, file model.Obj) (*model.Link, body["stoken"] = d.ShareToken _, err = d.requestAt(d.pcApi(), reqCookie, "/file/download", http.MethodPost, func(req *resty.Request) { req.SetBody(body) - }, &resp) + }, &resp, onPuus) } } } diff --git a/drivers/quark_uc_tv/extension.go b/drivers/quark_uc_tv/extension.go index 2b3fbecb8..d6529bee4 100644 --- a/drivers/quark_uc_tv/extension.go +++ b/drivers/quark_uc_tv/extension.go @@ -179,20 +179,13 @@ func (d *QuarkUCTV) request2(pathname string, method string, callback base.ReqCa if err != nil { return nil, err } + // 只认响应 Set-Cookie 里的新 __puus:请求头里的值是构建请求时的 d.Cookie 快照 + // (RestyClient 无 cookie jar),若按它回写,并发请求会把另一请求刚轮换的新值覆盖回旧值。 __puus := cookie.GetCookie(res.Cookies(), "__puus") if __puus != nil { log.Debugf("update __puus: %v", __puus) d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) d.SaveCookie(d.Cookie) - } else { - c := res.Request.Header.Get("Cookie") - v1 := cookie.GetStr(d.Cookie, "__puus") - v2 := cookie.GetStr(c, "__puus") - if v2 != "" && v1 != v2 { - d.Cookie = cookie.SetStr(d.Cookie, "__puus", v2) - log.Debugf("update cookie: %v %v %v", d.Cookie, v1, v2) - d.SaveCookie(d.Cookie) - } } if e.Status >= 400 || e.Code != 0 { diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 711f46ab5..253d3c0a3 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -229,9 +230,14 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj return nil, errs.NotImplement } path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) + contentType, _ := ctx.Value(conf.DirectUploadContentTypeKey).(string) + if contentType == "" { + contentType = "application/octet-stream" + } req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{ - Bucket: &d.Bucket, - Key: &path, + Bucket: &d.Bucket, + Key: &path, + ContentType: &contentType, }) if req == nil { return nil, fmt.Errorf("failed to create PutObject request") @@ -243,6 +249,7 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj return &model.HttpDirectUploadInfo{ UploadURL: link, Method: "PUT", + Headers: map[string]string{"Content-Type": contentType}, }, nil } diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index 4c8e39543..7a3f7233a 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -27,6 +27,7 @@ type Strm struct { supportSuffix map[string]struct{} downloadSuffix map[string]struct{} + minSizeBytes int64 } func (d *Strm) Config() driver.Config { @@ -122,6 +123,7 @@ func (d *Strm) Init(ctx context.Context) error { if len(d.SaveLocalMode) == 0 { d.SaveLocalMode = SaveLocalInsertMode } + d.minSizeBytes = d.MinFileSize * 1024 * 1024 return nil } diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go index 78ec1e7e6..24b31ee3e 100644 --- a/drivers/strm/hook.go +++ b/drivers/strm/hook.go @@ -8,6 +8,7 @@ import ( "io" "os" stdpath "path" + "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -26,6 +27,10 @@ func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) + if err := createLocalDirectory(localParentPath); err != nil { + log.Warnf("failed to create local strm directory %s: %v", localParentPath, err) + return + } for _, obj := range objs { localPath := stdpath.Join(localParentPath, obj.GetName()) generateStrm(ctx, driver, obj, localPath) @@ -92,55 +97,68 @@ func RemoveStrm(dstPath string, d *Strm) { } func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { - if !obj.IsDir() { - if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { - return - } - link, err := driver.Link(ctx, obj, model.LinkArgs{}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) - return - } - defer link.Close() - size := link.ContentLength - if size <= 0 { - size = obj.GetSize() - } - rrf, err := stream.GetRangeReaderFromLink(size, link) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) - return - } - rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) - return - } - defer rc.Close() - same, err := isSameContent(localPath, size, rc) - if err != nil { - log.Warnf("failed to compare content of obj %s: %v", localPath, err) - return - } - if same { - return - } - rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) - return - } - defer rc.Close() - file, err := utils.CreateNestedFile(localPath) - if err != nil { - log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) - return - } - defer file.Close() - if _, err := utils.CopyWithBuffer(file, rc); err != nil { - log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + if obj.IsDir() { + if err := createLocalDirectory(localPath); err != nil { + log.Warnf("failed to create local strm directory %s: %v", localPath, err) } + return + } + + if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { + return } + link, err := driver.Link(ctx, obj, model.LinkArgs{}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) + return + } + defer link.Close() + size := link.ContentLength + if size <= 0 { + size = obj.GetSize() + } + rrf, err := stream.GetRangeReaderFromLink(size, link) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) + return + } + rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) + return + } + defer rc.Close() + same, err := isSameContent(localPath, size, rc) + if err != nil { + log.Warnf("failed to compare content of obj %s: %v", localPath, err) + return + } + if same { + return + } + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) + return + } + defer rc.Close() + if err := createLocalDirectory(filepath.Dir(localPath)); err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create parent directory: %v", localPath, err) + return + } + file, err := os.Create(localPath) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) + return + } + defer file.Close() + if _, err := utils.CopyWithBuffer(file, rc); err != nil { + log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + } +} + +func createLocalDirectory(path string) error { + return os.MkdirAll(path, 0o777) } func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) { diff --git a/drivers/strm/hook_test.go b/drivers/strm/hook_test.go new file mode 100644 index 000000000..eccdf209b --- /dev/null +++ b/drivers/strm/hook_test.go @@ -0,0 +1,43 @@ +package strm + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestCreateLocalDirectory(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + if _, err := os.Stat(nested); err != nil { + t.Fatalf("stat created directory: %v", err) + } +} + +func TestCreateLocalDirectoryPreservesExistingPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not supported on Windows") + } + + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + if err := os.MkdirAll(nested, 0o700); err != nil { + t.Fatalf("create existing directory: %v", err) + } + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + info, err := os.Stat(nested) + if err != nil { + t.Fatalf("stat existing directory: %v", err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("existing directory permissions = %#o, want %#o", got, 0o700) + } +} diff --git a/drivers/strm/hook_umask_unix_test.go b/drivers/strm/hook_umask_unix_test.go new file mode 100644 index 000000000..b693f9f18 --- /dev/null +++ b/drivers/strm/hook_umask_unix_test.go @@ -0,0 +1,30 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package strm + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestCreateLocalDirectoryRespectsUmask(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "library", "movie") + + originalUmask := syscall.Umask(0o027) + defer syscall.Umask(originalUmask) + + if err := createLocalDirectory(nested); err != nil { + t.Fatalf("createLocalDirectory() error = %v", err) + } + + info, err := os.Stat(nested) + if err != nil { + t.Fatalf("stat created directory: %v", err) + } + if got := info.Mode().Perm(); got != 0o750 { + t.Errorf("created directory permissions = %#o, want %#o", got, 0o750) + } +} diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index 843ff7c66..2e2e8a08e 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -17,6 +17,7 @@ type Addition struct { PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix"` DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"` FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"` + MinFileSize int64 `json:"minFileSize" type:"number" required:"false" default:"0" help:"Filter out files smaller than this size (unit: MB, 0 to disable)"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` WithSign bool `json:"withSign" default:"false"` diff --git a/drivers/strm/util.go b/drivers/strm/util.go index b9a40da5e..7dfc7dbee 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -73,6 +73,9 @@ func (d *Strm) convert2strmObjs(ctx context.Context, reqPath string, objs []mode if _, ok := d.downloadSuffix[ext]; ok { size = obj.GetSize() } else if _, ok := d.supportSuffix[ext]; ok { + if d.minSizeBytes > 0 && obj.GetSize() < d.minSizeBytes { + continue + } id = "strm" name = strings.TrimSuffix(name, sourceExt) + "strm" size = int64(len(d.getLink(ctx, path))) diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index ea0093b7c..849d9170e 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -55,10 +55,18 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.GetRootPath() + } + if dirPath == "" { + dirPath = "/" + } + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": dir.GetPath(), + "path": dirPath, "limit": "500", "page": "1", }) @@ -87,7 +95,7 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) var resp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": dir.GetPath(), + "path": dirPath, "limit": "500", "page": strconv.Itoa(page), }) @@ -114,7 +122,7 @@ func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ - Path: path.Join(dir.GetPath(), src.Name), + Path: path.Join(dirPath, src.Name), ID: src.ID, Name: src.Name, Size: func() int64 { diff --git a/drivers/teldrive/driver_test.go b/drivers/teldrive/driver_test.go index a5e4de9dc..3d073c697 100644 --- a/drivers/teldrive/driver_test.go +++ b/drivers/teldrive/driver_test.go @@ -4,9 +4,11 @@ import ( "context" "net/http" "net/http/httptest" + "sync" "testing" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) @@ -36,3 +38,44 @@ func TestListEmptyDir(t *testing.T) { t.Fatalf("expected no entries for an empty dir, got %d", len(objs)) } } + +func TestListRootUsesConfiguredRootPath(t *testing.T) { + var ( + mu sync.Mutex + paths []string + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Query().Get("path")) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"items":[{"id":"child","name":"child","type":"folder"}],"meta":{"count":1,"totalPages":1,"currentPage":1}}`)) + })) + defer srv.Close() + + oldClient := base.RestyClient + base.RestyClient = resty.New() + defer func() { base.RestyClient = oldClient }() + + d := &Teldrive{ + Addition: Addition{ + RootPath: driver.RootPath{RootFolderPath: "/configured-root"}, + }, + } + d.Address = srv.URL + + objs, err := d.List(context.Background(), &model.Object{}, model.ListArgs{}) + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(paths) != 1 || paths[0] != "/configured-root" { + t.Fatalf("expected request path %q, got %q", "/configured-root", paths) + } + if len(objs) != 1 || objs[0].GetPath() != "/configured-root/child" { + t.Fatalf("expected child path %q, got %#v", "/configured-root/child", objs) + } +} diff --git a/go.mod b/go.mod index c30c3d3e8..319a31010 100644 --- a/go.mod +++ b/go.mod @@ -1,36 +1,37 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.25.0 +go 1.26.0 -toolchain go1.26.4 +toolchain go1.27.1 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/KarpelesLab/reflink v1.0.2 github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 + github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 github.com/OpenListTeam/sftpd-openlist v1.0.1 github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 github.com/OpenListTeam/wopan-sdk-go v0.1.5 - github.com/ProtonMail/go-crypto v1.3.0 + github.com/ProtonMail/go-crypto v1.4.1 github.com/ProtonMail/gopenpgp/v2 v2.9.0 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.6 - github.com/antchfx/xpath v1.3.6 + github.com/antchfx/xpath v1.3.8 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.8 - github.com/blevesearch/bleve/v2 v2.6.0 - github.com/bmatcuk/doublestar/v4 v4.9.1 + github.com/blevesearch/bleve/v2 v2.6.1 + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.21.1 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e - github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 - github.com/coreos/go-oidc v2.3.0+incompatible - github.com/deckarep/golang-set/v2 v2.8.0 + github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c + github.com/coreos/go-oidc v2.5.0+incompatible + github.com/deckarep/golang-set/v2 v2.9.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.12.0 @@ -43,7 +44,7 @@ require ( github.com/glebarez/go-sqlite v1.21.2 github.com/glebarez/sqlite v1.11.0 github.com/go-resty/resty/v2 v2.17.2 - github.com/go-webauthn/webauthn v0.13.4 + github.com/go-webauthn/webauthn v0.18.0 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -51,8 +52,7 @@ require ( github.com/hekmon/transmissionrpc/v3 v3.0.0 github.com/henrybear327/go-proton-api v1.0.0 github.com/ipfs/go-ipfs-api v0.7.0 - github.com/itsHenry35/gofakes3 v0.0.8 - github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 + github.com/jlaffaye/ftp v0.2.4 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 github.com/maruel/natural v1.3.0 @@ -61,60 +61,63 @@ require ( github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.5 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.10 + github.com/pkg/sftp v1.13.11 github.com/power721/115driver v0.3.2 github.com/pquerna/otp v1.5.0 - github.com/quic-go/quic-go v0.60.0 - github.com/rclone/rclone v1.70.3 - github.com/shirou/gopsutil/v4 v4.25.5 - github.com/sirupsen/logrus v1.9.4 + github.com/quic-go/quic-go v0.61.0 + github.com/rclone/rclone v1.75.1 + github.com/shirou/gopsutil/v4 v4.26.8 + github.com/sirupsen/logrus v1.10.2 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 - github.com/stretchr/testify v1.11.1 - github.com/t3rm1n4l/go-mega v0.0.0-20260630094356-4de2f127f34a + github.com/stretchr/testify v1.12.1 + github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 github.com/tchap/go-patricia/v2 v2.3.3 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.51.0 - golang.org/x/image v0.29.0 - golang.org/x/net v0.55.0 + golang.org/x/crypto v0.56.0 + golang.org/x/image v0.45.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/time v0.14.0 + golang.org/x/time v0.15.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 - gorm.io/driver/mysql v1.5.7 - gorm.io/driver/postgres v1.6.0 + gorm.io/driver/mysql v1.6.0 + gorm.io/driver/postgres v1.6.2 gorm.io/driver/sqlite v1.6.0 - gorm.io/gorm v1.30.0 + gorm.io/gorm v1.31.2 ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect - github.com/PuerkitoBio/goquery v1.10.3 // indirect + github.com/PuerkitoBio/goquery v1.12.0 // indirect github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect - github.com/blevesearch/zapx/v17 v17.1.2 // indirect + github.com/adrg/xdg v0.5.3 // indirect + github.com/andybalholm/cascadia v1.3.4 // indirect + github.com/blevesearch/zapx/v17 v17.2.3 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/charmbracelet/colorprofile v0.4.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/clipperhouse/displaywidth v0.9.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cronokirby/saferith v0.33.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.8.4 // indirect + github.com/ebitengine/purego v0.10.2 // indirect github.com/emersion/go-message v0.18.2 // indirect - github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect + github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -124,18 +127,24 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/lanrat/extsort v1.0.2 // indirect + github.com/lanrat/extsort v1.4.2 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect - github.com/minio/minlz v1.0.1 // indirect + github.com/minio/minlz v1.2.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/relvacode/iso8601 v1.7.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/stangelandcl/ppmd v0.1.1 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/zeebo/blake3 v0.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.mongodb.org/mongo-driver v1.17.4 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect - golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect @@ -147,15 +156,15 @@ require ( github.com/OpenListTeam/115-sdk-go v0.2.4 github.com/STARRY-S/zip v0.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/blevesearch/go-faiss v1.1.0 // indirect + github.com/blevesearch/go-faiss v1.1.5 // indirect github.com/blevesearch/zapx/v16 v16.3.4 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.4 + github.com/bodgit/sevenzip v1.6.5 github.com/bodgit/windows v1.0.1 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/charmbracelet/x/ansi v0.11.5 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/cloudflare/circl v1.6.1 // indirect + github.com/cloudflare/circl v1.6.4 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect @@ -169,7 +178,7 @@ require ( github.com/klauspost/pgzip v1.2.6 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 - github.com/nwaples/rardecode/v2 v2.2.0 + github.com/nwaples/rardecode/v2 v2.4.1 github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.15 // indirect @@ -183,19 +192,19 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.8.0 // indirect - github.com/andybalholm/brotli v1.2.1 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.4 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blevesearch/bleve_index_api v1.3.11 // indirect - github.com/blevesearch/geo v0.2.5 // indirect + github.com/blevesearch/bleve_index_api v1.4.1 // indirect + github.com/blevesearch/geo v0.2.6 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.2.0 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.4.7 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.4.10 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect @@ -205,60 +214,56 @@ require ( github.com/blevesearch/zapx/v13 v13.4.3 // indirect github.com/blevesearch/zapx/v14 v14.4.3 // indirect github.com/blevesearch/zapx/v15 v15.4.3 // indirect - github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/boombuler/barcode v1.1.0 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/fxamacker/cbor/v2 v2.9.3 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-chi/chi/v5 v5.2.2 // indirect + github.com/go-chi/chi/v5 v5.3.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.30.1 // indirect - github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.23 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/golang-jwt/jwt/v5 v5.2.3 // indirect + github.com/go-playground/validator/v10 v10.30.3 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/go-webauthn/x v0.3.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect - github.com/google/go-tpm v0.9.5 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/henrybear327/Proton-API-Bridge v1.0.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/pgx/v5 v5.10.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.18.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect - github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect + github.com/mailru/easyjson v0.9.2 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.23 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -275,40 +280,38 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pierrec/lz4/v4 v4.1.26 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pierrec/lz4/v4 v4.1.27 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.64.0 // indirect - github.com/prometheus/procfs v0.16.1 // indirect - github.com/rfjakob/eme v1.1.2 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/rfjakob/eme v1.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/tklauser/go-sysconf v0.3.15 // indirect - github.com/tklauser/numcpus v0.10.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.etcd.io/bbolt v1.4.0 // indirect + go.etcd.io/bbolt v1.5.0 // indirect golang.org/x/arch v0.23.0 // indirect - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 - golang.org/x/tools v0.44.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.82.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.41.0 + golang.org/x/tools v0.48.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/grpc v1.85.0-dev google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index feb2fff5c..ec6bc6752 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,24 @@ cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= -cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= -cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= @@ -31,6 +33,8 @@ github.com/OpenListTeam/115-sdk-go v0.2.4 h1:dVoEz3Pm996n/ZCuRckQvz36w938uNHjPrS github.com/OpenListTeam/115-sdk-go v0.2.4/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= +github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4 h1:Zy7/qg6aCS0OF/FPIoJh9/d0IgcIxpWRvn79ACm2R/Y= +github.com/OpenListTeam/gofakes3 v0.8.2-0.20260911142347-cd3c030a83b4/go.mod h1:mS9Ywbo6aId6BrRzeYjOIOpK0QDVnMoKOIb0hpaQZ3U= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo= @@ -49,22 +53,24 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9 github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= -github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= -github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE= github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s= -github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= -github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= github.com/RoaringBitmap/roaring/v2 v2.14.5 h1:ckd0o545JqDPeVJDgeFoaM21eBixUnlWfYgjE5VnyWw= github.com/RoaringBitmap/roaring/v2 v2.14.5/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4= github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= +github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= +github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= @@ -72,45 +78,46 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/andreburgaud/crypt2go v1.8.0 h1:J73vGTb1P6XL69SSuumbKs0DWn3ulbl9L92ZXBjw6pc= github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAPJAF5fKOLB9SXg= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= -github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg= +github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM= github.com/antchfx/htmlquery v1.3.6 h1:RNHHL7YehO5XdO8IM8CynwLKONwRHWkrghbYhQIk9ag= github.com/antchfx/htmlquery v1.3.6/go.mod h1:kcVUqancxPygm26X2rceEcagZFFVkLEE7xgLkGSDl/4= -github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.3.8 h1:RQlkLaJDKk1Ew1H6CUPUTKM+IQxm+6HTyOgcrfqOU9c= +github.com/antchfx/xpath v1.3.8/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49 h1:7gss+6H2mrrFtBrkokJRR2TzQD9qkpGA4N6BvIP/pCM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.49/go.mod h1:30PBx0ENoUCJm2AxzgCue8j7KEjb9ci4enxy6CCOjbE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3 h1:WZOmJfCDV+4tYacLxpiojoAdT5sxTfB3nTqQNtZu+J4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.3/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= -github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= -github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2 v1.43.7 h1:msCzvkeYJA9ehbV8mRRmkZLo/zJg/+yDVLNtflg83hQ= +github.com/aws/aws-sdk-go-v2 v1.43.7/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 h1:MBMg0zJ6i4TkAJ0dVFLKKn2cOkY6FkicmUDM67BRr6g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38/go.mod h1:9MWuJbyiUyj6eA7W1/zm1zuePDPSB3g+xcgRQeMWsXc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 h1:lHm4jPf3k1Lz5ZWc+Vcn3MKVwym+26kWCba9FkJ4f0Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38/go.mod h1:Rn+P2XR+FbyZzjmWKjg/KUZNxmGfr5oZwh5jQiE+CzI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 h1:vo4xvMRs/F6h1E52qsgLqCQgWIQXgIJUauG6rlZEh4U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39/go.mod h1:jB03R1ij/A+OE2e1dz6vgj076gd7vlYcfstAzj3HcnU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 h1:uZOinZb+h7lZw8IYzP1z1IuEnueB76/EFkcf/fEW4Ag= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31/go.mod h1:NRtwAM/p5VRt03TlEUs0pH3TeWamWdf4YyJpSrzPYLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 h1:H/5TI1jqaHsNoDQ60UwvPvJBg4GURkinXI3Qga29t2w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38/go.mod h1:PTVFf+XH++7NJOky+RLBYQx0QA5NcaeEYFQ2fsi0nwo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 h1:HLPAVrlLDaN2boN0xJx7MgaQDNEO3Q+c9L6kl/8m47Q= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39/go.mod h1:Pg/dVfsNkm1hsIDK/gMvCKtmyNfNTV12mrgHqVE/6Oo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 h1:IKoCZqfWfZzSBi16QFQ+QcbQ3LRQ7QgB1S5tDAyPBQQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3/go.mod h1:RBpRcXiM4s2pOInVs32GsBonnje+fiAj4mcrStRmlCA= +github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY= +github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -127,22 +134,22 @@ github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoG github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.6.0 h1:Cyd3dd4q5tCbOV8MnKUVRUDYMHOir9xn12NZzXVSEd4= -github.com/blevesearch/bleve/v2 v2.6.0/go.mod h1:gLmI8lWgHgrIYf7UpUX7JISI1CaqC6VScu46mHThuAY= -github.com/blevesearch/bleve_index_api v1.3.11 h1:x29vbV8OjWfLcrDVd7Lr1q+BkLNS0JWNEig0MCVnKH4= -github.com/blevesearch/bleve_index_api v1.3.11/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko= -github.com/blevesearch/geo v0.2.5 h1:yJg9FX1oRwLnjXSXF+ECHfXFTF4diF02Ca/qUGVjJhE= -github.com/blevesearch/geo v0.2.5/go.mod h1:Jhq7WE2K6mJTx1xS44M2pUO6Io+wjCSHh1+co3YOgH4= -github.com/blevesearch/go-faiss v1.1.0 h1:xM7Jc0ZUCv5lssG9Ohj3Jv0SdTpxcUABU1dDt9XVsc4= -github.com/blevesearch/go-faiss v1.1.0/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/bleve/v2 v2.6.1 h1:47vLskRTqxvQEtxVPYHjf5KpOgzD2msslXFjvUQCgWQ= +github.com/blevesearch/bleve/v2 v2.6.1/go.mod h1:Dvvx6ZoEBTOj6RSzfk0lEz0wce/qhe2yOUubXeuzd2c= +github.com/blevesearch/bleve_index_api v1.4.1 h1:CYIyecFlI+/RYjzUm+NmDjYbSvk870Bb7f+Vl4b12q8= +github.com/blevesearch/bleve_index_api v1.4.1/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko= +github.com/blevesearch/geo v0.2.6 h1:7K1oyQKYlauC+mJuo2AfNPyjN/4mihEoJMfyClVH1Mo= +github.com/blevesearch/geo v0.2.6/go.mod h1:6qzVUiB4BK47QkSZcRqiXEP2W3EeXuzM5XFTF8AdZ8A= +github.com/blevesearch/go-faiss v1.1.5 h1:/IU5lkOahH9Ghfk9n3F6N0XD7PYVXZJWmNDc9TtXuco= +github.com/blevesearch/go-faiss v1.1.5/go.mod h1:w3W9AiWsFRGVaMG+/cmJi7iHEAuGyC6blsgO1EzCK/M= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= -github.com/blevesearch/scorch_segment_api/v2 v2.4.7 h1:GlMzW08hcsM3DnLUxhyF/1PcDal1qtvvIuytuph5djw= -github.com/blevesearch/scorch_segment_api/v2 v2.4.7/go.mod h1://IJ7tG3QCf0cWW/aVSXqy77tc1AvLu3fcJLYEvOAFs= +github.com/blevesearch/scorch_segment_api/v2 v2.4.10 h1:C3873+iWZ0YJM2ijaSHhJJzSvD4x1k+5UaQdGygZVhM= +github.com/blevesearch/scorch_segment_api/v2 v2.4.10/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -163,18 +170,19 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= -github.com/blevesearch/zapx/v17 v17.1.2 h1:avbOk2igaASNoiy0BE/jPgcxAnRI2PGeydeP4hg7Ikk= -github.com/blevesearch/zapx/v17 v17.1.2/go.mod h1:WQObxKrqUX7cd0G1GMvDfc/bmZzQvoy7APOPimx7DiI= -github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= -github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/blevesearch/zapx/v17 v17.2.3 h1:UYYJPAt5b2tVxldx5h0jmv23RMsg8/UZKFVya7v92po= +github.com/blevesearch/zapx/v17 v17.2.3/go.mod h1:r7mb4QWbDQSkbAnOjCb9iCfkcrzajB4yBdJpuBIo/fE= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.4 h1:iHiVJfxbrB6RF4X+snI2MpVgNBKmVfGaTqZGNlMQIU0= -github.com/bodgit/sevenzip v1.6.4/go.mod h1:ZtNi5KNgHXeXg1G7WiF0IWSuFE2eG6lt/cTGlvuirO0= +github.com/bodgit/sevenzip v1.6.5 h1:7H7BxgmeX0j6UX42lH+KXQ92WgMQJ49DoocFdfHbCng= +github.com/bodgit/sevenzip v1.6.5/go.mod h1:GhuB6Lq1xCpP1sps+horjZ8lgiKPJcy2zUX3prla9wc= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo= +github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -212,23 +220,23 @@ github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+ github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= -github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 h1:+KC2I+emnT6ccWmo7IHbtchGc/vFqAxUuovo2eUgMuM= -github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8/go.mod h1:1pQXB0vAlzRlqcY7LYKOOZMw0wKfJPFxTLsJRF2Gswo= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= +github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c h1:7VByb9X2X3LXbk89eMavoQO9uD5dcAjY6uPZhAR9Yso= +github.com/cloudsoda/go-smb2 v0.0.0-20260803221621-0b399b9d036c/go.mod h1:1pQXB0vAlzRlqcY7LYKOOZMw0wKfJPFxTLsJRF2Gswo= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs= github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/coreos/go-oidc v2.3.0+incompatible h1:+5vEsrgprdLjjQ9FzIKAzQz1wwPD+83hQRfUIPh7rO0= -github.com/coreos/go-oidc v2.3.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc v2.5.0+incompatible h1:6W0vGJR3Tu0r0PwfmjOrRZSlfxeEln8dsejt3ZWIvwo= +github.com/coreos/go-oidc v2.5.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -236,8 +244,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= -github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/deckarep/golang-set/v2 v2.9.0 h1:prva4eP9UysWagLyKrtn074ughi0NnkIf0A4M5yOCKI= +github.com/deckarep/golang-set/v2 v2.9.0/go.mod h1:EWknQXbs0mcFpat2QOoXV0Ee57cD+w6ZEN76BR2JVrM= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= @@ -255,29 +263,29 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= -github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= -github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= -github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 h1:B9YK+Tck5mTccyDhtxBzWyqGYcFxLyB6+noMNW4/VgI= +github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6/go.mod h1:MAsn6OKL24MLbGdCjt1t44XMGgX3sFqukYTKmTUOci8= github.com/fclairamb/go-log v0.6.0 h1:1V7BJ75P2PvanLHRyGBBFjncB6d4AgEmu+BPWKbMkaU= github.com/fclairamb/go-log v0.6.0/go.mod h1:cyXxOw4aJwO6lrZb8GRELSw+sxO6wwkLJdsjY5xYCWA= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA= github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= -github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q= +github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64= github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.7.7 h1:Oh9joP463x7Mw72vhvJ61YQm8ODh9b04YR7vsOErD0Q= @@ -290,8 +298,8 @@ github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9g github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= -github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= -github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -312,27 +320,28 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= -github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= +github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= -github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.13.4 h1:q68qusWPcqHbg9STSxBLBHnsKaLxNO0RnVKaAqMuAuQ= -github.com/go-webauthn/webauthn v0.13.4/go.mod h1:MglN6OH9ECxvhDqoq1wMoF6P6JRYDiQpC9nc5OomQmI= -github.com/go-webauthn/x v0.1.23 h1:9lEO0s+g8iTyz5Vszlg/rXTGrx3CjcD0RZQ1GPZCaxI= -github.com/go-webauthn/x v0.1.23/go.mod h1:AJd3hI7NfEp/4fI6T4CHD753u91l510lglU7/NMN6+E= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-webauthn/webauthn v0.18.0 h1:PC8R3PNLEmjZf++WwcQlo1Z39S9rf8ma69rlwkypZhA= +github.com/go-webauthn/webauthn v0.18.0/go.mod h1:ymzZQhx3D/PrDjznemBdQJ23gHTaSDxUchM7sH1lUCg= +github.com/go-webauthn/x v0.3.0 h1:Q2X9vbrlP0Ed+QGEzixh1hthGZlDnzVT0XH/9IIQ0kE= +github.com/go-webauthn/x v0.3.0/go.mod h1:5OkdSQdOy7taRXWqvNHggtaPffmW94ybu3rZEER4I+I= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= -github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -345,8 +354,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= -github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= @@ -355,10 +366,10 @@ github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= -github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -369,13 +380,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -400,14 +406,12 @@ github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q= github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= -github.com/itsHenry35/gofakes3 v0.0.8 h1:1AgOl04IgoUV5r/WSK7ycnvwfpgharYLfVTmnzk5miw= -github.com/itsHenry35/gofakes3 v0.0.8/go.mod h1:gQwOJ7LoH5QSpCVmjzC6oKp+MS71utLS7GHtonsvD0c= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -426,8 +430,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs= -github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI= +github.com/jlaffaye/ftp v0.2.4 h1:JqI85DdkfZj8ntaHk8W9U2SC3jNfiPUU70+wtIWmlfE= +github.com/jlaffaye/ftp v0.2.4/go.mod h1:Y1ZnkzxownGIuX7xQ1mQzzkZ21+DbjVIyeKL/V+IIz4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -444,12 +448,12 @@ github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= @@ -460,8 +464,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lanrat/extsort v1.0.2 h1:p3MLVpQEPwEGPzeLBb+1eSErzRl6Bgjgr+qnIs2RxrU= -github.com/lanrat/extsort v1.0.2/go.mod h1:ivzsdLm8Tv+88qbdpMElV6Z15StlzPUtZSKsGb51hnQ= +github.com/lanrat/extsort v1.4.2 h1:akbLIdo4PhNZtvjpaWnbXtGMmLtnGzXplkzfgl+XTTY= +github.com/lanrat/extsort v1.4.2/go.mod h1:hceP6kxKPKebjN1RVrDBXMXXECbaI41Y94tt6MDazc4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -470,24 +474,25 @@ github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFG github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.27.8 h1:IX5x/4yKwyPQeVS2AXHZ3J4YATM9oHBGH1gBc23jBAI= github.com/libp2p/go-libp2p v0.27.8/go.mod h1:eCFFtd0s5i/EVKR7+5Ki8bM7qwkNW3TPTTSSW9sz8NE= -github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= -github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 h1:YkjVPl/YH5XlJ+/NiwzJtPYXXKRcyjmEUhsDci6YK3c= +github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mailru/easyjson v0.9.2 h1:dX8U45hQsZpxd80nLvDGihsQ/OxlvTkVUXH2r/8cb2M= +github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.32.0 h1:cWcycpONSH3VLTZ5npUl1O5aXPkNM0vUx6bywnYqGbE= @@ -498,16 +503,14 @@ github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwX github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= -github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= -github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/minio/minlz v1.2.0 h1:6IOBuiHg04QxvbFfgFLT/9sMaO/UhL7S+ApW1mK8q5A= +github.com/minio/minlz v1.2.0/go.mod h1:Ls9H7nlkASeCcdl5thjVD5Eraj6z+zGa7xtq57jIKD4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU= github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -547,62 +550,64 @@ github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8= github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4= +github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= -github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= -github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= -github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= -github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE= +github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0= +github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= +github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/power721/115driver v0.3.1 h1:lTlWbzL9OcEUppXF8eMW2W52R8XJEuULVMlp2Z+BEY4= -github.com/power721/115driver v0.3.1/go.mod h1:eJ9K5Paj5hhxlA/AtsUE4saICzXLHFFMaigE0PV8vLw= github.com/power721/115driver v0.3.2 h1:dtQAUSmQrgTbjZaREdYbtLCcUaAxEFEDnu8j5jX+gvQ= github.com/power721/115driver v0.3.2/go.mod h1:eJ9K5Paj5hhxlA/AtsUE4saICzXLHFFMaigE0PV8vLw= github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= -github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= -github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= -github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= -github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= -github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= -github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA= +github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= +github.com/rclone/rclone v1.75.1 h1:kIxQcoDLj2Gke/gMSHK7OnxhX1Gu1cJBLP1kJZoaFp0= +github.com/rclone/rclone v1.75.1/go.mod h1:4zmMjGatCkSJPRZDpo+7y3xOl8S29EMUyKvZop5mHr4= +github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo= +github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= -github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= +github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI= +github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= @@ -610,10 +615,10 @@ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJ github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= -github.com/shirou/gopsutil/v4 v4.25.5 h1:rtd9piuSMGeU8g1RMXjZs9y9luK5BwtnG7dZaQUJAsc= -github.com/shirou/gopsutil/v4 v4.25.5/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo= +github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= @@ -635,8 +640,9 @@ github.com/stangelandcl/ppmd v0.1.1/go.mod h1:Rrv7M+/2P5jYr/GMLhBl7Ug3uJ1bUiVzr5 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -648,18 +654,20 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/t3rm1n4l/go-mega v0.0.0-20260630094356-4de2f127f34a h1:VQ9CmpYeTgdFvg748N6tIOHBZn+hXjJe2xNuOJHW07E= -github.com/t3rm1n4l/go-mega v0.0.0-20260630094356-4de2f127f34a/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 h1:s30A8dMuZ55lUOi5xUTh1hlfLqHpVBsRMfpk2iKCayk= +github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhgwZDDc= github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= -github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= @@ -681,34 +689,46 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= +github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -724,13 +744,13 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= -golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -750,8 +770,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -761,8 +781,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -784,8 +804,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -796,8 +816,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -810,11 +830,11 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -823,20 +843,20 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= -google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= +google.golang.org/api v0.279.0 h1:hsx2M2OaRcaKtVYK6vXEUnQvdjnend7ZYES+lYaot74= +google.golang.org/api v0.279.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.85.0-dev h1:HxkDyKIIZPpFnroC56tQv5gNuKTmVvi0t7TzOf5zt7g= +google.golang.org/grpc v1.85.0-dev/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -850,8 +870,8 @@ gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKK gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -860,15 +880,14 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= -gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= +gorm.io/driver/postgres v1.6.2 h1:BvXQ/cNUg63q5TFNg672DmDcowZSFrNLkkA3Xe6GXq4= +gorm.io/driver/postgres v1.6.2/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= -gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= +gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= @@ -879,6 +898,8 @@ modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/authn/authn.go b/internal/authn/authn.go index a57823d15..b94454998 100644 --- a/internal/authn/authn.go +++ b/internal/authn/authn.go @@ -6,13 +6,12 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/go-webauthn/webauthn/webauthn" ) func NewAuthnInstance(c *gin.Context) (*webauthn.WebAuthn, error) { - siteUrl, err := url.Parse(common.GetApiUrl(c.Request.Context())) + siteUrl, err := url.Parse(conf.GetApiUrl(c.Request.Context())) if err != nil { return nil, err } diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 770aad58e..a030c810f 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -211,6 +211,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, + {Key: conf.SSOPostMessageOrigin, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PUBLIC}, // ldap settings {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index b1b1c77f7..44f25440d 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -15,18 +15,28 @@ import ( ) func initUser() { - admin, err := op.GetAdmin() - adminPassword := random.String(8) - envpass := os.Getenv("OPENLIST_ADMIN_PASSWORD") - if flags.Dev { - adminPassword = "admin" - } else if len(envpass) > 0 { - adminPassword = envpass - } + initAdmin() + initGuest() +} + +func initAdmin() { + _, err := op.GetAdmin() if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + // 系统尚未初始化:仅在开发模式或显式配置 OPENLIST_ADMIN_PASSWORD 时 + // 自动创建管理员;否则交由 Web 安装向导(POST /api/public/init/setup)完成。 + adminPassword := "admin" + envpass := os.Getenv("OPENLIST_ADMIN_PASSWORD") + if flags.Dev { + adminPassword = "admin" + } else if len(envpass) > 0 { + adminPassword = envpass + } else { + // 未初始化:不自动创建管理员,等待 Web 安装向导。 + return + } salt := random.String(16) - admin = &model.User{ + admin := &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), @@ -47,7 +57,10 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - _, err = op.GetGuest() +} + +func initGuest() { + _, err := op.GetGuest() if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) diff --git a/internal/conf/config.go b/internal/conf/config.go index 0c77bcd18..aa1246899 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -83,9 +83,10 @@ type Cors struct { } type S3 struct { - Enable bool `json:"enable" env:"ENABLE"` - Port int `json:"port" env:"PORT"` - SSL bool `json:"ssl" env:"SSL"` + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` + SSL bool `json:"ssl" env:"SSL"` + MultipartTTL string `json:"multipart_ttl" env:"MULTIPART_TTL"` } type FTP struct { @@ -240,9 +241,10 @@ func DefaultConfig(dataDir string) *Config { AllowHeaders: []string{"*"}, }, S3: S3{ - Enable: false, - Port: 5246, - SSL: false, + Enable: false, + Port: 5246, + SSL: false, + MultipartTTL: "24h", }, FTP: FTP{ Enable: false, diff --git a/internal/conf/const.go b/internal/conf/const.go index 1c0da7b4a..8a44b528b 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -153,6 +153,7 @@ const ( SSODefaultDir = "sso_default_dir" SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" + SSOPostMessageOrigin = "sso_postmessage_origin" // ldap LdapLoginEnabled = "ldap_login_enabled" @@ -232,4 +233,5 @@ const ( PathKey SharingIDKey SkipHookKey + DirectUploadContentTypeKey ) diff --git a/internal/conf/context.go b/internal/conf/context.go new file mode 100644 index 000000000..5b921d90b --- /dev/null +++ b/internal/conf/context.go @@ -0,0 +1,8 @@ +package conf + +import "context" + +func GetApiUrl(ctx context.Context) string { + api, _ := ctx.Value(ApiUrlKey).(string) + return api +} diff --git a/internal/conf/context_test.go b/internal/conf/context_test.go new file mode 100644 index 000000000..9d92c3188 --- /dev/null +++ b/internal/conf/context_test.go @@ -0,0 +1,29 @@ +package conf_test + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestGetApiUrl(t *testing.T) { + const want = "https://openlist.example" + tests := []struct { + name string + ctx context.Context + want string + }{ + {name: "present", ctx: context.WithValue(context.Background(), conf.ApiUrlKey, want), want: want}, + {name: "absent", ctx: context.Background()}, + {name: "wrong type", ctx: context.WithValue(context.Background(), conf.ApiUrlKey, 1)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := conf.GetApiUrl(tt.ctx); got != tt.want { + t.Fatalf("origin = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/db/meta.go b/internal/db/meta.go index 32eec2c38..b609261a6 100644 --- a/internal/db/meta.go +++ b/internal/db/meta.go @@ -13,6 +13,14 @@ func GetMetaByPath(path string) (*model.Meta, error) { return &meta, nil } +func GetAllMetas() ([]model.Meta, error) { + var metas []model.Meta + if err := db.Find(&metas).Error; err != nil { + return nil, errors.Wrapf(err, "failed get all metas") + } + return metas, nil +} + func GetMetaById(id uint) (*model.Meta, error) { var u model.Meta if err := db.First(&u, id).Error; err != nil { diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go index 11d345a77..18aa93053 100644 --- a/internal/db/searchnode.go +++ b/internal/db/searchnode.go @@ -35,7 +35,7 @@ func DeleteSearchNodesByParent(path string) error { if err != nil { return err } - dir, name := stdpath.Split(path) + dir, name := stdpath.Dir(path), stdpath.Base(path) return db.Where(fmt.Sprintf("%s = ? AND %s = ?", columnName("parent"), columnName("name")), dir, name).Delete(&model.SearchNode{}).Error diff --git a/internal/errs/errors.go b/internal/errs/errors.go index fdf7f2189..44f7ff56e 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -18,6 +18,7 @@ var ( StorageNotInit = errors.New("storage not init") StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") StreamPeekFail = errors.New("StreamPeekFail") + TemporaryCapacity = errors.New("temporary capacity unavailable") UnknownArchiveFormat = errors.New("unknown archive format") WrongArchivePassword = errors.New("wrong archive password") diff --git a/internal/fs/archive.go b/internal/fs/archive.go index 784ba587a..021983562 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -21,7 +21,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -415,7 +414,7 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args return nil, err } else { tsk.Creator, _ = ctx.Value(conf.UserKey).(*model.User) - tsk.ApiUrl = common.GetApiUrl(ctx) + tsk.ApiUrl = conf.GetApiUrl(ctx) ArchiveDownloadTaskManager.Add(tsk) return tsk, nil } diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be8..1d171a9b9 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -14,7 +14,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" ) @@ -166,7 +165,7 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } t.Creator, _ = ctx.Value(conf.UserKey).(*model.User) - t.ApiUrl = common.GetApiUrl(ctx) + t.ApiUrl = conf.GetApiUrl(ctx) if taskType == copy || taskType == merge { CopyTaskManager.Add(t) } else { diff --git a/internal/fs/copy_queue_persistence_test.go b/internal/fs/copy_queue_persistence_test.go new file mode 100644 index 000000000..d3c559b09 --- /dev/null +++ b/internal/fs/copy_queue_persistence_test.go @@ -0,0 +1,65 @@ +package fs + +import ( + "encoding/json" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/tache" +) + +func TestMigratedCopyTaskRecoversNativeFields(t *testing.T) { + previousConf := conf.Conf + conf.Conf = &conf.Config{} + t.Cleanup(func() { conf.Conf = previousConf }) + + raw := []byte(`{ + "id":"task-one", + "state":0, + "retry":0, + "max_retry":0, + "Creator":{"id":1,"username":"admin","password":"","base_path":"/","role":2,"disabled":false,"permission":511,"sso_id":"","allow_ldap":true}, + "start_time":"2026-08-11T01:00:00Z", + "end_time":"2026-08-11T01:01:00Z", + "TotalBytes":42, + "ApiUrl":"http://openlist.test:5244", + "src_path":"/folder/file", + "dst_path":"/backup", + "src_storage_mp":"/source", + "dst_storage_mp":"/target", + "TaskType":0 + }`) + + var task FileTransferTask + if err := json.Unmarshal(raw, &task); err != nil { + t.Fatalf("unmarshal migrated task: %v", err) + } + if task.GetID() != "task-one" || task.GetState() != tache.StatePending { + t.Fatalf("unexpected base fields: id=%q state=%d", task.GetID(), task.GetState()) + } + if task.GetCreator() == nil || task.GetCreator().Username != "admin" { + t.Fatal("creator was not recovered") + } + if task.GetStartTime() == nil || task.GetEndTime() == nil { + t.Fatal("task timestamps were not recovered") + } + if task.TaskType != copy { + t.Fatalf("unexpected task type: %d", task.TaskType) + } + if task.SrcActualPath != "/folder/file" || task.DstActualPath != "/backup" { + t.Fatal("copy paths were not recovered") + } + + _, maxRetry := task.GetRetry() + if maxRetry != 0 { + t.Fatalf("migration must defer retry initialization, got %d", maxRetry) + } + task.SetRetry(0, 2) + _, maxRetry = task.GetRetry() + if maxRetry != 2 { + t.Fatalf("retry initialization failed, got %d", maxRetry) + } + if task.groupID != "/target/backup" { + t.Fatalf("task group was not rebuilt: %q", task.groupID) + } +} diff --git a/internal/fs/link.go b/internal/fs/link.go index d84090cfc..8e1dedfd1 100644 --- a/internal/fs/link.go +++ b/internal/fs/link.go @@ -4,9 +4,9 @@ import ( "context" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) @@ -20,7 +20,7 @@ func link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, m return nil, nil, errors.WithMessage(err, "failed link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { - l.URL = common.GetApiUrl(ctx) + l.URL + l.URL = conf.GetApiUrl(ctx) + l.URL } return l, obj, nil } diff --git a/internal/fs/put.go b/internal/fs/put.go index 0b905be08..9042ae88a 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -7,7 +7,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -81,7 +80,7 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) t := &UploadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, storage: storage, dstDirActualPath: dstDirActualPath, diff --git a/internal/model/args.go b/internal/model/args.go index b0994da40..8a4ab8181 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -30,7 +30,7 @@ type Link struct { Header http.Header `json:"header"` // needed header (for url) RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL - Expiration *time.Duration // local cache expire Duration + Expiration *time.Duration // local cache expiration; not transferred by Clone //for accelerating request, use multi-thread downloading Concurrency int `json:"concurrency"` @@ -54,12 +54,12 @@ type LinkSource struct { Header http.Header `json:"header,omitempty"` } +// Clone transfers ownership of l without inheriting its cache expiration. func (l *Link) Clone() *Link { cloned := &Link{ URL: l.URL, Header: l.Header, RangeReader: l.RangeReader, - Expiration: l.Expiration, Concurrency: l.Concurrency, PartSize: l.PartSize, ContentLength: l.ContentLength, @@ -132,21 +132,3 @@ type SharingLinkArgs struct { type RangeReaderIF interface { RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) } - -type RangeReadCloserIF interface { - RangeReaderIF - utils.ClosersIF -} - -var _ RangeReadCloserIF = (*RangeReadCloser)(nil) - -type RangeReadCloser struct { - RangeReader RangeReaderIF - utils.Closers -} - -func (r *RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - rc, err := r.RangeReader.RangeRead(ctx, httpRange) - r.Add(rc) - return rc, err -} diff --git a/internal/model/args_test.go b/internal/model/args_test.go new file mode 100644 index 000000000..a82ae44b6 --- /dev/null +++ b/internal/model/args_test.go @@ -0,0 +1,25 @@ +package model + +import ( + "testing" + "time" +) + +func TestLinkCloneTransfersOwnershipWithoutCachePolicy(t *testing.T) { + ttl := time.Minute + source := &Link{URL: "https://example.test/file", Expiration: &ttl} + + clone := source.Clone() + if clone.URL != source.URL { + t.Fatal("clone did not preserve transport data") + } + if clone.Expiration != nil { + t.Fatal("clone inherited source cache policy") + } + if err := clone.Close(); err != nil { + t.Fatal(err) + } + if !source.Expired() { + t.Fatal("closing clone did not release its source") + } +} diff --git a/internal/model/obj.go b/internal/model/obj.go index 0e50032dd..4a74f0eac 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -19,6 +19,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type ObjDisplayName interface { + GetDisplayName() string +} + type Obj interface { GetSize() int64 GetDuration() int @@ -133,13 +137,17 @@ func ExtractFolder(objs []Obj, extractFolder string) { }) } -func WrapObjName(objs Obj) Obj { - return &ObjWrapName{Name: utils.MappingName(objs.GetName()), Obj: objs} +func WrapObjName(obj Obj) Obj { + name := obj.GetName() + if displayName, ok := obj.(ObjDisplayName); ok { + name = displayName.GetDisplayName() + } + return &ObjWrapName{Name: utils.MappingName(name), Obj: obj} } func WrapObjsName(objs []Obj) { for i := range objs { - objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} + objs[i] = WrapObjName(objs[i]) } } diff --git a/internal/net/preconditions.go b/internal/net/preconditions.go new file mode 100644 index 000000000..dc4727134 --- /dev/null +++ b/internal/net/preconditions.go @@ -0,0 +1,271 @@ +package net + +import ( + "net/http" + "net/textproto" + "strings" + "time" +) + +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" +} + +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} + +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult { + values := r.Header.Values("If-Match") + if len(values) == 0 { + return condNone + } + im := strings.Join(values, ",") + r.Header.Del("If-Match") + if !exists { + return condFalse + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break + } + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } + im = remain + } + + return condFalse +} + +func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" { + return condNone + } + r.Header.Del("If-Unmodified-Since") + if isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ius) + if err != nil { + return condNone + } + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return condTrue + } + return condFalse +} + +func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, exists bool) condResult { + values := r.Header.Values("If-None-Match") + if len(values) == 0 { + return condNone + } + inm := strings.Join(values, ",") + r.Header.Del("If-None-Match") + if !exists { + return condTrue + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + continue + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) + if etag == "" { + break + } + if etagWeakMatch(etag, w.Header().Get("Etag")) { + return condFalse + } + buf = remain + } + return condTrue +} + +func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" { + return condNone + } + r.Header.Del("If-Modified-Since") + if isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ims) + if err != nil { + return condNone + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if ret := modtime.Compare(t); ret <= 0 { + return condFalse + } + return condTrue +} + +func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ir := r.Header.Get("If-Range") + if ir == "" { + return condNone + } + r.Header.Del("If-Range") + etag, _ := scanETag(ir) + if etag != "" { + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } + return condFalse + } + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return condFalse + } + t, err := http.ParseTime(ir) + if err != nil { + return condFalse + } + if t.Unix() == modtime.Unix() { + return condTrue + } + return condFalse +} + +var unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} + +func setLastModified(w http.ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } +} + +func writeNotModified(w http.ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + delete(h, "Content-Encoding") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(http.StatusNotModified) +} + +// CheckPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. The caller must +// set the current ETag response header and indicate whether the representation exists. +func CheckPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time, exists bool) (done bool, rangeHeader string) { + // Evaluate preconditions in the order specified by RFC 9110 section 13.2.2. + ch := checkIfMatch(w, r, exists) + if ch == condNone { + ch = checkIfUnmodifiedSince(r, modtime) + } + if ch == condFalse { + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + } + switch checkIfNoneMatch(w, r, exists) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true, "" + } + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + case condNone: + if checkIfModifiedSince(r, modtime) == condFalse { + writeNotModified(w) + return true, "" + } + } + + rangeHeader = r.Header.Get("Range") + if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { + rangeHeader = "" + } + return false, rangeHeader +} diff --git a/internal/net/preconditions_test.go b/internal/net/preconditions_test.go new file mode 100644 index 000000000..f1ab6966a --- /dev/null +++ b/internal/net/preconditions_test.go @@ -0,0 +1,114 @@ +package net + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestCheckPreconditions(t *testing.T) { + modified := time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC) + date := modified.Format(http.TimeFormat) + older := modified.Add(-time.Second).Format(http.TimeFormat) + for _, tt := range []struct { + name string + method string + exists bool + etag string + header http.Header + status int + }{ + {"unconditional create", "PUT", false, "", nil, 0}, + {"unconditional overwrite", "PUT", true, `"current"`, nil, 0}, + {"create only missing", "PUT", false, "", http.Header{"If-None-Match": {"*"}}, 0}, + {"create only existing", "PUT", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 412}, + {"create only existing without etag", "PUT", true, "", http.Header{"If-None-Match": {"*"}}, 412}, + {"match missing", "PUT", false, "", http.Header{"If-Match": {"*"}}, 412}, + {"match existing", "PUT", true, `"current"`, http.Header{"If-Match": {"*"}}, 0}, + {"match existing without etag", "PUT", true, "", http.Header{"If-Match": {"*"}}, 0}, + {"match current", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}}, 0}, + {"match stale", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412}, + {"match missing tag", "PUT", false, "", http.Header{"If-Match": {`"current"`}}, 412}, + {"match weak request", "PUT", true, `"current"`, http.Header{"If-Match": {`W/"current"`}}, 412}, + {"match weak representation", "PUT", true, `W/"current"`, http.Header{"If-Match": {`"current"`}}, 412}, + {"match list", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale", "current"`}}, 0}, + {"match multiple lines", "PUT", true, `"current"`, http.Header{"If-Match": {`"stale"`, `"current"`}}, 0}, + {"match empty list members", "PUT", true, `"current"`, http.Header{"If-Match": {`, , "current",`}}, 0}, + {"match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-Match": {`"stale", "one,two"`}}, 0}, + {"malformed match", "PUT", true, `"current"`, http.Header{"If-Match": {"current"}}, 412}, + {"empty match", "PUT", true, `"current"`, http.Header{"If-Match": {""}}, 412}, + {"none match current", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"current"`}}, 412}, + {"none match stale", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}}, 0}, + {"none match missing tag", "PUT", false, "", http.Header{"If-None-Match": {`"current"`}}, 0}, + {"none match weak request", "PUT", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 412}, + {"none match weak representation", "PUT", true, `W/"current"`, http.Header{"If-None-Match": {`"current"`}}, 412}, + {"none match multiple lines", "PUT", true, `"current"`, http.Header{"If-None-Match": {`"stale"`, `"current"`}}, 412}, + {"none match quoted comma", "PUT", true, `"one,two"`, http.Header{"If-None-Match": {`"stale", "one,two"`}}, 412}, + {"both conditions must pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"current"`}}, 412}, + {"both conditions pass", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-None-Match": {`"stale"`}}, 0}, + {"both wildcards missing", "PUT", false, "", http.Header{"If-Match": {"*"}, "If-None-Match": {"*"}}, 412}, + {"unmodified since older", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {older}}, 412}, + {"unmodified since same second", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {date}}, 0}, + {"unmodified since missing", "PUT", false, "", http.Header{"If-Unmodified-Since": {older}}, 0}, + {"invalid unmodified since", "PUT", true, `"current"`, http.Header{"If-Unmodified-Since": {"invalid"}}, 0}, + {"match overrides unmodified since", "PUT", true, `"current"`, http.Header{"If-Match": {`"current"`}, "If-Unmodified-Since": {older}}, 0}, + {"ignore modified since on put", "PUT", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 0}, + {"get not modified", "GET", true, `"current"`, http.Header{"If-None-Match": {`W/"current"`}}, 304}, + {"head not modified", "HEAD", true, `"current"`, http.Header{"If-None-Match": {"*"}}, 304}, + {"get stale match", "GET", true, `"current"`, http.Header{"If-Match": {`"stale"`}}, 412}, + {"get modified since", "GET", true, `"current"`, http.Header{"If-Modified-Since": {date}}, 304}, + {"none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {`"stale"`}, "If-Modified-Since": {date}}, 0}, + {"empty none match overrides modified since", "GET", true, `"current"`, http.Header{"If-None-Match": {""}, "If-Modified-Since": {date}}, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest(tt.method, "/file.txt", nil) + for name, values := range tt.header { + for _, value := range values { + r.Header.Add(name, value) + } + } + w := httptest.NewRecorder() + if tt.etag != "" { + w.Header().Set("Etag", tt.etag) + } + var modTime time.Time + if tt.exists { + modTime = modified + } + done, _ := CheckPreconditions(w, r, modTime, tt.exists) + if done != (tt.status != 0) { + t.Fatalf("done = %v, want status %d", done, tt.status) + } + if done && w.Code != tt.status { + t.Errorf("status = %d, want %d", w.Code, tt.status) + } + if w.Body.Len() != 0 { + t.Errorf("unexpected response body: %q", w.Body.String()) + } + }) + } +} + +func TestCheckPreconditionsIfRange(t *testing.T) { + for _, tt := range []struct { + etag string + want string + }{ + {`"current"`, "bytes=0-3"}, + {`"stale"`, ""}, + {`W/"current"`, ""}, + } { + t.Run(tt.etag, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/file.txt", nil) + r.Header.Set("Range", "bytes=0-3") + r.Header.Set("If-Range", tt.etag) + w := httptest.NewRecorder() + w.Header().Set("Etag", `"current"`) + done, got := CheckPreconditions(w, r, time.Time{}, true) + if done || got != tt.want { + t.Fatalf("CheckPreconditions() = (%v, %q), want (false, %q)", done, got, tt.want) + } + }) + } +} diff --git a/internal/net/request.go b/internal/net/request.go index e1f045120..a5155d019 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -122,15 +122,15 @@ type downloader struct { } type ConcurrencyLimit struct { - _m sync.Mutex + mu sync.Mutex Limit int // 需要大于0 } var ErrExceedMaxConcurrency = HttpStatusCodeError(http.StatusTooManyRequests) func (l *ConcurrencyLimit) sub() error { - l._m.Lock() - defer l._m.Unlock() + l.mu.Lock() + defer l.mu.Unlock() if l.Limit-1 < 0 { return ErrExceedMaxConcurrency } @@ -139,8 +139,8 @@ func (l *ConcurrencyLimit) sub() error { return nil } func (l *ConcurrencyLimit) add() { - l._m.Lock() - defer l._m.Unlock() + l.mu.Lock() + defer l.mu.Unlock() l.Limit++ // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) } @@ -202,7 +202,12 @@ func (d *downloader) download() (io.ReadCloser, error) { d.pos = d.params.Range.Start d.maxPos = d.params.Range.Start + d.params.Range.Length d.concurrency = d.cfg.Concurrency - _ = d.sendChunkTask(true) + if err := d.sendChunkTask(true); err != nil { + d.cancel(err) + d.concurrencyFinish() + _ = d.interrupt() + return nil, err + } var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) @@ -223,8 +228,7 @@ func (d *downloader) sendChunkTask(newConcurrency bool) error { return err } } - d.concurrency-- - go d.downloadPart() + // slot 扣减与 worker 启动移至分片入队之后(见下方 select),对齐官方 #3090 } var buf *Buf @@ -270,8 +274,17 @@ func (d *downloader) sendChunkTask(newConcurrency bool) error { } d.pos += finalSize d.nextChunk++ - d.chunkChannel <- ch - return nil + select { + case <-d.ctx.Done(): + return context.Cause(d.ctx) + case d.chunkChannel <- ch: + if newConcurrency { + // The worker owns the acquired slot only after its chunk is queued. + d.concurrency-- + go d.downloadPart() + } + return nil + } } return nil } diff --git a/internal/net/request_cancel_test.go b/internal/net/request_cancel_test.go new file mode 100644 index 000000000..83ff187ac --- /dev/null +++ b/internal/net/request_cancel_test.go @@ -0,0 +1,88 @@ +package net + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestDownloadCancelledAcquisitionReturnsErrorAndReleasesLimit(t *testing.T) { + const attempts = 32 + limits := make([]*ConcurrencyLimit, 0, attempts) + for range attempts { + limit := &ConcurrencyLimit{Limit: 1} + limits = append(limits, limit) + d := NewDownloader(func(d *Downloader) { + d.Concurrency = 2 + d.PartSize = 4 + d.ConcurrencyLimit = limit + d.HttpClient = func(ctx context.Context, _ *HttpRequestParams) (*http.Response, error) { + return nil, ctx.Err() + } + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + reader, err := d.Download(ctx, &HttpRequestParams{Size: 16, Range: http_range.Range{Length: -1}}) + if reader == nil && err == nil { + t.Error("cancelled download returned a nil reader and nil error") + } + if reader != nil { + _ = reader.Close() + } else if !errors.Is(err, context.Canceled) { + t.Errorf("cancelled download error = %v, want context.Canceled", err) + } + } + time.Sleep(50 * time.Millisecond) // allow any started workers to release their slots + for i, limit := range limits { + limit.mu.Lock() + got := limit.Limit + limit.mu.Unlock() + if got != 1 { + t.Errorf("attempt %d remaining concurrency = %d, want 1", i, got) + } + } +} + +func TestDownloadSinglePartFailureReleasesLimit(t *testing.T) { + upstreamErr := errors.New("upstream failure") + for _, tc := range []struct { + name string + ctx context.Context + want error + }{ + {name: "cancelled", ctx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }(), want: context.Canceled}, + {name: "upstream failure", ctx: context.Background(), want: upstreamErr}, + } { + t.Run(tc.name, func(t *testing.T) { + limit := &ConcurrencyLimit{Limit: 1} + d := NewDownloader(func(d *Downloader) { + d.PartSize = 32 + d.ConcurrencyLimit = limit + d.HttpClient = func(ctx context.Context, _ *HttpRequestParams) (*http.Response, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, upstreamErr + } + }) + reader, err := d.Download(tc.ctx, &HttpRequestParams{Size: 16, Range: http_range.Range{Length: -1}}) + if reader != nil || !errors.Is(err, tc.want) { + t.Fatalf("single-part failed download = %v, %v; want nil, %v", reader, err, tc.want) + } + limit.mu.Lock() + got := limit.Limit + limit.mu.Unlock() + if got != 1 { + t.Errorf("remaining concurrency = %d, want 1", got) + } + }) + } +} diff --git a/internal/net/serve.go b/internal/net/serve.go index 89a209d88..fe1a4a7e7 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -4,6 +4,7 @@ import ( "compress/gzip" "context" "crypto/tls" + stderrors "errors" "fmt" "io" "mime/multipart" @@ -15,7 +16,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -25,12 +25,8 @@ import ( //this file is inspired by GO_SDK net.http.ServeContent -//type RangeReadCloser struct { -// GetReaderForRange RangeReaderFunc -//} - // ServeHTTP replies to the request using the content in the -// provided RangeReadCloser. The main benefit of ServeHTTP over io.Copy +// provided range reader. The main benefit of ServeHTTP over io.Copy // is that it handles Range requests properly, sets the MIME type, and // handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, // and If-Range requests. @@ -47,15 +43,13 @@ import ( // request includes an If-Modified-Since header, ServeHTTP uses // modtime to decide whether the content needs to be sent at all. // -// The content's RangeReadCloser method must work: ServeHTTP gives a range, -// caller will give the reader for that Range. +// The content's RangeRead method must return a reader for the requested range. // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. -func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) error { - defer RangeReadCloser.Close() +func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, rangeReader model.RangeReaderIF) (err error) { setLastModified(w, modTime) - done, rangeReq := checkPreconditions(w, r, modTime) + done, rangeReq := CheckPreconditions(w, r, modTime, true) if done { return nil } @@ -113,10 +107,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time ctx := r.Context() switch { case len(ranges) == 0: - reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + reader, err := openRange(ctx, rangeReader, http_range.Range{Length: -1}) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -136,10 +131,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // does not request multiple parts might not support // multipart responses." ra := ranges[0] - sendContent, err = RangeReadCloser.RangeRead(ctx, ra) + sendContent, err = openRange(ctx, rangeReader, ra) if err != nil { code = http.StatusRequestedRangeNotSatisfiable - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } http.Error(w, err.Error(), code) @@ -159,7 +155,6 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time mw := multipart.NewWriter(pw) w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) sendContent = pr - defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. go func() { for _, ra := range ranges { part, err := mw.CreatePart(ra.MimeHeader(contentType, size)) @@ -167,21 +162,18 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - reader, err := RangeReadCloser.RangeRead(ctx, ra) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := utils.CopyWithBufferN(part, reader, ra.Length); err != nil { + if err := copyRange(ctx, part, rangeReader, ra); err != nil { pw.CloseWithError(err) return } } - mw.Close() - pw.Close() + _ = pw.CloseWithError(mw.Close()) }() } + defer func() { + err = closeWithError(err, sendContent) + }() w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { @@ -201,7 +193,8 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } code = http.StatusInternalServerError - if statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError); ok { + var statusCode HttpStatusCodeError + if errors.As(err, &statusCode) { code = int(statusCode) } w.WriteHeader(code) @@ -210,16 +203,86 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time } return nil } + +func copyRange(ctx context.Context, dst io.Writer, rangeReader model.RangeReaderIF, requested http_range.Range) (err error) { + reader, err := openRange(ctx, rangeReader, requested) + if err != nil { + return err + } + defer func() { + err = closeWithError(err, reader) + }() + _, err = utils.CopyWithBufferN(dst, reader, requested.Length) + return err +} + +func openRange(ctx context.Context, rangeReader model.RangeReaderIF, requested http_range.Range) (io.ReadCloser, error) { + reader, err := rangeReader.RangeRead(ctx, requested) + if err != nil { + if reader != nil { + err = closeWithError(err, reader) + } + return nil, err + } + if reader == nil { + return nil, errors.New("range reader returned a nil body") + } + return reader, nil +} + +func closeWithError(err error, closer io.Closer) error { + closeErr := closer.Close() + if err == nil { + return closeErr + } + if closeErr == nil { + return err + } + return stderrors.Join(err, closeErr) +} + +// unsafeProxyHeaders are never forwarded from the client request to the +// upstream storage, regardless of the proxy_ignore_headers setting. They either +// carry the caller's credentials, describe the hop to this server rather than +// the hop to upstream, or let the caller influence how upstream routes and +// authenticates the request. +var unsafeProxyHeaders = map[string]struct{}{ + "authorization": {}, + "cookie": {}, + "proxy-authorization": {}, + "www-authenticate": {}, + "host": {}, + "referer": {}, + "origin": {}, + "connection": {}, + "keep-alive": {}, + "proxy-connection": {}, + "te": {}, + "trailer": {}, + "transfer-encoding": {}, + "upgrade": {}, + "forwarded": {}, + "x-forwarded-for": {}, + "x-forwarded-host": {}, + "x-forwarded-proto": {}, + "x-real-ip": {}, +} + + func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} // client header for h, val := range origin { - if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) { + lower := strings.ToLower(h) + if _, unsafe := unsafeProxyHeaders[lower]; unsafe { + continue + } + if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], lower) { continue } result[h] = val } - // needed header + // needed header, produced by the storage driver rather than the client for h, val := range override { result[h] = val } @@ -239,20 +302,18 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea } // TODO clean header with blocklist or passlist res.Header.Del("set-cookie") - var reader io.Reader if res.StatusCode >= 400 { - // 根据 Content-Encoding 判断 Body 是否压缩 - switch res.Header.Get("Content-Encoding") { - case "gzip": - // 使用gzip.NewReader解压缩 - reader, _ = gzip.NewReader(res.Body) - defer reader.(*gzip.Reader).Close() - default: - // 没有Content-Encoding,直接读取 - reader = res.Body + defer res.Body.Close() + var reader io.Reader = res.Body + if res.Header.Get("Content-Encoding") == "gzip" { + gzipReader, err := gzip.NewReader(res.Body) + if err != nil { + return nil, fmt.Errorf("http request [%s] failure,status: %w decode gzip response: %v", URL, HttpStatusCodeError(res.StatusCode), err) + } + defer gzipReader.Close() + reader = gzipReader } all, _ := io.ReadAll(reader) - _ = res.Body.Close() msg := string(all) log.Debugln(msg) return nil, fmt.Errorf("http request [%s] failure,status: %w response:%s", URL, HttpStatusCodeError(res.StatusCode), msg) diff --git a/internal/net/serve_header_test.go b/internal/net/serve_header_test.go new file mode 100644 index 000000000..d1df43c14 --- /dev/null +++ b/internal/net/serve_header_test.go @@ -0,0 +1,67 @@ +package net + +import ( + "net/http" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// The client must not be able to smuggle credential or routing headers into the +// request that this server makes to the upstream storage, even when the +// proxy_ignore_headers setting has been emptied. +func TestProcessHeaderDropsUnsafeClientHeaders(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil + + origin := http.Header{} + origin.Set("Authorization", "Bearer victim-token") + origin.Set("Cookie", "session=victim") + origin.Set("X-Forwarded-For", "127.0.0.1") + origin.Set("Host", "internal.example") + origin.Set("Range", "bytes=0-1023") + + result := ProcessHeader(origin, nil) + + for _, h := range []string{"Authorization", "Cookie", "X-Forwarded-For", "Host"} { + if got := result.Get(h); got != "" { + t.Errorf("header %q must not be forwarded upstream, got %q", h, got) + } + } + if got := result.Get("Range"); got != "bytes=0-1023" { + t.Errorf("Range must be preserved, got %q", got) + } +} + +// Headers supplied by the storage driver still win, since they carry the +// credentials needed to reach upstream. +func TestProcessHeaderOverrideWins(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil + + origin := http.Header{} + origin.Set("Authorization", "Bearer victim-token") + + override := http.Header{} + override.Set("Authorization", "Bearer driver-token") + + result := ProcessHeader(origin, override) + if got := result.Get("Authorization"); got != "Bearer driver-token" { + t.Errorf("driver header must be used, got %q", got) + } +} + +func TestProcessHeaderStillHonoursIgnoreSetting(t *testing.T) { + conf.SlicesMap[conf.ProxyIgnoreHeaders] = []string{"x-custom"} + t.Cleanup(func() { conf.SlicesMap[conf.ProxyIgnoreHeaders] = nil }) + + origin := http.Header{} + origin.Set("X-Custom", "drop-me") + origin.Set("X-Keep", "keep-me") + + result := ProcessHeader(origin, nil) + if got := result.Get("X-Custom"); got != "" { + t.Errorf("configured ignore header must be dropped, got %q", got) + } + if got := result.Get("X-Keep"); got != "keep-me" { + t.Errorf("unrelated header must be preserved, got %q", got) + } +} diff --git a/internal/net/serve_test.go b/internal/net/serve_test.go new file mode 100644 index 000000000..d28d2ed1d --- /dev/null +++ b/internal/net/serve_test.go @@ -0,0 +1,236 @@ +package net + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestServeHTTPClosesMultipartRangeBeforeOpeningNext(t *testing.T) { + source := newSequentialRangeSource("abc") + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + request := httptest.NewRequest(http.MethodGet, "/file", nil).WithContext(ctx) + request.Header.Set("Range", "bytes=0-0,2-2") + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + response := recorder.Result() + defer response.Body.Close() + if response.StatusCode != http.StatusPartialContent { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusPartialContent) + } + + mediaType, params, err := mime.ParseMediaType(response.Header.Get("Content-Type")) + if err != nil { + t.Fatalf("parse Content-Type: %v", err) + } + if mediaType != "multipart/byteranges" { + t.Fatalf("Content-Type = %q, want multipart/byteranges", mediaType) + } + multipartReader := multipart.NewReader(response.Body, params["boundary"]) + var parts []string + for { + part, err := multipartReader.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("read multipart part: %v", err) + } + body, err := io.ReadAll(part) + if err != nil { + t.Fatalf("read multipart body: %v", err) + } + parts = append(parts, string(body)) + } + if want := []string{"a", "c"}; !reflect.DeepEqual(parts, want) { + t.Fatalf("multipart parts = %q, want %q", parts, want) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0", "open:2", "close:2"}, []int{1, 1}) +} + +func TestServeHTTPClosesSelectedRangeBody(t *testing.T) { + tests := []struct { + name string + method string + rangeValue string + wantStatus int + wantEvents []string + }{ + {name: "full", method: http.MethodGet, wantStatus: http.StatusOK, wantEvents: []string{"open:0", "close:0"}}, + {name: "single range", method: http.MethodGet, rangeValue: "bytes=1-1", wantStatus: http.StatusPartialContent, wantEvents: []string{"open:1", "close:1"}}, + {name: "head", method: http.MethodHead, wantStatus: http.StatusOK, wantEvents: []string{"open:0", "close:0"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := newSequentialRangeSource("abc") + request := httptest.NewRequest(test.method, "/file", nil) + if test.rangeValue != "" { + request.Header.Set("Range", test.rangeValue) + } + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + if recorder.Code != test.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, test.wantStatus) + } + assertRangeLifecycle(t, source, test.wantEvents, []int{1}) + }) + } +} + +func TestServeHTTPClosesRangeAfterWriteFailure(t *testing.T) { + writeErr := errors.New("write failed") + source := newSequentialRangeSource("abc") + request := httptest.NewRequest(http.MethodGet, "/file", nil) + writer := &failingResponseWriter{header: make(http.Header), err: writeErr} + + err := ServeHTTP(writer, request, "file.txt", time.Time{}, 3, source) + if !errors.Is(err, writeErr) { + t.Fatalf("ServeHTTP() error = %v, want %v", err, writeErr) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +func TestServeHTTPClosesBodyReturnedWithOpenError(t *testing.T) { + source := newSequentialRangeSource("abc") + source.openErr = HttpStatusCodeError(http.StatusServiceUnavailable) + source.closeErr = errors.New("close failed") + request := httptest.NewRequest(http.MethodGet, "/file", nil) + recorder := httptest.NewRecorder() + + if err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source); err != nil { + t.Fatalf("ServeHTTP() error = %v", err) + } + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusServiceUnavailable) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +func TestServeHTTPStopsMultipartAfterRangeCloseFailure(t *testing.T) { + closeErr := errors.New("close failed") + source := newSequentialRangeSource("abc") + source.closeErr = closeErr + request := httptest.NewRequest(http.MethodGet, "/file", nil) + request.Header.Set("Range", "bytes=0-0,2-2") + recorder := httptest.NewRecorder() + + err := ServeHTTP(recorder, request, "file.txt", time.Time{}, 3, source) + if !errors.Is(err, closeErr) { + t.Fatalf("ServeHTTP() error = %v, want %v", err, closeErr) + } + assertRangeLifecycle(t, source, []string{"open:0", "close:0"}, []int{1}) +} + +type sequentialRangeSource struct { + content []byte + permit chan struct{} + + mu sync.Mutex + events []string + closeCounts []int + closeErr error + openErr error +} + +func newSequentialRangeSource(content string) *sequentialRangeSource { + return &sequentialRangeSource{ + content: []byte(content), + permit: make(chan struct{}, 1), + } +} + +func (s *sequentialRangeSource) RangeRead(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + select { + case s.permit <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + start := int(requested.Start) + length := int(requested.Length) + if length < 0 || start+length > len(s.content) { + length = len(s.content) - start + } + end := start + length + s.mu.Lock() + index := len(s.closeCounts) + s.events = append(s.events, fmt.Sprintf("open:%d", requested.Start)) + s.closeCounts = append(s.closeCounts, 0) + s.mu.Unlock() + return &testReadCloser{ + Reader: bytes.NewReader(s.content[start:end]), + close: func() error { + s.mu.Lock() + s.closeCounts[index]++ + closeCalls := s.closeCounts[index] + if closeCalls == 1 { + s.events = append(s.events, fmt.Sprintf("close:%d", requested.Start)) + } + s.mu.Unlock() + if closeCalls != 1 { + return fmt.Errorf("body closed %d times", closeCalls) + } + <-s.permit + return s.closeErr + }, + }, s.openErr +} + +func (s *sequentialRangeSource) eventsSnapshot() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.events...) +} + +func (s *sequentialRangeSource) closeCountsSnapshot() []int { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int(nil), s.closeCounts...) +} + +func assertRangeLifecycle(t *testing.T, source *sequentialRangeSource, wantEvents []string, wantCloseCounts []int) { + t.Helper() + if got := source.eventsSnapshot(); !reflect.DeepEqual(got, wantEvents) { + t.Fatalf("range lifecycle = %v, want %v", got, wantEvents) + } + if got := source.closeCountsSnapshot(); !reflect.DeepEqual(got, wantCloseCounts) { + t.Fatalf("close counts = %v, want %v", got, wantCloseCounts) + } +} + +type failingResponseWriter struct { + header http.Header + err error +} + +func (w *failingResponseWriter) Header() http.Header { return w.header } +func (*failingResponseWriter) WriteHeader(int) {} +func (w *failingResponseWriter) Write([]byte) (int, error) { + return 0, w.err +} + +type testReadCloser struct { + io.Reader + close func() error +} + +func (b *testReadCloser) Close() error { return b.close() } diff --git a/internal/net/util.go b/internal/net/util.go index 4bf2ba6cf..f0f770331 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -4,10 +4,7 @@ import ( "io" "mime/multipart" "net/http" - "net/textproto" "net/url" - "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -18,260 +15,6 @@ import ( log "github.com/sirupsen/logrus" ) -// scanETag determines if a syntactically valid ETag is present at s. If so, -// the ETag and remaining text after consuming ETag is returned. Otherwise, -// it returns "", "". -func scanETag(s string) (etag string, remain string) { - s = textproto.TrimString(s) - start := 0 - if strings.HasPrefix(s, "W/") { - start = 2 - } - if len(s[start:]) < 2 || s[start] != '"' { - return "", "" - } - // ETag is either W/"text" or "text". - // See RFC 7232 2.3. - for i := start + 1; i < len(s); i++ { - c := s[i] - switch { - // Character values allowed in ETags. - case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: - case c == '"': - return s[:i+1], s[i+1:] - default: - return "", "" - } - } - return "", "" -} - -// etagStrongMatch reports whether a and b match using strong ETag comparison. -// Assumes a and b are valid ETags. -func etagStrongMatch(a, b string) bool { - return a == b && a != "" && a[0] == '"' -} - -// etagWeakMatch reports whether a and b match using weak ETag comparison. -// Assumes a and b are valid ETags. -func etagWeakMatch(a, b string) bool { - return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") -} - -// condResult is the result of an HTTP request precondition check. -// See https://tools.ietf.org/html/rfc7232 section 3. -type condResult int - -const ( - condNone condResult = iota - condTrue - condFalse -) - -func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { - im := r.Header.Get("If-Match") - if im == "" { - return condNone - } - r.Header.Del("If-Match") - for { - im = textproto.TrimString(im) - if len(im) == 0 { - break - } - if im[0] == ',' { - im = im[1:] - continue - } - if im[0] == '*' { - return condTrue - } - etag, remain := scanETag(im) - if etag == "" { - break - } - if etagStrongMatch(etag, w.Header().Get("Etag")) { - return condTrue - } - im = remain - } - - return condFalse -} - -func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { - ius := r.Header.Get("If-Unmodified-Since") - if ius == "" { - return condNone - } - r.Header.Del("If-Unmodified-Since") - if isZeroTime(modtime) { - return condNone - } - t, err := http.ParseTime(ius) - if err != nil { - return condNone - } - - // The Last-Modified header truncates sub-second precision so - // the modtime needs to be truncated too. - modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { - return condTrue - } - return condFalse -} - -func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { - inm := r.Header.Get("If-None-Match") - if inm == "" { - return condNone - } - r.Header.Del("If-None-Match") - buf := inm - for { - buf = textproto.TrimString(buf) - if len(buf) == 0 { - break - } - if buf[0] == ',' { - buf = buf[1:] - continue - } - if buf[0] == '*' { - return condFalse - } - etag, remain := scanETag(buf) - if etag == "" { - break - } - if etagWeakMatch(etag, w.Header().Get("Etag")) { - return condFalse - } - buf = remain - } - return condTrue -} - -func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { - if r.Method != "GET" && r.Method != "HEAD" { - return condNone - } - ims := r.Header.Get("If-Modified-Since") - if ims == "" { - return condNone - } - r.Header.Del("If-Modified-Since") - if isZeroTime(modtime) { - return condNone - } - t, err := http.ParseTime(ims) - if err != nil { - return condNone - } - // The Last-Modified header truncates sub-second precision so - // the modtime needs to be truncated too. - modtime = modtime.Truncate(time.Second) - if ret := modtime.Compare(t); ret <= 0 { - return condFalse - } - return condTrue -} - -func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { - if r.Method != "GET" && r.Method != "HEAD" { - return condNone - } - ir := r.Header.Get("If-Range") - if ir == "" { - return condNone - } - r.Header.Del("If-Range") - etag, _ := scanETag(ir) - if etag != "" { - if etagStrongMatch(etag, w.Header().Get("Etag")) { - return condTrue - } - return condFalse - } - // The If-Range value is typically the ETag value, but it may also be - // the modtime date. See golang.org/issue/8367. - if modtime.IsZero() { - return condFalse - } - t, err := http.ParseTime(ir) - if err != nil { - return condFalse - } - if t.Unix() == modtime.Unix() { - return condTrue - } - return condFalse -} - -var unixEpochTime = time.Unix(0, 0) - -// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). -func isZeroTime(t time.Time) bool { - return t.IsZero() || t.Equal(unixEpochTime) -} - -func setLastModified(w http.ResponseWriter, modtime time.Time) { - if !isZeroTime(modtime) { - w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) - } -} - -func writeNotModified(w http.ResponseWriter) { - // RFC 7232 section 4.1: - // a sender SHOULD NOT generate representation metadata other than the - // above listed fields unless said metadata exists for the purpose of - // guiding cache updates (e.g., Last-Modified might be useful if the - // response does not have an ETag field). - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - delete(h, "Content-Encoding") - if h.Get("Etag") != "" { - delete(h, "Last-Modified") - } - w.WriteHeader(http.StatusNotModified) -} - -// checkPreconditions evaluates request preconditions and reports whether a precondition -// resulted in sending StatusNotModified or StatusPreconditionFailed. -func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) { - // This function carefully follows RFC 7232 section 6. - ch := checkIfMatch(w, r) - if ch == condNone { - ch = checkIfUnmodifiedSince(r, modtime) - } - if ch == condFalse { - w.WriteHeader(http.StatusPreconditionFailed) - return true, "" - } - switch checkIfNoneMatch(w, r) { - case condFalse: - if r.Method == "GET" || r.Method == "HEAD" { - writeNotModified(w) - return true, "" - } - w.WriteHeader(http.StatusPreconditionFailed) - return true, "" - case condNone: - if checkIfModifiedSince(r, modtime) == condFalse { - writeNotModified(w) - return true, "" - } - } - - rangeHeader = r.Header.Get("Range") - if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { - rangeHeader = "" - } - return false, rangeHeader -} - func sumRangesSize(ranges []http_range.Range) (size int64) { for _, ra := range ranges { size += ra.Length diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index e147d159e..68a94f5f4 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -18,13 +18,13 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -69,7 +69,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } } // try putting url - if args.Tool == "SimpleHttp" { + if args.Tool == "SimpleHttp" && !isEd2kURL(args.URL) { if isSimpleHttpSchemeUnsupported(args.URL) { return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") } @@ -83,13 +83,17 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro // ed2k 链接自动路由:如果当前工具不支持 ed2k,自动尝试使用迅雷系工具 if isEd2kURL(args.URL) { if !isEd2kCapableTool(args.Tool) { - // 尝试找到一个可用的支持 ed2k 的工具 - fallbackTool, fallbackName := findEd2kCapableTool() - if fallbackTool != nil { - // 使用找到的迅雷工具替代 - args.Tool = fallbackName + if storageTool := ed2kToolForStorage(storage); storageTool != "" { + // Prefer the matching native tool when the destination storage supports ed2k. + args.Tool = storageTool } else { - return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + // Otherwise, try to find an available Thunder-family tool. + fallbackTool, fallbackName := findEd2kCapableTool() + if fallbackTool != nil { + args.Tool = fallbackName + } else { + return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + } } } } @@ -179,7 +183,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro t := &DownloadTask{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, Url: args.URL, DstDirPath: args.DstDirPath, @@ -217,8 +221,47 @@ func isEd2kURL(urlStr string) bool { return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") } -// ed2kCapableTools 支持 ed2k 协议的工具列表(迅雷系) -var ed2kCapableTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} +func ed2kToolForStorage(storage driver.Driver) string { + name := NativeToolName(storage) + switch name { + case "115 Cloud", "115 Open": + return name + default: + return "" + } +} + +// NativeToolName returns the offline-download tool implemented by storage. +func NativeToolName(storage driver.Driver) string { + switch storage.(type) { + case *_115.Pan115: + return "115 Cloud" + case *_115_open.Open115: + return "115 Open" + case *_123.Pan123: + return "123Pan" + case *_123_open.Open123: + return "123 Open" + case *guangyapan.GuangYaPan: + return "GuangYaPan" + case *pikpak.PikPak: + return "PikPak" + case *thunder.Thunder: + return "Thunder" + case *thunderx.ThunderX: + return "ThunderX" + case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: + return "ThunderBrowser" + default: + return "" + } +} + +// ed2kCapableTools 支持 ed2k 协议的工具列表 +var ed2kCapableTools = []string{"115 Cloud", "115 Open", "Thunder", "ThunderX", "ThunderBrowser"} + +// ed2kFallbackTools 是当前可用于自动接管 ed2k 任务的工具列表。 +var ed2kFallbackTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} // isEd2kCapableTool 检查工具是否支持 ed2k 协议 func isEd2kCapableTool(toolName string) bool { @@ -232,7 +275,7 @@ func isEd2kCapableTool(toolName string) bool { // findEd2kCapableTool 查找一个可用的支持 ed2k 的工具 func findEd2kCapableTool() (Tool, string) { - for _, name := range ed2kCapableTools { + for _, name := range ed2kFallbackTools { t, err := Tools.Get(name) if err != nil { continue diff --git a/internal/offline_download/tool/add_test.go b/internal/offline_download/tool/add_test.go new file mode 100644 index 000000000..e538ea2f3 --- /dev/null +++ b/internal/offline_download/tool/add_test.go @@ -0,0 +1,86 @@ +package tool + +import ( + "testing" + + _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" + _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" + "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" + "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" + "github.com/OpenListTeam/OpenList/v4/internal/driver" +) + +func TestIsEd2kCapableTool(t *testing.T) { + tests := []struct { + name string + want bool + }{ + {name: "115 Cloud", want: true}, + {name: "115 Open", want: true}, + {name: "Thunder", want: true}, + {name: "ThunderX", want: true}, + {name: "ThunderBrowser", want: true}, + {name: "aria2", want: false}, + {name: "SimpleHttp", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isEd2kCapableTool(tt.name); got != tt.want { + t.Fatalf("isEd2kCapableTool(%q) = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +func TestEd2kToolForStorage(t *testing.T) { + tests := []struct { + name string + storage driver.Driver + want string + }{ + {name: "115 Cloud", storage: &_115.Pan115{}, want: "115 Cloud"}, + {name: "115 Open", storage: &_115_open.Open115{}, want: "115 Open"}, + {name: "other", storage: nil, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ed2kToolForStorage(tt.storage); got != tt.want { + t.Fatalf("ed2kToolForStorage(%T) = %q, want %q", tt.storage, got, tt.want) + } + }) + } +} + +func TestNativeToolName(t *testing.T) { + tests := []struct { + name string + storage driver.Driver + want string + }{ + {name: "115 Cloud", storage: &_115.Pan115{}, want: "115 Cloud"}, + {name: "115 Open", storage: &_115_open.Open115{}, want: "115 Open"}, + {name: "123Pan", storage: &_123.Pan123{}, want: "123Pan"}, + {name: "123 Open", storage: &_123_open.Open123{}, want: "123 Open"}, + {name: "GuangYaPan", storage: &guangyapan.GuangYaPan{}, want: "GuangYaPan"}, + {name: "PikPak", storage: &pikpak.PikPak{}, want: "PikPak"}, + {name: "Thunder", storage: &thunder.Thunder{}, want: "Thunder"}, + {name: "ThunderX", storage: &thunderx.ThunderX{}, want: "ThunderX"}, + {name: "ThunderBrowser", storage: &thunder_browser.ThunderBrowser{}, want: "ThunderBrowser"}, + {name: "other", storage: nil, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NativeToolName(tt.storage); got != tt.want { + t.Fatalf("NativeToolName(%T) = %q, want %q", tt.storage, got, tt.want) + } + }) + } +} diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go index d5a90f504..57fccd069 100644 --- a/internal/offline_download/tool/tools.go +++ b/internal/offline_download/tool/tools.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" ) var ( @@ -35,6 +36,31 @@ func (t ToolsManager) Names() []string { return names } +// NamesForPath returns ready tools and the native tool for the destination storage. +// Native tools can write directly to their own storage even without a temporary path setting. +func (t ToolsManager) NamesForPath(path string) []string { + names := t.Names() + storage, _, err := op.GetStorageAndActualPath(path) + if err != nil { + return names + } + + name := NativeToolName(storage) + if name == "" { + return names + } + for _, existing := range names { + if existing == name { + return names + } + } + if _, ok := t[name]; ok { + names = append(names, name) + sort.Strings(names) + } + return names +} + func (t ToolsManager) Items() []model.SettingItem { var items []model.SettingItem for _, tool := range t { diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 7109669ee..7c5bd164a 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -20,7 +20,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -140,7 +139,7 @@ func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy D TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(tempDir, entry.Name()), DstActualPath: dstDirActualPath, @@ -276,7 +275,7 @@ func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy D TaskData: fs.TaskData{ TaskExtension: task.TaskExtension{ Creator: taskCreator, - ApiUrl: common.GetApiUrl(ctx), + ApiUrl: conf.GetApiUrl(ctx), }, SrcActualPath: stdpath.Join(srcObjActualPath, obj.GetName()), DstActualPath: dstDirActualPath, diff --git a/internal/op/archive.go b/internal/op/archive.go index d0a53a919..bb3de11a1 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -390,8 +390,9 @@ func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args mo } type objWithLink struct { - link *model.Link - obj model.Obj + link *model.Link + obj model.Obj + policy linkCachePolicy } var ( @@ -405,7 +406,7 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args } key := stdpath.Join(Key(storage, path), args.InnerPath) if ol, ok := extractCache.Get(key); ok { - if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -415,8 +416,8 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } - if ol.link.Expiration != nil { - extractCache.SetWithTTL(key, ol, *ol.link.Expiration) + if ol.policy.expiration != nil { + extractCache.SetWithTTL(key, ol, *ol.policy.expiration) } else { extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers) } @@ -428,7 +429,7 @@ func DriverExtract(ctx context.Context, storage driver.Driver, path string, args if err != nil { return nil, nil, err } - if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -450,7 +451,10 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) - return &objWithLink{link: link, obj: extracted}, err + if err != nil { + return nil, err + } + return admitLink(link, extracted) } type streamWithParent struct { diff --git a/internal/op/fs.go b/internal/op/fs.go index a628796fd..ef6b3524d 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -233,7 +233,10 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if mode == -1 { mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } - typeKey := args.Type + typeKey := "proxy/" + args.Type + if args.Redirect { + typeKey = "redirect/" + args.Type + } if mode&driver.LinkCacheIP != 0 { typeKey += "/" + args.IP } @@ -242,8 +245,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } key := Key(storage, path) if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { - if ol.link.Expiration != nil || - ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } @@ -261,9 +263,12 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil || link == nil { return nil, errors.Wrapf(err, "failed get link") } - ol := &objWithLink{link: link, obj: file} - if link.Expiration != nil { - Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) + ol, err := admitLink(link, file) + if err != nil { + return nil, err + } + if ol.policy.expiration != nil { + Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *ol.policy.expiration) } else { Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } @@ -277,7 +282,7 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if ol == nil || ol.link == nil { return nil, nil, errors.WithStack(errs.ObjectNotFound) } - if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + if ol.acquire() { return ol.link, ol.obj, nil } } diff --git a/internal/op/link_cache_test.go b/internal/op/link_cache_test.go new file mode 100644 index 000000000..6924bf837 --- /dev/null +++ b/internal/op/link_cache_test.go @@ -0,0 +1,71 @@ +package op + +import ( + "context" + "io" + "strings" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +type linkModeDriver struct { + driver.Driver + storage model.Storage + calls int +} + +func (d *linkModeDriver) Config() driver.Config { return driver.Config{} } + +func (d *linkModeDriver) GetStorage() *model.Storage { return &d.storage } + +func (d *linkModeDriver) Get(context.Context, string) (model.Obj, error) { + return &model.Object{Name: "file"}, nil +} + +func (d *linkModeDriver) Link(_ context.Context, _ model.Obj, args model.LinkArgs) (*model.Link, error) { + d.calls++ + expiration := time.Minute + if args.Redirect { + return &model.Link{URL: "https://example.com/file", Expiration: &expiration}, nil + } + return &model.Link{ + RangeReader: stream.RangeReaderFunc(func(context.Context, http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("file")), nil + }), + Expiration: &expiration, + }, nil +} + +func TestLinkCacheSeparatesRedirectAndProxy(t *testing.T) { + for _, tc := range []struct { + name string + firstRedirect bool + }{ + {name: "redirect then proxy", firstRedirect: true}, + {name: "proxy then redirect", firstRedirect: false}, + } { + t.Run(tc.name, func(t *testing.T) { + d := &linkModeDriver{storage: model.Storage{MountPath: "/" + t.Name()}} + for _, redirect := range []bool{tc.firstRedirect, !tc.firstRedirect, tc.firstRedirect, !tc.firstRedirect} { + link, _, err := Link(context.Background(), d, "/file", model.LinkArgs{Redirect: redirect}) + if err != nil { + t.Fatal(err) + } + if redirect && (link.URL == "" || link.RangeReader != nil) { + t.Fatalf("redirect link has wrong shape: %+v", link) + } + if !redirect && (link.URL != "" || link.RangeReader == nil) { + t.Fatalf("proxy link has wrong shape: %+v", link) + } + } + if d.calls != 2 { + t.Fatalf("expected one driver call per mode, got %d", d.calls) + } + }) + } +} diff --git a/internal/op/link_lifecycle.go b/internal/op/link_lifecycle.go new file mode 100644 index 000000000..71491c97b --- /dev/null +++ b/internal/op/link_lifecycle.go @@ -0,0 +1,34 @@ +package op + +import ( + "errors" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var errConflictingLinkLifecycle = errors.New("invalid link lifecycle: expiration cannot be combined with owned closers or RequireReference") + +type linkCachePolicy struct { + expiration *time.Duration + requireReference bool +} + +func admitLink(link *model.Link, obj model.Obj) (*objWithLink, error) { + if link.Expiration != nil && (link.RequireReference || link.SyncClosers.Length() > 0) { + return nil, errors.Join(errConflictingLinkLifecycle, link.Close()) + } + return &objWithLink{ + link: link, + obj: obj, + policy: linkCachePolicy{ + expiration: link.Expiration, + requireReference: link.RequireReference, + }, + }, nil +} + +func (ol *objWithLink) acquire() bool { + return ol.policy.expiration != nil || + ol.link.SyncClosers.AcquireReference() || !ol.policy.requireReference +} diff --git a/internal/op/link_lifecycle_test.go b/internal/op/link_lifecycle_test.go new file mode 100644 index 000000000..53bb0baf7 --- /dev/null +++ b/internal/op/link_lifecycle_test.go @@ -0,0 +1,143 @@ +package op + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type linkLifecycleDriver struct { + model.Storage + links func() *model.Link + calls atomic.Int32 +} + +func (d *linkLifecycleDriver) Config() driver.Config { return driver.Config{} } +func (d *linkLifecycleDriver) GetAddition() driver.Additional { return nil } +func (d *linkLifecycleDriver) Init(context.Context) error { return nil } +func (d *linkLifecycleDriver) Drop(context.Context) error { return nil } +func (d *linkLifecycleDriver) List(context.Context, model.Obj, model.ListArgs) ([]model.Obj, error) { + return nil, nil +} +func (d *linkLifecycleDriver) Get(context.Context, string) (model.Obj, error) { + return &model.Object{Name: "file", Path: "/file"}, nil +} +func (d *linkLifecycleDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + d.calls.Add(1) + return d.links(), nil +} + +func resetLinkLifecycleState(t *testing.T) { + t.Helper() + oldCache := Cache + Cache, linkG = NewCacheManager(), singleflight.Group[*objWithLink]{} + t.Cleanup(func() { Cache, linkG = oldCache, singleflight.Group[*objWithLink]{} }) +} + +func acquireTestLink(t *testing.T, d *linkLifecycleDriver) *model.Link { + t.Helper() + link, _, err := Link(context.Background(), d, "/file", model.LinkArgs{}) + if err != nil { + t.Fatal(err) + } + return link +} + +func TestLinkLifecycleModes(t *testing.T) { + t.Run("TTL descriptor remains reusable after close", func(t *testing.T) { + resetLinkLifecycleState(t) + ttl := time.Minute + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/ttl"}, + links: func() *model.Link { return &model.Link{URL: "https://example.test/file", Expiration: &ttl} }, + } + + first := acquireTestLink(t, d) + _ = first.Close() + second := acquireTestLink(t, d) + if second.URL != first.URL || d.calls.Load() != 1 { + t.Fatalf("TTL link was not reused: calls=%d", d.calls.Load()) + } + _ = second.Close() + }) + + t.Run("references keep shared resources alive until final close", func(t *testing.T) { + resetLinkLifecycleState(t) + var closes atomic.Int32 + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/reference"}, + links: func() *model.Link { + return &model.Link{ + URL: "https://example.test/file", + SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { closes.Add(1); return nil })), + RequireReference: true, + } + }, + } + + first := acquireTestLink(t, d) + second := acquireTestLink(t, d) + _ = first.Close() + if closes.Load() != 0 { + t.Fatal("shared resource closed while another reference was active") + } + _ = second.Close() + if closes.Load() != 1 { + t.Fatalf("final close count = %d, want 1", closes.Load()) + } + third := acquireTestLink(t, d) + _ = third.Close() + if d.calls.Load() != 2 || closes.Load() != 2 { + t.Fatalf("stale link was not replaced: calls=%d closes=%d", d.calls.Load(), closes.Load()) + } + }) + + t.Run("close-invalidated link is reacquired", func(t *testing.T) { + resetLinkLifecycleState(t) + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/close-invalidated"}, + links: func() *model.Link { + return &model.Link{SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { return nil }))} + }, + } + + first := acquireTestLink(t, d) + _ = first.Close() + second := acquireTestLink(t, d) + _ = second.Close() + if d.calls.Load() != 2 { + t.Fatalf("driver calls = %d, want 2", d.calls.Load()) + } + }) + + t.Run("TTL with owned resources is rejected and released", func(t *testing.T) { + resetLinkLifecycleState(t) + ttl := time.Minute + var closes atomic.Int32 + d := &linkLifecycleDriver{ + Storage: model.Storage{MountPath: "/conflict"}, + links: func() *model.Link { + return &model.Link{ + Expiration: &ttl, + SyncClosers: utils.NewSyncClosers(utils.CloseFunc(func() error { closes.Add(1); return nil })), + RequireReference: true, + } + }, + } + + _, _, err := Link(context.Background(), d, "/file", model.LinkArgs{}) + if err == nil || !strings.Contains(err.Error(), "expiration cannot be combined") { + t.Fatalf("unexpected error: %v", err) + } + if closes.Load() != 1 { + t.Fatalf("rejected link close count = %d, want 1", closes.Load()) + } + }) +} diff --git a/internal/op/meta.go b/internal/op/meta.go index b7d867307..526332b25 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,6 +2,8 @@ package op import ( stdpath "path" + "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/db" @@ -15,6 +17,9 @@ import ( ) var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) +var metaSnapshotMu sync.Mutex +var metaSnapshot []model.Meta +var metaSnapshotLoaded bool // metaG maybe not needed var metaG singleflight.Group[*model.Meta] @@ -23,17 +28,56 @@ func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } func getNearestMeta(path string) (*model.Meta, error) { - meta, err := GetMetaByPath(path) - if err == nil { - return meta, nil + var metas []model.Meta + loaded := false + for { + meta, err := GetMetaByPath(path) + if err == nil { + return meta, nil + } + if errors.Cause(err) != errs.MetaNotFound { + return nil, err + } + if !loaded { + metas, err = getMetaSnapshot() + if err != nil { + return nil, err + } + loaded = true + } + var matched *model.Meta + for i := range metas { + if !strings.EqualFold(utils.FixAndCleanPath(metas[i].Path), path) { + continue + } + if matched != nil { + return nil, errors.Errorf("multiple metas match %q case-insensitively", path) + } + matched = &metas[i] + } + if matched != nil { + return matched, nil + } + if path == "/" { + return nil, errs.MetaNotFound + } + path = stdpath.Dir(path) } - if errors.Cause(err) != errs.MetaNotFound { - return nil, err +} + +func getMetaSnapshot() ([]model.Meta, error) { + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + if metaSnapshotLoaded { + return metaSnapshot, nil } - if path == "/" { - return nil, errs.MetaNotFound + metas, err := db.GetAllMetas() + if err != nil { + return nil, err } - return getNearestMeta(stdpath.Dir(path)) + metaSnapshot = metas + metaSnapshotLoaded = true + return metaSnapshot, nil } func GetMetaByPath(path string) (*model.Meta, error) { @@ -68,7 +112,12 @@ func DeleteMetaById(id uint) error { return err } metaCache.Del(old.Path) - return db.DeleteMetaById(id) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err = db.DeleteMetaById(id) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func UpdateMeta(u *model.Meta) error { @@ -79,13 +128,23 @@ func UpdateMeta(u *model.Meta) error { } metaCache.Del(old.Path) metaCache.Del(u.Path) - return db.UpdateMeta(u) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err = db.UpdateMeta(u) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func CreateMeta(u *model.Meta) error { u.Path = utils.FixAndCleanPath(u.Path) metaCache.Del(u.Path) - return db.CreateMeta(u) + metaSnapshotMu.Lock() + defer metaSnapshotMu.Unlock() + err := db.CreateMeta(u) + metaSnapshot = nil + metaSnapshotLoaded = false + return err } func GetMetaById(id uint) (*model.Meta, error) { diff --git a/internal/op/path.go b/internal/op/path.go index 294261e89..9157e2f22 100644 --- a/internal/op/path.go +++ b/internal/op/path.go @@ -30,6 +30,20 @@ func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath return } +// GetStorageVirtualMountPath returns the deterministic virtual mount path +// without advancing the balanced-storage counter. +func GetStorageVirtualMountPath(rawPath string) (string, error) { + rawPath = utils.FixAndCleanPath(rawPath) + storages := getStoragesByPath(rawPath) + if len(storages) == 0 { + if rawPath == "/" { + return "", errs.NewErr(errs.StorageNotFound, "please add a storage first") + } + return "", errs.NewErr(errs.StorageNotFound, "rawPath: %s", rawPath) + } + return utils.FixAndCleanPath(utils.GetActualMountPath(storages[0].GetStorage().MountPath)), nil +} + // urlTreeSplitLineFormPath 分割path中分割真实路径和UrlTree定义字符串 func urlTreeSplitLineFormPath(path string) (pp string, file string) { // url.PathUnescape 会移除 // ,手动加回去 diff --git a/internal/op/storage.go b/internal/op/storage.go index 1537fcd3f..0f05badc9 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -242,14 +242,16 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { return nil } storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed get storage driver") + } + // Storage settings may change which objects are exposed at the same mount + // path, so cached entries must be discarded before reinitializing the driver. + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) - Cache.DeleteDirectoryTree(storageDriver, "/") - Cache.InvalidateStorageDetails(storageDriver) - } - if err != nil { - return errors.WithMessage(err, "failed get storage driver") } err = storageDriver.Drop(ctx) if err != nil { diff --git a/internal/sharing/link.go b/internal/sharing/link.go index 32ae6b836..e021f3c23 100644 --- a/internal/sharing/link.go +++ b/internal/sharing/link.go @@ -4,11 +4,11 @@ import ( "context" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" ) @@ -38,7 +38,7 @@ func link(ctx context.Context, sid, path string, args *LinkArgs) (*model.Sharing return nil, nil, nil, errors.WithMessage(err, "failed get sharing link") } if l.URL != "" && !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://") { - l.URL = common.GetApiUrl(ctx) + l.URL + l.URL = conf.GetApiUrl(ctx) + l.URL } return sharing, l, obj, nil } diff --git a/internal/stream/readat_test.go b/internal/stream/readat_test.go new file mode 100644 index 000000000..3698052c2 --- /dev/null +++ b/internal/stream/readat_test.go @@ -0,0 +1,179 @@ +package stream_test + +import ( + "bytes" + "context" + "io" + "math/rand" + "sync/atomic" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +// maxReuseGap mirrors the internal continuation-reuse window (4*utils.MB). +const maxReuseGap = 4 * 1024 * 1024 + +// newMockSeekableStream builds a SeekableStream whose range reads are served +// from data, counting every upstream range request in gets. +func newMockSeekableStream(t *testing.T, data []byte, gets *atomic.Int64) *stream.SeekableStream { + t.Helper() + rr := stream.RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + gets.Add(1) + if r.Length < 0 || r.Start+r.Length > int64(len(data)) { + r.Length = int64(len(data)) - r.Start + } + return io.NopCloser(io.NewSectionReader(bytes.NewReader(data), r.Start, r.Length)), nil + }) + ss, err := stream.NewSeekableStream(&stream.FileStream{ + Obj: &model.Object{Size: int64(len(data))}, + Ctx: context.Background(), + }, &model.Link{ + RangeReader: rr, + ContentLength: int64(len(data)), + }) + if err != nil { + t.Fatalf("NewSeekableStream() error = %v", err) + } + return ss +} + +// readAtFull reads len(p) bytes at off and fails the test on mismatch. +func readAtFull(t *testing.T, ra io.ReaderAt, data []byte, off int64, p []byte) { + t.Helper() + n, err := ra.ReadAt(p, off) + if err != nil { + t.Fatalf("ReadAt(off=%d) error = %v", off, err) + } + if !bytes.Equal(p, data[off:off+int64(n)]) { + t.Fatalf("ReadAt(off=%d) content mismatch", off) + } +} + +func randomData(size int) []byte { + data := make([]byte, size) + x := uint64(42) + for i := range data { + x = x*6364136223846793005 + 1 + data[i] = byte(x >> 33) + } + return data +} + +// Sequential reads must reuse a single upstream range request. +func TestReadAtSeekerSequentialReuse(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 128*1024) + for off := 0; off < len(data); off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + if n := gets.Load(); n != 1 { + t.Fatalf("sequential read issued %d range requests, want 1", n) + } +} + +// A read landing up to maxReuseGap bytes past a parked reader must be served +// by advancing that reader, without a new range request. +func TestReadAtSeekerSkipsAheadWithinWindow(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + // Park a continuation reader right after reading the first 2 MiB. + chunk := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(chunk) { + readAtFull(t, ra, data, int64(off), chunk) + } + skip := 512 * 1024 + off := int64(2*1024*1024 + skip) + readAtFull(t, ra, data, off, chunk) + if n := gets.Load(); n != 1 { + t.Fatalf("window skip issued %d range requests, want 1", n) + } + // A second skip deeper inside the window must also be free. + off = int64(4*1024*1024) - 128*1024 + readAtFull(t, ra, data, off, chunk) + if n := gets.Load(); n != 1 { + t.Fatalf("second window skip issued %d range requests, want 1", n) + } +} + +// A forward jump beyond the reuse window must open a new range request but +// keep the parked reader available for later window hits. +func TestReadAtSeekerFarJumpOpensNewRequest(t *testing.T) { + data := randomData(16 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + // 2 MiB -> 10 MiB is beyond the 4 MiB reuse window. + off := int64(10 * 1024 * 1024) + readAtFull(t, ra, data, off, buf) + if n := gets.Load(); n != 2 { + t.Fatalf("far jump issued %d range requests, want 2", n) + } + // Back within the window of the 10 MiB chain: free reuse again. + readAtFull(t, ra, data, off+maxReuseGap, buf) + if n := gets.Load(); n != 2 { + t.Fatalf("jump inside new window issued %d range requests, want 2", n) + } +} + +// Backward reads can never reuse a parked continuation and must open a new +// range request. +func TestReadAtSeekerBackwardJumpOpensNewRequest(t *testing.T) { + data := randomData(8 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + buf := make([]byte, 256*1024) + for off := 0; off < 2*1024*1024; off += len(buf) { + readAtFull(t, ra, data, int64(off), buf) + } + readAtFull(t, ra, data, int64(1024*1024), buf) + if n := gets.Load(); n != 2 { + t.Fatalf("backward jump issued %d range requests, want 2", n) + } +} + +// Random reads must return correct data and keep upstream requests bounded: +// each read is either a window hit or a fresh request, never more than one. +func TestReadAtSeekerRandomReads(t *testing.T) { + data := randomData(32 * 1024 * 1024) + var gets atomic.Int64 + ss := newMockSeekableStream(t, data, &gets) + ra, err := stream.NewReadAtSeeker(ss, 0, true) + if err != nil { + t.Fatalf("NewReadAtSeeker() error = %v", err) + } + const chunk = 8 * 1024 + buf := make([]byte, chunk) + r := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + off := r.Int63n(int64(len(data)) - chunk) + readAtFull(t, ra, data, off, buf) + } + if n := gets.Load(); n > 200 { + t.Fatalf("random reads issued %d range requests, want <= 200", n) + } +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 6d3b36072..45be2e5ad 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -7,6 +7,7 @@ import ( "io" "math" "os" + "sort" "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -419,10 +420,72 @@ func (r *ReaderUpdatingProgress) Close() error { type RangeReadReadAtSeeker struct { ss *SeekableStream masterOff int64 - readerMap sync.Map + readers orderedReaders headCache *headCache } +type orderedReaders struct { + mu sync.Mutex + m map[int64]io.Reader + keys []int64 +} + +func (o *orderedReaders) store(off int64, r io.Reader) { + o.mu.Lock() + defer o.mu.Unlock() + if _, ok := o.m[off]; ok { + o.m[off] = r + return + } + if o.m == nil { + o.m = make(map[int64]io.Reader) + } + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= off }) + o.keys = append(o.keys, 0) + copy(o.keys[i+1:], o.keys[i:]) + o.keys[i] = off + o.m[off] = r +} + +func (o *orderedReaders) takeExact(off int64) (io.Reader, bool) { + o.mu.Lock() + defer o.mu.Unlock() + r, ok := o.m[off] + if ok { + delete(o.m, off) + o.removeKey(off) + } + return r, ok +} + +func (o *orderedReaders) takeBest(off int64) (io.Reader, int64, bool) { + o.mu.Lock() + defer o.mu.Unlock() + if r, ok := o.m[off]; ok { + delete(o.m, off) + o.removeKey(off) + return r, off, true + } + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= off }) + if i == 0 { + return nil, 0, false + } + k := o.keys[i-1] + if off-k > 4*utils.MB { + return nil, 0, false + } + r := o.m[k] + delete(o.m, k) + o.removeKey(k) + return r, k, true +} + +func (o *orderedReaders) removeKey(k int64) { + i := sort.Search(len(o.keys), func(i int) bool { return o.keys[i] >= k }) + copy(o.keys[i:], o.keys[i+1:]) + o.keys = o.keys[:len(o.keys)-1] +} + type headCache struct { reader io.Reader bufs [][]byte @@ -457,7 +520,7 @@ func (r *headCache) Close() error { func (r *RangeReadReadAtSeeker) InitHeadCache() { if r.masterOff == 0 { - value, _ := r.readerMap.LoadAndDelete(int64(0)) + value, _ := r.readers.takeExact(0) r.headCache = &headCache{reader: value.(io.Reader)} r.ss.Closers.Add(r.headCache) } @@ -483,9 +546,9 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (mode if err != nil { return nil, err } - r.readerMap.Store(int64(offset), reader) + r.readers.store(offset, reader) } else { - r.readerMap.Store(int64(offset), ss) + r.readers.store(0, ss) } return r, nil } @@ -503,41 +566,15 @@ func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error) { - for { - var cur int64 = -1 - r.readerMap.Range(func(key, value any) bool { - k := key.(int64) - if off == k { - cur = k - return false - } - if off > k && off-k <= 4*utils.MB && k > cur { - cur = k - } - return true - }) - if cur < 0 { - break - } - v, ok := r.readerMap.LoadAndDelete(int64(cur)) - if !ok { - continue - } - rr := v.(io.Reader) - if off == int64(cur) { - // logrus.Debugf("getReaderAtOffset match_%d", off) + if rr, cur, ok := r.readers.takeBest(off); ok { + if cur == off { return rr, nil } n, _ := utils.CopyWithBufferN(io.Discard, rr, off-cur) - cur += n - if cur == off { - // logrus.Debugf("getReaderAtOffset old_%d", off) + if cur+n == off { return rr, nil } - break } - - // logrus.Debugf("getReaderAtOffset new_%d", off) reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) if err != nil { return nil, err @@ -562,7 +599,7 @@ func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { off += int64(n) switch err { case nil: - r.readerMap.Store(int64(off), rr) + r.readers.store(off, rr) case io.ErrUnexpectedEOF: err = io.EOF } diff --git a/internal/task/base.go b/internal/task/base.go index f969e6c27..6e778dd96 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -12,8 +12,8 @@ import ( type TaskExtension struct { tache.Base Creator *model.User - startTime *time.Time - endTime *time.Time + StartTime *time.Time `json:"start_time,omitempty"` + EndTime *time.Time `json:"end_time,omitempty"` TotalBytes int64 ApiUrl string } @@ -38,23 +38,23 @@ func (t *TaskExtension) GetCreator() *model.User { } func (t *TaskExtension) SetStartTime(startTime time.Time) { - t.startTime = &startTime + t.StartTime = &startTime } func (t *TaskExtension) GetStartTime() *time.Time { - return t.startTime + return t.StartTime } func (t *TaskExtension) SetEndTime(endTime time.Time) { - t.endTime = &endTime + t.EndTime = &endTime } func (t *TaskExtension) GetEndTime() *time.Time { - return t.endTime + return t.EndTime } func (t *TaskExtension) ClearEndTime() { - t.endTime = nil + t.EndTime = nil } func (t *TaskExtension) SetTotalBytes(totalBytes int64) { diff --git a/internal/task/base_test.go b/internal/task/base_test.go new file mode 100644 index 000000000..51ddc675b --- /dev/null +++ b/internal/task/base_test.go @@ -0,0 +1,20 @@ +package task_test + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +func TestTaskExtensionRestoresAPIURL(t *testing.T) { + const want = "https://openlist.example" + extension := task.TaskExtension{ApiUrl: want} + + extension.SetCtx(context.Background()) + + if got := conf.GetApiUrl(extension.Ctx()); got != want { + t.Fatalf("restored origin = %q, want %q", got, want) + } +} diff --git a/internal/token/token.go b/internal/token/token.go index 39fe61281..1b03c4ccd 100755 --- a/internal/token/token.go +++ b/internal/token/token.go @@ -1,6 +1,7 @@ package token import ( + "fmt" "strconv" "time" @@ -73,12 +74,27 @@ func saveAccountToken(prefix, refreshToken, accessToken string, accountId int) { func SyncTokens(id int, data base.Json) { url := "http://127.0.0.1:4567/api/pan/accounts/" + strconv.Itoa(id) + "/token" - _, err := base.RestyClient.R(). - SetHeader("X-API-KEY", setting.GetStr("atv_api_key")). - SetBody(data). - Post(url) - if err != nil { - log.Warnf("[%v] sync tokens failed: %v", id, err) + // 回写失败必须可见并重试:非 2xx 以前被静默吞掉,alist-tvbox 侧 cookie 停在旧值, + // 其下次重启会用旧值重建 storage,覆盖这里已轮换的新值(#1082)。 + for attempt := 1; attempt <= 2; attempt++ { + res, err := base.RestyClient.R(). + SetHeader("X-API-KEY", setting.GetStr("atv_api_key")). + SetBody(data). + Post(url) + if err == nil && res.IsSuccess() { + return + } + if err == nil { + body := res.String() + if len(body) > 200 { + body = body[:200] + } + err = fmt.Errorf("http %d: %s", res.StatusCode(), body) + } + log.Warnf("[%v] sync tokens failed (attempt %d/2): %v", id, attempt, err) + if attempt == 1 { + time.Sleep(time.Second) + } } } diff --git a/renovate.json b/renovate.json index 48515b9fc..d73a66196 100644 --- a/renovate.json +++ b/renovate.json @@ -2,5 +2,12 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:best-practices" + ], + "packageRules": [ + { + "matchManagers": ["dockerfile", "docker-compose"], + "matchPackageNames": ["alpine", "ghcr.io/openlistteam/openlist-base-image", "openlistteam/openlist:latest"], + "enabled": false + } ] } diff --git a/server/common/base.go b/server/common/base.go index 8aa669e3a..53ef633b4 100644 --- a/server/common/base.go +++ b/server/common/base.go @@ -31,6 +31,5 @@ func GetApiUrlFromRequest(r *http.Request) string { } func GetApiUrl(ctx context.Context) string { - api, _ := ctx.Value(conf.ApiUrlKey).(string) - return api + return conf.GetApiUrl(ctx) } diff --git a/server/common/check.go b/server/common/check.go index 2c8d3bc8b..728897013 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -44,7 +44,10 @@ func CanWriteContentBypassUserPerms(meta *model.Meta, path string) bool { if meta == nil || !meta.Write { return false } - return MetaCoversPath(meta.Path, path, meta.WSub) + if utils.PathEqual(meta.Path, path) { + return true + } + return utils.IsSubPath(meta.Path, path) && meta.WSub } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { @@ -78,10 +81,21 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri } func MetaCoversPath(metaPath, reqPath string, applyToSubFolder bool) bool { - if utils.PathEqual(metaPath, reqPath) { + metaPath = utils.FixAndCleanPath(metaPath) + reqPath = utils.FixAndCleanPath(reqPath) + if strings.EqualFold(metaPath, reqPath) { return true } - return utils.IsSubPath(metaPath, reqPath) && applyToSubFolder + if !applyToSubFolder { + return false + } + for reqPath != "/" { + reqPath = path.Dir(reqPath) + if strings.EqualFold(metaPath, reqPath) { + return true + } + } + return false } // ShouldProxy TODO need optimize diff --git a/server/common/check_test.go b/server/common/check_test.go index 18abca8e9..631cf2667 100644 --- a/server/common/check_test.go +++ b/server/common/check_test.go @@ -84,6 +84,27 @@ func TestCoversPath(t *testing.T) { applySub: true, want: false, }, + { + name: "case-insensitive exact path match", + metaPath: "/Folder", + reqPath: "/folder", + applySub: false, + want: true, + }, + { + name: "case-insensitive sub path match", + metaPath: "/Folder", + reqPath: "/folder/Subfolder", + applySub: true, + want: true, + }, + { + name: "case-insensitive sibling prefix does not match", + metaPath: "/Folder", + reqPath: "/folder-name", + applySub: true, + want: false, + }, } for _, tt := range tests { @@ -166,6 +187,17 @@ func TestCanWriteContentIgnoringUserPerms(t *testing.T) { want: false, reason: "non-sub path should deny write even with WSub=true", }, + { + name: "case-insensitive match does not broaden write bypass", + meta: &model.Meta{ + Path: "/Folder", + Write: true, + WSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "case-insensitive matching should only enforce restrictions, not grant write bypass", + }, } for _, tt := range tests { @@ -579,6 +611,23 @@ func TestCanAccessWithReadPermissions(t *testing.T) { want: false, reason: "user not in ReadUsers list should be denied", }, + { + name: "case-insensitive exact path still requires password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/Folder", + Password: "secret", + PSub: false, + }, + reqPath: "/folder", + password: "wrong", + want: false, + reason: "changing only path casing must not bypass an exact-path password", + }, } for _, tt := range tests { diff --git a/server/common/etag_test.go b/server/common/etag_test.go new file mode 100644 index 000000000..d4784727e --- /dev/null +++ b/server/common/etag_test.go @@ -0,0 +1,42 @@ +package common + +import ( + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func TestGetEtagStableAcrossCacheRefresh(t *testing.T) { + uploaded := &model.Object{Size: 4, Modified: time.Unix(1700000000, 123456789)} + before := GetEtag(uploaded, uploaded.Size) + refreshed := &model.Object{Size: uploaded.Size, Modified: time.Unix(1700000000, 0)} + if got := GetEtag(refreshed, refreshed.Size); got != before { + t.Fatalf("refreshed file ETag = %q, want %q", got, before) + } + refreshed.Modified = refreshed.Modified.Add(time.Second) + if got := GetEtag(refreshed, refreshed.Size); got == before { + t.Errorf("changed modification second retained ETag %q", got) + } + refreshed.Modified = uploaded.Modified + if got := GetEtag(refreshed, refreshed.Size+1); got == before { + t.Errorf("changed size retained ETag %q", got) + } +} + +func TestGetEtagContentHash(t *testing.T) { + file := &model.Object{ + Size: 4, + HashInfo: utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("data"))), + } + before := GetEtag(file, file.Size) + file.Modified = time.Unix(1700000000, 100) + if got := GetEtag(file, file.Size); got != before { + t.Errorf("unchanged content ETag = %q, want %q", got, before) + } + file.HashInfo = utils.NewHashInfo(utils.SHA256, utils.HashData(utils.SHA256, []byte("next"))) + if got := GetEtag(file, file.Size); got == before { + t.Errorf("changed content retained ETag %q", got) + } +} diff --git a/server/common/proxy.go b/server/common/proxy.go index be1a8a933..ce0e85222 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -60,9 +60,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. attachHeader(w, file, link) w.Header().Set("Content-Type", detected.ContentType) w.Header().Set("Etag", GetEtag(file, detected.Size)) - return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), detected.Size, &model.RangeReadCloser{ - RangeReader: detected.RangeReader, - }) + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), detected.Size, detected.RangeReader) } } } @@ -98,9 +96,7 @@ func proxyRangeReader(w http.ResponseWriter, r *http.Request, link *model.Link, w.Header().Set("Content-Type", detected.ContentType) w.Header().Set("Etag", GetEtag(file, size)) } - return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, &model.RangeReadCloser{ - RangeReader: rrf, - }) + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), size, rrf) } func attachHeader(w http.ResponseWriter, file model.Obj, link *model.Link) { @@ -129,7 +125,8 @@ func GetEtag(file model.Obj, size int64) string { if len(hash) > 0 { return fmt.Sprintf(`"%s"`, hash) } - // 参考nginx + // Storage backends may not preserve sub-second timestamps across cache + // refreshes, so only use the reproducible second precision here. return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), size) } diff --git a/server/common/proxy_cancel_test.go b/server/common/proxy_cancel_test.go new file mode 100644 index 000000000..04f37401e --- /dev/null +++ b/server/common/proxy_cancel_test.go @@ -0,0 +1,49 @@ +package common + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestProxyCancelledPartitionedReaderDoesNotPanic(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + t.Cleanup(func() { conf.Conf = oldConf }) + link := &model.Link{ + Concurrency: 2, + PartSize: 4, + RangeReader: stream.RangeReaderFunc(func(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader([]byte("0123456789abcdef")[requested.Start : requested.Start+requested.Length])), nil + }), + } + file := &model.Object{Name: "fixture.bin", Size: 16} + for range 32 { + func() { + defer func() { + if recovered := recover(); recovered != nil { + t.Errorf("Proxy panicked on cancelled partitioned read: %v", recovered) + } + }() + r := httptest.NewRequest(http.MethodGet, "/proxy/fixture.bin", nil) + ctx, cancel := context.WithCancel(r.Context()) + cancel() + w := httptest.NewRecorder() + _ = Proxy(w, r.WithContext(ctx), link, file) + if bytes.Contains(w.Body.Bytes(), []byte("0123456789abcdef")) { + t.Errorf("cancelled response contained file contents: %q", w.Body.String()) + } + }() + } +} diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 7a96a4f65..1894fc0ee 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -94,10 +94,11 @@ func (f *FileUploadProxy) Close() error { return err } arr := make([]byte, 512) - if _, err := f.buffer.Read(arr); err != nil { + n, err := f.buffer.Read(arr) + if err != nil && err != io.EOF { return err } - contentType := http.DetectContentType(arr) + contentType := http.DetectContentType(arr[:n]) if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { return err } diff --git a/server/handles/auth.go b/server/handles/auth.go index 780069091..2013a5c2b 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,14 +3,17 @@ package handles import ( "bytes" "encoding/base64" + "errors" "image/png" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" + "gorm.io/gorm" ) type LoginReq struct { @@ -111,11 +114,29 @@ func UpdateCurrent(c *gin.Context) { common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } + ssoID := req.SsoID + if req.SsoID != "" && req.SsoID != user.SsoID { + claims, err := parseSSOBindingToken(c, req.SsoID, ssoBindingProofPurpose) + if err != nil { + common.ErrorStrResp(c, "invalid or expired SSO binding proof", 400) + return + } + boundUser, err := db.GetUserBySSOID(claims.SsoID) + if err == nil && boundUser.ID != user.ID { + common.ErrorStrResp(c, "SSO account is already bound to another user", 409) + return + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + common.ErrorResp(c, err, 500) + return + } + ssoID = claims.SsoID + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) } - user.SsoID = req.SsoID + user.SsoID = ssoID if err := op.UpdateUser(user); err != nil { common.ErrorResp(c, err, 500) } else { diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go index d77c3044c..221b1afa4 100644 --- a/server/handles/direct_upload.go +++ b/server/handles/direct_upload.go @@ -1,6 +1,8 @@ package handles import ( + "context" + "mime" "net/url" stdpath "path" @@ -8,15 +10,18 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" + "github.com/pkg/errors" ) type FsGetDirectUploadInfoReq struct { - Path string `json:"path" form:"path"` - FileName string `json:"file_name" form:"file_name"` - FileSize int64 `json:"file_size" form:"file_size"` - Tool string `json:"tool" form:"tool"` + Path string `json:"path" form:"path"` + FileName string `json:"file_name" form:"file_name"` + FileSize int64 `json:"file_size" form:"file_size"` + ContentType string `json:"content_type" form:"content_type"` + Tool string `json:"tool" form:"tool"` } // FsGetDirectUploadInfo returns the direct upload info if supported by the driver @@ -44,8 +49,40 @@ func FsGetDirectUploadInfo(c *gin.Context) { common.ErrorResp(c, err, 403) return } - overwrite := c.GetHeader("Overwrite") != "false" + // Resolve the destination once so permission checks and the issued upload + // capability cannot target different paths. dstPath := stdpath.Join(path, req.FileName) + path = stdpath.Dir(dstPath) + req.FileName = stdpath.Base(dstPath) + parentMeta, err := op.GetNearestMeta(path) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, path) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !common.CanWrite(user, parentMeta, path) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + // A single-segment file name can still name a nested mount point. + parentMountPath, err := op.GetStorageVirtualMountPath(path) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + targetMountPath, err := op.GetStorageVirtualMountPath(dstPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if parentMountPath != targetMountPath { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + overwrite := c.GetHeader("Overwrite") != "false" if !overwrite { res, err := fs.Get(c.Request.Context(), dstPath, &fs.GetArgs{NoLog: true}) if err != nil && !errs.IsObjectNotFound(err) { @@ -57,7 +94,14 @@ func FsGetDirectUploadInfo(c *gin.Context) { return } } - directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize, overwrite) + if req.ContentType != "" { + if _, _, err := mime.ParseMediaType(req.ContentType); err != nil { + common.ErrorResp(c, err, 400) + return + } + } + ctx := context.WithValue(c, conf.DirectUploadContentTypeKey, req.ContentType) + directUploadInfo, err := fs.GetDirectUploadInfo(ctx, req.Tool, path, req.FileName, req.FileSize, overwrite) if err != nil { if !overwrite && errs.IsObjectAlreadyExists(err) { common.ErrorStrResp(c, "file exists", 403) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 9587b358e..2b99a8f4f 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -117,6 +117,10 @@ func FsMove(c *gin.Context) { // ensure req.Names are not relative/traversal paths safeNames := make([]string, 0, len(req.Names)) for _, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } srcPath := stdpath.Join(srcDir, name) if !strings.HasPrefix(srcPath+"/", srcDir) { continue @@ -222,6 +226,10 @@ func FsCopy(c *gin.Context) { // ensure req.Names are not relative/traversal paths safeNames := make([]string, 0, len(req.Names)) for _, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } srcPath := stdpath.Join(srcDir, name) if !strings.HasPrefix(srcPath+"/", srcDir) { continue @@ -379,6 +387,10 @@ func FsRemove(c *gin.Context) { return } for _, name := range req.Names { + if err := checkRelativePath(name); err != nil { + common.ErrorResp(c, err, 403) + return + } // Skip invalid item names (empty string, whitespace, ".", "/","\t\t","..") to prevent accidental removal of current directory if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { utils.Log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, reqDir) diff --git a/server/handles/fsmanage_test.go b/server/handles/fsmanage_test.go new file mode 100644 index 000000000..37096e97a --- /dev/null +++ b/server/handles/fsmanage_test.go @@ -0,0 +1,126 @@ +package handles + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func setupBackslashTraversalTest(t *testing.T, root string, permission int32) *model.User { + t.Helper() + database, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + conf.Conf = conf.DefaultConfig(t.TempDir()) + db.Init(database) + addition, err := utils.Json.MarshalToString(map[string]string{"root_folder_path": root}) + if err != nil { + t.Fatal(err) + } + if _, err = op.CreateStorage(context.Background(), model.Storage{ + Driver: "Local", MountPath: "/", Addition: addition, + }); err != nil { + t.Fatal(err) + } + return &model.User{ + Username: "restricted-user", BasePath: "/team/a", Role: model.GENERAL, + Permission: permission, + } +} + +func prepareBackslashTraversalFs(t *testing.T) (root string, secretPath string) { + t.Helper() + root = t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "team", "a", "writable"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "team", "ab"), 0o700); err != nil { + t.Fatal(err) + } + secretPath = filepath.Join(root, "team", "ab", "secret.txt") + if err := os.WriteFile(secretPath, []byte("synthetic-secret"), 0o600); err != nil { + t.Fatal(err) + } + return root, secretPath +} + +func invokeHandler(t *testing.T, user *model.User, payload any, handler gin.HandlerFunc) *httptest.ResponseRecorder { + t.Helper() + body, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + req := httptest.NewRequest(http.MethodPost, "/api/fs/remove", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(context.WithValue(req.Context(), conf.UserKey, user)) + ctx.Request = req + handler(ctx) + return recorder +} + +func TestFsRemoveRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<7) + + for _, name := range []string{"../../ab/secret.txt", `..\..\ab\secret.txt`} { + recorder := invokeHandler(t, user, map[string]any{"dir": "/writable", "names": []string{name}}, FsRemove) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("payload %q: got status=%d body=%s, want 403", name, recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("payload %q deleted sibling file: %v", name, err) + } + } +} + +func TestFsMoveRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<5) + + recorder := invokeHandler(t, user, map[string]any{ + "src_dir": "/writable", "dst_dir": "/", "names": []string{`..\..\ab\secret.txt`}, + }, FsMove) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("got status=%d body=%s, want 403", recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("backslash traversal moved sibling file: %v", err) + } +} + +func TestFsCopyRejectsBackslashTraversal(t *testing.T) { + gin.SetMode(gin.TestMode) + root, secretPath := prepareBackslashTraversalFs(t) + user := setupBackslashTraversalTest(t, root, 1<<3|1<<6) + + recorder := invokeHandler(t, user, map[string]any{ + "src_dir": "/writable", "dst_dir": "/", "names": []string{`..\..\ab\secret.txt`}, + }, FsCopy) + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"code":403`) { + t.Fatalf("got status=%d body=%s, want 403", recorder.Code, recorder.Body.String()) + } + if _, err := os.Stat(secretPath); err != nil { + t.Fatalf("backslash traversal affected sibling file: %v", err) + } +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index eb8e2362f..3991c2219 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -354,11 +354,13 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { } } } - var related []model.Obj parentPath := stdpath.Dir(reqPath) - sameLevelFiles, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) - if err == nil { - related = filterRelated(sameLevelFiles, obj) + var related []model.Obj + if !obj.IsDir() && utils.GetFileType(obj.GetName()) == conf.VIDEO { + sameLevelFiles, err := fs.List(c.Request.Context(), parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelated(sameLevelFiles, obj) + } } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) @@ -373,7 +375,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), Thumb: thumb, MountDetails: mountDetails, }, diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 339ad71c8..4f9abaf58 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -3,15 +3,7 @@ package handles import ( "strings" - _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" - _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" - _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" - _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" - "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" - "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -23,6 +15,44 @@ import ( "github.com/pkg/errors" ) +func saveAndInitOfflineDownloadTool(c *gin.Context, name string, items []model.SettingItem) (string, bool) { + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + downloadTool, err := tool.Tools.Get(name) + if err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + version, err := downloadTool.Init() + if err != nil { + common.ErrorResp(c, err, 500) + return "", false + } + return version, true +} + +func validateOfflineDownloadStorage(c *gin.Context, tempDir, nativeTool string) bool { + if tempDir == "" { + return true + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return false + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return false + } + if tool.NativeToolName(storage) != nativeTool { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only "+nativeTool+" is supported", 400) + return false + } + return true +} + type SetAria2Req struct { Uri string `json:"uri" form:"uri"` Secret string `json:"secret" form:"secret"` @@ -38,18 +68,8 @@ func SetAria2(c *gin.Context) { {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("aria2") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - version, err := _tool.Init() - if err != nil { - common.ErrorResp(c, err, 500) + version, ok := saveAndInitOfflineDownloadTool(c, "aria2", items) + if !ok { return } common.SuccessResp(c, version) @@ -70,17 +90,7 @@ func SetQbittorrent(c *gin.Context) { {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("qBittorrent") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "qBittorrent", items); !ok { return } common.SuccessResp(c, "ok") @@ -101,17 +111,7 @@ func SetTransmission(c *gin.Context) { {Key: conf.TransmissionUri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.TransmissionSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("Transmission") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "Transmission", items); !ok { return } common.SuccessResp(c, "ok") @@ -127,35 +127,13 @@ func Set115(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_115.Pan115); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "115 Cloud") { + return } items := []model.SettingItem{ {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("115 Cloud") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "115 Cloud", items); !ok { return } common.SuccessResp(c, "ok") @@ -171,35 +149,13 @@ func Set115Open(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_115_open.Open115); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Open is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "115 Open") { + return } items := []model.SettingItem{ {Key: conf.Pan115OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("115 Open") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "115 Open", items); !ok { return } common.SuccessResp(c, "ok") @@ -215,35 +171,13 @@ func Set123Pan(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_123.Pan123); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123Pan is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "123Pan") { + return } items := []model.SettingItem{ {Key: conf.Pan123TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("123Pan") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "123Pan", items); !ok { return } common.SuccessResp(c, "ok") @@ -260,36 +194,14 @@ func Set123Open(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*_123_open.Open123); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123 Open is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "123 Open") { + return } items := []model.SettingItem{ {Key: conf.Pan123OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, {Key: conf.Pan123OpenOfflineDownloadCallbackUrl, Value: req.CallbackUrl, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("123 Open") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "123 Open", items); !ok { return } common.SuccessResp(c, "ok") @@ -305,35 +217,13 @@ func SetPikPak(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*pikpak.PikPak); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "PikPak") { + return } items := []model.SettingItem{ {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("PikPak") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "PikPak", items); !ok { return } common.SuccessResp(c, "ok") @@ -393,35 +283,13 @@ func SetThunder(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*thunder.Thunder); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "Thunder") { + return } items := []model.SettingItem{ {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("Thunder") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "Thunder", items); !ok { return } common.SuccessResp(c, "ok") @@ -437,35 +305,13 @@ func SetThunderX(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - if _, ok := storage.(*thunderx.ThunderX); !ok { - common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderX is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "ThunderX") { + return } items := []model.SettingItem{ {Key: conf.ThunderXTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("ThunderX") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "ThunderX", items); !ok { return } common.SuccessResp(c, "ok") @@ -481,43 +327,19 @@ func SetThunderBrowser(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if req.TempDir != "" { - storage, _, err := op.GetStorageAndActualPath(req.TempDir) - if err != nil { - common.ErrorStrResp(c, "storage does not exists", 400) - return - } - if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { - common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) - return - } - switch storage.(type) { - case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: - default: - common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderBrowser is supported", 400) - return - } + if !validateOfflineDownloadStorage(c, req.TempDir, "ThunderBrowser") { + return } items := []model.SettingItem{ {Key: conf.ThunderBrowserTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - _tool, err := tool.Tools.Get("ThunderBrowser") - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if _, err := _tool.Init(); err != nil { - common.ErrorResp(c, err, 500) + if _, ok := saveAndInitOfflineDownloadTool(c, "ThunderBrowser", items); !ok { return } common.SuccessResp(c, "ok") } func OfflineDownloadTools(c *gin.Context) { - tools := tool.Tools.Names() + tools := tool.Tools.NamesForPath(c.Query("path")) common.SuccessResp(c, tools) } diff --git a/server/handles/offline_download_test.go b/server/handles/offline_download_test.go new file mode 100644 index 000000000..6abecbee4 --- /dev/null +++ b/server/handles/offline_download_test.go @@ -0,0 +1,155 @@ +package handles + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func init() { + dataDir, err := os.MkdirTemp("", "openlist-handles-*") + if err != nil { + panic(err) + } + conf.Conf = conf.DefaultConfig(dataDir) + database, err := gorm.Open(sqlite.Open("file:handles?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + panic(err) + } + db.Init(database) +} + +type settingsTool struct { + name string + version string + initCalls int +} + +func (t *settingsTool) Name() string { return t.name } +func (*settingsTool) Items() []model.SettingItem { return nil } +func (t *settingsTool) Init() (string, error) { t.initCalls++; return t.version, nil } +func (*settingsTool) IsReady() bool { return true } +func (*settingsTool) AddURL(*tool.AddUrlArgs) (string, error) { return "", nil } +func (*settingsTool) Remove(*tool.DownloadTask) error { return nil } +func (*settingsTool) Status(*tool.DownloadTask) (*tool.Status, error) { + return &tool.Status{}, nil +} +func (*settingsTool) Run(*tool.DownloadTask) error { return nil } + +func TestOfflineDownloadSettingsPreserveSuccessPayloads(t *testing.T) { + tests := []struct { + name string + toolName string + version string + body string + handler gin.HandlerFunc + wantData string + settingKey string + }{ + { + name: "aria2 returns version", + toolName: "aria2", + version: "v-test", + body: `{"uri":"http://aria2","secret":"secret"}`, + handler: SetAria2, + wantData: "v-test", + settingKey: conf.Aria2Uri, + }, + { + name: "qBittorrent returns ok", + toolName: "qBittorrent", + version: "ignored", + body: `{"url":"http://qbit","seedtime":"1"}`, + handler: SetQbittorrent, + wantData: "ok", + settingKey: conf.QbittorrentUrl, + }, + } + gin.SetMode(gin.TestMode) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fake := &settingsTool{name: test.toolName, version: test.version} + previous, existed := tool.Tools[test.toolName] + tool.Tools[test.toolName] = fake + t.Cleanup(func() { + if existed { + tool.Tools[test.toolName] = previous + } else { + delete(tool.Tools, test.toolName) + } + _ = db.DeleteSettingItemByKey(test.settingKey) + op.SettingCacheUpdate() + }) + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + ctx.Request = httptest.NewRequest(http.MethodPost, "/", strings.NewReader(test.body)) + ctx.Request.Header.Set("Content-Type", "application/json") + test.handler(ctx) + + var result common.Resp[string] + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result.Code != 200 || result.Data != test.wantData { + t.Fatalf("response = %#v, want code 200 and data %q", result, test.wantData) + } + if fake.initCalls != 1 { + t.Fatalf("Init calls = %d, want 1", fake.initCalls) + } + }) + } +} + +func TestValidateOfflineDownloadStorageRejectsWrongNativeTool(t *testing.T) { + root := t.TempDir() + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + }{RootFolderPath: root}) + if err != nil { + t.Fatal(err) + } + mount := "/" + strings.ReplaceAll(t.Name(), "/", "_") + storageID, err := op.CreateStorage(context.Background(), model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := op.DeleteStorageById(context.Background(), storageID); err != nil { + t.Errorf("delete fixture storage: %v", err) + } + }) + + response := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(response) + if validateOfflineDownloadStorage(ctx, mount, "Thunder") { + t.Fatal("Local storage unexpectedly accepted as Thunder") + } + var result common.Resp[any] + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + want := "unsupported storage driver for offline download, only Thunder is supported" + if result.Code != 400 || result.Message != want { + t.Fatalf("response = %#v, want code 400 and message %q", result, want) + } +} diff --git a/server/handles/search.go b/server/handles/search.go index 40b005ac1..8adab7aa4 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -53,11 +53,16 @@ func Search(c *gin.Context) { if !utils.IsSubPath(user.BasePath, node.Parent) { continue } - meta, err := op.GetNearestMeta(node.Parent) + nodePath := path.Join(node.Parent, node.Name) + metaPath := node.Parent + if node.IsDir { + metaPath = nodePath + } + meta, err := op.GetNearestMeta(metaPath) if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { continue } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { + if !common.CanAccess(user, meta, nodePath, req.Password) { continue } filteredNodes = append(filteredNodes, node) @@ -68,6 +73,22 @@ func Search(c *gin.Context) { }) } +func isSearchNodeAccessible(user *model.User, node model.SearchNode, password string, resolveMeta func(string) (*model.Meta, error)) bool { + if !utils.IsSubPath(user.BasePath, node.Parent) { + return false + } + nodePath := path.Join(node.Parent, node.Name) + metaPath := node.Parent + if node.IsDir { + metaPath = nodePath + } + meta, err := resolveMeta(metaPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return false + } + return common.CanAccess(user, meta, nodePath, password) +} + func nodeToSearchResp(node model.SearchNode) SearchResp { return SearchResp{ SearchNode: node, diff --git a/server/handles/search_test.go b/server/handles/search_test.go new file mode 100644 index 000000000..1c373ffe7 --- /dev/null +++ b/server/handles/search_test.go @@ -0,0 +1,84 @@ +package handles + +import ( + "path" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func fakeResolveMeta(metas map[string]*model.Meta) func(string) (*model.Meta, error) { + return func(p string) (*model.Meta, error) { + for { + if meta, ok := metas[p]; ok { + return meta, nil + } + if p == "/" { + return nil, errs.MetaNotFound + } + p = path.Dir(p) + } + } +} + +func TestIsSearchNodeAccessible(t *testing.T) { + tests := []struct { + name string + metas map[string]*model.Meta + node model.SearchNode + basePath string + want bool + wantMetaPath string + }{ + { + name: "restricted directory", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}}}, + node: model.SearchNode{Parent: "/", Name: "private", IsDir: true}, + want: false, + wantMetaPath: "/private", + }, + { + name: "restricted sub directory", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}, ReadUsersSub: true}}, + node: model.SearchNode{Parent: "/private", Name: "sub", IsDir: true}, + want: false, + wantMetaPath: "/private/sub", + }, + { + name: "file keeps parent scope", + metas: map[string]*model.Meta{"/private": {Path: "/private", ReadUsers: []uint{1}}}, + node: model.SearchNode{Parent: "/private", Name: "a.txt", IsDir: false}, + want: true, + wantMetaPath: "/private", + }, + { + name: "outside base path", + basePath: "/home", + node: model.SearchNode{Parent: "/other", Name: "private", IsDir: true}, + want: false, + wantMetaPath: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + basePath := tt.basePath + if basePath == "" { + basePath = "/" + } + user := &model.User{ID: 2, BasePath: basePath} + resolve := fakeResolveMeta(tt.metas) + var gotMetaPath string + spy := func(p string) (*model.Meta, error) { + gotMetaPath = p + return resolve(p) + } + if got := isSearchNodeAccessible(user, tt.node, "", spy); got != tt.want { + t.Fatalf("isSearchNodeAccessible() = %v, want %v", got, tt.want) + } + if tt.wantMetaPath != "" && gotMetaPath != tt.wantMetaPath { + t.Fatalf("meta resolved at %q, want %q", gotMetaPath, tt.wantMetaPath) + } + }) + } +} diff --git a/server/handles/setup.go b/server/handles/setup.go new file mode 100644 index 000000000..584d3c9d7 --- /dev/null +++ b/server/handles/setup.go @@ -0,0 +1,82 @@ +package handles + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// InitStatus 返回系统是否已完成初始化(即是否已存在管理员账号)。 +// 前端据此判断是否跳转到安装向导。 +func InitStatus(c *gin.Context) { + if _, err := op.GetAdmin(); err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + common.SuccessResp(c, gin.H{"initialized": false}) + return + } + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{"initialized": true}) +} + +// InitSetupReq 系统初始化请求体 +type InitSetupReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` + SiteTitle string `json:"site_title"` +} + +// InitSetup 执行系统初始化:创建管理员账号并设置站点名称等初始参数。 +// 仅在系统尚未初始化时允许调用;已初始化则返回错误。 +func InitSetup(c *gin.Context) { + var req InitSetupReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // 已初始化则拒绝 + if _, err := op.GetAdmin(); err == nil { + common.ErrorStrResp(c, "system has already been initialized", 400) + return + } else if !errors.Is(err, gorm.ErrRecordNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + // 密码最小长度校验 + if len(req.Password) < 4 { + common.ErrorStrResp(c, "password must be at least 4 characters", 400) + return + } + admin := &model.User{ + Username: req.Username, + Role: model.ADMIN, + BasePath: "/", + Authn: "[]", + Permission: 0x71FF, + } + admin.SetPassword(req.Password) + if err := op.CreateUser(admin); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + // 设置站点名称(如果提供) + if req.SiteTitle != "" { + item := model.SettingItem{ + Key: conf.SiteTitle, + Value: req.SiteTitle, + Type: conf.TypeString, + Group: model.SITE, + Flag: model.PUBLIC, + } + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + common.SuccessResp(c) +} diff --git a/server/handles/sharing.go b/server/handles/sharing.go index ded8a20df..221bc1277 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -54,7 +54,7 @@ func SharingGet(c *gin.Context, req *FsGetReq) { HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), Sign: "", - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), Thumb: thumb, }, RawURL: url, diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 4baabf6c1..3b152505c 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -1,6 +1,7 @@ package handles import ( + "crypto/sha256" "encoding/base64" "errors" "fmt" @@ -22,12 +23,25 @@ import ( "github.com/coreos/go-oidc" "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" + "github.com/golang-jwt/jwt/v4" "golang.org/x/oauth2" "gorm.io/gorm" ) const stateLength = 16 const stateExpire = time.Minute * 5 +const ssoBindingExpire = time.Minute * 5 +const ssoBindingCookie = "openlist_sso_binding" +const ssoBindingStatePurpose = "sso_binding_state" +const ssoBindingProofPurpose = "sso_binding_proof" + +type ssoBindingClaims struct { + Purpose string `json:"purpose"` + Method string `json:"method"` + SessionDigest string `json:"session_digest"` + SsoID string `json:"sso_id,omitempty"` + jwt.RegisteredClaims +} var stateCache = cache.NewMemCache[string](cache.WithShards[string](stateLength)) @@ -46,6 +60,115 @@ func verifyState(clientID, ip, state string) bool { return ok && value == ip } +func ssoBindingSessionDigest(session string) string { + digest := sha256.Sum256([]byte(session)) + return base64.RawURLEncoding.EncodeToString(digest[:]) +} + +func parseSSOBindingToken(c *gin.Context, rawToken, purpose string) (*ssoBindingClaims, error) { + claims := &ssoBindingClaims{} + token, err := jwt.ParseWithClaims(rawToken, claims, func(token *jwt.Token) (interface{}, error) { + if token.Method != jwt.SigningMethodHS256 { + return nil, errors.New("invalid SSO binding token algorithm") + } + return common.SecretKey, nil + }) + if err != nil || !token.Valid || claims.ExpiresAt == nil || claims.Purpose != purpose || + claims.Method != "get_sso_id" || len(claims.SessionDigest) != 43 { + return nil, errors.New("invalid or expired SSO binding token") + } + if (purpose == ssoBindingStatePurpose && claims.SsoID != "") || + (purpose == ssoBindingProofPurpose && claims.SsoID == "") { + return nil, errors.New("invalid SSO binding token payload") + } + session, err := c.Cookie(ssoBindingCookie) + if err != nil || ssoBindingSessionDigest(session) != claims.SessionDigest { + return nil, errors.New("invalid SSO binding session") + } + return claims, nil +} + +func generateSSOBindingToken(c *gin.Context, purpose, ssoID string) (string, error) { + session, err := c.Cookie(ssoBindingCookie) + expire := ssoBindingExpire + if purpose == ssoBindingStatePurpose { + session = random.String(32) + expire = stateExpire + c.SetSameSite(http.SameSiteLaxMode) + c.SetCookie( + ssoBindingCookie, + session, + int((stateExpire+ssoBindingExpire).Seconds()), + path.Join(conf.URL.Path, "/api"), + "", + strings.HasPrefix(common.GetApiUrl(c), "https://"), + true, + ) + } else if err != nil { + return "", errors.New("missing SSO binding session") + } + now := time.Now() + claims := ssoBindingClaims{ + Purpose: purpose, + Method: "get_sso_id", + SessionDigest: ssoBindingSessionDigest(session), + SsoID: ssoID, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(now.Add(expire)), + IssuedAt: jwt.NewNumericDate(now), + NotBefore: jwt.NewNumericDate(now), + }, + } + return jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString(common.SecretKey) +} + +// ssoTargetOrigin returns the origin that is allowed to receive the SSO result +// via postMessage. It honours the operator-configured sso_postmessage_origin so +// a frontend served from a different origin than the API can still receive the +// result; otherwise it falls back to the API origin, or "/" to restrict +// delivery to same-origin openers when that cannot be resolved. +func ssoTargetOrigin(c *gin.Context) string { + if configured := setting.GetStr(conf.SSOPostMessageOrigin); configured != "" { + if u, err := url.Parse(configured); err == nil && + (u.Scheme == "http" || u.Scheme == "https") && + u.Host != "" && u.User == nil && + (u.Path == "" || u.Path == "/") && + u.RawQuery == "" && u.Fragment == "" { + return u.Scheme + "://" + u.Host + } + } + u, err := url.Parse(common.GetApiUrl(c)) + if err != nil || u.Scheme == "" || u.Host == "" { + return "/" + } + return u.Scheme + "://" + u.Host +} + +// ssoPostMessage hands the SSO result back to the window that started the login. +// The target origin is pinned so that an arbitrary page cannot open the SSO +// endpoint in a popup and read the payload out of the message event. +func ssoPostMessage(c *gin.Context, payload map[string]string) { + data, err := utils.Json.MarshalToString(payload) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + origin, err := utils.Json.MarshalToString(ssoTargetOrigin(c)) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + html := fmt.Sprintf(` + + + + `, data, origin) + c.Data(200, "text/html; charset=utf-8", []byte(html)) +} + func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { if useCompatibility { return common.GetApiUrl(c) + "/api/auth/" + method @@ -74,6 +197,16 @@ func SSOLoginRedirect(c *gin.Context) { urlValues.Add("response_type", "code") urlValues.Add("redirect_uri", redirectUri) urlValues.Add("client_id", clientId) + bindingState := "" + var err error + if method == "get_sso_id" { + bindingState, err = generateSSOBindingToken(c, ssoBindingStatePurpose, "") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + urlValues.Add("state", bindingState) + } switch platform { case "Github": rUrl = "https://github.com/login/oauth/authorize?" @@ -94,15 +227,19 @@ func SSOLoginRedirect(c *gin.Context) { endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") rUrl = endpoint + "/login/oauth/authorize?" urlValues.Add("scope", "profile") - urlValues.Add("state", endpoint) + if bindingState == "" { + urlValues.Add("state", endpoint) + } case "OIDC": oauth2Config, err := GetOIDCClient(c, useCompatibility, redirectUri, method) if err != nil { common.ErrorStrResp(c, err.Error(), 400) return } - state := generateState(clientId, c.ClientIP()) - c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(state)) + if bindingState == "" { + bindingState = generateState(clientId, c.ClientIP()) + } + c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(bindingState)) return default: common.ErrorStrResp(c, "invalid platform", 400) @@ -201,11 +338,15 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if !verifyState(clientId, c.ClientIP(), c.Query("state")) { + if method == "get_sso_id" { + if _, err := parseSSOBindingToken(c, c.Query("state"), ssoBindingStatePurpose); err != nil { + common.ErrorStrResp(c, "incorrect or expired state parameter", 400) + return + } + } else if !verifyState(clientId, c.ClientIP(), c.Query("state")) { common.ErrorStrResp(c, "incorrect or expired state parameter", 400) return } - oauth2Token, err := oauth2Config.Exchange(c, c.Query("code")) if err != nil { common.ErrorResp(c, err, 400) @@ -235,19 +376,16 @@ func OIDCLoginCallback(c *gin.Context) { return } if method == "get_sso_id" { + bindingProof, err := generateSSOBindingToken(c, ssoBindingProofPurpose, userID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } if useCompatibility { - c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) + c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+bindingProof) return } - html := fmt.Sprintf(` - - - - `, userID) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"sso_id": bindingProof}) return } if method == "sso_get_token" { @@ -268,15 +406,7 @@ func OIDCLoginCallback(c *gin.Context) { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } - html := fmt.Sprintf(` - - - - `, token) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"token": token}) return } } @@ -352,6 +482,12 @@ func SSOLoginCallback(c *gin.Context) { common.ErrorStrResp(c, "No code provided", 400) return } + if argument == "get_sso_id" { + if _, err := parseSSOBindingToken(c, c.Query("state"), ssoBindingStatePurpose); err != nil { + common.ErrorStrResp(c, "incorrect or expired state parameter", 400) + return + } + } var resp *resty.Response var err error if platform == "Dingtalk" { @@ -402,19 +538,16 @@ func SSOLoginCallback(c *gin.Context) { return } if argument == "get_sso_id" { + bindingProof, err := generateSSOBindingToken(c, ssoBindingProofPurpose, userID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } if usecompatibility { - c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+userID) + c.Redirect(302, common.GetApiUrl(c)+"/@manage?sso_id="+bindingProof) return } - html := fmt.Sprintf(` - - - - `, userID) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"sso_id": bindingProof}) return } username := utils.Json.Get(resp.Body(), usernameField).ToString() @@ -435,13 +568,5 @@ func SSOLoginCallback(c *gin.Context) { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) return } - html := fmt.Sprintf(` - - - - `, token) - c.Data(200, "text/html; charset=utf-8", []byte(html)) + ssoPostMessage(c, map[string]string{"token": token}) } diff --git a/server/handles/ssologin_origin_test.go b/server/handles/ssologin_origin_test.go new file mode 100644 index 000000000..388ebc746 --- /dev/null +++ b/server/handles/ssologin_origin_test.go @@ -0,0 +1,102 @@ +package handles + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/gin-gonic/gin" +) + +func ssoTestContext(apiUrl string) (*gin.Context, *httptest.ResponseRecorder) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, engine := gin.CreateTestContext(rec) + // Matches server.Init, which is what lets GetApiUrl reach the value the + // middleware stored on the request context. + engine.ContextWithFallback = true + req := httptest.NewRequest(http.MethodGet, "/api/auth/sso?method=sso_get_token", nil) + if apiUrl != "" { + req = req.WithContext(context.WithValue(req.Context(), conf.ApiUrlKey, apiUrl)) + } + c.Request = req + // Keep setting lookups off the (uninitialised) database: ssoTargetOrigin + // reads sso_postmessage_origin through the setting cache. + op.Cache.SetSetting(conf.SSOPostMessageOrigin, &model.SettingItem{ + Key: conf.SSOPostMessageOrigin, + Value: "", + }) + return c, rec +} + +// A page that opens the SSO endpoint in a popup must not be able to read the +// token: the postMessage target origin has to name the site, never "*". +func TestSSOPostMessagePinsTargetOrigin(t *testing.T) { + c, rec := ssoTestContext("https://openlist.example.com/base") + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if strings.Contains(body, `"*"`) || strings.Contains(body, `, '*'`) { + t.Fatalf("wildcard target origin present in response:\n%s", body) + } + if !strings.Contains(body, `"https://openlist.example.com"`) { + t.Errorf("expected the site origin as target, got:\n%s", body) + } + if !strings.Contains(body, "secret-token") { + t.Errorf("payload should still reach a legitimate opener, got:\n%s", body) + } +} + +// A frontend served from a different origin than the API needs the operator to +// be able to point the target at the frontend origin. The configured origin +// must win over the API origin. +func TestSSOPostMessageUsesConfiguredOrigin(t *testing.T) { + c, rec := ssoTestContext("https://api.example.com/base") + op.Cache.SetSetting(conf.SSOPostMessageOrigin, &model.SettingItem{ + Key: conf.SSOPostMessageOrigin, + Value: "https://frontend.example.com", + }) + defer op.Cache.ClearAll() + + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if !strings.Contains(body, `"https://frontend.example.com"`) { + t.Errorf("expected the configured origin as target, got:\n%s", body) + } +} + +// If the site URL cannot be resolved the fallback must tighten delivery to +// same-origin openers, not widen it back to every origin. +func TestSSOPostMessageFallsBackToSameOrigin(t *testing.T) { + c, rec := ssoTestContext("") + ssoPostMessage(c, map[string]string{"token": "secret-token"}) + + body := rec.Body.String() + if strings.Contains(body, `"*"`) { + t.Fatalf("fallback must not be a wildcard origin:\n%s", body) + } + if !strings.Contains(body, `"/"`) { + t.Errorf(`expected "/" fallback origin, got:\n%s`, body) + } +} + +// userID comes from the identity provider, so it must be encoded rather than +// interpolated into the JS string literal it used to land in. +func TestSSOPostMessageEscapesProviderControlledValue(t *testing.T) { + c, rec := ssoTestContext("https://openlist.example.com") + ssoPostMessage(c, map[string]string{"sso_id": `"});alert(document.domain);//`}) + + body := rec.Body.String() + if strings.Contains(body, `alert(document.domain)`) && !strings.Contains(body, `\"`) { + t.Fatalf("provider value was not escaped:\n%s", body) + } + if !strings.Contains(body, `\"});alert`) { + t.Errorf("expected the injected quote to be escaped, got:\n%s", body) + } +} diff --git a/server/mcp/fs_get.go b/server/mcp/fs_get.go index 550cc036f..c975695fd 100644 --- a/server/mcp/fs_get.go +++ b/server/mcp/fs_get.go @@ -68,9 +68,11 @@ func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) parentPath := stdpath.Dir(reqPath) var related []model.Obj - sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) - if err == nil { - related = filterRelatedObjs(sameLevelFiles, obj) + if !obj.IsDir() && utils.GetFileType(obj.GetName()) == conf.VIDEO { + sameLevelFiles, err := fs.List(ctx, parentPath, &fs.ListArgs{}) + if err == nil { + related = filterRelatedObjs(sameLevelFiles, obj) + } } parentMeta, _ := op.GetNearestMeta(parentPath) @@ -85,7 +87,7 @@ func (s *Server) callFSGet(c *gin.Context, raw json.RawMessage) (any, *rpcError) Created: obj.CreateTime(), Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), Thumb: thumb, - Type: utils.GetFileType(obj.GetName()), + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), HashInfoStr: obj.GetHash().String(), HashInfo: obj.GetHash().Export(), MountDetails: mountDetails, diff --git a/server/middlewares/check_test.go b/server/middlewares/check_test.go new file mode 100644 index 000000000..a92136378 --- /dev/null +++ b/server/middlewares/check_test.go @@ -0,0 +1,62 @@ +package middlewares + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/gin-gonic/gin" +) + +func TestStoragesLoadedAdmitsRequestOrigin(t *testing.T) { + originalMode := gin.Mode() + gin.SetMode(gin.TestMode) + originalConf := conf.Conf + originalLoaded := conf.StoragesLoaded + t.Cleanup(func() { + gin.SetMode(originalMode) + conf.Conf = originalConf + conf.StoragesLoaded = originalLoaded + }) + conf.StoragesLoaded = true + + router := gin.New() + router.Use(StoragesLoaded) + router.GET("/", func(c *gin.Context) { + c.String(http.StatusOK, conf.GetApiUrl(c.Request.Context())) + }) + + assertOrigin := func(name, siteURL, target string, header http.Header, want string) { + t.Run(name, func(t *testing.T) { + conf.Conf = &conf.Config{SiteURL: siteURL} + + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header = header + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if got := rec.Body.String(); got != want { + t.Fatalf("origin = %q, want %q", got, want) + } + }) + } + + assertOrigin( + "configured site URL", + "https://openlist.example/base/", + "http://ignored.example/", + nil, + "https://openlist.example/base", + ) + assertOrigin( + "forwarded request", + "", + "http://internal.example/", + http.Header{ + "X-Forwarded-Proto": {"https"}, + "X-Forwarded-Host": {"public.example"}, + }, + "https://public.example", + ) +} diff --git a/server/middlewares/down.go b/server/middlewares/down.go index d71be00b3..787936d66 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -60,7 +60,7 @@ func needSign(meta *model.Meta, path string) bool { if meta == nil || meta.Password == "" { return false } - if !meta.PSub && path != meta.Path { + if !meta.PSub && !common.MetaCoversPath(meta.Path, path, false) { return false } return true diff --git a/server/router.go b/server/router.go index 61f38cc37..7c6491143 100644 --- a/server/router.go +++ b/server/router.go @@ -101,6 +101,8 @@ func Init(e *gin.Engine) { public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + public.Any("/init_status", handles.InitStatus) + public.POST("/init/setup", handles.InitSetup) _fs(auth.Group("/fs")) _index115(auth.Group("/index115")) @@ -245,7 +247,7 @@ func _fs(g *gin.RouterGroup) { // 创建 115 永久分享(share/send + updateshare -1,快照语义) g.POST("/share/create", handles.FsShareCreate) // Direct upload (client-side upload to storage) - g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) + g.POST("/get_direct_upload_info", handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) { diff --git a/server/s3/backend.go b/server/s3/backend.go index f00d2149c..de6413ce9 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -23,7 +23,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" "github.com/ncw/swift/v2" log "github.com/sirupsen/logrus" ) @@ -33,17 +33,25 @@ var ( timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" ) -// s3Backend implements the gofacess3.Backend interface to make an S3 -// backend for gofakes3 +// s3Backend implements the gofakes3.Backend interface to make an S3 +// backend for gofakes3. It also implements gofakes3.MultipartBackend so that +// multipart uploads are streamed to local temp files part-by-part and +// assembled into storage on completion, instead of being buffered in memory. type s3Backend struct { - meta *sync.Map + meta *sync.Map + listDir func(context.Context, string) ([]model.Obj, error) + uploads *sync.Map // map[gofakes3.UploadID]*multipartState } // newBackend creates a new SimpleBucketBackend. func newBackend() gofakes3.Backend { - return &s3Backend{ - meta: new(sync.Map), + b := &s3Backend{ + meta: new(sync.Map), + uploads: new(sync.Map), + listDir: getDirEntries, } + b.startReaper() + return b } // ListBuckets always returns the default bucket. @@ -84,10 +92,9 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *g prefix.HasDelimiter = false } - response := gofakes3.NewObjectList() path, remaining := prefixParser(prefix) - err = b.entryListR(bucketPath, path, remaining, prefix.HasDelimiter, response) + response, err := b.listPage(ctx, bucketPath, path, remaining, prefix.HasDelimiter, page) if err == gofakes3.ErrNoSuchKey { // AWS just returns an empty list response = gofakes3.NewObjectList() @@ -95,7 +102,7 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *g return nil, err } - return b.pager(response, page) + return response, nil } // HeadObject returns the fileinfo for the given object name. @@ -145,6 +152,8 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin // GetObject fetchs the object from the filesystem. func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (s3Obj *gofakes3.Object, err error) { + defer func() { err = mapBackendError(err) }() + bucket, err := getBucketByName(bucketName) if err != nil { return nil, err @@ -186,7 +195,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") } - var rd io.Reader + var rd io.ReadCloser if rnge != nil { rd, err = rrf.RangeRead(ctx, http_range.Range(*rnge)) } else { @@ -208,6 +217,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string meta[k] = v } } + closers := utils.NewClosers(rd, link) return &gofakes3.Object{ // Name: gofakes3.URLEncode(objectName), @@ -216,7 +226,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string Metadata: meta, Size: size, Range: rnge, - Contents: utils.ReadCloser{Reader: rd, Closer: link}, + Contents: utils.ReadCloser{Reader: rd, Closer: &closers}, }, nil } @@ -232,9 +242,20 @@ func (b *s3Backend) PutObject( meta map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { + return result, b.putStream(ctx, bucketName, objectName, meta, input, size) +} + +// putStream stores the given object into the underlying storage. It is shared +// by PutObject and the multipart-upload Complete step so both paths apply the +// same directory creation, metadata and ignore rules. +func (b *s3Backend) putStream( + ctx context.Context, bucketName, objectName string, + meta map[string]string, + input io.Reader, size int64, +) error { bucket, err := getBucketByName(bucketName) if err != nil { - return result, err + return err } bucketPath := bucket.Path @@ -260,15 +281,15 @@ func (b *s3Backend) PutObject( log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) err = fs.MakeDir(ctx, reqPath) if err != nil { - return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) + return errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) } } else { - return result, gofakes3.KeyNotFound(objectName) + return gofakes3.KeyNotFound(objectName) } } if isDir { - return result, nil + return nil } var ti time.Time @@ -294,28 +315,25 @@ func (b *s3Backend) PutObject( } // Check if system file should be ignored if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { - return result, errs.IgnoredSystemFile + return errs.IgnoredSystemFile } stream := &stream.FileStream{ Obj: &obj, Reader: input, Mimetype: meta["Content-Type"], } + if stream.Mimetype == "" { + stream.Mimetype = "application/octet-stream" + } err = fs.PutDirectly(ctx, reqPath, stream) if err != nil { - return result, err + return err } - // if err := stream.Close(); err != nil { - // // remove file when close error occurred (FsPutErr) - // _ = fs.Remove(ctx, fp) - // return result, err - // } - b.meta.Store(fp, meta) - return result, nil + return nil } // DeleteMulti deletes multiple objects in a single request. diff --git a/server/s3/errors_test.go b/server/s3/errors_test.go new file mode 100644 index 000000000..85df7fe81 --- /dev/null +++ b/server/s3/errors_test.go @@ -0,0 +1,66 @@ +package s3 + +import ( + "context" + "encoding/xml" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/gofakes3" + "github.com/OpenListTeam/gofakes3/s3mem" +) + +func TestMapBackendErrorMapsOnlyTemporaryCapacity(t *testing.T) { + capacity := errs.NewErr(errs.TemporaryCapacity, "callback admission timed out") + if got := mapBackendError(capacity); got != gofakes3.ErrSlowDown { + t.Fatalf("capacity error mapped to %v, want %v", got, gofakes3.ErrSlowDown) + } + + permanent := errors.New("permission denied") + if got := mapBackendError(permanent); got != permanent { + t.Fatalf("permanent error mapped to %v, want original error", got) + } + if got := mapBackendError(nil); got != nil { + t.Fatalf("nil error mapped to %v", got) + } +} + +type capacityBackend struct { + gofakes3.Backend +} + +func (b capacityBackend) GetObject(context.Context, string, string, *gofakes3.ObjectRangeRequest) (*gofakes3.Object, error) { + return nil, mapBackendError(errs.NewErr(errs.TemporaryCapacity, "callback admission timed out")) +} + +func TestTemporaryCapacityProducesS3SlowDownResponse(t *testing.T) { + memory := s3mem.New() + if err := memory.CreateBucket(t.Context(), "bucket"); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(gofakes3.New(capacityBackend{Backend: memory}).Server()) + defer server.Close() + + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL+"/bucket/object", nil) + if err != nil { + t.Fatal(err) + } + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusServiceUnavailable) + } + var result gofakes3.ErrorResult + if err := xml.NewDecoder(response.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if result.Code != gofakes3.ErrSlowDown || result.Message != gofakes3.ErrSlowDown.Message() { + t.Fatalf("S3 error = %#v, want SlowDown with standard message", result) + } +} diff --git a/server/s3/get_object_close_test.go b/server/s3/get_object_close_test.go new file mode 100644 index 000000000..00ef34208 --- /dev/null +++ b/server/s3/get_object_close_test.go @@ -0,0 +1,132 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "gorm.io/gorm" +) + +const closeTrackingDriverName = "S3CloseTrackingLocal" + +type closeTrackingDriver struct { + local.Local + closed *[]string +} + +func (d *closeTrackingDriver) Config() driver.Config { + c := d.Local.Config() + c.Name = closeTrackingDriverName + return c +} + +func (d *closeTrackingDriver) Link(context.Context, model.Obj, model.LinkArgs) (*model.Link, error) { + link := &model.Link{ + ContentLength: 4, + RangeReader: stream.RangeReaderFunc(func(context.Context, http_range.Range) (io.ReadCloser, error) { + return utils.NewReadCloser(strings.NewReader("body"), func() error { + *d.closed = append(*d.closed, "body") + return nil + }), nil + }), + RequireReference: true, + } + link.SyncClosers.Add(utils.CloseFunc(func() error { + *d.closed = append(*d.closed, "link") + return nil + })) + return link, nil +} + +func TestGetObjectClosesRangeBodyBeforeLink(t *testing.T) { + ctx := context.Background() + var closed []string + op.RegisterDriver(func() driver.Driver { + return &closeTrackingDriver{closed: &closed} + }) + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "fixture.txt"), []byte("body"), 0o600); err != nil { + t.Fatal(err) + } + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + }{RootFolderPath: root}) + if err != nil { + t.Fatal(err) + } + mount := "/" + sanitizeTestName(t.Name()) + storageID, err := op.CreateStorage(ctx, model.Storage{ + Driver: closeTrackingDriverName, + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := op.DeleteStorageById(ctx, storageID); err != nil { + t.Errorf("delete fixture storage: %v", err) + } + }) + + previousBuckets, previousBucketsErr := op.GetSettingItemByKey(conf.S3Buckets) + if previousBucketsErr != nil && !errors.Is(previousBucketsErr, gorm.ErrRecordNotFound) { + t.Fatal(previousBucketsErr) + } + if err := op.SaveSettingItem(&model.SettingItem{ + Key: conf.S3Buckets, + Value: `[{"name":"close","path":"` + mount + `"}]`, + }); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if previousBucketsErr == nil { + if err := op.SaveSettingItem(previousBuckets); err != nil { + t.Errorf("restore S3 buckets: %v", err) + } + return + } + if err := db.DeleteSettingItemByKey(conf.S3Buckets); err != nil { + t.Errorf("delete fixture S3 buckets: %v", err) + } + op.SettingCacheUpdate() + }) + + object, err := newBackend().(*s3Backend).GetObject(ctx, "close", "fixture.txt", nil) + if err != nil { + t.Fatal(err) + } + contents, err := io.ReadAll(object.Contents) + if err != nil { + t.Fatal(err) + } + if string(contents) != "body" { + t.Fatalf("contents = %q, want body", contents) + } + if err := object.Contents.Close(); err != nil { + t.Fatal(err) + } + if err := object.Contents.Close(); err != nil { + t.Fatal(err) + } + if !slices.Equal(closed, []string{"body", "link"}) { + t.Fatalf("close order = %v, want [body link] exactly once", closed) + } +} diff --git a/server/s3/list.go b/server/s3/list.go index d783099a5..73a8bcaa2 100644 --- a/server/s3/list.go +++ b/server/s3/list.go @@ -3,24 +3,119 @@ package s3 import ( + "context" "path" + "slices" + "sort" "strings" "time" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/gofakes3" log "github.com/sirupsen/logrus" ) -func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { - fp := path.Join(bucket, fdPath) +// S3 ListObjects responses contain at most 1,000 keys. +const maxListPageKeys int64 = 1000 + +type objectPage struct { + result *gofakes3.ObjectList + marker string + maxKeys int64 + count int64 + lastKey string +} + +func newObjectPage(page gofakes3.ListBucketPage) *objectPage { + maxKeys := page.MaxKeys + if maxKeys <= 0 || maxKeys > maxListPageKeys { + maxKeys = maxListPageKeys + } + marker := "" + if page.HasMarker { + marker = page.Marker + } + return &objectPage{ + result: gofakes3.NewObjectList(), + marker: marker, + maxKeys: maxKeys, + } +} + +func (p *objectPage) addContent(item *gofakes3.Content) bool { + if item.Key <= p.marker { + return true + } + if p.count >= p.maxKeys { + p.result.IsTruncated = true + return false + } + p.result.Add(item) + p.count++ + p.lastKey = item.Key + return true +} - dirEntries, err := getDirEntries(fp) +func (p *objectPage) addPrefix(prefix string) bool { + if prefix <= p.marker { + return true + } + if p.count >= p.maxKeys { + p.result.IsTruncated = true + return false + } + p.result.AddPrefix(prefix) + p.count++ + p.lastKey = prefix + return true +} + +func (p *objectPage) finish() *gofakes3.ObjectList { + if p.result.IsTruncated { + p.result.NextMarker = p.lastKey + } + return p.result +} + +func (b *s3Backend) listPage( + ctx context.Context, + bucket, fdPath, name string, + addPrefix bool, + page gofakes3.ListBucketPage, +) (*gofakes3.ObjectList, error) { + result := newObjectPage(page) + _, err := b.walkPage(ctx, bucket, fdPath, name, addPrefix, result) if err != nil { - return err + return nil, err + } + return result.finish(), nil +} + +// walkPage returns false after finding an entry beyond the requested page. +func (b *s3Backend) walkPage( + ctx context.Context, + bucket, fdPath, name string, + addPrefix bool, + page *objectPage, +) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + + fp := path.Join(bucket, fdPath) + dirEntries, err := b.listDir(ctx, fp) + if err != nil { + return false, err + } + if err := ctx.Err(); err != nil { + return false, err } // workaround as s3 can't have empty files in directories, useful in deletions if len(dirEntries) == 0 { + if !strings.HasPrefix(emptyObjectName, name) { + return true, nil + } item := &gofakes3.Content{ // Key: gofakes3.URLEncode(path.Join(fdPath, emptyObjectName)), Key: path.Join(fdPath, emptyObjectName), @@ -29,11 +124,19 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp Size: 0, StorageClass: gofakes3.StorageStandard, } - response.Add(item) log.Debugf("Adding empty object %s to response", item.Key) - return nil + return page.addContent(item), nil } + // Emit entries in the order their keys have in a flat S3 keyspace. A + // directory sorts as if it carried its trailing slash, so "b.txt" comes + // before "b/..." as it does in a real bucket. Without this a listing + // resumed from a marker could silently skip or repeat keys. + dirEntries = slices.Clone(dirEntries) + sort.Slice(dirEntries, func(i, j int) bool { + return listSortKey(dirEntries[i]) < listSortKey(dirEntries[j]) + }) + for _, entry := range dirEntries { object := entry.GetName() @@ -45,14 +148,23 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp } if entry.IsDir() { + // S3 common prefixes include the delimiter, e.g. "base/20261009T020008/". + // Clients such as barman-cloud ignore prefixes without it. + subtreePrefix := objectPath + "/" if addPrefix { - // response.AddPrefix(gofakes3.URLEncode(objectPath)) - response.AddPrefix(objectPath) + // response.AddPrefix(gofakes3.URLEncode(subtreePrefix)) + if !page.addPrefix(subtreePrefix) { + return false, nil + } + continue + } + // A marker beyond this subtree lets us avoid an upstream directory read. + if subtreePrefix <= page.marker && !strings.HasPrefix(page.marker, subtreePrefix) { continue } - err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response) - if err != nil { - return err + keepGoing, err := b.walkPage(ctx, bucket, objectPath, "", false, page) + if err != nil || !keepGoing { + return keepGoing, err } } else { item := &gofakes3.Content{ @@ -63,8 +175,19 @@ func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, resp Size: entry.GetSize(), StorageClass: gofakes3.StorageStandard, } - response.Add(item) + if !page.addContent(item) { + return false, nil + } } } - return nil + return true, nil +} + +// listSortKey returns the name used to order directory entries so that the +// emitted keys follow S3 lexicographic order. +func listSortKey(entry model.Obj) string { + if entry.IsDir() { + return entry.GetName() + "/" + } + return entry.GetName() } diff --git a/server/s3/list_test.go b/server/s3/list_test.go new file mode 100644 index 000000000..fbfb2bc0e --- /dev/null +++ b/server/s3/list_test.go @@ -0,0 +1,346 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/gofakes3" +) + +func TestListPageBoundsRecursiveTraversal(t *testing.T) { + listCalls := 0 + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + listCalls++ + if dir == "bucket/data" { + entries := make([]model.Obj, 256) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("%02x", i), IsFolder: true} + } + return entries, nil + } + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 3}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + + wantKeys := []string{"data/00/pack", "data/01/pack", "data/02/pack"} + if len(got.Contents) != len(wantKeys) { + t.Fatalf("len(Contents) = %d, want %d", len(got.Contents), len(wantKeys)) + } + for i, want := range wantKeys { + if got.Contents[i].Key != want { + t.Errorf("Contents[%d].Key = %q, want %q", i, got.Contents[i].Key, want) + } + } + if !got.IsTruncated { + t.Error("IsTruncated = false, want true") + } + if got.NextMarker != wantKeys[len(wantKeys)-1] { + t.Errorf("NextMarker = %q, want %q", got.NextMarker, wantKeys[len(wantKeys)-1]) + } + if listCalls > 5 { + t.Errorf("list calls = %d, want at most 5 for a three-key page", listCalls) + } +} + +func TestListPageContinuesWithoutGapsOrDuplicates(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + if dir == "bucket/data" { + entries := make([]model.Obj, 16) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("%02x", 15-i), IsFolder: true} + } + return entries, nil + } + entries := make([]model.Obj, 20) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("pack-%02d", 19-i), Size: 1} + } + return entries, nil + }, + } + + var gotKeys []string + page := gofakes3.ListBucketPage{MaxKeys: 37} + for pageNumber := 0; pageNumber < 10; pageNumber++ { + got, err := b.listPage(context.Background(), "bucket", "data", "", false, page) + if err != nil { + t.Fatalf("listPage() page %d error = %v", pageNumber, err) + } + for _, item := range got.Contents { + gotKeys = append(gotKeys, item.Key) + } + if !got.IsTruncated { + break + } + page.Marker = got.NextMarker + page.HasMarker = true + } + + wantKeys := make([]string, 0, 16*20) + for shard := 0; shard < 16; shard++ { + for pack := 0; pack < 20; pack++ { + wantKeys = append(wantKeys, fmt.Sprintf("data/%02x/pack-%02d", shard, pack)) + } + } + sort.Strings(wantKeys) + if len(gotKeys) != len(wantKeys) { + t.Fatalf("listed %d keys, want %d", len(gotKeys), len(wantKeys)) + } + for i, want := range wantKeys { + if gotKeys[i] != want { + t.Fatalf("key %d = %q, want %q", i, gotKeys[i], want) + } + } +} + +func TestListPagePaginatesContentsAndCommonPrefixesTogether(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return []model.Obj{ + &model.Object{Name: "d", IsFolder: true}, + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "a", Size: 1}, + }, nil + }, + } + + first, err := b.listPage(context.Background(), "bucket", "data", "", true, gofakes3.ListBucketPage{MaxKeys: 2}) + if err != nil { + t.Fatalf("first listPage() error = %v", err) + } + if len(first.Contents) != 1 || first.Contents[0].Key != "data/a" { + t.Fatalf("first contents = %#v, want data/a", first.Contents) + } + if len(first.CommonPrefixes) != 1 || first.CommonPrefixes[0].Prefix != "data/b/" { + t.Fatalf("first prefixes = %#v, want data/b/", first.CommonPrefixes) + } + if !first.IsTruncated || first.NextMarker != "data/b/" { + t.Fatalf("first page truncated = %v, next marker = %q", first.IsTruncated, first.NextMarker) + } + + second, err := b.listPage(context.Background(), "bucket", "data", "", true, gofakes3.ListBucketPage{ + Marker: first.NextMarker, + HasMarker: true, + MaxKeys: 2, + }) + if err != nil { + t.Fatalf("second listPage() error = %v", err) + } + if len(second.Contents) != 1 || second.Contents[0].Key != "data/c" { + t.Fatalf("second contents = %#v, want data/c", second.Contents) + } + if len(second.CommonPrefixes) != 1 || second.CommonPrefixes[0].Prefix != "data/d/" { + t.Fatalf("second prefixes = %#v, want data/d/", second.CommonPrefixes) + } + if second.IsTruncated || second.NextMarker != "" { + t.Fatalf("second page truncated = %v, next marker = %q", second.IsTruncated, second.NextMarker) + } +} + +func TestListPageSkipsCompletedSubtrees(t *testing.T) { + var calls []string + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + calls = append(calls, dir) + if dir == "bucket/data" { + return []model.Obj{ + &model.Object{Name: "c", IsFolder: true}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "a", IsFolder: true}, + }, nil + } + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{ + Marker: "data/b/pack", + HasMarker: true, + MaxKeys: 1, + }) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != 1 || got.Contents[0].Key != "data/c/pack" { + t.Fatalf("contents = %#v, want data/c/pack", got.Contents) + } + for _, call := range calls { + if call == "bucket/data/a" { + t.Fatalf("completed subtree was listed: calls = %v", calls) + } + } +} + +func TestListPageUsesS3MaximumAndProbesForTruncation(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + entries := make([]model.Obj, maxListPageKeys+1) + for i := range entries { + entries[i] = &model.Object{Name: fmt.Sprintf("pack-%04d", i), Size: 1} + } + return entries, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 5000}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != int(maxListPageKeys) { + t.Fatalf("len(Contents) = %d, want %d", len(got.Contents), maxListPageKeys) + } + if !got.IsTruncated || got.NextMarker != "data/pack-0999" { + t.Fatalf("truncated = %v, next marker = %q", got.IsTruncated, got.NextMarker) + } +} + +func TestListPageDoesNotReturnEmptyDirectoryMarkerForDifferentPrefix(t *testing.T) { + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return nil, nil + }, + } + + got, err := b.listPage(context.Background(), "bucket", "empty", "not-the-marker", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if err != nil { + t.Fatalf("listPage() error = %v", err) + } + if len(got.Contents) != 0 { + t.Fatalf("contents = %#v, want empty", got.Contents) + } +} + +func TestListPageHonorsCanceledContext(t *testing.T) { + listCalls := 0 + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + listCalls++ + return nil, nil + }, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := b.listPage(ctx, "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("listPage() error = %v, want context.Canceled", err) + } + if listCalls != 0 { + t.Fatalf("list calls = %d, want 0", listCalls) + } +} + +func TestListPageStopsWhenDirectoryReadCancelsContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + cancel() + return []model.Obj{&model.Object{Name: "pack", Size: 1}}, nil + }, + } + + _, err := b.listPage(ctx, "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("listPage() error = %v, want context.Canceled", err) + } +} + +func TestListPageDoesNotReorderDirectoryCacheEntries(t *testing.T) { + entries := []model.Obj{ + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "a", Size: 1}, + &model.Object{Name: "b", Size: 1}, + } + b := &s3Backend{ + listDir: func(_ context.Context, _ string) ([]model.Obj, error) { + return entries, nil + }, + } + + if _, err := b.listPage(context.Background(), "bucket", "data", "", false, gofakes3.ListBucketPage{MaxKeys: 10}); err != nil { + t.Fatalf("listPage() error = %v", err) + } + wantOrder := []string{"c", "a", "b"} + for i, want := range wantOrder { + if entries[i].GetName() != want { + t.Fatalf("entry %d = %q, want original order %q", i, entries[i].GetName(), want) + } + } +} + +func TestListPageMixedDirectoryAndFilePaginatesInKeyOrder(t *testing.T) { + tree := map[string][]model.Obj{ + "bucket/data": { + &model.Object{Name: "c", Size: 1}, + &model.Object{Name: "b", IsFolder: true}, + &model.Object{Name: "b.txt", Size: 1}, + &model.Object{Name: "b-c", IsFolder: true}, + &model.Object{Name: "a", Size: 1}, + }, + "bucket/data/b": {&model.Object{Name: "y", Size: 1}, &model.Object{Name: "x", Size: 1}}, + "bucket/data/b-c": {&model.Object{Name: "k", Size: 1}}, + } + b := &s3Backend{ + listDir: func(_ context.Context, dir string) ([]model.Obj, error) { + return tree[dir], nil + }, + } + tests := []struct { + name string + delimiter bool + want []string + }{ + {"delimiter", true, []string{"data/a", "data/b-c/", "data/b.txt", "data/b/", "data/c"}}, + {"recursive", false, []string{"data/a", "data/b-c/k", "data/b.txt", "data/b/x", "data/b/y", "data/c"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !sort.StringsAreSorted(tt.want) { + t.Fatalf("expectation is not in key order: %v", tt.want) + } + for maxKeys := int64(1); maxKeys <= int64(len(tt.want))+1; maxKeys++ { + var got []string + page := gofakes3.ListBucketPage{MaxKeys: maxKeys} + for i := 0; i <= len(tt.want); i++ { + list, err := b.listPage(context.Background(), "bucket", "data", "", tt.delimiter, page) + if err != nil { + t.Fatalf("MaxKeys=%d: listPage() error = %v", maxKeys, err) + } + var keys []string + for _, item := range list.Contents { + keys = append(keys, item.Key) + } + for _, prefix := range list.CommonPrefixes { + if !strings.HasSuffix(prefix.Prefix, "/") { + t.Fatalf("common prefix %q does not end with /", prefix.Prefix) + } + keys = append(keys, prefix.Prefix) + } + sort.Strings(keys) + got = append(got, keys...) + if !list.IsTruncated { + break + } + page.Marker, page.HasMarker = list.NextMarker, true + } + if !slices.Equal(got, tt.want) { + t.Fatalf("MaxKeys=%d: listing = %v, want %v", maxKeys, got, tt.want) + } + } + }) + } +} diff --git a/server/s3/logger.go b/server/s3/logger.go index daf0aec24..1715a4004 100644 --- a/server/s3/logger.go +++ b/server/s3/logger.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) // logger output formatted message diff --git a/server/s3/multipart.go b/server/s3/multipart.go new file mode 100644 index 000000000..7261ea361 --- /dev/null +++ b/server/s3/multipart.go @@ -0,0 +1,383 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for openlist +package s3 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/gofakes3" + log "github.com/sirupsen/logrus" +) + +// Compile-time assertions that s3Backend implements both the base Backend and +// the optional MultipartBackend interface from gofakes3. +var ( + _ gofakes3.Backend = (*s3Backend)(nil) + _ gofakes3.MultipartBackend = (*s3Backend)(nil) +) + +// multipartPart records a single uploaded part on disk. +type multipartPart struct { + path string + size int64 + md5hex string // unquoted lowercase hex + updated time.Time +} + +// multipartState tracks one in-progress multipart upload. +// +// Concurrency: gofakes3 does not serialize operations for the same uploadID, +// so concurrent UploadPart/Complete/Abort calls may overlap. The parts map is +// protected by mu. Each part is written to its own file inside dir, so +// concurrent UploadPart calls for different part numbers are safe without +// additional locking. lastActivity is updated under mu on create and on each +// part upload so the reaper can make a consistent expiry decision. +type multipartState struct { + bucket string + object string + meta map[string]string + dir string + created time.Time + lastActivity time.Time // updated under mu on create and each part upload + + mu sync.Mutex + parts map[int]*multipartPart +} + +// CreateMultipartUpload begins a new multipart upload. Parts are streamed to +// local temp files so that large uploads do not need to be buffered in memory. +// +// It implements gofakes3.MultipartBackend. +func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (gofakes3.UploadID, error) { + if _, err := getBucketByName(bucket); err != nil { + return "", err + } + + tempDir := conf.Conf.TempDir + if tempDir == "" { + tempDir = os.TempDir() + } + dir, err := os.MkdirTemp(tempDir, "s3-multipart-*") + if err != nil { + return "", fmt.Errorf("create multipart upload dir: %w", err) + } + + uploadID := gofakes3.UploadID(strings.ReplaceAll(uuid.NewString(), "-", "")) + now := time.Now() + state := &multipartState{ + bucket: bucket, + object: object, + meta: meta, + dir: dir, + created: now, + lastActivity: now, + parts: map[int]*multipartPart{}, + } + + b.uploads.Store(uploadID, state) + log.Debugf("s3 multipart: created upload %s for %s/%s", uploadID, bucket, object) + return uploadID, nil +} + +// UploadPart writes a single part to disk and returns its (quoted) MD5 etag. +// +// It implements gofakes3.MultipartBackend. The body must contain exactly +// contentLength bytes; a short read is reported as ErrIncompleteBody so a +// truncated client request is never silently stored. +func (b *s3Backend) UploadPart(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID, partNumber int, contentLength int64, body io.Reader) (string, error) { + if partNumber <= 0 || partNumber > gofakes3.MaxUploadPartNumber { + return "", gofakes3.ErrInvalidPart + } + + val, ok := b.uploads.Load(uploadID) + if !ok { + return "", gofakes3.ErrNoSuchUpload + } + state := val.(*multipartState) + + partPath := filepath.Join(state.dir, fmt.Sprintf("part-%05d", partNumber)) + f, err := os.Create(partPath) + if err != nil { + return "", fmt.Errorf("create part file: %w", err) + } + // Remove a half-written file on any failure path. + partFailed := true + defer func() { + if partFailed { + _ = f.Close() + _ = os.Remove(partPath) + } + }() + + hash := md5.New() + // io.TeeReader feeds the hasher while the part is streamed to disk, so the + // etag costs no extra pass over the data. + n, err := utils.CopyWithBuffer(io.MultiWriter(f, hash), io.LimitReader(body, contentLength)) + if err != nil { + return "", fmt.Errorf("write part %d: %w", partNumber, err) + } + if err := f.Close(); err != nil { + return "", fmt.Errorf("close part %d: %w", partNumber, err) + } + if n != contentLength { + // The client under-delivered (truncated/aborted request). gofakes3 does + // not validate this for streaming backends, so we must. + return "", gofakes3.ErrIncompleteBody + } + + md5hex := hex.EncodeToString(hash.Sum(nil)) + etag := fmt.Sprintf("%q", md5hex) + + now := time.Now() + state.mu.Lock() + if old := state.parts[partNumber]; old != nil && old.path != partPath { + _ = os.Remove(old.path) + } + state.parts[partNumber] = &multipartPart{ + path: partPath, + size: n, + md5hex: md5hex, + updated: now, + } + state.lastActivity = now + state.mu.Unlock() + + partFailed = false + log.Debugf("s3 multipart: stored part %d for %s (%d bytes)", partNumber, uploadID, n) + return etag, nil +} + +// CompleteMultipartUpload assembles the uploaded parts in ascending part-number +// order and streams the result into storage via the shared putStream path. +// +// It implements gofakes3.MultipartBackend. Part ordering and etags are +// validated against the parts actually received. +func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID, input *gofakes3.CompleteMultipartUploadRequest) (gofakes3.VersionID, string, error) { + val, ok := b.uploads.Load(uploadID) + if !ok { + return "", "", gofakes3.ErrNoSuchUpload + } + state := val.(*multipartState) + + if input == nil || len(input.Parts) == 0 { + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrMalformedXML, "complete multipart upload has no parts") + } + // S3 requires the parts in a CompleteMultipartUpload request to be listed + // in ascending part-number order. + for i := 1; i < len(input.Parts); i++ { + if input.Parts[i].PartNumber <= input.Parts[i-1].PartNumber { + return "", "", gofakes3.ErrInvalidPartOrder + } + } + + // Validate every requested part exists with a matching etag, and collect + // them in the order requested by the client (which is sorted ascending). + state.mu.Lock() + ordered := make([]*multipartPart, 0, len(input.Parts)) + var concat []byte + for _, p := range input.Parts { + stored := state.parts[p.PartNumber] + if stored == nil { + state.mu.Unlock() + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrInvalidPart, "unexpected part number %d in complete request", p.PartNumber) + } + if strings.Trim(p.ETag, "\"") != stored.md5hex { + state.mu.Unlock() + return "", "", gofakes3.ErrorMessagef(gofakes3.ErrInvalidPart, "unexpected part etag for number %d in complete request", p.PartNumber) + } + ordered = append(ordered, stored) + // S3 multipart etag = hex(md5(concat(part_md5_digests)))-N + concat = append(concat, stored.md5Bytes()...) + } + // Hold the lock until the part files are opened so an abort racing with + // complete cannot delete them out from under us. + readers := make([]io.Reader, 0, len(ordered)) + closers := make([]io.Closer, 0, len(ordered)) + var total int64 + for _, part := range ordered { + f, err := os.Open(part.path) + if err != nil { + for _, c := range closers { + _ = c.Close() + } + state.mu.Unlock() + return "", "", fmt.Errorf("open part %s: %w", part.path, err) + } + readers = append(readers, f) + closers = append(closers, f) + total += part.size + } + state.mu.Unlock() + + combined := utils.NewReadCloser(io.MultiReader(readers...), func() error { + var firstErr error + for _, c := range closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr + }) + + defer combined.Close() + + err := b.putStream(ctx, bucket, object, state.meta, combined, total) + if err != nil { + // Leave the upload in place so the client may retry completion, per + // the gofakes3 MultipartBackend contract. + return "", "", err + } + + // Success: drop bookkeeping and clean up part files. + b.removeUpload(uploadID) + + sum := md5.Sum(concat) + etag := fmt.Sprintf("%q", fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:]), len(ordered))) + log.Debugf("s3 multipart: completed upload %s -> %s/%s (%d bytes)", uploadID, bucket, object, total) + return "", etag, nil +} + +// AbortMultipartUpload discards an in-progress upload and its parts. +// +// It implements gofakes3.MultipartBackend and is idempotent: aborting an +// unknown upload succeeds so retries do not fail. +func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucket, object string, uploadID gofakes3.UploadID) error { + b.removeUpload(uploadID) + return nil +} + +// removeUpload deletes the upload's temp directory and drops its bookkeeping. +// Missing uploads are ignored to keep abort/complete idempotent. +func (b *s3Backend) removeUpload(uploadID gofakes3.UploadID) { + val, ok := b.uploads.LoadAndDelete(uploadID) + if !ok { + return + } + state := val.(*multipartState) + if state.dir != "" { + if err := os.RemoveAll(state.dir); err != nil { + log.Warnf("s3 multipart: failed to clean up %s: %v", state.dir, err) + } + } +} + +// md5Bytes returns the raw 16-byte MD5 digest of the part. +func (p *multipartPart) md5Bytes() []byte { + b, _ := hex.DecodeString(p.md5hex) + return b +} + +// Defaults for reaping abandoned multipart uploads. A client that never sends +// CompleteMultipartUpload or AbortMultipartUpload would otherwise leave part +// files on disk forever; the reaper drops uploads inactive for longer than the +// TTL. +const ( + defaultMultipartTTL = 24 * time.Hour + multipartDirPrefix = "s3-multipart-" +) + +// multipartTTL returns the configured max idle time for an upload before the +// reaper reclaims it. It parses conf.Conf.S3.MultipartTTL as a Go duration +// (e.g. "24h", "30m"); an empty or invalid value falls back to the default. +func multipartTTL() time.Duration { + if v := conf.Conf.S3.MultipartTTL; v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultMultipartTTL +} + +// reapInterval derives the reaper tick interval from the TTL: a quarter of the +// TTL, clamped to [10s, 1h]. +func reapInterval(ttl time.Duration) time.Duration { + d := ttl / 4 + if d < 10*time.Second { + d = 10 * time.Second + } + if d > time.Hour { + d = time.Hour + } + return d +} + +// startReaper removes leftover part directories from a previous process crash +// and then launches a background goroutine that periodically reclaims uploads +// inactive for longer than the TTL. The goroutine runs for the lifetime of the +// process; NewServer is called once at startup, so there is one reaper per +// backend instance. +func (b *s3Backend) startReaper() { + b.cleanupStaleDirs(time.Now(), multipartTTL()) + interval := reapInterval(multipartTTL()) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for now := range ticker.C { + b.reapExpired(now, multipartTTL()) + } + }() +} + +// reapExpired removes uploads whose lastActivity is older than ttl. It is safe +// to call concurrently with UploadPart/Complete/Abort: each candidate is +// re-checked under its own lock and removed atomically via removeUpload. +func (b *s3Backend) reapExpired(now time.Time, ttl time.Duration) { + b.uploads.Range(func(key, val any) bool { + state := val.(*multipartState) + state.mu.Lock() + expired := now.Sub(state.lastActivity) > ttl + state.mu.Unlock() + if !expired { + return true + } + b.removeUpload(key.(gofakes3.UploadID)) + log.Infof("s3 multipart: reaped abandoned upload %s (%s/%s)", key, state.bucket, state.object) + return true + }) +} + +// cleanupStaleDirs removes s3-multipart-* directories under TempDir that are +// older than ttl. This reclaims part files left behind by a previous process +// crash; dirs younger than ttl are left alone so a concurrently-starting +// sibling backend instance is never disturbed. +func (b *s3Backend) cleanupStaleDirs(now time.Time, ttl time.Duration) { + tempDir := conf.Conf.TempDir + if tempDir == "" { + tempDir = os.TempDir() + } + entries, err := os.ReadDir(tempDir) + if err != nil { + return + } + cutoff := now.Add(-ttl) + for _, e := range entries { + if !e.IsDir() || !strings.HasPrefix(e.Name(), multipartDirPrefix) { + continue + } + info, err := e.Info() + if err != nil { + continue + } + if info.ModTime().After(cutoff) { + continue + } + if err := os.RemoveAll(filepath.Join(tempDir, e.Name())); err != nil { + log.Warnf("s3 multipart: failed to clean up stale dir %s: %v", e.Name(), err) + } else { + log.Infof("s3 multipart: removed stale multipart dir %s", e.Name()) + } + } +} diff --git a/server/s3/multipart_test.go b/server/s3/multipart_test.go new file mode 100644 index 000000000..a7349583d --- /dev/null +++ b/server/s3/multipart_test.go @@ -0,0 +1,332 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/OpenListTeam/OpenList/v4/drivers/local" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + + "github.com/OpenListTeam/gofakes3" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func init() { + dataDir, err := os.MkdirTemp("", "openlist-s3-mp-*") + if err != nil { + panic(err) + } + conf.Conf = conf.DefaultConfig(dataDir) + if err := os.MkdirAll(conf.Conf.TempDir, 0o755); err != nil { + panic("mkdir temp dir: " + err.Error()) + } + dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + panic("failed to connect database: " + err.Error()) + } + db.Init(dB) +} + +// s3ErrorCode extracts the gofakes3 ErrorCode from an error returned by the +// MultipartBackend methods. +func s3ErrorCode(err error) gofakes3.ErrorCode { + if err == nil { + return gofakes3.ErrNone + } + var s3err interface{ ErrorCode() gofakes3.ErrorCode } + if errors.As(err, &s3err) { + return s3err.ErrorCode() + } + return gofakes3.ErrNone +} + +// setupMultipartBackend prepares a Local storage mounted at /mpbucket and an +// s3Backend with an "mp" bucket pointing at it. It returns the backend, the +// local root directory on disk, and a cleanup function. +func setupMultipartBackend(t *testing.T) (*s3Backend, string) { + t.Helper() + ctx := context.Background() + + // Unique mount path and bucket per test: the in-memory sqlite is shared + // across tests in this package, so a fixed mount path would clash. + mount := "/" + sanitizeTestName(t.Name()) + bucket := "mp" + + localRoot, err := os.MkdirTemp("", "openlist-s3-local-*") + if err != nil { + t.Fatalf("mkdir local root: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(localRoot) }) + addition, err := json.Marshal(struct { + RootFolderPath string `json:"root_folder_path"` + Thumbnail bool `json:"thumbnail"` + }{ + RootFolderPath: localRoot, + Thumbnail: false, + }) + if err != nil { + t.Fatalf("marshal local storage addition: %v", err) + } + + _, err = op.CreateStorage(ctx, model.Storage{ + Driver: "Local", + MountPath: mount, + Addition: string(addition), + }) + if err != nil { + t.Fatalf("create local storage: %+v", err) + } + + if err := op.SaveSettingItem(&model.SettingItem{ + Key: conf.S3Buckets, + Value: `[{"name":"` + bucket + `","path":"` + mount + `"}]`, + }); err != nil { + t.Fatalf("save s3 buckets setting: %+v", err) + } + + return newBackend().(*s3Backend), localRoot +} + +func sanitizeTestName(name string) string { + r := strings.NewReplacer("/", "_", " ", "_") + return r.Replace(name) +} + +func TestMultipartUploadEndToEnd(t *testing.T) { + ctx := context.Background() + b, localRoot := setupMultipartBackend(t) + + meta := map[string]string{"Content-Type": "text/plain"} + uploadID, err := b.CreateMultipartUpload(ctx, "mp", "dir/hello.txt", meta) + if err != nil { + t.Fatalf("CreateMultipartUpload: %+v", err) + } + if uploadID == "" { + t.Fatal("empty upload id") + } + + part := func(n int, body string) string { + t.Helper() + etag, err := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, n, int64(len(body)), strings.NewReader(body)) + if err != nil { + t.Fatalf("UploadPart %d: %+v", n, err) + } + return etag + } + + etag1 := part(1, "Hello, ") + etag2 := part(2, "multipart ") + etag3 := part(3, "world!") + + // Re-uploading the same part number overwrites it and returns a fresh etag. + if e := part(2, "multipart "); e != etag2 { + t.Fatalf("re-upload part 2 etag = %q, want %q", e, etag2) + } + + // Short read (body smaller than declared Content-Length) must fail. + _, shortErr := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, 4, 10, strings.NewReader("abc")) + shortErr = s3ErrorCode(shortErr) + if shortErr != gofakes3.ErrIncompleteBody { + t.Fatalf("short read error = %v, want IncompleteBody", shortErr) + } + + // Unknown upload id. + _, uerr := b.UploadPart(ctx, "mp", "dir/hello.txt", "does-not-exist", 1, 1, strings.NewReader("x")) + if code := s3ErrorCode(uerr); code != gofakes3.ErrNoSuchUpload { + t.Fatalf("unknown upload error = %v, want NoSuchUpload", code) + } + // Out-of-range part number. + _, perr := b.UploadPart(ctx, "mp", "dir/hello.txt", uploadID, 0, 1, strings.NewReader("x")) + if code := s3ErrorCode(perr); code != gofakes3.ErrInvalidPart { + t.Fatalf("part 0 error = %v, want InvalidPart", code) + } + + // Parts out of order. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 2, ETag: etag2}, + {PartNumber: 1, ETag: etag1}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPartOrder { + t.Fatalf("out-of-order complete error = %v, want InvalidPartOrder", code) + } + + // Wrong etag. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 2, ETag: `"deadbeef"`}, + {PartNumber: 3, ETag: etag3}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPart { + t.Fatalf("wrong etag complete error = %v, want InvalidPart", code) + } + + // Missing part number. + _, _, err = b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 99, ETag: etag3}, + }, + }) + if code := s3ErrorCode(err); code != gofakes3.ErrInvalidPart { + t.Fatalf("missing part complete error = %v, want InvalidPart", code) + } + + // The failed completes must leave the upload available for retry. + if _, ok := b.uploads.Load(uploadID); !ok { + t.Fatal("upload was removed after a failed complete") + } + + // Successful complete. + _, etag, err := b.CompleteMultipartUpload(ctx, "mp", "dir/hello.txt", uploadID, &gofakes3.CompleteMultipartUploadRequest{ + Parts: []gofakes3.CompletedPart{ + {PartNumber: 1, ETag: etag1}, + {PartNumber: 2, ETag: etag2}, + {PartNumber: 3, ETag: etag3}, + }, + }) + if err != nil { + t.Fatalf("CompleteMultipartUpload: %+v", err) + } + if !strings.HasSuffix(etag, `-3"`) || !strings.HasPrefix(etag, `"`) { + t.Fatalf("complete etag = %q, want a quoted \"-3\" multipart etag", etag) + } + + // The object must exist on disk with the concatenated content. + got, err := os.ReadFile(filepath.Join(localRoot, "dir", "hello.txt")) + if err != nil { + t.Fatalf("read resulting file: %+v", err) + } + want := []byte("Hello, multipart world!") + if !bytes.Equal(got, want) { + t.Fatalf("resulting file content = %q, want %q", got, want) + } + + // Bookkeeping and temp files must be cleaned up on success. + if _, ok := b.uploads.Load(uploadID); ok { + t.Fatal("upload still tracked after successful complete") + } +} + +func TestMultipartAbort(t *testing.T) { + ctx := context.Background() + b, _ := setupMultipartBackend(t) + + uploadID, err := b.CreateMultipartUpload(ctx, "mp", "abort.txt", nil) + if err != nil { + t.Fatalf("CreateMultipartUpload: %+v", err) + } + if _, err := b.UploadPart(ctx, "mp", "abort.txt", uploadID, 1, 3, strings.NewReader("abc")); err != nil { + t.Fatalf("UploadPart: %+v", err) + } + + state, _ := b.uploads.Load(uploadID) + dir := state.(*multipartState).dir + if _, err := os.Stat(dir); err != nil { + t.Fatalf("temp dir missing before abort: %+v", err) + } + + if err := b.AbortMultipartUpload(ctx, "mp", "abort.txt", uploadID); err != nil { + t.Fatalf("AbortMultipartUpload: %+v", err) + } + if _, ok := b.uploads.Load(uploadID); ok { + t.Fatal("upload still tracked after abort") + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("temp dir still exists after abort (err=%v)", err) + } + + // Aborting an unknown upload must be idempotent. + if err := b.AbortMultipartUpload(ctx, "mp", "abort.txt", "nope"); err != nil { + t.Fatalf("abort unknown upload returned error: %+v", err) + } +} + +func TestMultipartReapExpired(t *testing.T) { + ctx := context.Background() + b, _ := setupMultipartBackend(t) + + // An active upload (fresh lastActivity) must be kept. + freshID, err := b.CreateMultipartUpload(ctx, "mp", "fresh.txt", nil) + if err != nil { + t.Fatalf("create fresh upload: %+v", err) + } + + // An abandoned upload (stale lastActivity) must be reaped. + staleID, err := b.CreateMultipartUpload(ctx, "mp", "stale.txt", nil) + if err != nil { + t.Fatalf("create stale upload: %+v", err) + } + if _, err := b.UploadPart(ctx, "mp", "stale.txt", staleID, 1, 3, strings.NewReader("abc")); err != nil { + t.Fatalf("upload stale part: %+v", err) + } + staleState, _ := b.uploads.Load(staleID) + staleDir := staleState.(*multipartState).dir + if _, err := os.Stat(staleDir); err != nil { + t.Fatalf("stale temp dir missing: %+v", err) + } + + // Force the stale upload's lastActivity well into the past. + ttl := 30 * time.Minute + now := time.Now() + staleState.(*multipartState).mu.Lock() + staleState.(*multipartState).lastActivity = now.Add(-2 * ttl) + staleState.(*multipartState).mu.Unlock() + + b.reapExpired(now, ttl) + + if _, ok := b.uploads.Load(staleID); ok { + t.Fatal("stale upload still tracked after reap") + } + if _, err := os.Stat(staleDir); !os.IsNotExist(err) { + t.Fatalf("stale temp dir still exists after reap (err=%v)", err) + } + if _, ok := b.uploads.Load(freshID); !ok { + t.Fatal("fresh upload was reaped, should have been kept") + } +} + +func TestMultipartCleanupStaleDirs(t *testing.T) { + b, _ := setupMultipartBackend(t) + + tempDir := conf.Conf.TempDir + staleDir, err := os.MkdirTemp(tempDir, multipartDirPrefix+"*") + if err != nil { + t.Fatalf("mkdir stale dir: %v", err) + } + freshDir, err := os.MkdirTemp(tempDir, multipartDirPrefix+"*") + if err != nil { + t.Fatalf("mkdir fresh dir: %v", err) + } + // Age the stale dir beyond the TTL; leave the fresh dir young. + ttl := 30 * time.Minute + now := time.Now() + past := now.Add(-2 * ttl) + if err := os.Chtimes(staleDir, past, past); err != nil { + t.Fatalf("chtimes stale dir: %v", err) + } + + b.cleanupStaleDirs(now, ttl) + + if _, err := os.Stat(staleDir); !os.IsNotExist(err) { + t.Fatalf("stale dir should have been removed (err=%v)", err) + } + if _, err := os.Stat(freshDir); err != nil { + t.Fatalf("fresh dir should have been kept (err=%v)", err) + } + _ = os.RemoveAll(freshDir) +} diff --git a/server/s3/pager.go b/server/s3/pager.go deleted file mode 100644 index a607b1db5..000000000 --- a/server/s3/pager.go +++ /dev/null @@ -1,67 +0,0 @@ -// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 -// Package s3 implements a fake s3 server for openlist -package s3 - -import ( - "sort" - - "github.com/itsHenry35/gofakes3" -) - -// pager splits the object list into multiple pages. -func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { - // sort by alphabet - sort.Slice(list.CommonPrefixes, func(i, j int) bool { - return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix - }) - // sort by modtime - sort.Slice(list.Contents, func(i, j int) bool { - return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) - }) - tokens := page.MaxKeys - if tokens == 0 { - tokens = 1000 - } - if page.HasMarker { - for i, obj := range list.Contents { - if obj.Key == page.Marker { - list.Contents = list.Contents[i+1:] - break - } - } - for i, obj := range list.CommonPrefixes { - if obj.Prefix == page.Marker { - list.CommonPrefixes = list.CommonPrefixes[i+1:] - break - } - } - } - - response := gofakes3.NewObjectList() - for _, obj := range list.CommonPrefixes { - if tokens <= 0 { - break - } - response.AddPrefix(obj.Prefix) - tokens-- - } - - for _, obj := range list.Contents { - if tokens <= 0 { - break - } - response.Add(obj) - tokens-- - } - - if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { - response.IsTruncated = true - if len(response.Contents) > 0 { - response.NextMarker = response.Contents[len(response.Contents)-1].Key - } else { - response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix - } - } - - return response, nil -} diff --git a/server/s3/range.go b/server/s3/range.go new file mode 100644 index 000000000..7cc3f1cb0 --- /dev/null +++ b/server/s3/range.go @@ -0,0 +1,47 @@ +package s3 + +import "net/http" + +// gofakes3 sets Content-Range without selecting the partial-content status. +func rangeStatusHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.Header.Get("Range") == "" { + next.ServeHTTP(w, r) + return + } + rw := &rangeResponseWriter{ResponseWriter: w} + next.ServeHTTP(rw, r) + if !rw.wroteHeader { + rw.WriteHeader(http.StatusOK) + } + }) +} + +type rangeResponseWriter struct { + http.ResponseWriter + wroteHeader bool +} + +func (w *rangeResponseWriter) WriteHeader(status int) { + if w.wroteHeader { + return + } + if status >= 200 { + w.wroteHeader = true + } + if status == http.StatusOK && w.Header().Get("Content-Range") != "" { + status = http.StatusPartialContent + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *rangeResponseWriter) Write(p []byte) (int, error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(p) +} + +func (w *rangeResponseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} diff --git a/server/s3/range_test.go b/server/s3/range_test.go new file mode 100644 index 000000000..9d1a76830 --- /dev/null +++ b/server/s3/range_test.go @@ -0,0 +1,55 @@ +package s3 + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRangeStatusHandler(t *testing.T) { + for _, tt := range []struct { + name string + method string + rangeHeader string + contentRange string + status int + body string + want int + }{ + {"partial", "GET", "bytes=2-5", "bytes 2-5/16", 0, "2345", 206}, + {"explicit OK", "GET", "bytes=2-5", "bytes 2-5/16", 200, "2345", 206}, + {"already partial", "GET", "bytes=2-5", "bytes 2-5/16", 206, "2345", 206}, + {"full", "GET", "", "", 0, "0123456789abcdef", 200}, + {"range ignored", "GET", "bytes=2-5", "", 0, "0123456789abcdef", 200}, + {"not modified", "GET", "bytes=2-5", "", 304, "", 304}, + {"invalid range", "GET", "bytes=20-", "bytes */16", 416, "error", 416}, + {"forbidden", "GET", "bytes=2-5", "", 403, "error", 403}, + {"empty body", "GET", "bytes=2-5", "bytes 2-5/16", 0, "", 206}, + {"head", "HEAD", "bytes=2-5", "", 0, "", 200}, + } { + t.Run(tt.name, func(t *testing.T) { + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tt.contentRange != "" { + w.Header().Set("Content-Range", tt.contentRange) + } + if tt.status != 0 { + w.WriteHeader(tt.status) + } + if tt.body != "" { + _, _ = io.WriteString(w, tt.body) + } + }) + r := httptest.NewRequest(tt.method, "/bucket/object", nil) + r.Header.Set("Range", tt.rangeHeader) + w := httptest.NewRecorder() + rangeStatusHandler(next).ServeHTTP(w, r) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d", w.Code, tt.want) + } + if w.Body.String() != tt.body || w.Header().Get("Content-Range") != tt.contentRange { + t.Fatalf("response changed: body=%q, range=%q", w.Body.String(), w.Header().Get("Content-Range")) + } + }) + } +} diff --git a/server/s3/redirect.go b/server/s3/redirect.go index 4031e171d..6d8d430bc 100644 --- a/server/s3/redirect.go +++ b/server/s3/redirect.go @@ -14,7 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" - "github.com/itsHenry35/gofakes3/signature" + "github.com/OpenListTeam/gofakes3/signature" ) func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler { diff --git a/server/s3/server.go b/server/s3/server.go index d33b63ddd..f1ed5ba38 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -7,7 +7,7 @@ import ( "math/rand" "net/http" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) // Make a new S3 Server to serve the remote @@ -24,5 +24,5 @@ func NewServer(ctx context.Context) (h http.Handler, err error) { gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) - return redirectHandler(faker.Server(), authPairs), nil + return redirectHandler(rangeStatusHandler(faker.Server()), authPairs), nil } diff --git a/server/s3/utils.go b/server/s3/utils.go index eb3dd5685..6624a07fc 100644 --- a/server/s3/utils.go +++ b/server/s3/utils.go @@ -5,6 +5,7 @@ package s3 import ( "context" "encoding/json" + stderrors "errors" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -13,7 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/itsHenry35/gofakes3" + "github.com/OpenListTeam/gofakes3" ) type Bucket struct { @@ -21,6 +22,13 @@ type Bucket struct { Path string `json:"path"` } +func mapBackendError(err error) error { + if stderrors.Is(err, errs.TemporaryCapacity) { + return gofakes3.ErrSlowDown + } + return err +} + const emptyObjectName = "ThisIsAnEmptyFolderInTheS3Bucket" func getAndParseBuckets() ([]Bucket, error) { @@ -42,10 +50,12 @@ func getBucketByName(name string) (Bucket, error) { return Bucket{}, gofakes3.BucketNotFound(name) } -func getDirEntries(path string) ([]model.Obj, error) { - ctx := context.Background() +func getDirEntries(ctx context.Context, path string) ([]model.Obj, error) { meta, _ := op.GetNearestMeta(path) fi, err := fs.Get(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.GetArgs{}) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if errs.IsNotFoundError(err) { return nil, gofakes3.ErrNoSuchKey } else if err != nil { @@ -57,6 +67,9 @@ func getDirEntries(path string) ([]model.Obj, error) { } dirEntries, err := fs.List(context.WithValue(ctx, conf.MetaKey, meta), path, &fs.ListArgs{}) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { return nil, err } diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 06d1431ac..5729543e6 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -355,8 +355,6 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, return status, err } defer release() - // TODO(rost): Support the If-Match, If-None-Match headers? See bradfitz' - // comments in http.checkEtag. ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) @@ -394,6 +392,30 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if !common.CanWrite(user, parentMeta, parentPath) { return http.StatusForbidden, errs.PermissionDenied } + fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + exists := err == nil + if err != nil && !errs.IsObjectNotFound(err) { + if errs.IsNotFoundError(err) { + return http.StatusNotFound, err + } + return http.StatusInternalServerError, err + } + var modTime time.Time + if exists { + if fi.IsDir() { + return http.StatusMethodNotAllowed, nil + } + etag, err := findETag(ctx, h.LockSystem, reqPath, fi) + if err != nil { + return http.StatusInternalServerError, err + } + w.Header().Set("Etag", etag) + modTime = fi.ModTime() + } + if done, _ := net.CheckPreconditions(w, r, modTime, exists); done { + return 0, nil + } + w.Header().Del("Etag") fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, @@ -411,16 +433,21 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if err != nil { return http.StatusMethodNotAllowed, err } - fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + fi, err = fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { - fi = &obj + return http.StatusInternalServerError, err } etag, err := findETag(ctx, h.LockSystem, reqPath, fi) if err != nil { return http.StatusInternalServerError, err } w.Header().Set("Etag", etag) - return http.StatusCreated, nil + if exists { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusCreated) + } + return 0, nil } func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status int, err error) {