From b680a3562f3ccf05e2e85d6a67c4b768936cfc7e Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 28 Dec 2025 18:17:05 +0800 Subject: [PATCH 1/6] fix(driver): fix file copy failure to 123pan due to incorrect etag --- drivers/123_open/driver.go | 41 ++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index ac75e51d7..f2e9e69d4 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -2,7 +2,9 @@ package _123_open import ( "context" + "encoding/hex" "fmt" + "io" "strconv" "time" @@ -10,7 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -156,20 +158,36 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } - // etag 文件md5 - etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { - _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) + + // 1. 流式计算MD5 + md5Hash := utils.MD5.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation + var offset int64 = 0 + for offset < size { + readSize := min(chunkSize, size-offset) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) if err != nil { - return nil, err + return nil, fmt.Errorf("range read for MD5 calculation failed: %w", err) + } + if _, err := io.Copy(md5Hash, reader); err != nil { + return nil, fmt.Errorf("calculate MD5 failed: %w", err) } + offset += readSize + + progress := 40 * float64(offset) / float64(size) + up(progress) } + + etag := hex.EncodeToString(md5Hash.Sum(nil)) + + // 2. 创建上传任务 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -188,13 +206,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 From 16b29e837dc333d64c93afba4be7dac2f25427a1 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 29 Dec 2025 00:24:08 +0800 Subject: [PATCH 2/6] fix(driver): improve etag handling for file uploads --- drivers/123_open/driver.go | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index f2e9e69d4..90112b37a 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -165,7 +165,31 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre return nil, fmt.Errorf("parse parentFileID error: %v", err) } - // 1. 流式计算MD5 + // etag 文件md5 + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) >= utils.MD5.Width { + // 有etag时，先尝试秒传 + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return nil, err + } + // 是否秒传 + if createResp.Data.Reuse { + // 秒传成功才会返回正确的 FileID，否则为 0 + if createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + } + // 秒传失败，etag可能不可靠，继续流式计算真实MD5 + } + + // 流式计算MD5 md5Hash := utils.MD5.NewFunc() size := file.GetSize() chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation @@ -185,7 +209,7 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre up(progress) } - etag := hex.EncodeToString(md5Hash.Sum(nil)) + etag = hex.EncodeToString(md5Hash.Sum(nil)) // 2. 创建上传任务 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) From 88a72e744119f7c6ef78decfe1450564e43c7857 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 29 Dec 2025 02:46:00 +0800 Subject: [PATCH 3/6] fix(driver): optimize SHA1 calculation for file uploads using chunked reading --- drivers/115_open/driver.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 909bf4a99..5b920a4ac 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,7 +2,9 @@ package _115_open import ( "context" + "encoding/hex" "fmt" + "io" "net/http" "strconv" "strings" @@ -14,7 +16,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" @@ -228,10 +229,30 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } sha1 := file.GetHash().GetHash(utils.SHA1) if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) - if err != nil { - return err + // 流式计算SHA1 + sha1Hash := utils.SHA1.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for SHA1 calculation + var offset int64 = 0 + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + if err != nil { + return fmt.Errorf("range read for SHA1 calculation failed: %w", err) + } + if _, err := io.Copy(sha1Hash, reader); err != nil { + return fmt.Errorf("calculate SHA1 failed: %w", err) + } + offset += readSize + + progress := 10 * float64(offset) / float64(size) + up(progress) } + + sha1 = hex.EncodeToString(sha1Hash.Sum(nil)) } const PreHashSize int64 = 128 * utils.KB hashSize := PreHashSize From 40f66870cb4f21d815994797b39cb599156fd41b Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 29 Dec 2025 02:53:51 +0800 Subject: [PATCH 4/6] fix(driver): streamline SHA1 and MD5 calculations using streaming to reduce memory usage --- drivers/115_open/driver.go | 28 ++++------------------------ drivers/123_open/driver.go | 28 +++++----------------------- internal/stream/util.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 47 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 5b920a4ac..94c863315 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,9 +2,7 @@ package _115_open import ( "context" - "encoding/hex" "fmt" - "io" "net/http" "strconv" "strings" @@ -16,6 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" @@ -230,29 +229,10 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre sha1 := file.GetHash().GetHash(utils.SHA1) if len(sha1) != utils.SHA1.Width { // 流式计算SHA1 - sha1Hash := utils.SHA1.NewFunc() - size := file.GetSize() - chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for SHA1 calculation - var offset int64 = 0 - for offset < size { - readSize := chunkSize - if size-offset < chunkSize { - readSize = size - offset - } - reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) - if err != nil { - return fmt.Errorf("range read for SHA1 calculation failed: %w", err) - } - if _, err := io.Copy(sha1Hash, reader); err != nil { - return fmt.Errorf("calculate SHA1 failed: %w", err) - } - offset += readSize - - progress := 10 * float64(offset) / float64(size) - up(progress) + sha1, err = stream.StreamHashFile(file, utils.SHA1, 10, &up) + if err != nil { + return err } - - sha1 = hex.EncodeToString(sha1Hash.Sum(nil)) } const PreHashSize int64 = 128 * utils.KB hashSize := PreHashSize diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 90112b37a..42bd36dcb 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -2,9 +2,7 @@ package _123_open import ( "context" - "encoding/hex" "fmt" - "io" "strconv" "time" @@ -12,7 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -189,28 +187,12 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre // 秒传失败，etag可能不可靠，继续流式计算真实MD5 } - // 流式计算MD5 - md5Hash := utils.MD5.NewFunc() - size := file.GetSize() - chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation - var offset int64 = 0 - for offset < size { - readSize := min(chunkSize, size-offset) - reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) - if err != nil { - return nil, fmt.Errorf("range read for MD5 calculation failed: %w", err) - } - if _, err := io.Copy(md5Hash, reader); err != nil { - return nil, fmt.Errorf("calculate MD5 failed: %w", err) - } - offset += readSize - - progress := 40 * float64(offset) / float64(size) - up(progress) + // 流式MD5计算 + etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + if err != nil { + return nil, err } - etag = hex.EncodeToString(md5Hash.Sum(nil)) - // 2. 创建上传任务 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d..3e1bf8bd0 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -174,6 +174,40 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + var offset int64 = 0 + + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + if err != nil { + return "", fmt.Errorf("range read for hash calculation failed: %w", err) + } + if _, err := io.Copy(hashFunc, reader); err != nil { + return "", fmt.Errorf("calculate hash failed: %w", err) + } + offset += readSize + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) From 31a004acc7c8402960d4b718edb28acef9271bd2 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 29 Dec 2025 18:02:44 +0800 Subject: [PATCH 5/6] test nil --- drivers/baidu_netdisk/types.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 03e84b396..35886ce76 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -7,7 +7,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) var ( @@ -76,9 +75,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.ServerMtime, 0), Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, - - // 直接获取的MD5是错误的 - HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + // 百度API返回的MD5不可信，不使用HashInfo }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } From 27f20404d4bc318fe2c0e886001c739eb95045a8 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 28 Dec 2025 23:16:46 +0800 Subject: [PATCH 6/6] refactor(build): restrict builds to x64 architecture and simplify Docker workflow --- .github/workflows/test_docker.yml | 48 +++++++++++-------------------- build.sh | 20 ++++++------- 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..1110599aa 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,5 +1,4 @@ name: Beta Release (Docker) - on: workflow_dispatch: push: @@ -7,51 +6,51 @@ on: - main pull_request: branches: - - main + - fix # 👈 允许你的 fix 分支触发 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' IMAGE_TAGS_BETA: | type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + type=raw,value=beta jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64 Only) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: '1.25.0' + # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build go binary (beta) + - name: Build go binary + # 这里还是跑 docker-multiplatform，虽然会多编译一些架构，但这是兼容 Dockerfile 路径最稳妥的方法 run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -69,12 +68,13 @@ jobs: release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + # 你可以选择只构建 latest，或者保留全部变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" @@ -102,46 +102,32 @@ jobs: with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + flavor: ${{ matrix.tag_favor }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} diff --git a/build.sh b/build.sh index 26e5a301b..0e8f4b85d 100644 --- a/build.sh +++ b/build.sh @@ -186,8 +186,8 @@ BuildDockerMultiplatform() { docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -205,14 +205,14 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - done + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # done } BuildRelease() {