diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..1110599aa 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,5 +1,4 @@ name: Beta Release (Docker) - on: workflow_dispatch: push: @@ -7,51 +6,51 @@ on: - main pull_request: branches: - - main + - fix # 👈 允许你的 fix 分支触发 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' IMAGE_TAGS_BETA: | type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + type=raw,value=beta jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64 Only) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: '1.25.0' + # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build go binary (beta) + - name: Build go binary + # 这里还是跑 docker-multiplatform，虽然会多编译一些架构，但这是兼容 Dockerfile 路径最稳妥的方法 run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -69,12 +68,13 @@ jobs: release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + # 你可以选择只构建 latest，或者保留全部变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" @@ -102,46 +102,32 @@ jobs: with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + flavor: ${{ matrix.tag_favor }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} diff --git a/build.sh b/build.sh index 26e5a301b..0e8f4b85d 100644 --- a/build.sh +++ b/build.sh @@ -186,8 +186,8 @@ BuildDockerMultiplatform() { docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -205,14 +205,14 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - done + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # done } BuildRelease() { diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 909bf4a99..94c863315 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -228,7 +228,8 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } sha1 := file.GetHash().GetHash(utils.SHA1) if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + // 流式计算SHA1 + sha1, err = stream.StreamHashFile(file, utils.SHA1, 10, &up) if err != nil { return err } diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index ac75e51d7..42bd36dcb 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -156,20 +156,44 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { - _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if len(etag) >= utils.MD5.Width { + // 有etag时，先尝试秒传 + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } + // 是否秒传 + if createResp.Data.Reuse { + // 秒传成功才会返回正确的 FileID，否则为 0 + if createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + } + // 秒传失败，etag可能不可靠，继续流式计算真实MD5 + } + + // 流式MD5计算 + etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + if err != nil { + return nil, err } + + // 2. 创建上传任务 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -188,13 +212,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 03e84b396..35886ce76 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -7,7 +7,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) var ( @@ -76,9 +75,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.ServerMtime, 0), Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, - - // 直接获取的MD5是错误的 - HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + // 百度API返回的MD5不可信，不使用HashInfo }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d..3e1bf8bd0 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -174,6 +174,40 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + var offset int64 = 0 + + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + if err != nil { + return "", fmt.Errorf("range read for hash calculation failed: %w", err) + } + if _, err := io.Copy(hashFunc, reader); err != nil { + return "", fmt.Errorf("calculate hash failed: %w", err) + } + offset += readSize + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error)