download: cdn friendly & getfile support sign & optimize listbucket2 & support intelligent tiering

cmd/asyncfetch.go

@@ -33,7 +33,7 @@ func asyncFetchCmdBuilder(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.CallbackBody, "callback-body", "b", "", "callback body")
 	cmd.Flags().StringVarP(&info.CallbackHost, "callback-host", "T", "", "callback HOST")
 
-	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "storage type, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage")
+	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "storage type, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().IntVarP(&info.FileType, "storage-type", "g", 0, "storage type, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage-type", "use --file-type instead") // 废弃 storage-type
 

cmd/bucket.go

@@ -137,7 +137,7 @@ var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.StartDate, "start", "s", "", "start date with format yyyy-mm-dd-hh-MM-ss, all files will be listed according to the prefix and then filtered.")
 	cmd.Flags().StringVarP(&info.EndDate, "end", "e", "", "end date with format yyyy-mm-dd-hh-MM-ss, all files will be listed according to the prefix and then filtered.")
 
-	cmd.Flags().StringVarP(&info.FileTypes, "file-types", "", "", "Specify storage type, separated by comma, all files will be listed according to the prefix and then filtered. 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage 4:ARCHIVE_IR storage")
+	cmd.Flags().StringVarP(&info.FileTypes, "file-types", "", "", "Specify storage type, separated by comma, all files will be listed according to the prefix and then filtered. 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().StringVarP(&info.FileTypes, "storages", "", "", "Specify storage type, same to --file-types")
 	_ = cmd.Flags().MarkDeprecated("storages", "use --file-types instead") // 废弃 storages
 

cmd/download.go

@@ -99,6 +99,9 @@ have already in local disk and need to skip download or not.`,
 	cmd.Flags().StringVarP(&LogFile, "log-file", "", "", "the output file of the download log is output to the file specified by record_root by default, and the specific file path can be seen in the terminal output")
 	cmd.Flags().IntVarP(&LogRotate, "log-rotate", "", 7, "the switching period of the download log file, the unit is day,")
 
+	// 标记废弃
+	_ = cmd.Flags().MarkDeprecated("public", "deprecated")
+
 	return cmd
 }
 
@@ -130,6 +133,10 @@ var getCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().Int64VarP(&info.SliceSize, "slice-size", "", 4*utils.MB, "slice size that download using slices. when you use --enable-slice option, the file will be cut into data blocks according to the slice size, then the data blocks will be downloaded concurrently, and finally these data blocks will be spliced into a file. Unit: B")
 	cmd.Flags().IntVarP(&info.SliceConcurrentCount, "slice-concurrent-count", "", 10, "the count of concurrently downloaded slices.")
 	cmd.Flags().BoolVarP(&info.RemoveTempWhileError, "remove-temp-while-error", "", false, "remove download temp file while error happened, default is false")
+
+	// 标记废弃
+	_ = cmd.Flags().MarkDeprecated("public", "deprecated")
+
 	return cmd
 }
 

cmd/rs.go

@@ -70,7 +70,7 @@ var deleteCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 var changeLifecycleCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	var info = &operations.ChangeLifecycleInfo{}
 	var cmd = &cobra.Command{
-		Use:   "chlifecycle <Bucket> <Key> [--to-ia-after-days <ToIAAfterDays>] [--to-archive-after-days <ToArchiveAfterDays>] [--to-deep-archive-after-days <ToDeepArchiveAfterDays>] [--delete-after-days <DeleteAfterDays>]",
+		Use:   "chlifecycle <Bucket> <Key> [--to-ia-after-days <ToIAAfterDays>] [--to-archive-after-days <ToArchiveAfterDays>] [--to-deep-archive-after-days <ToDeepArchiveAfterDays>] [--to-intelligent-tiering-after-days <ToIntelligentTieringAfterDays>] [--delete-after-days <DeleteAfterDays>]",
 		Short: "Set the lifecycle of a file.",
 		Long: `Set the lifecycle of a file. Lifecycle value must great than or equal to -1, unit: day.
 * less than  -1: there's no point and it won't trigger any effect
@@ -92,6 +92,7 @@ var changeLifecycleCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().IntVarP(&info.ToArchiveIRAfterDays, "to-archive-ir-after-days", "", 0, "to ARCHIVE_IR storage after some days. the range is -1 or bigger than 0. -1 means cancel to ARCHIVE_IR storage")
 	cmd.Flags().IntVarP(&info.ToArchiveAfterDays, "to-archive-after-days", "", 0, "to ARCHIVE storage after some days. the range is -1 or bigger than 0. -1 means cancel to ARCHIVE storage")
 	cmd.Flags().IntVarP(&info.ToDeepArchiveAfterDays, "to-deep-archive-after-days", "", 0, "to DEEP_ARCHIVE storage after some days. the range is -1 or bigger than 0. -1 means cancel to DEEP_ARCHIVE storage")
+	cmd.Flags().IntVarP(&info.ToIntelligentTieringAfterDays, "to-intelligent-tiering-after-days", "", 0, "to INTELLIGENT_TIERING storage after some days. the range is -1 or bigger than 0. -1 means cancel to INTELLIGENT_TIERING storage")
 	cmd.Flags().IntVarP(&info.DeleteAfterDays, "delete-after-days", "", 0, "delete after some days. the range is -1 or bigger than 0. -1 means cancel to delete")
 	return cmd
 }
@@ -245,7 +246,8 @@ And 0 means STANDARD storage,
 while 1 means IA storage,
 while 2 means ARCHIVE storage.
 while 3 means DEEP_ARCHIVE storage.
-while 4 means ARCHIVE_IR storage.`,
+while 4 means ARCHIVE_IR storage.
+while 5 means INTELLIGENT_TIERING storage`,
 		Example: `change storage type of A.png(bucket:bucketA key:A.png) to ARCHIVE storage
 	qshell chtype bucketA A.png 2
 and you can check result by command:

cmd/rsbatch.go

@@ -179,6 +179,7 @@ Lifecycle value must great than or equal to -1, unit: day.
 	cmd.Flags().IntVarP(&info.ToArchiveIRAfterDays, "to-archive-ir-after-days", "", 0, "to ARCHIVE_IR storage after some days. the range is -1 or bigger than 0. -1 means cancel to ARCHIVE_IR storage")
 	cmd.Flags().IntVarP(&info.ToArchiveAfterDays, "to-archive-after-days", "", 0, "to ARCHIVE storage after some days. the range is -1 or bigger than 0. -1 means cancel to ARCHIVE storage")
 	cmd.Flags().IntVarP(&info.ToDeepArchiveAfterDays, "to-deep-archive-after-days", "", 0, "to DEEP_ARCHIVE storage after some days. the range is -1 or bigger than 0. -1 means cancel to DEEP_ARCHIVE storage")
+	cmd.Flags().IntVarP(&info.ToIntelligentTieringAfterDays, "to-intelligent-tiering-after-days", "", 0, "to INTELLIGENT_TIERING storage after some days. the range is -1 or bigger than 0. -1 means cancel to INTELLIGENT_TIERING storage")
 	cmd.Flags().IntVarP(&info.DeleteAfterDays, "delete-after-days", "", 0, "delete after some days. the range is -1 or bigger than 0. -1 means cancel to delete")
 	setBatchCmdDefaultFlags(cmd, &info.BatchInfo)
 	return cmd

cmd/upload.go

@@ -96,7 +96,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVar(&LogLevel, "log-level", "debug", "log level")
 	cmd.Flags().IntVar(&LogRotate, "log-rotate", 7, "log rotate days")
 
-	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage")
+	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().IntVarP(&info.FileType, "storage", "", 0, "set storage type of file, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 
@@ -149,7 +149,7 @@ var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "upload host")
 	cmd.Flags().BoolVarP(&info.Accelerate, "accelerate", "", false, "enable uploading acceleration")
 
-	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage")
+	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().IntVarP(&info.FileType, "storage", "s", 0, "set storage type of file, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 
@@ -200,7 +200,7 @@ var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket")
 	cmd.Flags().StringVarP(&info.MimeType, "mimetype", "t", "", "file mime type")
 
-	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage")
+	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().IntVarP(&info.FileType, "storage", "s", 0, "set storage type of file, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 
@@ -259,7 +259,7 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().Int64VarP(&info.ChunkSize, "v2-part-size", "", data.BLOCK_SIZE, "the part size when use resumable upload v2 APIs to upload, same to --resumable-api-v2-part-size")
 	_ = cmd.Flags().MarkDeprecated("v2-part-size", "use --resumable-api-v2-part-size instead")
 
-	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage")
+	cmd.Flags().IntVarP(&info.FileType, "file-type", "", 0, "set storage type of file, 0:STANDARD storage, 1:IA storage, 2:ARCHIVE storage, 3:DEEP_ARCHIVE storage, 4:ARCHIVE_IR storage, 5:INTELLIGENT_TIERING")
 	cmd.Flags().IntVarP(&info.FileType, "storage", "s", 0, "set storage type of file, same to --file-type")
 	_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
 

cmd_test/bucket_domain_test.go

@@ -3,9 +3,10 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 func TestBucketDomain(t *testing.T) {
@@ -15,7 +16,7 @@ func TestBucketDomain(t *testing.T) {
 	}
 
 	if !strings.Contains(result, test.BucketDomain) {
-		t.Fatal("no expected domain:%", test.BucketDomain)
+		t.Fatal("no expected domain:", result)
 	}
 
 	return

cmd_test/change_type_test.go

@@ -4,10 +4,11 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 func TestChangeType(t *testing.T) {
@@ -100,6 +101,19 @@ func TestBatchChangeType(t *testing.T) {
 	defer func() {
 		test.RemoveFile(successLogPath)
 		test.RemoveFile(failLogPath)
+
+		//back
+		batchConfig = ""
+		for _, key := range test.Keys {
+			batchConfig += key + "\t" + "1" + "\n"
+		}
+
+		path, err = test.CreateFileWithContent("batch_chtype.txt", batchConfig)
+		if err != nil {
+			t.Fatal("create chtype config file error:", err)
+		}
+
+		test.RunCmdWithError("batchchtype", test.Bucket, "-i", path, "-y")
 	}()
 
 	if !test.IsFileHasContent(successLogPath) {
@@ -109,19 +123,6 @@ func TestBatchChangeType(t *testing.T) {
 	if !test.IsFileHasContent(failLogPath) {
 		t.Fatal("batch result: fail log  to file error: file empty")
 	}
-
-	//back
-	batchConfig = ""
-	for _, key := range test.Keys {
-		batchConfig += key + "\t" + "1" + "\n"
-	}
-
-	path, err = test.CreateFileWithContent("batch_chtype.txt", batchConfig)
-	if err != nil {
-		t.Fatal("create chtype config file error:", err)
-	}
-
-	test.RunCmdWithError("batchchtype", test.Bucket, "-i", path, "-y")
 }
 
 func TestBatchChangeTypeRecord(t *testing.T) {
@@ -139,6 +140,21 @@ func TestBatchChangeTypeRecord(t *testing.T) {
 		t.Fatal("create batch move config file error:", err)
 	}
 
+	defer func() {
+		//back
+		batchConfig = ""
+		for _, key := range test.Keys {
+			batchConfig += key + "\t" + "1" + "\n"
+		}
+
+		path, err = test.CreateFileWithContent("batch_chtype.txt", batchConfig)
+		if err != nil {
+			t.Fatal("create chtype config file error:", err)
+		}
+
+		test.RunCmdWithError("batchchtype", test.Bucket, "-i", path, "-y")
+	}()
+
 	test.RunCmdWithError("batchchtype", test.Bucket,
 		"-i", path,
 		"--enable-record",
@@ -183,19 +199,6 @@ func TestBatchChangeTypeRecord(t *testing.T) {
 		fmt.Println("=========================== result   end ===========================")
 		t.Fatal("batch result: should redo because set --record-redo-while-error")
 	}
-
-	//back
-	batchConfig = ""
-	for _, key := range test.Keys {
-		batchConfig += key + "\t" + "1" + "\n"
-	}
-
-	path, err = test.CreateFileWithContent("batch_chtype.txt", batchConfig)
-	if err != nil {
-		t.Fatal("create chtype config file error:", err)
-	}
-
-	test.RunCmdWithError("batchchtype", test.Bucket, "-i", path, "-y")
 }
 
 func TestBatchChangeTypeDocument(t *testing.T) {

cmd_test/copy_test.go

@@ -4,10 +4,11 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 func TestCopy(t *testing.T) {
@@ -96,7 +97,6 @@ func TestBatchCopy(t *testing.T) {
 		"--worker", "4",
 		"--min-worker", "10",
 		"--worker-count-increase-period", "50",
-		"-w",
 		"-y")
 	defer func() {
 		test.RemoveFile(successLogPath)

cmd_test/download_get_test.go

@@ -17,18 +17,14 @@ func TestGetImage(t *testing.T) {
 		t.Fatal("get result path error:", err)
 	}
 	path := filepath.Join(resultPath, test.ImageKey)
-	ret, errs := test.RunCmdWithError("get", test.Bucket, test.ImageKey,
-		"--public",
+	_, errs := test.RunCmdWithError("get", test.Bucket, test.ImageKey,
 		"-o", path,
 		"-d")
 	defer test.RemoveFile(path)
 
 	if len(errs) > 0 {
 		t.Fail()
 	}
-	if !strings.Contains(ret, ".qiniucs.com") {
-		t.Fatal("get file: should get io src domain")
-	}
 	if !test.IsFileHasContent(path) {
 		t.Fatal("get file content can't be empty")
 	}
@@ -44,7 +40,6 @@ func TestGetImageAndCheck(t *testing.T) {
 	// 因为有源站域名，所以经过重试下载会成功
 	result, errs := test.RunCmdWithError("get", test.Bucket, test.ImageKey,
 		"--check-size",
-		"--public",
 		"-d",
 		"-o", path)
 	defer test.RemoveFile(path)
@@ -186,7 +181,7 @@ func TestGetNoExistDomain(t *testing.T) {
 
 func TestGetNoExistBucket(t *testing.T) {
 	_, errs := test.RunCmdWithError("get", test.BucketNotExist, test.Key)
-	if !strings.Contains(errs, "no such bucket") {
+	if !(strings.Contains(errs, "no such bucket") || strings.Contains(errs, "no such entry")) {
 		t.Fail()
 	}
 }

cmd_test/download_test.go

@@ -33,7 +33,6 @@ func TestDownloadWithKeyFile(t *testing.T) {
 			Prefix:     "hell",
 			Suffixes:   ".json",
 			IoHost:     "",
-			Public:     true,
 			CheckHash:  true,
 			Referer:    "",
 			CdnDomain:  "",
@@ -73,7 +72,6 @@ func TestDownloadFromBucket(t *testing.T) {
 			Prefix:          "hello3,hello5,hello7",
 			Suffixes:        "",
 			IoHost:          utils.Endpoint(false, test.BucketDomain),
-			Public:          true,
 			CheckSize:       true,
 			Referer:         "",
 			CdnDomain:       "",
@@ -114,7 +112,6 @@ func TestDownloadWithDomain(t *testing.T) {
 			Prefix:          "hello3,hello5,hello7",
 			Suffixes:        "",
 			IoHost:          utils.Endpoint(false, test.BucketDomain),
-			Public:          true,
 			CheckSize:       true,
 			Referer:         "",
 			Domain:          utils.Endpoint(false, test.BucketDomain),
@@ -186,7 +183,6 @@ func TestDownloadNoDomain(t *testing.T) {
 			Bucket:     test.Bucket,
 			Prefix:     "hello3,hello5,hello7",
 			Suffixes:   "",
-			Public:     true,
 			CheckHash:  true,
 			Referer:    "",
 			CdnDomain:  "",
@@ -283,7 +279,6 @@ func TestDownload2AllFilesFromBucket(t *testing.T) {
 		"--bucket", test.Bucket,
 		"--dest-dir", destDir,
 		"--suffixes", ".json",
-		"--public",
 		"--log-file", logPath,
 		"--log-level", "info",
 		"-c", "4",
@@ -355,10 +350,6 @@ func TestDownload2WithKeyFile(t *testing.T) {
 	}
 
 	logContent := test.FileContent(logPath)
-	if !strings.Contains(logContent, "?e=") {
-		t.Fatal("download url should private")
-	}
-
 	if !strings.Contains(logContent, "work consumer 3 start") {
 		t.Fatal("download should have consumer 3")
 	}
@@ -397,7 +388,6 @@ func TestDownload2PublicWithKeyFile(t *testing.T) {
 		"--key-file", keysFilePath,
 		"--log-file", logPath,
 		"--log-level", "debug",
-		"--public",
 		"-c", "4",
 		"-d")
 	if test.FileCountInDir(destDir) < 2 {
@@ -408,11 +398,6 @@ func TestDownload2PublicWithKeyFile(t *testing.T) {
 		t.Fatal("log file should has content")
 	}
 
-	logContent := test.FileContent(logPath)
-	if strings.Contains(logContent, "?e=") {
-		t.Fatal("download url should public")
-	}
-
 	return
 }
 
@@ -445,7 +430,6 @@ func TestDownload2PublicWithDomain(t *testing.T) {
 		"--key-file", keysFilePath,
 		"--log-file", logPath,
 		"--log-level", "debug",
-		"--public",
 		"-c", "4",
 		"-d")
 	if test.FileCountInDir(destDir) < 2 {
@@ -456,11 +440,6 @@ func TestDownload2PublicWithDomain(t *testing.T) {
 		t.Fatal("log file should has content")
 	}
 
-	logContent := test.FileContent(logPath)
-	if strings.Contains(logContent, "?e=") {
-		t.Fatal("download url should public")
-	}
-
 	return
 }
 

cmd_test/fetch_test.go

@@ -3,10 +3,11 @@
 package cmd
 
 import (
-	"github.com/qiniu/qshell/v2/cmd_test/test"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/qshell/v2/cmd_test/test"
 )
 
 func TestFetch(t *testing.T) {

docs/abfetch.md

@@ -37,7 +37,7 @@ $ qshell abfetch --doc
 - -b/--callback-body：回调的 http Body。 【可选】          
 - -T/--callback-host：回调时的 HOST 头。 【可选】
 - -a/--callback-url：回调的请求地址。 【可选】
-- --file-type：抓取的资源存储在七牛存储空间的类型，0:普通存储 1:低频存储 2:归档存储 3:深度归档 4:归档直读存储, 默认为: 0。 【可选】
+- --file-type：抓取的资源存储在七牛存储空间的类型，0:标准存储 1:低频存储 2:归档存储 3:深度归档 4:归档直读存储 5:智能分层存储, 默认为: 0。 【可选】
 - -c/--thread-count：指定抓取时使用的线程数目，默认：20。 【可选】
 - --overwrite：是否覆盖空间已有文件，默认为 `false`。 【可选】
 - -s/--success-list：指定一个文件的路径，如果资源抓取成功，则将资源信息写入此文件；默认不导出。 【可选】