fix: Sync with upstream/master (Commit b9b3605)

gen3-client/g3cmd/root.go

@@ -54,8 +54,9 @@ func initConfig() {
 	}
 
 	// version checker
-	if os.Getenv("GEN3_CLIENT_VERSION_CHECK") != "false" &&
-	gitversion != "" && gitversion != "N/A" {
+	// Temp. disabling version check message to reduce confusion
+	if false && os.Getenv("GEN3_CLIENT_VERSION_CHECK") != "false" &&
+		gitversion != "" && gitversion != "N/A" {
 		githubTag := &latest.GithubTag{
 			Owner:      "uc-cdis",
 			Repository: "cdis-data-client",
@@ -78,6 +79,7 @@ func initConfig() {
 				return true
 			},
 		}
+
 		res, err := latest.Check(githubTag, gitversion)
 		if err != nil {
 			log.Println("Error occurred when checking for latest version: " + err.Error())

gen3-client/g3cmd/upload-multiple.go

@@ -76,47 +76,16 @@
 
 			uploadPath, err := commonUtils.GetAbsolutePath(uploadPath)
 			if err != nil {
 				log.Fatalf("Error when parsing file paths: " + err.Error())
 			}
 
-			for i, furObject := range furObjects {
-				if batch {
-					if len(batchFURObjects) < workers {
-						batchFURObjects = append(batchFURObjects, furObject)
-					} else {
-						batchUpload(gen3Interface, batchFURObjects, workers, respCh, errCh, bucketName)
-						batchFURObjects = make([]commonUtils.FileUploadRequestObject, 0)
-						batchFURObjects = append(batchFURObjects, furObject)
-					}
-					if i == len(furObjects)-1 { // upload remainders
-						batchUpload(gen3Interface, batchFURObjects, workers, respCh, errCh, bucketName)
-					}
-				} else {
-					file, err := os.Open(furObject.FilePath)
-					if err != nil {
-						log.Println("File open error: " + err.Error())
-						logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true)
-						logs.IncrementScore(logs.ScoreBoardLen - 1)
-						continue
-					}
-					defer file.Close()
-
-					furObject, err := GenerateUploadRequest(gen3Interface, furObject, file)
-					if err != nil {
-						file.Close()
-						logs.AddToFailedLog(furObject.FilePath, furObject.Filename, commonUtils.FileMetadata{}, furObject.GUID, 0, false, true)
-						logs.IncrementScore(logs.ScoreBoardLen - 1)
-						log.Printf("Error occurred during request generation: %s", err.Error())
-						continue
-					}
-					err = uploadFile(furObject, 0)
-					if err != nil {
-						log.Println(err.Error())
-						logs.IncrementScore(logs.ScoreBoardLen - 1)
-					} else {
-						logs.IncrementScore(0)
-					}
-					file.Close()
+			filePaths := make([]string, 0)
+			for _, object := range objects {
+				// Here we are assuming the local filename will be the same as GUID
+				filePath, err := getFullFilePath(uploadPath, object.ObjectID)
+				if err != nil {
+					log.Println(err.Error())
+					continue
 				}
 				filePaths = append(filePaths, filePath)
 			}
@@ -167,7 +136,9 @@
 	uploadMultipleCmd.MarkFlagRequired("upload-path") //nolint:errcheck
 	uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload in parallel")
 	uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel")
-	uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded")
+	uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.")
+	uploadMultipleCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload when possible (file size >= 5MB)")
+	uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", false, "Include subdirectory names in file name")
 	RootCmd.AddCommand(uploadMultipleCmd)
 }
 

gen3-client/g3cmd/upload-single.go

@@ -95,6 +95,6 @@ func init() {
 	uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck
 	uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file")
 	uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck
-	uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded")
+	uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.")
 	RootCmd.AddCommand(uploadSingleCmd)
 }

gen3-client/g3cmd/upload.go

@@ -55,7 +55,7 @@
 			uploadPath, _ = commonUtils.GetAbsolutePath(uploadPath)
 			filePaths, err := commonUtils.ParseFilePaths(uploadPath, hasMetadata)
 			if err != nil {
 				log.Fatalf("Error when parsing file paths: " + err.Error())
 			}
 			if len(filePaths) == 0 {
 				log.Println("No file has been found in the provided location \"" + uploadPath + "\"")
@@ -154,26 +154,7 @@
 			if len(multipartFilePaths) > 0 {
 				// NOTE(@mpingram) - For the moment Shepherd doesn't support multipart uploads.
 				// Throw an error if Shepherd is enabled and user attempts to multipart upload.
-				profileConfig := conf.ParseConfig(profile)
-				if profileConfig.UseShepherd == "true" ||
-					profileConfig.UseShepherd == "" && commonUtils.DefaultUseShepherd == true {
-					log.Fatalf("Error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n	$ gen3-client configure --profile=%v --use-shepherd=false\nand try again.\n", profile)
-				}
-				log.Println("Multipart uploading....")
-				for _, filePath := range multipartFilePaths {
-					fileInfo, err := ProcessFilename(uploadPath, filePath, includeSubDirName, false)
-					if err != nil {
-						logs.AddToFailedLog(filePath, filepath.Base(filePath), commonUtils.FileMetadata{}, "", 0, false, true)
-						log.Println("Process filename error for file: " + err.Error())
-						continue
-					}
-					err = multipartUpload(gen3Interface, fileInfo, 0, bucketName)
-					if err != nil {
-						log.Println(err.Error())
-					} else {
-						logs.IncrementScore(0)
-					}
-				}
+				processMultipartUpload(gen3Interface, multipartFilePaths, bucketName, includeSubDirName, uploadPath)
 			}
 
 			if !logs.IsFailedLogMapEmpty() {
@@ -193,6 +174,6 @@
 	uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", false, "Include subdirectory names in file name")
 	uploadCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload if possible")
 	uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file")
-	uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded")
+	uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.")
 	RootCmd.AddCommand(uploadCmd)
 }

gen3-client/g3cmd/utils.go

@@ -25,7 +25,7 @@
 	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
 // go:generate mockgen -destination=./gen3-client/mocks/mock_gen3interface.go -package=mocks github.com/uc-cdis/gen3-client/gen3-client/g3cmd Gen3Interface
 
 // ManifestObject represents an object from manifest that downloaded from windmill / data-portal
 type ManifestObject struct {
@@ -537,37 +537,6 @@
 	}
 }
 
-func validateObject(objects []ManifestObject, uploadPath string) []commonUtils.FileUploadRequestObject {
-	furObjects := make([]commonUtils.FileUploadRequestObject, 0)
-	for _, object := range objects {
-		guid := object.ObjectID
-		var fileName = ""
-
-		if object.Filename != "" {
-		    // conform to fence naming convention
-		    fileName = object.Filename
-		} else {
-			// Otherwise, here we are assuming the local filename will be the same as GUID
-			fileName = object.ObjectID
-		}
-
-		filePath, err := getFullFilePath(uploadPath, fileName)
-		if err != nil {
-			log.Println(err.Error())
-			continue
-		}
-
-		if _, err := os.Stat(filePath); os.IsNotExist(err) {
-			log.Printf("The file you specified \"%s\" does not exist locally.\n", filePath)
-			continue
-		}
-
-		furObject := commonUtils.FileUploadRequestObject{FilePath: filePath, Filename: fileName, GUID: guid}
-		furObjects = append(furObjects, furObject)
-	}
-	return furObjects
-}
-
 func uploadFile(furObject commonUtils.FileUploadRequestObject, retryCount int) error {
 	log.Println("Uploading data ...")
 	furObject.Bar.Start()