From da4a55de46b301cd6773e54892969963590406a8 Mon Sep 17 00:00:00 2001 From: Lix Guru Date: Tue, 21 May 2019 15:54:27 -0700 Subject: [PATCH] added download operation --- sup.go | 8 ++++- supfile.go | 27 ++++++++++----- tar.go | 26 ++++++++++++++ task.go | 100 +++++++++++++++++++++++++---------------------------- 4 files changed, 98 insertions(+), 63 deletions(-) diff --git a/sup.go b/sup.go index d815068..ee150f4 100644 --- a/sup.go +++ b/sup.go @@ -142,7 +142,13 @@ func (sup *Stackup) Run(network *Network, envVars EnvList, commands ...*Command) wg.Add(1) go func(c Client) { defer wg.Done() - _, err := io.Copy(os.Stdout, prefixer.New(c.Stdout(), prefix)) + var err error + if task.Output == nil { + _, err = io.Copy(os.Stdout, prefixer.New(c.Stdout(), prefix)) + } else { + // for download task, remote task stdout piped into local stdin of tar process + _, err = io.Copy(task.Output, c.Stdout()) + } if err != nil && err != io.EOF { // TODO: io.Copy() should not return io.EOF at all. // Upstream bug? Or prefixer.WriteTo() bug? diff --git a/supfile.go b/supfile.go index 2cf88b5..2f456b9 100644 --- a/supfile.go +++ b/supfile.go @@ -67,15 +67,16 @@ func (n *Networks) Get(name string) (Network, bool) { // Command represents command(s) to be run remotely. type Command struct { - Name string `yaml:"-"` // Command name. - Desc string `yaml:"desc"` // Command description. - Local string `yaml:"local"` // Command(s) to be run locally. - Run string `yaml:"run"` // Command(s) to be run remotelly. - Script string `yaml:"script"` // Load command(s) from script and run it remotelly. - Upload []Upload `yaml:"upload"` // See Upload struct. - Stdin bool `yaml:"stdin"` // Attach localhost STDOUT to remote commands' STDIN? - Once bool `yaml:"once"` // The command should be run "once" (on one host only). - Serial int `yaml:"serial"` // Max number of clients processing a task in parallel. + Name string `yaml:"-"` // Command name. + Desc string `yaml:"desc"` // Command description. + Local string `yaml:"local"` // Command(s) to be run locally. + Run string `yaml:"run"` // Command(s) to be run remotelly. + Script string `yaml:"script"` // Load command(s) from script and run it remotelly. + Upload []Upload `yaml:"upload"` // See Upload struct. + Download []Download `yaml:"download"` // See Download struct. + Stdin bool `yaml:"stdin"` // Attach localhost STDOUT to remote commands' STDIN? + Once bool `yaml:"once"` // The command should be run "once" (on one host only). + Serial int `yaml:"serial"` // Max number of clients processing a task in parallel. // API backward compatibility. Will be deprecated in v1.0. RunOnce bool `yaml:"run_once"` // The command should be run once only. @@ -151,6 +152,14 @@ type Upload struct { Exc string `yaml:"exclude"` } +// Download represents file copy operation from remote SrcFolder/Src to local Dst +type Download struct { + SrcFolder string `yaml:"src_folder"` + Src string `yaml:"src"` + Dst string `yaml:"dst"` + Exc string `yaml:"exclude"` // todo: to support Exclude setting +} + // EnvVar represents an environment variable type EnvVar struct { Key string diff --git a/tar.go b/tar.go index 10582f5..a75137d 100644 --- a/tar.go +++ b/tar.go @@ -10,7 +10,10 @@ import ( ) // Copying dirs/files over SSH using TAR. +// upload: // tar -C . -cvzf - $SRC | ssh $HOST "tar -C $DST -xvzf -" +// download: +// ssh $HOST "tar -C $SRC_DIR -czvf - $SRC_FILE" | tar -C $DST -xzvf - // RemoteTarCommand returns command to be run on remote SSH host // to properly receive the created TAR stream. @@ -51,3 +54,26 @@ func NewTarStreamReader(cwd, path, exclude string) (io.Reader, error) { return stdout, nil } + +// RemoteTarCreateCommand forms "tar -C $SRC_DIR -czvf - $SRC_FILE" +// which is the remote part of download task +func RemoteTarCreateCommand(dir, src string) string { + return fmt.Sprintf("tar -C \"%s\" -czvf - \"%s\"", dir, src) +} + +// NewTarStreamWriter creates a tar stream writer to local path +// by calling tar -C $DST -xzvf - +// which is the local part of download task +func NewTarStreamWriter(dst string) (io.Writer, error) { + cmd := exec.Command("tar", "-C", dst, "-xzvf", "-") + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Wrap(err, "tar: stdin pipe failed") + } + + if err := cmd.Start(); err != nil { + return nil, errors.Wrap(err, "tar: starting cmd failed") + } + + return stdin, nil +} diff --git a/task.go b/task.go index eebc3c7..462fcc8 100644 --- a/task.go +++ b/task.go @@ -13,6 +13,7 @@ import ( type Task struct { Run string Input io.Reader + Output io.Writer Clients []Client TTY bool } @@ -42,24 +43,28 @@ func (sup *Stackup) createTasks(cmd *Command, clients []Client, env string) ([]* TTY: false, } - if cmd.Once { - task.Clients = []Client{clients[0]} - tasks = append(tasks, &task) - } else if cmd.Serial > 0 { - // Each "serial" task client group is executed sequentially. - for i := 0; i < len(clients); i += cmd.Serial { - j := i + cmd.Serial - if j > len(clients) { - j = len(clients) - } - copy := task - copy.Clients = clients[i:j] - tasks = append(tasks, ©) - } - } else { - task.Clients = clients - tasks = append(tasks, &task) + addTask(task, cmd, clients, &tasks) + } + + // todo: impl download support + for _, download := range cmd.Download { + dst, err := ResolveLocalPath(cwd, download.Dst, env) + if err != nil { + return nil, errors.Wrap(err, "download: "+download.Dst) + } + tarWriter, err := NewTarStreamWriter(dst) + if err != nil { + return nil, errors.Wrap(err, "download: "+download.Dst) } + + // todo: support download.Exclude + task := Task{ + Run: RemoteTarCreateCommand(download.SrcFolder, download.Src), + Output: tarWriter, + TTY: false, + } + + addTask(task, cmd, clients, &tasks) } // Script. Read the file as a multiline input command. @@ -83,24 +88,8 @@ func (sup *Stackup) createTasks(cmd *Command, clients []Client, env string) ([]* if cmd.Stdin { task.Input = os.Stdin } - if cmd.Once { - task.Clients = []Client{clients[0]} - tasks = append(tasks, &task) - } else if cmd.Serial > 0 { - // Each "serial" task client group is executed sequentially. - for i := 0; i < len(clients); i += cmd.Serial { - j := i + cmd.Serial - if j > len(clients) { - j = len(clients) - } - copy := task - copy.Clients = clients[i:j] - tasks = append(tasks, ©) - } - } else { - task.Clients = clients - tasks = append(tasks, &task) - } + + addTask(task, cmd, clients, &tasks) } // Local command. @@ -135,29 +124,34 @@ func (sup *Stackup) createTasks(cmd *Command, clients []Client, env string) ([]* if cmd.Stdin { task.Input = os.Stdin } - if cmd.Once { - task.Clients = []Client{clients[0]} - tasks = append(tasks, &task) - } else if cmd.Serial > 0 { - // Each "serial" task client group is executed sequentially. - for i := 0; i < len(clients); i += cmd.Serial { - j := i + cmd.Serial - if j > len(clients) { - j = len(clients) - } - copy := task - copy.Clients = clients[i:j] - tasks = append(tasks, ©) - } - } else { - task.Clients = clients - tasks = append(tasks, &task) - } + + addTask(task, cmd, clients, &tasks) } return tasks, nil } +func addTask(task Task, cmd *Command, clients []Client, tasks *[]*Task) { + if cmd.Once { + task.Clients = []Client{clients[0]} + *tasks = append(*tasks, &task) + } else if cmd.Serial > 0 { + // Each "serial" task client group is executed sequentially. + for i := 0; i < len(clients); i += cmd.Serial { + j := i + cmd.Serial + if j > len(clients) { + j = len(clients) + } + copy := task + copy.Clients = clients[i:j] + *tasks = append(*tasks, ©) + } + } else { + task.Clients = clients + *tasks = append(*tasks, &task) + } +} + type ErrTask struct { Task *Task Reason string