From 84e4866ef1e0e06bac1119ec738239b281c9db91 Mon Sep 17 00:00:00 2001 From: windvalley Date: Fri, 29 Dec 2023 15:42:27 +0800 Subject: [PATCH] perf(push): increase files transfer efficiency (#32) --- .golangci.yaml | 2 +- CHANGELOG.md | 11 ++ README.md | 2 - internal/cmd/push.go | 66 +++++++--- internal/pkg/sshtask/sshtask.go | 9 +- pkg/batchssh/batchssh.go | 218 +++++++++----------------------- pkg/batchssh/push_zip.go | 64 ++++++++++ pkg/batchssh/pushv1.go | 128 +++++++++++++++++++ pkg/batchssh/pushv2.go | 180 ++++++++++++++++++++++++++ scripts/makefiles/go.makefile | 12 +- 10 files changed, 502 insertions(+), 190 deletions(-) create mode 100644 pkg/batchssh/push_zip.go create mode 100644 pkg/batchssh/pushv1.go create mode 100644 pkg/batchssh/pushv2.go diff --git a/.golangci.yaml b/.golangci.yaml index ad9d1ac..1cd670d 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -68,7 +68,6 @@ linters: enable: - bodyclose - deadcode - - depguard - dogsled - dupl - errcheck @@ -118,6 +117,7 @@ linters: # - wsl # - gochecknoinits # - gocritic + # - depguard issues: # Excluding configuration per-path, per-linter, per-text and per-source diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ef2c4..aba9cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.13.0] + +### Added + +Add flag `-z, --zip` for subcommand `push`. + +### Changed + +Improve the files transfer efficiency of the subcommand `push`. +The subcommand `push` no longer uses zip compression by default. If you want to continue using zip compression, you can add the `-z` flag to the command line. + ## [1.12.0] ### Changed diff --git a/README.md b/README.md index 65e38f5..142bb8f 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,6 @@ [![Version](https://img.shields.io/github/v/release/windvalley/gossh?include_prereleases)](https://github.com/windvalley/gossh/releases) [![LICENSE](https://img.shields.io/github/license/windvalley/gossh)](LICENSE)
![Page Views](https://views.whatilearened.today/views/github/windvalley/gossh.svg) -[![Traffic Clones Total](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.sre.im%2Fv1%2Fgithub%2Ftraffic%2Fclones%2Ftotal%3Fgit_user%3Dwindvalley%26git_repo%3Dgossh%26type%3Dcount%26label%3Dclones-total)](https://github.com/windvalley/traffic-clones-api) -[![Traffic Clones Uniques](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.sre.im%2Fv1%2Fgithub%2Ftraffic%2Fclones%2Ftotal%3Fgit_user%3Dwindvalley%26git_repo%3Dgossh%26type%3Duniques%26label%3Dclones-uniques)](https://github.com/windvalley/traffic-clones-api) [![Release Download Total](https://img.shields.io/github/downloads/windvalley/gossh/total)](https://github.com/windvalley/gossh/releases) Gossh is a high-performance and high-concurrency ssh tool written in Go. diff --git a/internal/cmd/push.go b/internal/cmd/push.go index e1516e5..e416d62 100644 --- a/internal/cmd/push.go +++ b/internal/cmd/push.go @@ -34,6 +34,7 @@ import ( "github.com/windvalley/gossh/internal/pkg/configflags" "github.com/windvalley/gossh/internal/pkg/sshtask" + "github.com/windvalley/gossh/pkg/log" "github.com/windvalley/gossh/pkg/util" ) @@ -41,6 +42,7 @@ var ( files []string fileDstPath string allowOverwrite bool + enableZip bool ) // pushCmd represents the push command @@ -50,12 +52,15 @@ var pushCmd = &cobra.Command{ Long: ` Copy local files and dirs to target hosts.`, Example: ` - # Copy a local file or dir to /tmp/ of the target hosts by default. + Copy a local file or dir to /tmp/ of the target hosts by default. $ gossh push host[1-2] -f /path/foo -k - # Copy local files and dirs to /home/user/ of the target hosts. + Copy local files and dirs to /home/user/ of the target hosts. $ gossh push host[1-2] -f /path/foo.txt,/path/bar/ -d /home/user -k + Enable zip files feature (zip first, then push). + $ gossh push host[1-2] -f /path/foo.txt,/path/bar/ -d /home/user -k -z + Find more examples at: https://github.com/windvalley/gossh/blob/main/docs/push.md`, PreRun: func(cmd *cobra.Command, args []string) { if errs := configflags.Config.Validate(); len(errs) != 0 { @@ -76,34 +81,49 @@ Copy local files and dirs to target hosts.`, var zipFiles []string - workDir, err := os.Getwd() - if err != nil { - util.CheckErr(err) - } + if enableZip { + log.Debugf("enabled zip files feature") - for _, f := range files { - fileName := filepath.Base(f) - zipName := "." + fileName + "." + fmt.Sprintf("%d", time.Now().UnixMicro()) - zipFile := path.Join(workDir, zipName) + zipTimeStart := time.Now() - if err := util.Zip(strings.TrimSuffix(f, string(os.PathSeparator)), zipFile); err != nil { + workDir, err := os.Getwd() + if err != nil { util.CheckErr(err) } - zipFiles = append(zipFiles, zipFile) - } + for _, f := range files { + fileName := filepath.Base(f) + zipName := "." + fileName + "." + fmt.Sprintf("%d", time.Now().UnixMicro()) + zipFile := path.Join(workDir, zipName) - defer func() { - for _, f := range zipFiles { - if err := os.Remove(f); err != nil { - fmt.Printf("Warning: %v\n", err) + if err := util.Zip(strings.TrimSuffix(f, string(os.PathSeparator)), zipFile); err != nil { + util.CheckErr(err) + } + + stat, err := os.Stat(zipFile) + if err != nil { + util.CheckErr(err) } + //nolint:gomnd + log.Debugf("zip file '%s' size: %d MB", zipFile, stat.Size()/1024/1024) + + zipFiles = append(zipFiles, zipFile) } - }() + + defer func() { + for _, f := range zipFiles { + if err := os.Remove(f); err != nil { + fmt.Printf("Warning: %v\n", err) + } + } + }() + + log.Debugf("zip files cost %v", time.Since(zipTimeStart)) + } task.SetTargetHosts(args) task.SetPushfiles(files, zipFiles) - task.SetPushOptions(fileDstPath, allowOverwrite) + task.SetPushOptions(fileDstPath, allowOverwrite, enableZip) task.Start() @@ -128,6 +148,14 @@ func init() { "allow overwrite files/dirs if they already exist on target hosts", ) + pushCmd.Flags().BoolVarP( + &enableZip, + "zip", + "z", + false, + "enable zip files ('unzip' must be installed on target hosts)", + ) + pushCmd.SetHelpFunc(func(command *cobra.Command, strings []string) { util.CobraMarkHiddenGlobalFlags( command, diff --git a/internal/pkg/sshtask/sshtask.go b/internal/pkg/sshtask/sshtask.go index 01615eb..7a6ccba 100644 --- a/internal/pkg/sshtask/sshtask.go +++ b/internal/pkg/sshtask/sshtask.go @@ -114,6 +114,7 @@ type Task struct { tmpDir string remove bool allowOverwrite bool + enableZip bool taskOutput chan taskResult detailOutput chan detailResult @@ -204,9 +205,10 @@ func (t *Task) SetScriptOptions(destPath string, remove, allowOverwrite bool) { } // SetPushOptions ... -func (t *Task) SetPushOptions(destPath string, allowOverwrite bool) { +func (t *Task) SetPushOptions(destPath string, allowOverwrite, enableZip bool) { t.dstDir = destPath t.allowOverwrite = allowOverwrite + t.enableZip = enableZip } // SetFetchOptions ... @@ -227,7 +229,7 @@ func (t *Task) RunSSH(host *batchssh.Host) (string, error) { case ScriptTask: return t.sshClient.ExecuteScript(host, t.scriptFile, t.dstDir, lang, runAs, sudo, t.remove, t.allowOverwrite) case PushTask: - return t.sshClient.PushFiles(host, t.pushFiles.files, t.pushFiles.zipFiles, t.dstDir, t.allowOverwrite) + return t.sshClient.PushFiles(host, t.pushFiles.files, t.pushFiles.zipFiles, t.dstDir, t.allowOverwrite, t.enableZip) case FetchTask: return t.sshClient.FetchFiles(host, t.fetchFiles, t.dstDir, t.tmpDir, sudo, runAs) default: @@ -235,8 +237,9 @@ func (t *Task) RunSSH(host *batchssh.Host) (string, error) { } } -//nolint:gocyclo // BatchRun ... +// +//nolint:gocyclo func (t *Task) BatchRun() { timeNow := time.Now() diff --git a/pkg/batchssh/batchssh.go b/pkg/batchssh/batchssh.go index 5e2d740..2f915aa 100644 --- a/pkg/batchssh/batchssh.go +++ b/pkg/batchssh/batchssh.go @@ -26,7 +26,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "os" "path" @@ -216,7 +215,7 @@ func (c *Client) ExecuteScript( } defer ftpC.Close() - file, err := c.pushFile(ftpC, srcFile, dstDir, allowOverwrite) + file, err := c.pushFile(ftpC, srcFile, dstDir, host.Host, allowOverwrite) if err != nil { return "", err } @@ -260,7 +259,7 @@ func (c *Client) PushFiles( host *Host, srcFiles, srcZipFiles []string, dstDir string, - allowOverwrite bool, + allowOverwrite, enableZip bool, ) (string, error) { client, err := c.getClient(host) if err != nil { @@ -274,52 +273,63 @@ func (c *Client) PushFiles( } defer ftpC.Close() - for i, f := range srcZipFiles { - srcFile := srcFiles[i] + if !enableZip { + for _, srcFile := range srcFiles { + err = c.pushFileOrDirV2(client, ftpC, srcFile, dstDir, host.Host, allowOverwrite) + if err != nil { + return "", err + } + } + } else { + for i, f := range srcZipFiles { + srcFile := srcFiles[i] - dstZipFile := filepath.Base(f) + dstZipFile := filepath.Base(f) - done := make(chan struct{}) - var ( - err error - file *sftp.File - ) - go func() { - defer close(done) + done := make(chan struct{}) + var ( + err error + file *sftp.File + ) + go func() { + defer close(done) - file, err = c.pushZipFile(ftpC, f, filepath.Base(srcFile), dstDir, allowOverwrite) - if err == nil { - file.Close() - } - }() + file, err = pushZipFile(ftpC, f, filepath.Base(srcFile), dstDir, host.Host, allowOverwrite) + if err == nil { + file.Close() + } + }() - <-done + <-done - if err != nil { - return "", err - } + if err != nil { + return "", err + } - session, err := client.NewSession() - if err != nil { - return "", err - } - defer session.Close() + session, err := client.NewSession() + if err != nil { + return "", err + } + defer session.Close() - _, err = c.executeCmd( - session, - fmt.Sprintf( - `which unzip &>/dev/null && { cd %s;unzip -o %s;rm %s;} || + _, err = c.executeCmd( + session, + fmt.Sprintf( + `which unzip &>/dev/null && { cd %s;unzip -o %s;rm %s;} || { echo "need install 'unzip' command";cd %s;rm %s;exit 1;}`, - dstDir, - dstZipFile, - dstZipFile, - dstDir, - dstZipFile, - ), - host.Password, - ) - if err != nil { - return "", err + dstDir, + dstZipFile, + dstZipFile, + dstDir, + dstZipFile, + ), + host.Password, + ) + if err != nil { + return "", err + } + + log.Debugf("%s: %s unzipped", host.Host, f) } } @@ -331,8 +341,9 @@ func (c *Client) PushFiles( return fmt.Sprintf("'%s' %s been copied to '%s'", strings.Join(srcFiles, ","), hasOrHave, dstDir), nil } -//nolint:funlen,gocyclo // FetchFiles from remote host. +// +//nolint:funlen,gocyclo func (c *Client) FetchFiles( host *Host, srcFiles []string, @@ -551,119 +562,6 @@ func (c *Client) executeCmd(session *ssh.Session, command, password string) (str return outputStr, nil } -func (c *Client) pushFile( - ftpC *sftp.Client, - srcFile, dstDir string, - allowOverwrite bool, -) (*sftp.File, error) { - homeDir := os.Getenv("HOME") - if strings.HasPrefix(srcFile, "~/") { - srcFile = strings.Replace(srcFile, "~", homeDir, 1) - } - - content, err := ioutil.ReadFile(srcFile) - if err != nil { - return nil, err - } - - fileStat, err := os.Stat(srcFile) - if err != nil { - return nil, err - } - - srcFileBaseName := filepath.Base(srcFile) - dstFile := path.Join(dstDir, srcFileBaseName) - - if !allowOverwrite { - dstFileInfo, _ := ftpC.Stat(dstFile) - if dstFileInfo != nil { - return nil, fmt.Errorf( - "%s alreay exists, you can add '-F' flag to overwrite it", - dstFile, - ) - } - } - - file, err := ftpC.Create(dstFile) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("dest dir '%s' not exist", dstDir) - } - - if err, ok := err.(*sftp.StatusError); ok && err.Code == uint32(sftp.ErrSshFxPermissionDenied) { - return nil, fmt.Errorf("no permission to write to dest dir '%s'", dstDir) - } - - return nil, err - } - - _, err = file.Write(content) - if err != nil { - return nil, err - } - - if err := file.Chmod(fileStat.Mode()); err != nil { - return nil, err - } - - if err := ftpC.Chtimes(dstFile, time.Now(), fileStat.ModTime()); err != nil { - return nil, err - } - - return file, nil -} - -func (c *Client) pushZipFile( - ftpC *sftp.Client, - srcZipFile, srcFileName, dstDir string, - allowOverwrite bool, -) (*sftp.File, error) { - homeDir := os.Getenv("HOME") - if strings.HasPrefix(srcZipFile, "~/") { - srcZipFile = strings.Replace(srcZipFile, "~", homeDir, 1) - } - - content, err := ioutil.ReadFile(srcZipFile) - if err != nil { - return nil, err - } - - srcZipFileName := filepath.Base(srcZipFile) - dstZipFile := path.Join(dstDir, srcZipFileName) - - dstFile := path.Join(dstDir, srcFileName) - - if !allowOverwrite { - dstFileInfo, _ := ftpC.Stat(dstFile) - if dstFileInfo != nil { - return nil, fmt.Errorf( - "%s alreay exists, you can add '-F' flag to overwrite it", - dstFile, - ) - } - } - - file, err := ftpC.Create(dstZipFile) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("dest dir '%s' not exist", dstDir) - } - - if err, ok := err.(*sftp.StatusError); ok && err.Code == uint32(sftp.ErrSshFxPermissionDenied) { - return nil, fmt.Errorf("no permission to write to dest dir '%s'", dstDir) - } - - return nil, err - } - - _, err = file.Write(content) - if err != nil { - return nil, err - } - - return file, nil -} - func (c *Client) fetchZipFile( ftpC *sftp.Client, srcZipFile, dstDir string, @@ -831,3 +729,13 @@ func WithProxyServer(proxyServer, user string, port int, auths []ssh.AuthMethod) c.Proxy.SSHClient = proxyClient } } + +// 判断是否是目录 +func isDir(path string) bool { + stat, err := os.Stat(path) + if err != nil { + return false + } + + return stat.IsDir() +} diff --git a/pkg/batchssh/push_zip.go b/pkg/batchssh/push_zip.go new file mode 100644 index 0000000..43d188b --- /dev/null +++ b/pkg/batchssh/push_zip.go @@ -0,0 +1,64 @@ +package batchssh + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/pkg/sftp" + + "github.com/windvalley/gossh/pkg/log" +) + +func pushZipFile( + ftpC *sftp.Client, + srcZipFile, srcFileName, dstDir, host string, + allowOverwrite bool, +) (*sftp.File, error) { + homeDir := os.Getenv("HOME") + if strings.HasPrefix(srcZipFile, "~/") { + srcZipFile = strings.Replace(srcZipFile, "~", homeDir, 1) + } + + content, err := os.ReadFile(srcZipFile) + if err != nil { + return nil, err + } + + srcZipFileName := filepath.Base(srcZipFile) + dstZipFile := path.Join(dstDir, srcZipFileName) + + dstFile := path.Join(dstDir, srcFileName) + + if !allowOverwrite { + err = checkAllowOverWrite(ftpC, host, dstFile) + if err != nil { + return nil, err + } + } + + file, err := ftpC.Create(dstZipFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("dest dir '%s' not exist", dstDir) + } + + if err, ok := err.(*sftp.StatusError); ok && err.Code == uint32(sftp.ErrSshFxPermissionDenied) { + return nil, fmt.Errorf("no permission to write to dest dir '%s'", dstDir) + } + + return nil, err + } + + _, err = file.Write(content) + if err != nil { + return nil, err + } + + log.Debugf("%s: zip file '%s' pushed", host, srcZipFile) + + return file, nil +} diff --git a/pkg/batchssh/pushv1.go b/pkg/batchssh/pushv1.go new file mode 100644 index 0000000..cafbfd4 --- /dev/null +++ b/pkg/batchssh/pushv1.go @@ -0,0 +1,128 @@ +package batchssh + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/pkg/sftp" + + "github.com/windvalley/gossh/pkg/log" +) + +// pushFileOrDir is less efficient than pushFileOrDirV2. +// +//nolint:unused +func (c *Client) pushFileOrDir( + ftpC *sftp.Client, + srcFile, dstDir, host string, + allowOverwrite bool, +) error { + if !isDir(srcFile) { + _, err := c.pushFile(ftpC, srcFile, dstDir, host, allowOverwrite) + if err != nil { + return err + } + + log.Debugf("%s: '%s' -> '%s", host, srcFile, dstDir) + + return nil + } + + localFiles, err := os.ReadDir(srcFile) + if err != nil { + return fmt.Errorf("read dir '%s' failed: %w", srcFile, err) + } + + remoteFilePath := path.Join(dstDir, filepath.Base(srcFile)) + err = ftpC.MkdirAll(remoteFilePath) + if err != nil { + log.Errorf("%s: mkdir '%s' failed", host, remoteFilePath) + return err + } + + for _, item := range localFiles { + localFilePath := path.Join(srcFile, item.Name()) + + if item.IsDir() { + err = c.pushFileOrDir(ftpC, localFilePath, remoteFilePath, host, allowOverwrite) + if err != nil { + log.Errorf("%s: pushFileOrDir '%s' failed", host, localFilePath) + return err + } + } else { + _, err = c.pushFile(ftpC, localFilePath, remoteFilePath, host, allowOverwrite) + if err != nil { + log.Errorf("%s: pushFile '%s' failed", host, localFilePath) + return err + } + } + } + + return nil +} + +func (c *Client) pushFile( + ftpC *sftp.Client, + srcFile, dstDir, host string, + allowOverwrite bool, +) (*sftp.File, error) { + homeDir := os.Getenv("HOME") + if strings.HasPrefix(srcFile, "~/") { + srcFile = strings.Replace(srcFile, "~", homeDir, 1) + } + + content, err := os.ReadFile(srcFile) + if err != nil { + return nil, err + } + + fileStat, err := os.Stat(srcFile) + if err != nil { + return nil, err + } + + srcFileBasename := filepath.Base(srcFile) + dstFile := path.Join(dstDir, srcFileBasename) + + if !allowOverwrite { + err = checkAllowOverWrite(ftpC, host, dstFile) + if err != nil { + return nil, err + } + } + + file, err := ftpC.Create(dstFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("dest dir '%s' not exist", dstDir) + } + + if err, ok := err.(*sftp.StatusError); ok && err.Code == uint32(sftp.ErrSshFxPermissionDenied) { + return nil, fmt.Errorf("no permission to write to dest dir '%s'", dstDir) + } + + return nil, err + } + + _, err = file.Write(content) + if err != nil { + return nil, err + } + + if err := file.Chmod(fileStat.Mode()); err != nil { + return nil, err + } + + if err := ftpC.Chtimes(dstFile, time.Now(), fileStat.ModTime()); err != nil { + return nil, err + } + + log.Debugf("%s: '%s' -> '%s", host, srcFile, dstFile) + + return file, nil +} diff --git a/pkg/batchssh/pushv2.go b/pkg/batchssh/pushv2.go new file mode 100644 index 0000000..a8684b0 --- /dev/null +++ b/pkg/batchssh/pushv2.go @@ -0,0 +1,180 @@ +package batchssh + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + + "github.com/windvalley/gossh/pkg/log" +) + +const ( + PushBeginFile = "C" + PushBeginFolder = "D" + PushBeginEndFolder = "0" + PushEndFolder = "E" + PushEnd = "\x00" +) + +func (c *Client) pushFileOrDirV2( + client *ssh.Client, + ftpC *sftp.Client, + srcFile, dstDir, host string, + allowOverwrite bool, +) error { + if !allowOverwrite { + dstFile := path.Join(dstDir, filepath.Base(srcFile)) + if err := checkAllowOverWrite(ftpC, host, dstFile); err != nil { + return err + } + } + + session, err := client.NewSession() + if err != nil { + return err + } + defer session.Close() + + if !isDir(srcFile) { + return pushFileV2(session, srcFile, dstDir, host) + } + + return pushDirV2(session, srcFile, dstDir, host) +} + +func pushFileV2(session *ssh.Session, src, dest, host string) error { + go func() { + w, err := session.StdinPipe() + if err != nil { + log.Errorf("%s: failed to open stdin, error: %v", host, err) + return + } + defer w.Close() + + fileinfo, err := os.Stat(src) + if err != nil { + log.Errorf("%s: failed to get file stat, error: %v", host, err) + return + } + + if err := createFile(w, host, src, fileinfo); err != nil { + return + } + }() + + return session.Run("scp -rt " + dest) +} + +func pushDirV2(session *ssh.Session, src, dest, host string) error { + go func() { + w, err := session.StdinPipe() + if err != nil { + log.Errorf("failed to open stdin, error: %v", err) + return + } + defer w.Close() + + fileinfo, err := os.Stat(src) + if err != nil { + log.Errorf("failed to get file stat, error: %v", err) + return + } + + fmt.Fprintln(w, PushBeginFolder+getMode(fileinfo), PushBeginEndFolder, fileinfo.Name()) + + if err := walkDir(w, src, host); err != nil { + log.Errorf("failed to walk dir, error: %v", err) + return + } + + fmt.Fprintln(w, PushEndFolder) + }() + + return session.Run("scp -rt " + dest) +} + +func walkDir(w io.WriteCloser, dir, host string) error { + files, err := os.ReadDir(dir) + if err != nil { + return err + } + + for _, file := range files { + name := file.Name() + path := path.Join(dir, name) + + fileinfo, err := file.Info() + if err != nil { + return err + } + + if file.IsDir() { + fmt.Fprintln(w, PushBeginFolder+getMode(fileinfo), PushBeginEndFolder, name) + + if err := walkDir(w, path, host); err != nil { + return err + } + + fmt.Fprintln(w, PushEndFolder) + } else { + if err := createFile(w, host, path, fileinfo); err != nil { + return err + } + } + } + + return nil +} + +func createFile(w io.WriteCloser, host, path string, fileinfo fs.FileInfo) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + fmt.Fprintln(w, PushBeginFile+getMode(fileinfo), fileinfo.Size(), fileinfo.Name()) + + if _, err := io.Copy(w, f); err != nil { + return err + } + + fmt.Fprint(w, PushEnd) + + log.Debugf("%s: %s pushed", host, path) + + return nil +} + +func getMode(f fs.FileInfo) string { + mod := f.Mode() + if mod > (1 << 9) { + mod = mod % (1 << 9) + } + + return fmt.Sprintf("%#o", uint32(mod)) +} + +func checkAllowOverWrite(ftpC *sftp.Client, host, dstFile string) error { + f, err := ftpC.Stat(dstFile) + if err != nil && !os.IsNotExist(err) { + log.Errorf("%s: failed to stat %s, error: %v", host, dstFile, err) + return err + } + + if f != nil { + return fmt.Errorf( + "%s: %s alreay exists, you can add '-F' flag to overwrite it", + host, + dstFile, + ) + } + + return nil +} diff --git a/scripts/makefiles/go.makefile b/scripts/makefiles/go.makefile index 57e6a44..f765561 100644 --- a/scripts/makefiles/go.makefile +++ b/scripts/makefiles/go.makefile @@ -1,8 +1,5 @@ # go.makefile -# Supports Go versions. -GO_SUPPORTED_VERSIONS ?= 1.13|1.14|1.15|1.16|1.17 - # The project package name. ROOT_PACKAGE = github.com/windvalley/gossh # The project version package name. @@ -44,11 +41,6 @@ endif EXCLUDE_TESTS = ${ROOT_PACKAGE}/test ${ROOT_PACKAGE}/pkg/log ${ROOT_PACKAGE}/third_party -.PHONY: go.build.verify -go.build.verify: -ifneq ($(shell ${GO} version | grep -q -E '\bgo(${GO_SUPPORTED_VERSIONS})\b' && echo 0 || echo 1), 0) - $(error unsupported Go version. Supported versions: '${GO_SUPPORTED_VERSIONS}') -endif .PHONY: go.build.% go.build.%: @@ -63,10 +55,10 @@ go.build.%: @echo "${OUTPUT_DIR}/platforms/${OS}/${ARCH}/${COMMAND}${GO_BIN_EXT}" .PHONY: go.build -go.build: go.build.verify go.tidy $(addprefix go.build., $(addprefix ${PLATFORM}., ${BINS})) +go.build: go.tidy $(addprefix go.build., $(addprefix ${PLATFORM}., ${BINS})) .PHONY: go.build.multiarch -go.build.multiarch: go.build.verify go.tidy $(foreach p,${PLATFORMS},$(addprefix go.build., $(addprefix ${p}., ${BINS}))) +go.build.multiarch: go.tidy $(foreach p,${PLATFORMS},$(addprefix go.build., $(addprefix ${p}., ${BINS}))) .PHONY: go.lint go.lint: tools.verify.golangci-lint