From 0cf0beb0a35172297d029169a1b85efba17a7cad Mon Sep 17 00:00:00 2001 From: Tony Blyler Date: Wed, 15 Jun 2016 20:32:15 -0400 Subject: [PATCH] Fix resuming to be more reliable --- easysftp.go | 75 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/easysftp.go b/easysftp.go index ece8ec0..f4359e2 100644 --- a/easysftp.go +++ b/easysftp.go @@ -11,6 +11,9 @@ import ( "time" ) +// resumeBufferSize is the size of writes when downloading via sftp (32KiB) * 2 +const resumeBufferSize = 1 << 16 + // ClientConfig maintains all of the configuration info to connect to a SSH host type ClientConfig struct { Username string @@ -134,7 +137,7 @@ func (c *Client) Download(path string, output io.Writer, offset int64) error { } // Mirror downloads an entire folder (recursively) or file underneath the given localParentPath -// resume will continue downloading interrupted files +// resume will try to continue downloading interrupted files func (c *Client) Mirror(path string, localParentPath string, resume bool) error { sftpClient, err := c.newSftpClient() if err != nil { @@ -162,10 +165,7 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error flags := os.O_RDWR | os.O_CREATE - if resume { - // append to the end of the file - flags |= os.O_APPEND - } else { + if !resume { // truncate the file flags |= os.O_TRUNC } @@ -188,8 +188,34 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error return err } - // we assume that the size of the file is the resume point - offset = info.Size() + offset = info.Size() - resumeBufferSize + if offset <= 0 { + offset = 0 + } else { + _, err = file.Seek(offset, 0) + if err != nil { + return err + } + + buf := make([]byte, resumeBufferSize) + _, err = file.Read(buf) + if err != nil { + return err + } + + for _, val := range buf { + if val == 0 { + break + } + + offset++ + } + + _, err = file.Seek(offset, 0) + if err != nil { + return err + } + } } return c.Download(path, file, offset) @@ -249,9 +275,7 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error flags := os.O_RDWR | os.O_CREATE - if resume { - flags |= os.O_APPEND - } else { + if !resume { flags |= os.O_TRUNC } @@ -267,7 +291,36 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error return err } - _, err = remoteFile.Seek(info.Size(), 0) + offset := info.Size() - resumeBufferSize + if offset <= 0 { + offset = 0 + } else { + _, err = localFile.Seek(offset, 0) + if err != nil { + return err + } + + buf := make([]byte, resumeBufferSize) + _, err = localFile.Read(buf) + if err != nil { + return err + } + + for _, val := range buf { + if val == 0 { + break + } + + offset++ + } + + _, err = localFile.Seek(offset, 0) + if err != nil { + return err + } + } + + _, err = remoteFile.Seek(offset, 0) if err != nil { return err }