Fix resuming to be more reliable

This commit is contained in:
Tony Blyler 2016-06-15 20:32:15 -04:00
parent f6eb173a4f
commit 0cf0beb0a3
No known key found for this signature in database
GPG key ID: 25C9D3A655D1A65C

View file

@ -11,6 +11,9 @@ import (
"time" "time"
) )
// resumeBufferSize is the size of writes when downloading via sftp (32KiB) * 2
const resumeBufferSize = 1 << 16
// ClientConfig maintains all of the configuration info to connect to a SSH host // ClientConfig maintains all of the configuration info to connect to a SSH host
type ClientConfig struct { type ClientConfig struct {
Username string Username string
@ -134,7 +137,7 @@ func (c *Client) Download(path string, output io.Writer, offset int64) error {
} }
// Mirror downloads an entire folder (recursively) or file underneath the given localParentPath // Mirror downloads an entire folder (recursively) or file underneath the given localParentPath
// resume will continue downloading interrupted files // resume will try to continue downloading interrupted files
func (c *Client) Mirror(path string, localParentPath string, resume bool) error { func (c *Client) Mirror(path string, localParentPath string, resume bool) error {
sftpClient, err := c.newSftpClient() sftpClient, err := c.newSftpClient()
if err != nil { if err != nil {
@ -162,10 +165,7 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error
flags := os.O_RDWR | os.O_CREATE flags := os.O_RDWR | os.O_CREATE
if resume { if !resume {
// append to the end of the file
flags |= os.O_APPEND
} else {
// truncate the file // truncate the file
flags |= os.O_TRUNC flags |= os.O_TRUNC
} }
@ -188,8 +188,34 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error
return err return err
} }
// we assume that the size of the file is the resume point offset = info.Size() - resumeBufferSize
offset = info.Size() if offset <= 0 {
offset = 0
} else {
_, err = file.Seek(offset, 0)
if err != nil {
return err
}
buf := make([]byte, resumeBufferSize)
_, err = file.Read(buf)
if err != nil {
return err
}
for _, val := range buf {
if val == 0 {
break
}
offset++
}
_, err = file.Seek(offset, 0)
if err != nil {
return err
}
}
} }
return c.Download(path, file, offset) return c.Download(path, file, offset)
@ -249,9 +275,7 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error
flags := os.O_RDWR | os.O_CREATE flags := os.O_RDWR | os.O_CREATE
if resume { if !resume {
flags |= os.O_APPEND
} else {
flags |= os.O_TRUNC flags |= os.O_TRUNC
} }
@ -267,7 +291,36 @@ func (c *Client) Mirror(path string, localParentPath string, resume bool) error
return err return err
} }
_, err = remoteFile.Seek(info.Size(), 0) offset := info.Size() - resumeBufferSize
if offset <= 0 {
offset = 0
} else {
_, err = localFile.Seek(offset, 0)
if err != nil {
return err
}
buf := make([]byte, resumeBufferSize)
_, err = localFile.Read(buf)
if err != nil {
return err
}
for _, val := range buf {
if val == 0 {
break
}
offset++
}
_, err = localFile.Seek(offset, 0)
if err != nil {
return err
}
}
_, err = remoteFile.Seek(offset, 0)
if err != nil { if err != nil {
return err return err
} }