- djm@cvs.openbsd.org 2013/07/25 00:56:52
[sftp-client.c sftp-client.h sftp.1 sftp.c]
sftp support for resuming partial downloads; patch mostly by Loganaden
Velvindron/AfriNIC with some tweaks by me; feedback and ok dtucker@
diff --git a/ChangeLog b/ChangeLog
index f799663..9552a9b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,10 @@
[ssh.c]
daemonise backgrounded (ControlPersist'ed) multiplexing master to ensure
it is fully detached from its controlling terminal. based on debugging
+ - djm@cvs.openbsd.org 2013/07/25 00:56:52
+ [sftp-client.c sftp-client.h sftp.1 sftp.c]
+ sftp support for resuming partial downloads; patch mostly by Loganaden
+ Velvindron/AfriNIC with some tweaks by me; feedback and ok dtucker@
20130720
- (djm) OpenBSD CVS Sync
diff --git a/sftp-client.c b/sftp-client.c
index ab035c7..cb4efd3 100644
--- a/sftp-client.c
+++ b/sftp-client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.c,v 1.100 2013/06/01 22:34:50 dtucker Exp $ */
+/* $OpenBSD: sftp-client.c,v 1.101 2013/07/25 00:56:51 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
*
@@ -112,7 +112,7 @@
iov[1].iov_len = buffer_len(m);
if (atomiciov6(writev, conn->fd_out, iov, 2,
- conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) !=
+ conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) !=
buffer_len(m) + sizeof(mlen))
fatal("Couldn't send packet: %s", strerror(errno));
@@ -988,16 +988,17 @@
int
do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
- Attrib *a, int pflag)
+ Attrib *a, int pflag, int resume)
{
Attrib junk;
Buffer msg;
char *handle;
- int local_fd, status = 0, write_error;
- int read_error, write_errno;
- u_int64_t offset, size;
+ int local_fd = -1, status = 0, write_error;
+ int read_error, write_errno, reordered = 0;
+ u_int64_t offset = 0, size, highwater;
u_int handle_len, mode, type, id, buflen, num_req, max_req;
off_t progress_counter;
+ struct stat st;
struct request {
u_int id;
u_int len;
@@ -1050,21 +1051,36 @@
return(-1);
}
- local_fd = open(local_path, O_WRONLY | O_CREAT | O_TRUNC,
+ local_fd = open(local_path, O_WRONLY | O_CREAT | (resume ? : O_TRUNC),
mode | S_IWUSR);
if (local_fd == -1) {
error("Couldn't open local file \"%s\" for writing: %s",
local_path, strerror(errno));
- do_close(conn, handle, handle_len);
- buffer_free(&msg);
- free(handle);
- return(-1);
+ goto fail;
+ }
+ offset = highwater = 0;
+ if (resume) {
+ if (fstat(local_fd, &st) == -1) {
+ error("Unable to stat local file \"%s\": %s",
+ local_path, strerror(errno));
+ goto fail;
+ }
+ if ((size_t)st.st_size > size) {
+ error("Unable to resume download of \"%s\": "
+ "local file is larger than remote", local_path);
+ fail:
+ do_close(conn, handle, handle_len);
+ buffer_free(&msg);
+ free(handle);
+ return -1;
+ }
+ offset = highwater = st.st_size;
}
/* Read from remote and write to local */
- write_error = read_error = write_errno = num_req = offset = 0;
+ write_error = read_error = write_errno = num_req = 0;
max_req = 1;
- progress_counter = 0;
+ progress_counter = offset;
if (showprogress && size != 0)
start_progress_meter(remote_path, size, &progress_counter);
@@ -1139,6 +1155,10 @@
write_error = 1;
max_req = 0;
}
+ else if (!reordered && req->offset <= highwater)
+ highwater = req->offset + len;
+ else if (!reordered && req->offset > highwater)
+ reordered = 1;
progress_counter += len;
free(data);
@@ -1187,7 +1207,15 @@
/* Sanity check */
if (TAILQ_FIRST(&requests) != NULL)
fatal("Transfer complete, but requests still in queue");
-
+ /* Truncate at highest contiguous point to avoid holes on interrupt */
+ if (read_error || write_error || interrupted) {
+ if (reordered && resume) {
+ error("Unable to resume download of \"%s\": "
+ "server reordered requests", local_path);
+ }
+ debug("truncating at %llu", (unsigned long long)highwater);
+ ftruncate(local_fd, highwater);
+ }
if (read_error) {
error("Couldn't read from remote file \"%s\" : %s",
remote_path, fx2txt(status));
@@ -1199,7 +1227,8 @@
do_close(conn, handle, handle_len);
} else {
status = do_close(conn, handle, handle_len);
-
+ if (interrupted)
+ status = -1;
/* Override umask and utimes if asked */
#ifdef HAVE_FCHMOD
if (pflag && fchmod(local_fd, mode) == -1)
@@ -1227,7 +1256,7 @@
static int
download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
- Attrib *dirattrib, int pflag, int printflag, int depth)
+ Attrib *dirattrib, int pflag, int printflag, int depth, int resume)
{
int i, ret = 0;
SFTP_DIRENT **dir_entries;
@@ -1280,11 +1309,11 @@
continue;
if (download_dir_internal(conn, new_src, new_dst,
&(dir_entries[i]->a), pflag, printflag,
- depth + 1) == -1)
+ depth + 1, resume) == -1)
ret = -1;
} else if (S_ISREG(dir_entries[i]->a.perm) ) {
if (do_download(conn, new_src, new_dst,
- &(dir_entries[i]->a), pflag) == -1) {
+ &(dir_entries[i]->a), pflag, resume) == -1) {
error("Download of file %s to %s failed",
new_src, new_dst);
ret = -1;
@@ -1317,7 +1346,7 @@
int
download_dir(struct sftp_conn *conn, char *src, char *dst,
- Attrib *dirattrib, int pflag, int printflag)
+ Attrib *dirattrib, int pflag, int printflag, int resume)
{
char *src_canon;
int ret;
@@ -1328,7 +1357,7 @@
}
ret = download_dir_internal(conn, src_canon, dst,
- dirattrib, pflag, printflag, 0);
+ dirattrib, pflag, printflag, 0, resume);
free(src_canon);
return ret;
}
@@ -1553,7 +1582,7 @@
a.perm &= 01777;
if (!pflag)
a.flags &= ~SSH2_FILEXFER_ATTR_ACMODTIME;
-
+
status = do_mkdir(conn, dst, &a, 0);
/*
* we lack a portable status for errno EEXIST,
@@ -1563,7 +1592,7 @@
if (status != SSH2_FX_OK) {
if (status != SSH2_FX_FAILURE)
return -1;
- if (do_stat(conn, dst, 0) == NULL)
+ if (do_stat(conn, dst, 0) == NULL)
return -1;
}
@@ -1571,7 +1600,7 @@
error("Failed to open dir \"%s\": %s", src, strerror(errno));
return -1;
}
-
+
while (((dp = readdir(dirp)) != NULL) && !interrupted) {
if (dp->d_ino == 0)
continue;
diff --git a/sftp-client.h b/sftp-client.h
index aef54ef..111a998 100644
--- a/sftp-client.h
+++ b/sftp-client.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.h,v 1.20 2010/12/04 00:18:01 djm Exp $ */
+/* $OpenBSD: sftp-client.h,v 1.21 2013/07/25 00:56:51 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
@@ -106,13 +106,13 @@
* Download 'remote_path' to 'local_path'. Preserve permissions and times
* if 'pflag' is set
*/
-int do_download(struct sftp_conn *, char *, char *, Attrib *, int);
+int do_download(struct sftp_conn *, char *, char *, Attrib *, int, int);
/*
* Recursively download 'remote_directory' to 'local_directory'. Preserve
* times if 'pflag' is set
*/
-int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int);
+int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int, int);
/*
* Upload 'local_path' to 'remote_path'. Preserve permissions and times
diff --git a/sftp.1 b/sftp.1
index bcb4721..2577fe8 100644
--- a/sftp.1
+++ b/sftp.1
@@ -1,4 +1,4 @@
-.\" $OpenBSD: sftp.1,v 1.91 2011/09/05 05:56:13 djm Exp $
+.\" $OpenBSD: sftp.1,v 1.92 2013/07/25 00:56:51 djm Exp $
.\"
.\" Copyright (c) 2001 Damien Miller. All rights reserved.
.\"
@@ -22,7 +22,7 @@
.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd $Mdocdate: September 5 2011 $
+.Dd $Mdocdate: July 25 2013 $
.Dt SFTP 1
.Os
.Sh NAME
@@ -129,7 +129,7 @@
.Nm
will abort if any of the following
commands fail:
-.Ic get , put , rename , ln ,
+.Ic get , put , reget , rename , ln ,
.Ic rm , mkdir , chdir , ls ,
.Ic lchdir , chmod , chown ,
.Ic chgrp , lpwd , df , symlink ,
@@ -343,7 +343,7 @@
Quit
.Nm sftp .
.It Xo Ic get
-.Op Fl Ppr
+.Op Fl aPpr
.Ar remote-path
.Op Ar local-path
.Xc
@@ -363,6 +363,14 @@
.Ar local-path
must specify a directory.
.Pp
+If the
+.Fl a
+flag is specified, then attempt to resume partial transfers of existing files.
+Note that resumption assumes that any partial copy of the local file matches
+the remote copy.
+If the remote file differs from the partial local copy then the resultant file
+is likely to be corrupt.
+.Pp
If either the
.Fl P
or
@@ -503,6 +511,18 @@
.It Ic quit
Quit
.Nm sftp .
+.It Xo Ic reget
+.Op Fl Ppr
+.Ar remote-path
+.Op Ar local-path
+.Xc
+Resume download of
+.Ar remote-path .
+Equivalent to
+.Ic get
+with the
+.Fl a
+flag set.
.It Ic rename Ar oldpath Ar newpath
Rename remote file from
.Ar oldpath
diff --git a/sftp.c b/sftp.c
index f0daaef..969328d 100644
--- a/sftp.c
+++ b/sftp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp.c,v 1.147 2013/07/12 00:20:00 djm Exp $ */
+/* $OpenBSD: sftp.c,v 1.148 2013/07/25 00:56:52 djm Exp $ */
/*
* Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
*
@@ -88,6 +88,9 @@
/* When this option is set, we always recursively download/upload directories */
int global_rflag = 0;
+/* When this option is set, we resume download if possible */
+int global_aflag = 0;
+
/* When this option is set, the file transfers will always preserve times */
int global_pflag = 0;
@@ -151,6 +154,7 @@
#define I_SYMLINK 21
#define I_VERSION 22
#define I_PROGRESS 23
+#define I_REGET 26
struct CMD {
const char *c;
@@ -190,6 +194,7 @@
{ "put", I_PUT, LOCAL },
{ "pwd", I_PWD, REMOTE },
{ "quit", I_QUIT, NOARGS },
+ { "reget", I_REGET, REMOTE },
{ "rename", I_RENAME, REMOTE },
{ "rm", I_RM, REMOTE },
{ "rmdir", I_RMDIR, REMOTE },
@@ -239,6 +244,7 @@
" filesystem containing 'path'\n"
"exit Quit sftp\n"
"get [-Ppr] remote [local] Download file\n"
+ "reget remote [local] Resume download file\n"
"help Display this help text\n"
"lcd path Change local directory to 'path'\n"
"lls [ls-options [path]] Display local directory listing\n"
@@ -350,8 +356,8 @@
}
static int
-parse_getput_flags(const char *cmd, char **argv, int argc, int *pflag,
- int *rflag)
+parse_getput_flags(const char *cmd, char **argv, int argc,
+ int *aflag, int *pflag, int *rflag)
{
extern int opterr, optind, optopt, optreset;
int ch;
@@ -359,9 +365,12 @@
optind = optreset = 1;
opterr = 0;
- *rflag = *pflag = 0;
- while ((ch = getopt(argc, argv, "PpRr")) != -1) {
+ *aflag = *rflag = *pflag = 0;
+ while ((ch = getopt(argc, argv, "aPpRr")) != -1) {
switch (ch) {
+ case 'a':
+ *aflag = 1;
+ break;
case 'p':
case 'P':
*pflag = 1;
@@ -519,7 +528,7 @@
static int
process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
- int pflag, int rflag)
+ int pflag, int rflag, int resume)
{
char *abs_src = NULL;
char *abs_dst = NULL;
@@ -571,15 +580,18 @@
}
free(tmp);
- if (!quiet)
+ resume |= global_aflag;
+ if (!quiet && resume)
+ printf("Resuming %s to %s\n", g.gl_pathv[i], abs_dst);
+ else if (!quiet && !resume)
printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
if (pathname_is_dir(g.gl_pathv[i]) && (rflag || global_rflag)) {
- if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL,
- pflag || global_pflag, 1) == -1)
+ if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL,
+ pflag || global_pflag, 1, resume) == -1)
err = -1;
} else {
if (do_download(conn, g.gl_pathv[i], abs_dst, NULL,
- pflag || global_pflag) == -1)
+ pflag || global_pflag, resume) == -1)
err = -1;
}
free(abs_dst);
@@ -1118,8 +1130,9 @@
}
static int
-parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
- int *hflag, int *sflag, unsigned long *n_arg, char **path1, char **path2)
+parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
+ int *pflag, int *rflag, int *sflag, unsigned long *n_arg,
+ char **path1, char **path2)
{
const char *cmd, *cp = *cpp;
char *cp2, **argv;
@@ -1163,14 +1176,15 @@
}
/* Get arguments and parse flags */
- *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
+ *aflag = *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
*path1 = *path2 = NULL;
optidx = 1;
switch (cmdnum) {
case I_GET:
+ case I_REGET:
case I_PUT:
if ((optidx = parse_getput_flags(cmd, argv, argc,
- pflag, rflag)) == -1)
+ aflag, pflag, rflag)) == -1)
return -1;
/* Get first pathname (mandatory) */
if (argc - optidx < 1) {
@@ -1185,6 +1199,11 @@
/* Destination is not globbed */
undo_glob_escape(*path2);
}
+ if (*aflag && cmdnum == I_PUT) {
+ /* XXX implement resume for uploads */
+ error("Resume is not supported for uploads");
+ return -1;
+ }
break;
case I_LINK:
if ((optidx = parse_link_flags(cmd, argv, argc, sflag)) == -1)
@@ -1293,7 +1312,8 @@
int err_abort)
{
char *path1, *path2, *tmp;
- int pflag = 0, rflag = 0, lflag = 0, iflag = 0, hflag = 0, sflag = 0;
+ int aflag = 0, hflag = 0, iflag = 0, lflag = 0, pflag = 0;
+ int rflag = 0, sflag = 0;
int cmdnum, i;
unsigned long n_arg = 0;
Attrib a, *aa;
@@ -1302,9 +1322,8 @@
glob_t g;
path1 = path2 = NULL;
- cmdnum = parse_args(&cmd, &pflag, &rflag, &lflag, &iflag, &hflag,
- &sflag, &n_arg, &path1, &path2);
-
+ cmdnum = parse_args(&cmd, &aflag, &hflag, &iflag, &lflag, &pflag,
+ &rflag, &sflag, &n_arg, &path1, &path2);
if (iflag != 0)
err_abort = 0;
@@ -1319,8 +1338,12 @@
/* Unrecognized command */
err = -1;
break;
+ case I_REGET:
+ aflag = 1;
+ /* FALLTHROUGH */
case I_GET:
- err = process_get(conn, path1, path2, *pwd, pflag, rflag);
+ err = process_get(conn, path1, path2, *pwd, pflag,
+ rflag, aflag);
break;
case I_PUT:
err = process_put(conn, path1, path2, *pwd, pflag, rflag);
@@ -1949,12 +1972,10 @@
}
} else {
/* XXX this is wrong wrt quoting */
- if (file2 == NULL)
- snprintf(cmd, sizeof cmd, "get %s", dir);
- else
- snprintf(cmd, sizeof cmd, "get %s %s", dir,
- file2);
-
+ snprintf(cmd, sizeof cmd, "get%s %s%s%s",
+ global_aflag ? " -a" : "", dir,
+ file2 == NULL ? "" : " ",
+ file2 == NULL ? "" : file2);
err = parse_dispatch_command(conn, cmd,
&remote_path, 1);
free(dir);
@@ -2143,7 +2164,7 @@
infile = stdin;
while ((ch = getopt(argc, argv,
- "1246hpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
+ "1246ahpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
switch (ch) {
/* Passed through to ssh(1) */
case '4':
@@ -2183,6 +2204,9 @@
case '2':
sshver = 2;
break;
+ case 'a':
+ global_aflag = 1;
+ break;
case 'B':
copy_buffer_len = strtol(optarg, &cp, 10);
if (copy_buffer_len == 0 || *cp != '\0')