From 713f8f357dc5eb512f736ad97cb092dfbd1dbfe9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 28 Feb 2021 11:36:56 +0000 Subject: [PATCH] sftp: fix "file not found" errors for read once servers - fixes #5077 It introduces a new flag --sftp-disable-concurrent-reads to stop the problematic behaviour in the SFTP library for read-once servers. This upgrades the sftp library to v1.13.0 which has the fix. --- backend/sftp/sftp.go | 69 +++++++++++++++++++++++++++++--------------- go.mod | 6 ++-- go.sum | 6 ++++ 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index c039aa045..8bbd50fe2 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -204,6 +204,25 @@ Fstat instead of Stat which is called on an already open file handle. It has been found that this helps with IBM Sterling SFTP servers which have "extractability" level set to 1 which means only 1 file can be opened at any given time. +`, + Advanced: true, + }, { + Name: "disable_concurrent_reads", + Default: false, + Help: `If set don't use concurrent reads + +Normally concurrent reads are safe to use and not using them will +degrade performance, so this option is disabled by default. + +Some servers limit the amount number of times a file can be +downloaded. Using concurrent reads can trigger this limit, so if you +have a server which returns + + Failed to copy: file does not exist + +Then you may need to enable this flag. + +If concurrent reads are disabled, the use_fstat option is ignored. `, Advanced: true, }, { @@ -224,28 +243,29 @@ Set to 0 to keep connections indefinitely. // Options defines the configuration for this backend type Options struct { - Host string `config:"host"` - User string `config:"user"` - Port string `config:"port"` - Pass string `config:"pass"` - KeyPem string `config:"key_pem"` - KeyFile string `config:"key_file"` - KeyFilePass string `config:"key_file_pass"` - PubKeyFile string `config:"pubkey_file"` - KnownHostsFile string `config:"known_hosts_file"` - KeyUseAgent bool `config:"key_use_agent"` - UseInsecureCipher bool `config:"use_insecure_cipher"` - DisableHashCheck bool `config:"disable_hashcheck"` - AskPassword bool `config:"ask_password"` - PathOverride string `config:"path_override"` - SetModTime bool `config:"set_modtime"` - Md5sumCommand string `config:"md5sum_command"` - Sha1sumCommand string `config:"sha1sum_command"` - SkipLinks bool `config:"skip_links"` - Subsystem string `config:"subsystem"` - ServerCommand string `config:"server_command"` - UseFstat bool `config:"use_fstat"` - IdleTimeout fs.Duration `config:"idle_timeout"` + Host string `config:"host"` + User string `config:"user"` + Port string `config:"port"` + Pass string `config:"pass"` + KeyPem string `config:"key_pem"` + KeyFile string `config:"key_file"` + KeyFilePass string `config:"key_file_pass"` + PubKeyFile string `config:"pubkey_file"` + KnownHostsFile string `config:"known_hosts_file"` + KeyUseAgent bool `config:"key_use_agent"` + UseInsecureCipher bool `config:"use_insecure_cipher"` + DisableHashCheck bool `config:"disable_hashcheck"` + AskPassword bool `config:"ask_password"` + PathOverride string `config:"path_override"` + SetModTime bool `config:"set_modtime"` + Md5sumCommand string `config:"md5sum_command"` + Sha1sumCommand string `config:"sha1sum_command"` + SkipLinks bool `config:"skip_links"` + Subsystem string `config:"subsystem"` + ServerCommand string `config:"server_command"` + UseFstat bool `config:"use_fstat"` + DisableConcurrentReads bool `config:"disable_concurrent_reads"` + IdleTimeout fs.Duration `config:"idle_timeout"` } // Fs stores the interface to the remote SFTP files @@ -373,7 +393,10 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C } } opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts - opts = append(opts, sftp.UseFstat(f.opt.UseFstat)) + opts = append(opts, + sftp.UseFstat(f.opt.UseFstat), + sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads), + ) return sftp.NewClientPipe(pr, pw, opts...) } diff --git a/go.mod b/go.mod index f4972eab9..65e44fc1f 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/nsf/termbox-go v0.0.0-20210114135735-d04385b850e8 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.12.0 + github.com/pkg/sftp v1.13.0 github.com/prometheus/client_golang v1.9.0 github.com/prometheus/procfs v0.3.0 // indirect github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 @@ -66,11 +66,11 @@ require ( go.opencensus.io v0.22.6 // indirect go.uber.org/zap v1.16.0 // indirect goftp.io/server v0.4.1 - golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c + golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect golang.org/x/text v0.3.5 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 diff --git a/go.sum b/go.sum index 893b696f2..6c18a4324 100644 --- a/go.sum +++ b/go.sum @@ -524,6 +524,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.12.0 h1:/f3b24xrDhkhddlaobPe2JgBqfdt+gC/NYl0QY9IOuI= github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8= +github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI= +github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -736,6 +738,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0a golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -902,6 +906,8 @@ golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 h1:jxr7/dEo+rR29uEBoLSWJ1tRHCFAMwFbGUU9nRqzpds= +golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=