Remove Authorization: headers from --dump-headers output

Add in `--dump-auth` flag to put it back.

docs/content/docs.md

@@ -543,6 +543,13 @@ here which are used for testing.  These start with remote name eg
 
 Write CPU profile to file.  This can be analysed with `go tool pprof`.
 
+### --dump-auth ###
+
+Dump HTTP headers - will contain sensitive info such as
+`Authorization:` headers - use `--dump-headers` to dump without
+`Authorization:` headers.  Can be very verbose.  Useful for debugging
+only.
+
 ### --dump-bodies ###
 
 Dump HTTP headers and bodies - may contain sensitive info.  Can be
@@ -555,8 +562,11 @@ and exclude options are filtering on.
 
 ### --dump-headers ###
 
-Dump HTTP headers - may contain sensitive info.  Can be very verbose.
-Useful for debugging only.
+Dump HTTP headers with `Authorization:` lines removed. May still
+contain sensitive info.  Can be very verbose.  Useful for debugging
+only.
+
+Use `--dump-auth` if you do want the `Authorization:` headers.
 
 ### --memprofile=FILE ###
 

fs/config.go

@@ -76,6 +76,7 @@ var (
 	timeout         = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
 	dumpHeaders     = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
 	dumpBodies      = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
+	dumpAuth        = pflag.BoolP("dump-auth", "", false, "Dump HTTP headers with auth info")
 	skipVerify      = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
 	AskPassword     = pflag.BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
 	deleteBefore    = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
@@ -287,6 +288,7 @@ type ConfigInfo struct {
 	Timeout            time.Duration // Data channel timeout
 	DumpHeaders        bool
 	DumpBodies         bool
+	DumpAuth           bool
 	Filter             *Filter
 	InsecureSkipVerify bool // Skip server certificate verification
 	DeleteBefore       bool // Delete before checking
@@ -341,6 +343,7 @@ func LoadConfig() {
 	Config.IgnoreExisting = *ignoreExisting
 	Config.DumpHeaders = *dumpHeaders
 	Config.DumpBodies = *dumpBodies
+	Config.DumpAuth = *dumpAuth
 	Config.InsecureSkipVerify = *skipVerify
 	Config.LowLevelRetries = *lowLevelRetries
 	Config.UpdateOlder = *updateOlder

fs/http.go

@@ -3,6 +3,7 @@
 package fs
 
 import (
+	"bytes"
 	"crypto/tls"
 	"net"
 	"net/http"
@@ -112,7 +113,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper {
 		//   t.ExpectContinueTimeout
 		ci.initTransport(t)
 		// Wrap that http.Transport in our own transport
-		transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies)
+		transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies, ci.DumpAuth)
 	})
 	return transport
 }
@@ -131,15 +132,17 @@ type Transport struct {
 	*http.Transport
 	logHeader bool
 	logBody   bool
+	logAuth   bool
 }
 
 // NewTransport wraps the http.Transport passed in and logs all
 // roundtrips including the body if logBody is set.
-func NewTransport(transport *http.Transport, logHeader, logBody bool) *Transport {
+func NewTransport(transport *http.Transport, logHeader, logBody, logAuth bool) *Transport {
 	return &Transport{
 		Transport: transport,
 		logHeader: logHeader,
 		logBody:   logBody,
+		logAuth:   logAuth,
 	}
 }
 
@@ -180,13 +183,48 @@ func checkServerTime(req *http.Request, resp *http.Response) {
 	checkedHostMu.Unlock()
 }
 
+var authBuf = []byte("Authorization: ")
+
+// cleanAuth gets rid of one Authorization: header within the first 4k
+func cleanAuth(buf []byte) []byte {
+	// Find how much buffer to check
+	n := 4096
+	if len(buf) < n {
+		n = len(buf)
+	}
+	// See if there is an Authorization: header
+	i := bytes.Index(buf[:n], authBuf)
+	if i < 0 {
+		return buf
+	}
+	i += len(authBuf)
+	// Overwrite the next 4 chars with 'X'
+	for j := 0; i < len(buf) && j < 4; j++ {
+		if buf[i] == '\n' {
+			break
+		}
+		buf[i] = 'X'
+		i++
+	}
+	// Snip out to the next '\n'
+	j := bytes.IndexByte(buf[i:], '\n')
+	if j < 0 {
+		return buf[:i]
+	}
+	n = copy(buf[i:], buf[i+j:])
+	return buf[:i+n]
+}
+
 // RoundTrip implements the RoundTripper interface.
 func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
 	// Force user agent
 	req.Header.Set("User-Agent", UserAgent)
 	// Log request
-	if t.logHeader || t.logBody {
+	if t.logHeader || t.logBody || t.logAuth {
 		buf, _ := httputil.DumpRequestOut(req, t.logBody)
+		if !t.logAuth {
+			buf = cleanAuth(buf)
+		}
 		Debug(nil, "%s", separatorReq)
 		Debug(nil, "%s (req %p)", "HTTP REQUEST", req)
 		Debug(nil, "%s", string(buf))
@@ -195,7 +233,7 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Log response
-	if t.logHeader || t.logBody {
+	if t.logHeader || t.logBody || t.logAuth {
 		Debug(nil, "%s", separatorResp)
 		Debug(nil, "%s (req %p)", "HTTP RESPONSE", req)
 		if err != nil {

fs/http_test.go

@@ -38,3 +38,27 @@ func TestSetDefaults(t *testing.T) {
 	assert.Equal(t, old.TLSNextProto, new.TLSNextProto, "when checking .TLSNextProto")
 	assert.Equal(t, old.MaxResponseHeaderBytes, new.MaxResponseHeaderBytes, "when checking .MaxResponseHeaderBytes")
 }
+
+func TestCleanAuth(t *testing.T) {
+	for _, test := range []struct {
+		in   string
+		want string
+	}{
+		{"", ""},
+		{"floo", "floo"},
+		{"Authorization: ", "Authorization: "},
+		{"Authorization: \n", "Authorization: \n"},
+		{"Authorization: A", "Authorization: X"},
+		{"Authorization: A\n", "Authorization: X\n"},
+		{"Authorization: AAAA", "Authorization: XXXX"},
+		{"Authorization: AAAA\n", "Authorization: XXXX\n"},
+		{"Authorization: AAAAA", "Authorization: XXXX"},
+		{"Authorization: AAAAA\n", "Authorization: XXXX\n"},
+		{"Authorization: AAAA\n", "Authorization: XXXX\n"},
+		{"Authorization: AAAAAAAAA\nPotato: Help\n", "Authorization: XXXX\nPotato: Help\n"},
+		{"Sausage: 1\nAuthorization: AAAAAAAAA\nPotato: Help\n", "Sausage: 1\nAuthorization: XXXX\nPotato: Help\n"},
+	} {
+		got := string(cleanAuth([]byte(test.in)))
+		assert.Equal(t, test.want, got, test.in)
+	}
+}