cmd/gitannex: Add support for different layouts

This commit adds support for the same repo layouts supported by
git-annex-remote-rclone. This should enable git-annex users with remotes
of type "rclone" to switch to a "rclone-builtin" without needing to
retransfer content.

Issue #7625

cmd/gitannex/gitannex.go

@@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string {
 // configDefinition describes a configuration value required by this command. We
 // use "GETCONFIG" messages to query git-annex for these values at runtime.
 type configDefinition struct {
-	name        string
-	description string
-	destination *string
+	name         string
+	description  string
+	destination  *string
+	defaultValue *string
 }
 
 // server contains this command's current state.
@@ -132,6 +133,7 @@ type server struct {
 	configsDone            bool
 	configPrefix           string
 	configRcloneRemoteName string
+	configRcloneLayout     string
 }
 
 func (s *server) sendMsg(msg string) {
@@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error {
 
 // Get a list of configs with pointers to fields of `s`.
 func (s *server) getRequiredConfigs() []configDefinition {
+	defaultRclonePrefix := "git-annex-rclone"
+	defaultRcloneLayout := "nodir"
+
 	return []configDefinition{
 		{
 			"rcloneremotename",
@@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition {
 				"Must match a remote known to rclone. " +
 				"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
 			&s.configRcloneRemoteName,
+			nil,
 		},
 		{
 			"rcloneprefix",
 			"Directory where rclone will write git-annex content. " +
-				"If not specified, defaults to \"git-annex-rclone\". " +
-				"This directory be created on init if it does not exist.",
+				fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) +
+				"This directory will be created on init if it does not exist.",
 			&s.configPrefix,
+			&defaultRclonePrefix,
+		},
+		{
+			"rclonelayout",
+			"Defines where, within the rcloneprefix directory, rclone will write git-annex content. " +
+				fmt.Sprintf("Must be one of %v. ", allLayoutModes()) +
+				fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout),
+			&s.configRcloneLayout,
+			&defaultRcloneLayout,
 		},
 	}
 }
@@ -307,10 +322,13 @@ func (s *server) queryConfigs() error {
 		}
 
 		value := message.finalParameter()
-		if value == "" {
+		if value == "" && config.defaultValue == nil {
 			return fmt.Errorf("config value of %q must not be empty", config.name)
 		}
-
+		if value == "" {
+			*config.destination = *config.defaultValue
+			continue
+		}
 		*config.destination = value
 	}
 
@@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error {
 		return fmt.Errorf("error getting configs: %w", err)
 	}
 
-	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	layout := parseLayoutMode(s.configRcloneLayout)
+	if layout == layoutModeUnknown {
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
+		return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
+	}
+
+	remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
+		return fmt.Errorf("error building fs string: %w", err)
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), remoteFsString)
 	if err != nil {
 		s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
 		return err
@@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error {
 		return fmt.Errorf("error getting configs: %s", err)
 	}
 
-	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	layout := parseLayoutMode(s.configRcloneLayout)
+	if layout == layoutModeUnknown {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+		return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
+	}
+
+	remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+		return fmt.Errorf("error building fs string: %w", err)
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), remoteFsString)
 	if err != nil {
 		s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
 		return err
@@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error {
 	return nil
 }
 
+func (s *server) queryDirhash(msg string) (string, error) {
+	s.sendMsg(msg)
+	parser, err := s.getMsg()
+	if err != nil {
+		return "", err
+	}
+	keyword, err := parser.nextSpaceDelimitedParameter()
+	if err != nil {
+		return "", err
+	}
+	if keyword != "VALUE" {
+		return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword)
+	}
+	dirhash, err := parser.nextSpaceDelimitedParameter()
+	if err != nil {
+		return "", fmt.Errorf("failed to parse dirhash: %w", err)
+	}
+	return dirhash, nil
+}
+
 func (s *server) handleRemove(message *messageParser) error {
 	argKey := message.finalParameter()
 	if argKey == "" {
 		return errors.New("failed to parse key for REMOVE")
 	}
 
-	if err != nil {
-		return err
+	layout := parseLayoutMode(s.configRcloneLayout)
+	if layout == layoutModeUnknown {
+		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+		return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
 	}
 
-	remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+	remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
+	if err != nil {
+		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+		return fmt.Errorf("error building fs string: %w", err)
+	}
+
+	remoteFs, err := cache.Get(context.TODO(), remoteFsString)
 	if err != nil {
 		s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
 		return fmt.Errorf("error getting remote fs: %w", err)

cmd/gitannex/gitannex_test.go

@@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) {
 func (h *testState) preconfigureServer() {
 	h.server.configPrefix = h.localFsDir
 	h.server.configRcloneRemoteName = h.remoteName
+	h.server.configRcloneLayout = string(layoutModeNodir)
 	h.server.configsDone = true
 }
 
@@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{
 			h.requireWriteLine("VALUE " + h.remoteName)
 			h.requireReadLineExact("GETCONFIG rcloneprefix")
 			h.requireWriteLine("VALUE " + h.localFsDir)
+			h.requireReadLineExact("GETCONFIG rclonelayout")
+			h.requireWriteLine("VALUE foo")
 			h.requireReadLineExact("PREPARE-SUCCESS")
 
 			require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
@@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{
 			localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
 
 			h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
-			h.requireReadLineExact("GETCONFIG rcloneprefix")
 
+			h.requireReadLineExact("GETCONFIG rcloneprefix")
 			h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+
+			h.requireReadLineExact("GETCONFIG rclonelayout")
+			h.requireWriteLine("VALUE")
+
 			h.requireReadLineExact("PREPARE-SUCCESS")
 
 			require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
@@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{
 
 			// Note the whitespace following the key.
 			h.requireWriteLine("TRANSFER STORE Key ")
-			h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
+			h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path")
 
 			require.NoError(t, h.mockStdinW.Close())
 		},
-		expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
+		expectedError: "failed to parse file",
 	},
 	// Repeated EXTENSIONS messages add to each other rather than overriding
 	// prior advertised extensions. This behavior is not mandated by the

cmd/gitannex/layout.go

@@ -0,0 +1,72 @@
+package gitannex
+
+import (
+	"fmt"
+	"strings"
+)
+
+type layoutMode string
+
+// All layout modes from git-annex-remote-rclone are supported.
+const (
+	layoutModeLower       layoutMode = "lower"
+	layoutModeDirectory   layoutMode = "directory"
+	layoutModeNodir       layoutMode = "nodir"
+	layoutModeMixed       layoutMode = "mixed"
+	layoutModeFrankencase layoutMode = "frankencase"
+	layoutModeUnknown     layoutMode = ""
+)
+
+func allLayoutModes() []layoutMode {
+	return []layoutMode{
+		layoutModeLower,
+		layoutModeDirectory,
+		layoutModeNodir,
+		layoutModeMixed,
+		layoutModeFrankencase,
+	}
+}
+
+func parseLayoutMode(mode string) layoutMode {
+	for _, knownMode := range allLayoutModes() {
+		if mode == string(knownMode) {
+			return knownMode
+		}
+	}
+	return layoutModeUnknown
+}
+
+type queryDirhashFunc func(msg string) (string, error)
+
+func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
+	if mode == layoutModeNodir {
+		return fmt.Sprintf("%s:%s", remoteName, prefix), nil
+	}
+
+	var dirhash string
+	var err error
+	switch mode {
+	case layoutModeLower, layoutModeDirectory:
+		dirhash, err = queryDirhash("DIRHASH-LOWER " + key)
+	case layoutModeMixed, layoutModeFrankencase:
+		dirhash, err = queryDirhash("DIRHASH " + key)
+	default:
+		panic("unreachable")
+	}
+	if err != nil {
+		return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err)
+	}
+
+	switch mode {
+	case layoutModeLower:
+		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+	case layoutModeDirectory:
+		return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
+	case layoutModeMixed:
+		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+	case layoutModeFrankencase:
+		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
+	default:
+		panic("unreachable")
+	}
+}