mirror of https://github.com/rclone/rclone
bisync: Add new --ignore-listing-checksum flag to distinguish from --ignore-checksum
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes
This commit is contained in:
nielash
Nick Craig-Wood
parent
5ca61ab705
commit
f01a50eb47
|
|
@ -27,18 +27,19 @@ import (
|
|||
|
||||
// Options keep bisync options
|
||||
type Options struct {
|
||||
Resync bool
|
||||
CheckAccess bool
|
||||
CheckFilename string
|
||||
CheckSync CheckSyncMode
|
||||
RemoveEmptyDirs bool
|
||||
MaxDelete int // percentage from 0 to 100
|
||||
Force bool
|
||||
FiltersFile string
|
||||
Workdir string
|
||||
DryRun bool
|
||||
NoCleanup bool
|
||||
SaveQueues bool // save extra debugging files (test only flag)
|
||||
Resync bool
|
||||
CheckAccess bool
|
||||
CheckFilename string
|
||||
CheckSync CheckSyncMode
|
||||
RemoveEmptyDirs bool
|
||||
MaxDelete int // percentage from 0 to 100
|
||||
Force bool
|
||||
FiltersFile string
|
||||
Workdir string
|
||||
DryRun bool
|
||||
NoCleanup bool
|
||||
SaveQueues bool // save extra debugging files (test only flag)
|
||||
IgnoreListingChecksum bool
|
||||
}
|
||||
|
||||
// Default values
|
||||
|
|
@ -108,6 +109,7 @@ func init() {
|
|||
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
|
||||
flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
|
||||
}
|
||||
|
||||
// bisync command definition
|
||||
|
|
@ -213,9 +215,9 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
|
|||
if opt.DryRun {
|
||||
fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile)
|
||||
} else {
|
||||
fs.Infof(nil, "Storing filters file hash to %s", hashFile)
|
||||
if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
|
||||
return ctx, err
|
||||
fs.Infof(nil, "Storing filters file hash to %s", hashFile)
|
||||
if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
|
||||
return ctx, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -253,8 +253,9 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
|
|||
ci := fs.GetConfig(ctx)
|
||||
depth := ci.MaxDepth
|
||||
hashType := hash.None
|
||||
if !ci.IgnoreChecksum {
|
||||
// Currently bisync just honors --ignore-checksum
|
||||
if !b.opt.IgnoreListingChecksum {
|
||||
// Currently bisync just honors --ignore-listing-checksum
|
||||
// (note that this is different from --ignore-checksum)
|
||||
// TODO add full support for checksums and related flags
|
||||
hashType = f.Hashes().GetOne()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,6 +95,8 @@ Optional Flags:
|
|||
-1, --resync Performs the resync run.
|
||||
Warning: Path1 files may overwrite Path2 versions.
|
||||
Consider using `--verbose` or `--dry-run` first.
|
||||
--ignore-listing-checksum Do not use checksums for listings
|
||||
(add --ignore-checksum to additionally skip post-copy checksum checks)
|
||||
--localtime Use local time in listings (default: UTC)
|
||||
--no-cleanup Retain working files (useful for troubleshooting and testing).
|
||||
--workdir PATH Use custom working directory (useful for testing).
|
||||
|
|
@ -255,6 +257,34 @@ sync run times for very large numbers of files.
|
|||
The check may be run manually with `--check-sync=only`. It runs only the
|
||||
integrity check and terminates without actually synching.
|
||||
|
||||
|
||||
#### --ignore-listing-checksum
|
||||
|
||||
By default, bisync will retrieve (or generate) checksums (for backends that support them)
|
||||
when creating the listings for both paths, and store the checksums in the listing files.
|
||||
`--ignore-listing-checksum` will disable this behavior, which may speed things up considerably,
|
||||
especially on backends (such as [local](/local/)) where hashes must be computed on the fly instead of retrieved.
|
||||
Please note the following:
|
||||
|
||||
* While checksums are (by default) generated and stored in the listing files,
|
||||
they are NOT currently used for determining diffs (deltas).
|
||||
It is anticipated that full checksum support will be added in a future version.
|
||||
* `--ignore-listing-checksum` is NOT the same as [`--ignore-checksum`](/docs/#ignore-checksum),
|
||||
and you may wish to use one or the other, or both. In a nutshell:
|
||||
`--ignore-listing-checksum` controls whether checksums are considered when scanning for diffs,
|
||||
while `--ignore-checksum` controls whether checksums are considered during the copy/sync operations that follow,
|
||||
if there ARE diffs.
|
||||
* Unless `--ignore-listing-checksum` is passed, bisync currently computes hashes for one path
|
||||
*even when there's no common hash with the other path*
|
||||
(for example, a [crypt](/crypt/#modified-time-and-hashes) remote.)
|
||||
* If both paths support checksums and have a common hash,
|
||||
AND `--ignore-listing-checksum` was not specified when creating the listings,
|
||||
`--check-sync=only` can be used to compare Path1 vs. Path2 checksums (as of the time the previous listings were created.)
|
||||
However, `--check-sync=only` will NOT include checksums if the previous listings
|
||||
were generated on a run using `--ignore-listing-checksum`. For a more robust integrity check of the current state,
|
||||
consider using [`check`](commands/rclone_check/)
|
||||
(or [`cryptcheck`](/commands/rclone_cryptcheck/), if at least one path is a `crypt` remote.)
|
||||
|
||||
## Operation
|
||||
|
||||
### Runtime flow details
|
||||
|
|
@ -1134,3 +1164,5 @@ causing dry runs to inadvertently commit filter changes
|
|||
causing bisync to consider more files than necessary due to overbroad filters during delete operations
|
||||
* [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides)
|
||||
(identical files are now left alone instead of renamed)
|
||||
* Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes)
|
||||
to distinguish from `--ignore-checksum`
|
||||
|
|
|
|||
Loading…
Reference in New Issue