From e9ae4f89a44b1cac8e83ede1572f15dace692c67 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 18 Nov 2012 17:32:31 +0000 Subject: [PATCH] Initial commit - some small parts working --- .gitignore | 5 ++ COPYING | 20 ++++++ README.md | 53 ++++++++++++++++ notes.txt | 32 ++++++++++ swiftsync.go | 155 ++++++++++++++++++++++++++++++++++++++++++++++ swiftsync_test.go | 112 +++++++++++++++++++++++++++++++++ 6 files changed, 377 insertions(+) create mode 100644 .gitignore create mode 100644 COPYING create mode 100644 README.md create mode 100644 notes.txt create mode 100644 swiftsync.go create mode 100644 swiftsync_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..3d63fd7a2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*~ +*.pyc +test-env* +junk/ +swiftsync diff --git a/COPYING b/COPYING new file mode 100644 index 000000000..8c27c67fd --- /dev/null +++ b/COPYING @@ -0,0 +1,20 @@ +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 000000000..0e416a57e --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +Swiftsync +========== + +Sync files and directories to and from swift + +FIXME + + +Install +------- + +Swiftsync is a Go program and comes as a single binary file. + +Download the relevant binary from + +- http://www.craig-wood.com/nick/pub/swiftsync/ + +Or alternatively if you have Go installed use + + go get github.com/ncw/swiftsync + +and this will build the binary in `$GOPATH/bin`. You can then modify +the source and submit patches. + +Usage +----- + +FIXME + +License +------- + +This is free software under the terms of MIT the license (check the +COPYING file included in this package). + +Contact and support +------------------- + +The project website is at: + +- https://github.com/ncw/swiftsync + +There you can file bug reports, ask for help or contribute patches. + +Authors +------- + +- Nick Craig-Wood + +Contributors +------------ + +- Your name goes here! diff --git a/notes.txt b/notes.txt new file mode 100644 index 000000000..4878cade4 --- /dev/null +++ b/notes.txt @@ -0,0 +1,32 @@ +make 100% compatible with swift.py? + +Make Env vars compatible with st? + +Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file? + +st is setting this +'x-object-meta-mtime' + + getmtime(filename) + Return the last modification time of a file, reported by os.stat(). + +>>> f = os.path.getmtime("z") +1347717491.343554 +>>> print f +1347717491.34 +>>> str(f) +'1347717491.34' +>>> "%d" % f +'1347717491' +>>> + +swift.py appears to be doing it wrong with str(float) which isn't a +good way of stringifying floats... + +Make + +This also puts meta-mtime +https://github.com/gholt/swiftly + +As an integer, but it does parse it as a float +subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_)) diff --git a/swiftsync.go b/swiftsync.go new file mode 100644 index 000000000..b3e05f9ca --- /dev/null +++ b/swiftsync.go @@ -0,0 +1,155 @@ +// Sync files and directories to and from swift +// +// Nick Craig-Wood +package main + +import ( + //"bytes" + "flag" + "fmt" + //"io" + //"io/ioutil" + "log" + //"math/rand" + "os" + //"os/signal" + //"path/filepath" + //"regexp" + //"runtime" + "runtime/pprof" + "strconv" + "strings" + //"sync" + //"syscall" + //"time" + "github.com/ncw/swift" +) + +// Globals +var ( + // Flags + //fileSize = flag.Int64("s", 1E9, "Size of the check files") + cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file") + //duration = flag.Duration("duration", time.Hour*24, "Duration to run test") + //statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats") + //logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore") + + snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented + verbose = flag.Bool("verbose", false, "Print lots more stuff") + quiet = flag.Bool("quiet", false, "Print as little stuff as possible") + // FIXME make these part of swift so we get a standard set of flags? + authUrl = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.") + userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.") + apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.") +) + +// Turns a number of ns into a floating point string in seconds +// +// Trims trailing zeros and guaranteed to be perfectly accurate +func nsToFloatString(ns int64) string { + if ns < 0 { + return "-" + nsToFloatString(-ns) + } + result := fmt.Sprintf("%010d", ns) + split := len(result) - 9 + result, decimals := result[:split], result[split:] + decimals = strings.TrimRight(decimals, "0") + if decimals != "" { + result += "." + result += decimals + } + return result +} + +// Turns a floating point string in seconds into a ns integer +// +// Guaranteed to be perfectly accurate +func floatStringToNs(s string) (ns int64, err error) { + if s != "" && s[0] == '-' { + ns, err = floatStringToNs(s[1:]) + return -ns, err + } + point := strings.IndexRune(s, '.') + if point >= 0 { + tail := s[point+1:] + if len(tail) > 0 { + if len(tail) > 9 { + tail = tail[:9] + } + uns, err := strconv.ParseUint(tail, 10, 64) + if err != nil { + return 0, err + } + ns = int64(uns) + for i := 9 - len(tail); i > 0; i-- { + ns *= 10 + } + } + s = s[:point] + } + secs, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, err + } + ns += int64(1000000000) * secs + return ns, nil +} + +// syntaxError prints the syntax +func syntaxError() { + fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift + +FIXME + +Full options: +`) + flag.PrintDefaults() +} + +// Exit with the message +func fatal(message string, args ...interface{}) { + syntaxError() + fmt.Fprintf(os.Stderr, message, args...) + os.Exit(1) +} + +func main() { + flag.Usage = syntaxError + flag.Parse() + //args := flag.Args() + //runtime.GOMAXPROCS(3) + + // Setup profiling if desired + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + + // if len(args) < 1 { + // fatal("No command supplied\n") + // } + + if *userName == "" { + log.Fatal("Need --user or environmental variable ST_USER") + } + if *apiKey == "" { + log.Fatal("Need --key or environmental variable ST_KEY") + } + if *authUrl == "" { + log.Fatal("Need --auth or environmental variable ST_AUTH") + } + c := swift.Connection{ + UserName: *userName, + ApiKey: *apiKey, + AuthUrl: *authUrl, + } + err := c.Authenticate() + if err != nil { + log.Fatal("Failed to authenticate", err) + } + +} diff --git a/swiftsync_test.go b/swiftsync_test.go new file mode 100644 index 000000000..a453b5f10 --- /dev/null +++ b/swiftsync_test.go @@ -0,0 +1,112 @@ +// Tests for swiftsync +package main + +import ( + "testing" +) + +func TestNsToFloatString(t *testing.T) { + for _, d := range []struct { + ns int64 + fs string + }{ + {0, "0"}, + {1, "0.000000001"}, + {1000, "0.000001"}, + {1000000, "0.001"}, + {100000000, "0.1"}, + {1000000000, "1"}, + {10000000000, "10"}, + {12345678912, "12.345678912"}, + {12345678910, "12.34567891"}, + {12345678900, "12.3456789"}, + {12345678000, "12.345678"}, + {12345670000, "12.34567"}, + {12345600000, "12.3456"}, + {12345000000, "12.345"}, + {12340000000, "12.34"}, + {12300000000, "12.3"}, + {12000000000, "12"}, + {10000000000, "10"}, + {1347717491123123123, "1347717491.123123123"}, + } { + if nsToFloatString(d.ns) != d.fs { + t.Error("Failed", d.ns, "!=", d.fs) + } + if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs { + t.Error("Failed on negative", d.ns, "!=", d.fs) + } + } +} + +func TestFloatStringToNs(t *testing.T) { + for _, d := range []struct { + ns int64 + fs string + }{ + {0, "0"}, + {0, "0."}, + {0, "0.0"}, + {0, "0.0000000001"}, + {1, "0.000000001"}, + {1000, "0.000001"}, + {1000000, "0.001"}, + {100000000, "0.1"}, + {100000000, "0.10"}, + {100000000, "0.1000000001"}, + {1000000000, "1"}, + {1000000000, "1."}, + {1000000000, "1.0"}, + {10000000000, "10"}, + {12345678912, "12.345678912"}, + {12345678912, "12.3456789129"}, + {12345678912, "12.34567891299"}, + {12345678910, "12.34567891"}, + {12345678900, "12.3456789"}, + {12345678000, "12.345678"}, + {12345670000, "12.34567"}, + {12345600000, "12.3456"}, + {12345000000, "12.345"}, + {12340000000, "12.34"}, + {12300000000, "12.3"}, + {12000000000, "12"}, + {10000000000, "10"}, + // This is a typical value which has more bits in than a float64 + {1347717491123123123, "1347717491.123123123"}, + } { + ns, err := floatStringToNs(d.fs) + if err != nil { + t.Error("Failed conversion", err) + } + if ns != d.ns { + t.Error("Failed", d.fs, "!=", d.ns, "was", ns) + } + if d.ns > 0 { + ns, err := floatStringToNs("-" + d.fs) + if err != nil { + t.Error("Failed conversion", err) + } + if ns != -d.ns { + t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs) + } + } + } + + // These are expected to produce errors + for _, fs := range []string{ + "", + ".0", + " 1", + "- 1", + "- 1", + "1.-1", + "1.0.0", + "1x0", + } { + ns, err := floatStringToNs(fs) + if err == nil { + t.Error("Didn't produce expected error", fs, ns) + } + } + +}