mirror of https://github.com/rclone/rclone
Initial commit - some small parts working
This commit is contained in:
Nick Craig-Wood
commit
e9ae4f89a4
|
|
@ -0,0 +1,5 @@
|
|||
*~
|
||||
*.pyc
|
||||
test-env*
|
||||
junk/
|
||||
swiftsync
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
Swiftsync
|
||||
==========
|
||||
|
||||
Sync files and directories to and from swift
|
||||
|
||||
FIXME
|
||||
|
||||
|
||||
Install
|
||||
-------
|
||||
|
||||
Swiftsync is a Go program and comes as a single binary file.
|
||||
|
||||
Download the relevant binary from
|
||||
|
||||
- http://www.craig-wood.com/nick/pub/swiftsync/
|
||||
|
||||
Or alternatively if you have Go installed use
|
||||
|
||||
go get github.com/ncw/swiftsync
|
||||
|
||||
and this will build the binary in `$GOPATH/bin`. You can then modify
|
||||
the source and submit patches.
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
FIXME
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
This is free software under the terms of MIT the license (check the
|
||||
COPYING file included in this package).
|
||||
|
||||
Contact and support
|
||||
-------------------
|
||||
|
||||
The project website is at:
|
||||
|
||||
- https://github.com/ncw/swiftsync
|
||||
|
||||
There you can file bug reports, ask for help or contribute patches.
|
||||
|
||||
Authors
|
||||
-------
|
||||
|
||||
- Nick Craig-Wood <nick@craig-wood.com>
|
||||
|
||||
Contributors
|
||||
------------
|
||||
|
||||
- Your name goes here!
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
make 100% compatible with swift.py?
|
||||
|
||||
Make Env vars compatible with st?
|
||||
|
||||
Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
|
||||
|
||||
st is setting this
|
||||
'x-object-meta-mtime'
|
||||
|
||||
getmtime(filename)
|
||||
Return the last modification time of a file, reported by os.stat().
|
||||
|
||||
>>> f = os.path.getmtime("z")
|
||||
1347717491.343554
|
||||
>>> print f
|
||||
1347717491.34
|
||||
>>> str(f)
|
||||
'1347717491.34'
|
||||
>>> "%d" % f
|
||||
'1347717491'
|
||||
>>>
|
||||
|
||||
swift.py appears to be doing it wrong with str(float) which isn't a
|
||||
good way of stringifying floats...
|
||||
|
||||
Make
|
||||
|
||||
This also puts meta-mtime
|
||||
https://github.com/gholt/swiftly
|
||||
|
||||
As an integer, but it does parse it as a float
|
||||
subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
// Sync files and directories to and from swift
|
||||
//
|
||||
// Nick Craig-Wood <nick@craig-wood.com>
|
||||
package main
|
||||
|
||||
import (
|
||||
//"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
//"io"
|
||||
//"io/ioutil"
|
||||
"log"
|
||||
//"math/rand"
|
||||
"os"
|
||||
//"os/signal"
|
||||
//"path/filepath"
|
||||
//"regexp"
|
||||
//"runtime"
|
||||
"runtime/pprof"
|
||||
"strconv"
|
||||
"strings"
|
||||
//"sync"
|
||||
//"syscall"
|
||||
//"time"
|
||||
"github.com/ncw/swift"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
//fileSize = flag.Int64("s", 1E9, "Size of the check files")
|
||||
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
|
||||
//duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
|
||||
//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
|
||||
//logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
|
||||
|
||||
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
|
||||
verbose = flag.Bool("verbose", false, "Print lots more stuff")
|
||||
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
|
||||
// FIXME make these part of swift so we get a standard set of flags?
|
||||
authUrl = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.")
|
||||
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
|
||||
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
|
||||
)
|
||||
|
||||
// Turns a number of ns into a floating point string in seconds
|
||||
//
|
||||
// Trims trailing zeros and guaranteed to be perfectly accurate
|
||||
func nsToFloatString(ns int64) string {
|
||||
if ns < 0 {
|
||||
return "-" + nsToFloatString(-ns)
|
||||
}
|
||||
result := fmt.Sprintf("%010d", ns)
|
||||
split := len(result) - 9
|
||||
result, decimals := result[:split], result[split:]
|
||||
decimals = strings.TrimRight(decimals, "0")
|
||||
if decimals != "" {
|
||||
result += "."
|
||||
result += decimals
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Turns a floating point string in seconds into a ns integer
|
||||
//
|
||||
// Guaranteed to be perfectly accurate
|
||||
func floatStringToNs(s string) (ns int64, err error) {
|
||||
if s != "" && s[0] == '-' {
|
||||
ns, err = floatStringToNs(s[1:])
|
||||
return -ns, err
|
||||
}
|
||||
point := strings.IndexRune(s, '.')
|
||||
if point >= 0 {
|
||||
tail := s[point+1:]
|
||||
if len(tail) > 0 {
|
||||
if len(tail) > 9 {
|
||||
tail = tail[:9]
|
||||
}
|
||||
uns, err := strconv.ParseUint(tail, 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ns = int64(uns)
|
||||
for i := 9 - len(tail); i > 0; i-- {
|
||||
ns *= 10
|
||||
}
|
||||
}
|
||||
s = s[:point]
|
||||
}
|
||||
secs, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ns += int64(1000000000) * secs
|
||||
return ns, nil
|
||||
}
|
||||
|
||||
// syntaxError prints the syntax
|
||||
func syntaxError() {
|
||||
fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
|
||||
|
||||
FIXME
|
||||
|
||||
Full options:
|
||||
`)
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
// Exit with the message
|
||||
func fatal(message string, args ...interface{}) {
|
||||
syntaxError()
|
||||
fmt.Fprintf(os.Stderr, message, args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Usage = syntaxError
|
||||
flag.Parse()
|
||||
//args := flag.Args()
|
||||
//runtime.GOMAXPROCS(3)
|
||||
|
||||
// Setup profiling if desired
|
||||
if *cpuprofile != "" {
|
||||
f, err := os.Create(*cpuprofile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
pprof.StartCPUProfile(f)
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
// if len(args) < 1 {
|
||||
// fatal("No command supplied\n")
|
||||
// }
|
||||
|
||||
if *userName == "" {
|
||||
log.Fatal("Need --user or environmental variable ST_USER")
|
||||
}
|
||||
if *apiKey == "" {
|
||||
log.Fatal("Need --key or environmental variable ST_KEY")
|
||||
}
|
||||
if *authUrl == "" {
|
||||
log.Fatal("Need --auth or environmental variable ST_AUTH")
|
||||
}
|
||||
c := swift.Connection{
|
||||
UserName: *userName,
|
||||
ApiKey: *apiKey,
|
||||
AuthUrl: *authUrl,
|
||||
}
|
||||
err := c.Authenticate()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to authenticate", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
// Tests for swiftsync
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNsToFloatString(t *testing.T) {
|
||||
for _, d := range []struct {
|
||||
ns int64
|
||||
fs string
|
||||
}{
|
||||
{0, "0"},
|
||||
{1, "0.000000001"},
|
||||
{1000, "0.000001"},
|
||||
{1000000, "0.001"},
|
||||
{100000000, "0.1"},
|
||||
{1000000000, "1"},
|
||||
{10000000000, "10"},
|
||||
{12345678912, "12.345678912"},
|
||||
{12345678910, "12.34567891"},
|
||||
{12345678900, "12.3456789"},
|
||||
{12345678000, "12.345678"},
|
||||
{12345670000, "12.34567"},
|
||||
{12345600000, "12.3456"},
|
||||
{12345000000, "12.345"},
|
||||
{12340000000, "12.34"},
|
||||
{12300000000, "12.3"},
|
||||
{12000000000, "12"},
|
||||
{10000000000, "10"},
|
||||
{1347717491123123123, "1347717491.123123123"},
|
||||
} {
|
||||
if nsToFloatString(d.ns) != d.fs {
|
||||
t.Error("Failed", d.ns, "!=", d.fs)
|
||||
}
|
||||
if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
|
||||
t.Error("Failed on negative", d.ns, "!=", d.fs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFloatStringToNs(t *testing.T) {
|
||||
for _, d := range []struct {
|
||||
ns int64
|
||||
fs string
|
||||
}{
|
||||
{0, "0"},
|
||||
{0, "0."},
|
||||
{0, "0.0"},
|
||||
{0, "0.0000000001"},
|
||||
{1, "0.000000001"},
|
||||
{1000, "0.000001"},
|
||||
{1000000, "0.001"},
|
||||
{100000000, "0.1"},
|
||||
{100000000, "0.10"},
|
||||
{100000000, "0.1000000001"},
|
||||
{1000000000, "1"},
|
||||
{1000000000, "1."},
|
||||
{1000000000, "1.0"},
|
||||
{10000000000, "10"},
|
||||
{12345678912, "12.345678912"},
|
||||
{12345678912, "12.3456789129"},
|
||||
{12345678912, "12.34567891299"},
|
||||
{12345678910, "12.34567891"},
|
||||
{12345678900, "12.3456789"},
|
||||
{12345678000, "12.345678"},
|
||||
{12345670000, "12.34567"},
|
||||
{12345600000, "12.3456"},
|
||||
{12345000000, "12.345"},
|
||||
{12340000000, "12.34"},
|
||||
{12300000000, "12.3"},
|
||||
{12000000000, "12"},
|
||||
{10000000000, "10"},
|
||||
// This is a typical value which has more bits in than a float64
|
||||
{1347717491123123123, "1347717491.123123123"},
|
||||
} {
|
||||
ns, err := floatStringToNs(d.fs)
|
||||
if err != nil {
|
||||
t.Error("Failed conversion", err)
|
||||
}
|
||||
if ns != d.ns {
|
||||
t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
|
||||
}
|
||||
if d.ns > 0 {
|
||||
ns, err := floatStringToNs("-" + d.fs)
|
||||
if err != nil {
|
||||
t.Error("Failed conversion", err)
|
||||
}
|
||||
if ns != -d.ns {
|
||||
t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// These are expected to produce errors
|
||||
for _, fs := range []string{
|
||||
"",
|
||||
".0",
|
||||
" 1",
|
||||
"- 1",
|
||||
"- 1",
|
||||
"1.-1",
|
||||
"1.0.0",
|
||||
"1x0",
|
||||
} {
|
||||
ns, err := floatStringToNs(fs)
|
||||
if err == nil {
|
||||
t.Error("Didn't produce expected error", fs, ns)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue