github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2025-01-20 01:07:29 +01:00
Code Releases Activity

Update to AWS SDK 0.10.0

Browse Source 
Tested with S3 and Dreamhost

Here is a link to the release notes:

http://aws.amazon.com/releasenotes/5476699172355228
This commit is contained in:
klauspost 2015-10-30 12:50:45 +01:00 committed by Nick Craig-Wood
parent b872ff0237
commit 345c98ed62
1 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
s3/s3.go
View File

@ -27,6 +27,7 @@ import (
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/ncw/rclone/fs"
@ -129,12 +130,13 @@ const (
// FsS3 represents a remote s3 server
type FsS3 struct {
name string // the name of the remote
c *s3.S3 // the connection to the s3 server
bucket string // the bucket we are working on
perm string // permissions for new buckets / objects
root string // root of the bucket - ignore all objects above this
locationConstraint string // location constraint of new buckets
name string // the name of the remote
c *s3.S3 // the connection to the s3 server
ses *session.Session // the s3 session
bucket string // the bucket we are working on
perm string // permissions for new buckets / objects
root string // root of the bucket - ignore all objects above this
locationConstraint string // location constraint of new buckets
}
// FsObjectS3 describes a s3 object
@ -190,7 +192,7 @@ func s3ParsePath(path string) (bucket, directory string, err error) {
}
// s3Connection makes a connection to s3
func s3Connection(name string) (*s3.S3, error) {
func s3Connection(name string) (*s3.S3, *session.Session, error) {
// Make the auth
accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
@ -200,9 +202,9 @@ func s3Connection(name string) (*s3.S3, error) {
fs.Debug(name, "Using anonymous access for S3")
auth = credentials.AnonymousCredentials
case accessKeyID == "":
return nil, errors.New("access_key_id not found")
return nil, nil, errors.New("access_key_id not found")
case secretAccessKey == "":
return nil, errors.New("secret_access_key not found")
return nil, nil, errors.New("secret_access_key not found")
default:
auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
}
@ -223,12 +225,13 @@ func s3Connection(name string) (*s3.S3, error) {
WithHTTPClient(fs.Config.Client()).
WithS3ForcePathStyle(true)
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
c := s3.New(awsConfig)
ses := session.New()
c := s3.New(ses, awsConfig)
if region == "other-v2-signature" {
fs.Debug(name, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Service.Config.Credentials == credentials.AnonymousCredentials {
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}
sign(accessKeyID, secretAccessKey, req.HTTPRequest)
@ -241,7 +244,7 @@ func s3Connection(name string) (*s3.S3, error) {
c.Handlers.Build.PushBack(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
})
return c, nil
return c, ses, nil
}
// NewFs contstructs an FsS3 from the path, bucket:path
@ -250,7 +253,7 @@ func NewFs(name, root string) (fs.Fs, error) {
if err != nil {
return nil, err
}
c, err := s3Connection(name)
c, ses, err := s3Connection(name)
if err != nil {
return nil, err
}
@ -258,6 +261,7 @@ func NewFs(name, root string) (fs.Fs, error) {
name: name,
c: c,
bucket: bucket,
ses: ses,
// FIXME perm: s3.Private, // FIXME need user to specify
root: directory,
locationConstraint: fs.ConfigFile.MustValue(name, "location_constraint"),
@ -674,13 +678,11 @@ func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
// Update the Object from in with modTime and size
func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
opts := s3manager.UploadOptions{
// PartSize: 64 * 1024 * 1024, use default
Concurrency: 2, // limit concurrency
LeavePartsOnError: false,
S3: o.s3.c,
}
uploader := s3manager.NewUploader(&opts)
uploader := s3manager.NewUploader(o.s3.ses, func(u *s3manager.Uploader) {
u.Concurrency = 2
u.LeavePartsOnError = false
u.S3 = o.s3.c
})
// Set the mtime in the meta data
metadata := map[string]*string{