2013-01-08 19:53:35 +01:00
// S3 interface
2013-06-27 21:13:07 +02:00
package s3
2013-01-08 19:53:35 +01:00
2013-01-23 23:43:20 +01:00
// FIXME need to prevent anything but ListDir working for s3://
2013-01-08 19:53:35 +01:00
import (
2013-06-27 21:13:07 +02:00
"../fs"
2013-01-08 19:53:35 +01:00
"errors"
"flag"
"fmt"
"github.com/ncw/swift"
"io"
"launchpad.net/goamz/aws"
"launchpad.net/goamz/s3"
"log"
"mime"
"net/http"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
)
2013-06-27 21:13:07 +02:00
// Pattern to match a s3 url
var Match = regexp . MustCompile ( ` ^s3://([^/]*)(.*)$ ` )
// Register with Fs
func init ( ) {
fs . Register ( Match , NewFs )
}
2013-01-08 19:53:35 +01:00
// Constants
const (
metaMtime = "X-Amz-Meta-Mtime" // the meta key to store mtime in
)
// FsS3 represents a remote s3 server
type FsS3 struct {
c * s3 . S3 // the connection to the s3 server
b * s3 . Bucket // the connection to the bucket
bucket string // the bucket we are working on
perm s3 . ACL // permissions for new buckets / objects
}
// FsObjectS3 describes a s3 object
type FsObjectS3 struct {
// Will definitely have everything but meta which may be nil
//
// List will read everything but meta - to fill that in need to call
// readMetaData
s3 * FsS3 // what this object is part of
remote string // The remote path
etag string // md5sum of the object
bytes int64 // size of the object
lastModified time . Time // Last modified
meta s3 . Headers // The object metadata if known - may be nil
}
// ------------------------------------------------------------
// Globals
var (
// Flags
awsAccessKeyId = flag . String ( "aws-access-key-id" , os . Getenv ( "AWS_ACCESS_KEY_ID" ) , "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID." )
awsSecretAccessKey = flag . String ( "aws-secret-access-key" , os . Getenv ( "AWS_SECRET_ACCESS_KEY" ) , "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY." )
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
s3Endpoint = flag . String ( "s3-endpoint" , os . Getenv ( "S3_ENDPOINT" ) , "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/." )
s3LocationConstraint = flag . String ( "s3-location-constraint" , os . Getenv ( "S3_LOCATION_CONSTRAINT" ) , "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT." )
)
// String converts this FsS3 to a string
func ( f * FsS3 ) String ( ) string {
return fmt . Sprintf ( "S3 bucket %s" , f . bucket )
}
// parseParse parses a s3 'url'
func s3ParsePath ( path string ) ( bucket , directory string , err error ) {
2013-06-27 21:13:07 +02:00
parts := Match . FindAllStringSubmatch ( path , - 1 )
2013-01-08 19:53:35 +01:00
if len ( parts ) != 1 || len ( parts [ 0 ] ) != 3 {
err = fmt . Errorf ( "Couldn't parse s3 url %q" , path )
} else {
bucket , directory = parts [ 0 ] [ 1 ] , parts [ 0 ] [ 2 ]
directory = strings . Trim ( directory , "/" )
}
return
}
// s3Connection makes a connection to s3
func s3Connection ( ) ( * s3 . S3 , error ) {
// Make the auth
if * awsAccessKeyId == "" {
return nil , errors . New ( "Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID" )
}
if * awsSecretAccessKey == "" {
return nil , errors . New ( "Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY" )
}
2013-01-17 22:37:28 +01:00
auth := aws . Auth { AccessKey : * awsAccessKeyId , SecretKey : * awsSecretAccessKey }
2013-01-08 19:53:35 +01:00
// FIXME look through all the regions by name and use one of them if found
// Synthesize the region
if * s3Endpoint == "" {
* s3Endpoint = "https://s3.amazonaws.com/"
}
region := aws . Region {
Name : "s3" ,
S3Endpoint : * s3Endpoint ,
S3LocationConstraint : false ,
}
if * s3LocationConstraint != "" {
region . Name = * s3LocationConstraint
region . S3LocationConstraint = true
}
c := s3 . New ( auth , region )
return c , nil
}
// NewFsS3 contstructs an FsS3 from the path, bucket:path
2013-06-27 21:13:07 +02:00
func NewFs ( path string ) ( fs . Fs , error ) {
2013-01-08 19:53:35 +01:00
bucket , directory , err := s3ParsePath ( path )
if err != nil {
return nil , err
}
if directory != "" {
return nil , fmt . Errorf ( "Directories not supported yet in %q: %q" , path , directory )
}
c , err := s3Connection ( )
if err != nil {
return nil , err
}
f := & FsS3 {
c : c ,
bucket : bucket ,
b : c . Bucket ( bucket ) ,
perm : s3 . Private , // FIXME need user to specify
}
return f , nil
}
// Return an FsObject from a path
//
// May return nil if an error occurred
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) NewFsObjectWithInfo ( remote string , info * s3 . Key ) fs . FsObject {
o := & FsObjectS3 {
2013-01-08 19:53:35 +01:00
s3 : f ,
remote : remote ,
}
if info != nil {
// Set info but not meta
var err error
2013-06-27 21:13:07 +02:00
o . lastModified , err = time . Parse ( time . RFC3339 , info . LastModified )
2013-01-08 19:53:35 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . FsLog ( o , "Failed to read last modified: %s" , err )
o . lastModified = time . Now ( )
2013-01-08 19:53:35 +01:00
}
2013-06-27 21:13:07 +02:00
o . etag = info . ETag
o . bytes = info . Size
2013-01-08 19:53:35 +01:00
} else {
2013-06-27 21:13:07 +02:00
err := o . readMetaData ( ) // reads info and meta, returning an error
2013-01-08 19:53:35 +01:00
if err != nil {
// logged already FsDebug("Failed to read info: %s", err)
return nil
}
}
2013-06-27 21:13:07 +02:00
return o
2013-01-08 19:53:35 +01:00
}
// Return an FsObject from a path
//
// May return nil if an error occurred
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) NewFsObject ( remote string ) fs . FsObject {
2013-01-08 19:53:35 +01:00
return f . NewFsObjectWithInfo ( remote , nil )
}
// Walk the path returning a channel of FsObjects
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) List ( ) fs . FsObjectsChan {
out := make ( fs . FsObjectsChan , fs . Config . Checkers )
2013-01-08 19:53:35 +01:00
go func ( ) {
// FIXME need to implement ALL loop
objects , err := f . b . List ( "" , "" , "" , 10000 )
if err != nil {
2013-06-27 21:13:07 +02:00
fs . Stats . Error ( )
2013-01-08 19:53:35 +01:00
log . Printf ( "Couldn't read bucket %q: %s" , f . bucket , err )
} else {
for i := range objects . Contents {
object := & objects . Contents [ i ]
if fs := f . NewFsObjectWithInfo ( object . Key , object ) ; fs != nil {
out <- fs
}
}
}
close ( out )
} ( )
return out
}
2013-01-23 23:43:20 +01:00
// Lists the buckets
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) ListDir ( ) fs . FsDirChan {
out := make ( fs . FsDirChan , fs . Config . Checkers )
2013-01-23 23:43:20 +01:00
go func ( ) {
defer close ( out )
2013-02-07 00:31:42 +01:00
buckets , err := f . c . ListBuckets ( )
2013-01-23 23:43:20 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . Stats . Error ( )
2013-01-23 23:43:20 +01:00
log . Printf ( "Couldn't list buckets: %s" , err )
} else {
2013-02-07 00:31:42 +01:00
for _ , bucket := range buckets {
2013-06-27 21:13:07 +02:00
out <- & fs . FsDir {
2013-01-23 23:43:20 +01:00
Name : bucket . Name ,
2013-06-27 20:46:31 +02:00
When : bucket . CreationDate ,
2013-01-23 23:43:20 +01:00
Bytes : - 1 ,
Count : - 1 ,
}
}
}
} ( )
return out
}
2013-01-08 19:53:35 +01:00
// Put the FsObject into the bucket
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) Put ( in io . Reader , remote string , modTime time . Time , size int64 ) ( fs . FsObject , error ) {
2013-01-08 19:53:35 +01:00
// Temporary FsObject under construction
2013-01-10 22:58:46 +01:00
fs := & FsObjectS3 { s3 : f , remote : remote }
2013-01-08 19:53:35 +01:00
// Set the mtime in the headers
headers := s3 . Headers {
2013-01-10 22:58:46 +01:00
metaMtime : swift . TimeToFloatString ( modTime ) ,
2013-01-08 19:53:35 +01:00
}
// Guess the content type
2013-01-10 22:58:46 +01:00
contentType := mime . TypeByExtension ( path . Ext ( remote ) )
2013-01-08 19:53:35 +01:00
if contentType == "" {
contentType = "application/octet-stream"
}
2013-01-10 22:58:46 +01:00
_ , err := fs . s3 . b . PutReaderHeaders ( remote , in , size , contentType , f . perm , headers )
return fs , err
2013-01-08 19:53:35 +01:00
}
// Mkdir creates the bucket if it doesn't exist
func ( f * FsS3 ) Mkdir ( ) error {
2013-01-08 23:31:16 +01:00
err := f . b . PutBucket ( f . perm )
if err , ok := err . ( * s3 . Error ) ; ok {
if err . Code == "BucketAlreadyOwnedByYou" {
return nil
}
}
return err
2013-01-08 19:53:35 +01:00
}
// Rmdir deletes the bucket
//
// Returns an error if it isn't empty
func ( f * FsS3 ) Rmdir ( ) error {
return f . b . DelBucket ( )
}
2013-01-19 00:21:02 +01:00
// Return the precision
2013-06-27 21:13:07 +02:00
func ( f * FsS3 ) Precision ( ) time . Duration {
2013-01-19 00:21:02 +01:00
return time . Nanosecond
}
2013-01-08 19:53:35 +01:00
// ------------------------------------------------------------
// Return the remote path
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Remote ( ) string {
return o . remote
2013-01-08 19:53:35 +01:00
}
// Md5sum returns the Md5sum of an object returning a lowercase hex string
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Md5sum ( ) ( string , error ) {
return strings . Trim ( strings . ToLower ( o . etag ) , ` " ` ) , nil
2013-01-08 19:53:35 +01:00
}
// Size returns the size of an object in bytes
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Size ( ) int64 {
return o . bytes
2013-01-08 19:53:35 +01:00
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) readMetaData ( ) ( err error ) {
if o . meta != nil {
2013-01-08 19:53:35 +01:00
return nil
}
2013-06-27 21:13:07 +02:00
headers , err := o . s3 . b . Head ( o . remote , nil )
2013-01-08 19:53:35 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . FsDebug ( o , "Failed to read info: %s" , err )
2013-01-08 19:53:35 +01:00
return err
}
size , err := strconv . ParseInt ( headers [ "Content-Length" ] , 10 , 64 )
if err != nil {
2013-06-27 21:13:07 +02:00
fs . FsDebug ( o , "Failed to read size from: %q" , headers )
2013-01-08 19:53:35 +01:00
return err
}
2013-06-27 21:13:07 +02:00
o . etag = headers [ "Etag" ]
o . bytes = size
o . meta = headers
if o . lastModified , err = time . Parse ( http . TimeFormat , headers [ "Last-Modified" ] ) ; err != nil {
fs . FsLog ( o , "Failed to read last modified from HEAD: %s" , err )
o . lastModified = time . Now ( )
2013-01-08 19:53:35 +01:00
}
return nil
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) ModTime ( ) time . Time {
err := o . readMetaData ( )
2013-01-08 19:53:35 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . FsLog ( o , "Failed to read metadata: %s" , err )
2013-01-08 19:53:35 +01:00
return time . Now ( )
}
// read mtime out of metadata if available
2013-06-27 21:13:07 +02:00
d , ok := o . meta [ metaMtime ]
2013-01-08 19:53:35 +01:00
if ! ok {
2013-06-27 21:13:07 +02:00
// fs.FsDebug(o, "No metadata")
return o . lastModified
2013-01-08 19:53:35 +01:00
}
modTime , err := swift . FloatStringToTime ( d )
if err != nil {
2013-06-27 21:13:07 +02:00
fs . FsLog ( o , "Failed to read mtime from object: %s" , err )
return o . lastModified
2013-01-08 19:53:35 +01:00
}
return modTime
}
// Sets the modification time of the local fs object
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) SetModTime ( modTime time . Time ) {
err := o . readMetaData ( )
2013-01-08 23:31:16 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . Stats . Error ( )
fs . FsLog ( o , "Failed to read metadata: %s" , err )
2013-01-08 23:31:16 +01:00
return
}
2013-06-27 21:13:07 +02:00
o . meta [ metaMtime ] = swift . TimeToFloatString ( modTime )
_ , err = o . s3 . b . Update ( o . remote , o . s3 . perm , o . meta )
2013-01-08 23:31:16 +01:00
if err != nil {
2013-06-27 21:13:07 +02:00
fs . Stats . Error ( )
fs . FsLog ( o , "Failed to update remote mtime: %s" , err )
2013-01-08 23:31:16 +01:00
}
2013-01-08 19:53:35 +01:00
}
// Is this object storable
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Storable ( ) bool {
2013-01-08 19:53:35 +01:00
return true
}
// Open an object for read
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Open ( ) ( in io . ReadCloser , err error ) {
in , err = o . s3 . b . GetReader ( o . remote )
2013-01-08 19:53:35 +01:00
return
}
// Remove an object
2013-06-27 21:13:07 +02:00
func ( o * FsObjectS3 ) Remove ( ) error {
return o . s3 . b . Del ( o . remote )
2013-01-08 19:53:35 +01:00
}
// Check the interfaces are satisfied
2013-06-27 21:13:07 +02:00
var _ fs . Fs = & FsS3 { }
var _ fs . FsObject = & FsObjectS3 { }