Add Scaleway object storage provider (#93)

* Add Scaleway object storage provider

* Handle scaleway keys from env

README.md

@@ -101,6 +101,7 @@ SRC and DST must be an URI of the following object storage:
 - space: Digital Ocean Space
 - obs: Huawei Object Storage Service
 - oos: CTYun OOS
+- scw: Scaleway Object Storage
 
 SRC and DST should be in the following format:
 
@@ -125,7 +126,7 @@ Note:
 
 - It's recommended to run juicesync in the target region to have better performance.
 - Auto discover endpoint for bucket of S3, OSS, COS, OBS, BOS, `SRC` and `DST` can use format `NAME://[ACCESS_KEY:SECRET_KEY@]BUCKET[/PREFIX]` . `ACCESS_KEY` and `SECRET_KEY` can be provided by corresponding environment variables (see below).
-- S3: 
+- S3:
   * The access key and secret key for S3 could be provided by `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, or *IAM* role.
 - Wasb(Windows Azure Storage Blob)
   * The account name and account key can be provided as [connection string](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string#configure-a-connection-string-for-an-azure-storage-account) by `AZURE_STORAGE_CONNECTION_STRING`.
@@ -143,3 +144,5 @@ Note:
   The S3 endpoint should be used for Qiniu, for example, abc.cn-north-1-s3.qiniu.com.
   If there are keys starting with "/", the domain should be provided as `QINIU_DOMAIN`.
 - sftp: if your target machine uses SSH certificates instead of password, you should pass the path to your private key file to the environment variable `SSH_PRIVATE_KEY_PATH`, like ` SSH_PRIVATE_KEY_PATH=/home/someuser/.ssh/id_rsa juicesync [src] [dst]`.
+- Scaleway:
+  * The credential can be provided by environment variable `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` .

object/object_storage_test.go

@@ -371,3 +371,11 @@ func TestOOS(t *testing.T) {
 		os.Getenv("OOS_ACCESS_KEY"), os.Getenv("OOS_SECRET_KEY"))
 	testStorage(t, b)
 }
+
+func TestScw(t *testing.T) {
+	if os.Getenv("SCW_ACCESS_KEY") == "" {
+		t.SkipNow()
+	}
+	b, _ := newScw(fmt.Sprintf("https://%s", os.Getenv("SCW_TEST_BUCKET")), os.Getenv("SCW_ACCESS_KEY"), os.Getenv("SCW_SECRET_KEY"))
+	testStorage(t, b)
+}

object/scw.go

+// Copyright (C) 2018-present Juicedata Inc.
+
+package object
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+type scw struct {
+	s3client
+}
+
+func (s *scw) String() string {
+	return fmt.Sprintf("scw://%s", s.s3client.bucket)
+}
+
+func (s *scw) Create() error {
+	if _, err := s.List("", "", 1); err == nil {
+		return nil
+	}
+	_, err := s.s3.CreateBucket(&s3.CreateBucketInput{Bucket: &s.bucket})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeBucketAlreadyExists:
+				err = nil
+			case s3.ErrCodeBucketAlreadyOwnedByYou:
+				err = nil
+			}
+		}
+	}
+	return err
+}
+
+func newScw(endpoint, accessKey, secretKey string) (ObjectStorage, error) {
+	uri, err := url.ParseRequestURI(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid endpoint %s: %s", endpoint, err)
+	}
+	ssl := strings.ToLower(uri.Scheme) == "https"
+	hostParts := strings.Split(uri.Host, ".")
+	bucket := hostParts[0]
+	region := hostParts[2]
+	endpoint = uri.Host[len(bucket)+1:]
+
+	if accessKey == "" {
+		accessKey = os.Getenv("SCW_ACCESS_KEY")
+	}
+	if secretKey == "" {
+		secretKey = os.Getenv("SCW_SECRET_KEY")
+	}
+
+	awsConfig := &aws.Config{
+		Region:           &region,
+		Endpoint:         &endpoint,
+		DisableSSL:       aws.Bool(!ssl),
+		S3ForcePathStyle: aws.Bool(false),
+		HTTPClient:       httpClient,
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+	}
+
+	ses := session.New(awsConfig) //.WithLogLevel(aws.LogDebugWithHTTPBody))
+	return &scw{s3client{bucket, s3.New(ses), ses}}, nil
+}
+
+func init() {
+	Register("scw", newScw)
+}