Support RAM role for alicloud OSS (#55)

* add ecs ram role support when using alicloud oss

* add doc

* Update oss.go
Co-authored-by: Nchnliyong <liyong@juicedata.io>
Co-authored-by: NDavies Liu <davies@juicedata.io>

README.md

@@ -115,7 +115,8 @@ Note:
 - S3: The access key and secret key for S3 could be provided by AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or IAM role.
 - COS: The AppID should be part of the bucket name.
 - GCS: The machine should be authorized to access Google Cloud Storage.
+- OSS: The credential can be provided by RAM role.
 - Qiniu:
   The S3 endpoint should be used for Qiniu, for example, abc.cn-north-1-s3.qiniu.com.
   If there are keys starting with "/", the domain should be provided as QINIU_DOMAIN.
-- sftp: if your target machine uses SSH certificates instead of password, you should pass the path to your private key file to the environment variable `SSH_PRIVATE_KEY_PATH`, like ` SSH_PRIVATE_KEY_PATH=/home/someuser/.ssh/id_rsa juicesync [src] [dst]`, and then leave password empty.
\ No newline at end of file
+- sftp: if your target machine uses SSH certificates instead of password, you should pass the path to your private key file to the environment variable `SSH_PRIVATE_KEY_PATH`, like ` SSH_PRIVATE_KEY_PATH=/home/someuser/.ssh/id_rsa juicesync [src] [dst]`, and then leave password empty.

object/oss.go

@@ -4,10 +4,14 @@ package object
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 )
@@ -129,6 +133,47 @@ func (o *ossClient) ListUploads(marker string) ([]*PendingPart, string, error) {
 	return parts, result.NextKeyMarker, nil
 }
 
+type stsCred struct {
+	AccessKeyId     string
+	AccessKeySecret string
+	Expiration      string
+	SecurityToken   string
+	LastUpdated     string
+	Code            string
+}
+
+func fetchStsCred() (*stsCred, error) {
+	url := "http://100.100.100.200/latest/meta-data/Ram/security-credentials/"
+	req, _ := http.NewRequest("GET", url, nil)
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	role := string(d)
+	req, err = http.NewRequest("GET", url+role, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err = httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	d, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var cred stsCred
+	err = json.Unmarshal(d, &cred)
+	if err != nil {
+		return nil, err
+	}
+	return &cred, nil
+}
+
 func newOSS(endpoint, accessKey, secretKey string) ObjectStorage {
 	uri, err := url.ParseRequestURI(endpoint)
 	if err != nil {
@@ -138,14 +183,48 @@ func newOSS(endpoint, accessKey, secretKey string) ObjectStorage {
 	bucketName := hostParts[0]
 	domain := uri.Scheme + "://" + hostParts[1]
 
-	client, err := oss.New(domain, accessKey, secretKey)
+	var client *oss.Client
+	if accessKey != "" {
+		client, err = oss.New(domain, accessKey, secretKey)
+	} else {
+		cred, err := fetchStsCred()
+		if err != nil {
+			logger.Fatalf("No credential provided for OSS")
+		}
+		client, err = oss.New(domain, cred.AccessKeyId, cred.AccessKeySecret,
+			oss.SecurityToken(cred.SecurityToken))
+		go func() {
+			for {
+				cred, err := fetchStsCred()
+				if err == nil {
+					client.Config.AccessKeyID = cred.AccessKeyId
+					client.Config.AccessKeySecret = cred.AccessKeySecret
+					client.Config.SecurityToken = cred.SecurityToken
+					logger.Debugf("Refreshed STS, will be expired at %s", cred.Expiration)
+					expire, err := time.Parse("2006-01-02T15:04:05Z", cred.Expiration)
+					if err == nil {
+						time.Sleep(expire.Sub(time.Now()) / 2)
+					}
+				}
+			}
+		}()
+	}
 	if err != nil {
 		logger.Fatalf("Cannot create OSS client with endpoint %s: %s", endpoint, err)
 	}
+
+	client.Config.Timeout = 10
+	client.Config.RetryTimes = 1
+	client.Config.HTTPTimeout.ConnectTimeout = time.Second * 2   // 30s
+	client.Config.HTTPTimeout.ReadWriteTimeout = time.Second * 5 // 60s
+	client.Config.HTTPTimeout.HeaderTimeout = time.Second * 5    // 60s
+	client.Config.HTTPTimeout.LongTimeout = time.Second * 30     // 300s
+
 	bucket, err := client.Bucket(bucketName)
 	if err != nil {
 		logger.Fatalf("Cannot create bucket %s: %s", bucketName, err)
 	}
+
 	return &ossClient{client: client, bucket: bucket}
 }