add auto find endpoint suffix for azure blob storage (#83)

README.md

@@ -113,6 +113,7 @@ Some examples:
 - hdfs://hdfs@namenode1:9000,namenode2:9000/user/
 - s3://my-bucket/
 - s3://access-key:secret-key-id@my-bucket/prefix
+- wasb://account-name:account-key@my-container/prefix
 - gcs://my-bucket.us-west1.googleapi.com/
 - oss://test
 - cos://test-1234
@@ -125,12 +126,14 @@ Note:
 - Auto discover endpoint for bucket of S3, OSS, COS, OBS, BOS, `SRC` and `DST` can use format `NAME://[ACCESS_KEY:SECRET_KEY@]BUCKET[/PREFIX]` . `ACCESS_KEY` and `SECRET_KEY` can be provided by corresponding environment variables (see below).
 - S3: 
   * The access key and secret key for S3 could be provided by `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, or *IAM* role.
-- COS: 
-  * The AppID should be part of the bucket name.
-  * The credential can be provided by environment variable `COS_SECRETID` and `COS_SECRETKEY`.
+- Wasb(Windows Azure Storage Blob)
+  * The account name and account key can be provided as [connection string](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string#configure-a-connection-string-for-an-azure-storage-account) by `AZURE_STORAGE_CONNECTION_STRING`.
 - GCS: The machine should be authorized to access Google Cloud Storage.
 - OSS:
   * The credential can be provided by environment variable `ALICLOUD_ACCESS_KEY_ID` and `ALICLOUD_ACCESS_KEY_SECRET` , RAM role, [EMR MetaService](https://help.aliyun.com/document_detail/43966.html).
+- COS:
+  * The AppID should be part of the bucket name.
+  * The credential can be provided by environment variable `COS_SECRETID` and `COS_SECRETKEY`.
 - OBS:
   * The credential can be provided by environment variable `HWCLOUD_ACCESS_KEY` and `HWCLOUD_SECRET_KEY` .
 - BOS:

object/azure.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -105,14 +106,78 @@ func (b *wasb) List(prefix, marker string, limit int64) ([]*Object, error) {
 
 // TODO: support multipart upload
 
-func newWabs(endpoint, account, key string) ObjectStorage {
+func autoWasbEndpoint(containerName, accountName, accountKey string, useHTTPS bool) (string, error) {
+	baseURLs := []string{"core.windows.net", "core.chinacloudapi.cn"}
+	endpoint := ""
+	for _, baseURL := range baseURLs {
+		client, err := storage.NewClient(accountName, accountKey, baseURL, "2017-04-17", useHTTPS)
+		if err != nil {
+			log.Fatalf("Failed to create client: %v", err)
+		}
+		blobService := client.GetBlobService()
+		resp, err := blobService.ListContainers(storage.ListContainersParameters{Prefix: containerName, MaxResults: 1})
+		if err != nil {
+			logger.Debugf("Try to list containers at %s failed: %s", baseURL, err)
+			continue
+		}
+		if len(resp.Containers) == 1 {
+			endpoint = baseURL
+			break
+		}
+	}
+
+	if endpoint == "" {
+		return "", fmt.Errorf("fail to get endpoint for container %s", containerName)
+	}
+	return endpoint, nil
+}
+
+func newWabs(endpoint, accountName, accountKey string) ObjectStorage {
 	uri, err := url.ParseRequestURI(endpoint)
 	if err != nil {
 		log.Fatalf("Invalid endpoint: %v, error: %v", endpoint, err)
 	}
 	hostParts := strings.SplitN(uri.Host, ".", 2)
+
+	scheme := ""
+	domain := ""
+	// Connection string support: DefaultEndpointsProtocol=[http|https];AccountName=***;AccountKey=***;EndpointSuffix=[core.windows.net|core.chinacloudapi.cn]
+	if connString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING"); connString != "" {
+		items := strings.Split(connString, ";")
+		for _, item := range items {
+			if item = strings.TrimSpace(item); item == "" {
+				continue
+			}
+			parts := strings.SplitN(item, "=", 2)
+			if len(parts) != 2 {
+				logger.Fatalf("Invalid connection string item: %s", item)
+			}
+			// Arguments from command line take precedence
+			if parts[0] == "DefaultEndpointsProtocol" && scheme == "" {
+				scheme = parts[1]
+			} else if parts[0] == "AccountName" && accountName == "" {
+				accountName = parts[1]
+			} else if parts[0] == "AccountKey" && accountKey == "" {
+				accountKey = parts[1]
+			} else if parts[0] == "EndpointSuffix" && domain == "" {
+				domain = parts[1]
+			}
+		}
+	}
+	if scheme == "" {
+		scheme = "https"
+	}
 	name := hostParts[0]
-	client, err := storage.NewClient(account, key, hostParts[1], "2017-04-17", true)
+	if len(hostParts) > 1 {
+		// Arguments from command line take precedence
+		domain = hostParts[1]
+	} else if domain == "" {
+		if domain, err = autoWasbEndpoint(name, accountName, accountKey, scheme == "https"); err != nil {
+			logger.Fatalf("Unable to get endpoint of container %s: %s", name, err)
+		}
+	}
+
+	client, err := storage.NewClient(accountName, accountKey, domain, "2017-04-17", scheme == "https")
 	if err != nil {
 		log.Fatalf("Failed to create client: %v", err)
 	}