Feat: COS credential / upload callback

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/stretchr/testify v1.4.0
+	github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
 	github.com/upyun/go-sdk v2.1.0+incompatible
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
 	gopkg.in/go-playground/validator.v8 v8.18.2

go.sum

@@ -4,6 +4,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
@@ -73,8 +74,11 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE=
 github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
@@ -126,6 +130,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 h1:daZqE/T/yEoKIQNd3rwNeLsiS0VpZFfJulR0t/rtgAE=
 github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQCKEc5bDujxKRmbT6/vTnTt5CjStQ8bRfPWUuz/iY=
+github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
+github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
@@ -165,6 +171,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
+github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=

middleware/auth.go

@@ -293,3 +293,19 @@ func OneDriveCallbackAuth() gin.HandlerFunc {
 		c.Next()
 	}
 }
+
+// COSCallbackAuth 腾讯云COS回调签名验证
+// TODO 解耦 测试
+func COSCallbackAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// 验证key并查找用户
+		resp, _ := uploadCallbackCheck(c)
+		if resp.Code != 0 {
+			c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg})
+			c.Abort()
+			return
+		}
+
+		c.Next()
+	}
+}

models/policy.go

@@ -145,7 +145,7 @@ func (policy Policy) getOriginNameRule(origin string) string {
 			return "$(fname)"
 		case "local", "remote":
 			return origin
-		case "oss":
+		case "oss", "cos":
 			// OSS会将${filename}自动替换为原始文件名
 			return "${filename}"
 		case "upyun":
@@ -201,7 +201,7 @@ func (policy *Policy) GetUploadURL() string {
 		controller, _ = url.Parse("/api/v3/file/upload")
 	case "remote":
 		controller, _ = url.Parse("/api/v3/slave/upload")
-	case "oss":
+	case "oss", "cos":
 		return policy.BaseURL
 	case "upyun":
 		return "http://v0.api.upyun.com/" + policy.BucketName

pkg/filesystem/driver/cos/handller.go

+package cos
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	model "github.com/HFO4/cloudreve/models"
+	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
+	"github.com/HFO4/cloudreve/pkg/filesystem/response"
+	"github.com/HFO4/cloudreve/pkg/serializer"
+	cossdk "github.com/tencentyun/cos-go-sdk-v5"
+	"io"
+	"net/url"
+	"time"
+)
+
+// UploadPolicy 腾讯云COS上传策略
+type UploadPolicy struct {
+	Expiration string        `json:"expiration"`
+	Conditions []interface{} `json:"conditions"`
+}
+
+// MetaData 文件元信息
+type MetaData struct {
+	Size        uint64
+	CallbackKey string
+	CallbackURL string
+}
+
+// Driver 腾讯云COS适配器模板
+type Driver struct {
+	Policy *model.Policy
+	Client *cossdk.Client
+}
+
+// Get 获取文件
+func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+	return nil, errors.New("未实现")
+}
+
+// Put 将文件流保存到指定目录
+func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
+	return errors.New("未实现")
+}
+
+// Delete 删除一个或多个文件，
+// 返回未删除的文件，及遇到的最后一个错误
+func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+	return []string{}, errors.New("未实现")
+}
+
+// Thumb 获取文件缩略图
+func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+	return nil, errors.New("未实现")
+}
+
+// Source 获取外链URL
+func (handler Driver) Source(
+	ctx context.Context,
+	path string,
+	baseURL url.URL,
+	ttl int64,
+	isDownload bool,
+	speed int,
+) (string, error) {
+	return "", errors.New("未实现")
+}
+
+// Token 获取上传策略和认证Token
+func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
+	// 读取上下文中生成的存储路径
+	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
+	if !ok {
+		return serializer.UploadCredential{}, errors.New("无法获取存储路径")
+	}
+
+	// 生成回调地址
+	siteURL := model.GetSiteURL()
+	apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + key)
+	apiURL := siteURL.ResolveReference(apiBaseURI).String()
+
+	// 上传策略
+	startTime := time.Now()
+	endTime := startTime.Add(time.Duration(TTL) * time.Second)
+	keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
+	postPolicy := UploadPolicy{
+		Expiration: endTime.UTC().Format(time.RFC3339),
+		Conditions: []interface{}{
+			map[string]string{"bucket": handler.Policy.BucketName},
+			map[string]string{"$key": savePath},
+			map[string]string{"x-cos-meta-callback": apiURL},
+			map[string]string{"x-cos-meta-key": key},
+			[]interface{}{"content-length-range", 0, handler.Policy.MaxSize},
+			map[string]string{"q-sign-algorithm": "sha1"},
+			map[string]string{"q-ak": handler.Policy.AccessKey},
+			map[string]string{"q-sign-time": keyTime},
+		},
+	}
+
+	res, err := handler.getUploadCredential(ctx, postPolicy, keyTime)
+	if err == nil {
+		res.Callback = apiURL
+		res.Key = key
+	}
+
+	return res, err
+
+}
+
+// Meta 获取文件信息
+func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
+	res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return &MetaData{
+		Size:        uint64(res.ContentLength),
+		CallbackKey: res.Header.Get("x-cos-meta-key"),
+		CallbackURL: res.Header.Get("x-cos-meta-callback"),
+	}, nil
+}
+
+func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string) (serializer.UploadCredential, error) {
+	// 读取上下文中生成的存储路径
+	savePath, ok := ctx.Value(fsctx.SavePathCtx).(string)
+	if !ok {
+		return serializer.UploadCredential{}, errors.New("无法获取存储路径")
+	}
+
+	// 编码上传策略
+	policyJSON, err := json.Marshal(policy)
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
+
+	// 签名上传策略
+	hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
+	_, err = io.WriteString(hmacSign, keyTime)
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
+
+	sha1Sign := sha1.New()
+	_, err = sha1Sign.Write(policyJSON)
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
+
+	// 最终签名
+	hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
+	_, err = hmacFinalSign.Write([]byte(stringToSign))
+	if err != nil {
+		return serializer.UploadCredential{}, err
+	}
+	signature := hmacFinalSign.Sum(nil)
+
+	return serializer.UploadCredential{
+		Policy:    policyEncoded,
+		Path:      savePath,
+		AccessKey: handler.Policy.AccessKey,
+		Token:     fmt.Sprintf("%x", signature),
+		KeyTime:   keyTime,
+	}, nil
+}

pkg/filesystem/filesystem.go

@@ -6,6 +6,7 @@ import (
 	"github.com/HFO4/cloudreve/models"
 	"github.com/HFO4/cloudreve/pkg/auth"
 	"github.com/HFO4/cloudreve/pkg/conf"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/cos"
 	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive"
 	"github.com/HFO4/cloudreve/pkg/filesystem/driver/oss"
@@ -16,7 +17,9 @@ import (
 	"github.com/HFO4/cloudreve/pkg/request"
 	"github.com/HFO4/cloudreve/pkg/serializer"
 	"github.com/gin-gonic/gin"
+	cossdk "github.com/tencentyun/cos-go-sdk-v5"
 	"io"
+	"net/http"
 	"net/url"
 	"sync"
 )
@@ -191,6 +194,19 @@ func (fs *FileSystem) DispatchHandler() error {
 			HTTPClient: request.HTTPClient{},
 		}
 		return err
+	case "cos":
+		u, _ := url.Parse(currentPolicy.Server)
+		b := &cossdk.BaseURL{BucketURL: u}
+		fs.Handler = cos.Driver{
+			Policy: currentPolicy,
+			Client: cossdk.NewClient(b, &http.Client{
+				Transport: &cossdk.AuthorizationTransport{
+					SecretID:  currentPolicy.AccessKey,
+					SecretKey: currentPolicy.SecretKey,
+				},
+			}),
+		}
+		return nil
 	default:
 		return ErrUnknownPolicyType
 	}

pkg/filesystem/upload.go

@@ -165,6 +165,7 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
 	err = cache.Set(
 		"callback_"+callbackKey,
 		serializer.UploadSession{
+			Key:         callbackKey,
 			UID:         fs.User.ID,
 			PolicyID:    fs.User.GetPolicyID(),
 			VirtualPath: path,

pkg/serializer/upload.go

@@ -20,12 +20,16 @@ type UploadPolicy struct {
 type UploadCredential struct {
 	Token     string `json:"token"`
 	Policy    string `json:"policy"`
-	Path      string `json:"path"`
+	Path      string `json:"path"` // 存储路径
 	AccessKey string `json:"ak"`
+	KeyTime   string `json:"key_time,omitempty"` // COS用有效期
+	Callback  string `json:"callback,omitempty"` // 回调地址
+	Key       string `json:"key,omitempty"`      // 文件标识符，通常为回调key
 }
 
 // UploadSession 上传会话
 type UploadSession struct {
+	Key         string
 	UID         uint
 	PolicyID    uint
 	VirtualPath string

routers/controllers/callback.go

@@ -76,3 +76,14 @@ func OneDriveCallback(c *gin.Context) {
 		c.JSON(200, ErrorResponse(err))
 	}
 }
+
+// COSCallback COS上传完成客户端回调
+func COSCallback(c *gin.Context) {
+	var callbackBody callback.COSCallback
+	if err := c.ShouldBindQuery(&callbackBody); err == nil {
+		res := callbackBody.PreProcess(c)
+		c.JSON(200, res)
+	} else {
+		c.JSON(200, ErrorResponse(err))
+	}
+}

routers/router.go

@@ -159,6 +159,12 @@ func InitMasterRouter() *gin.Engine {
 					controllers.OneDriveCallback,
 				)
 			}
+			// 腾讯云COS策略上传回调
+			callback.GET(
+				"cos/:key",
+				middleware.COSCallbackAuth(),
+				controllers.COSCallback,
+			)
 		}
 
 		// 需要登录保护的

service/callback/upload.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/HFO4/cloudreve/pkg/filesystem"
+	"github.com/HFO4/cloudreve/pkg/filesystem/driver/cos"
 	"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
 	"github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive"
 	"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
@@ -52,6 +53,12 @@ type OneDriveCallback struct {
 	Meta *onedrive.FileInfo
 }
 
+// COSCallback COS 客户端回调正文
+type COSCallback struct {
+	Bucket string `form:"bucket"`
+	Etag   string `form:"etag"`
+}
+
 // GetBody 返回回调正文
 func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback {
 	res := serializer.UploadCallback{
@@ -90,6 +97,16 @@ func (service OneDriveCallback) GetBody(session *serializer.UploadSession) seria
 	}
 }
 
+// GetBody 返回回调正文
+func (service COSCallback) GetBody(session *serializer.UploadSession) serializer.UploadCallback {
+	return serializer.UploadCallback{
+		Name:       session.Name,
+		SourceName: session.SavePath,
+		PicInfo:    "",
+		Size:       session.Size,
+	}
+}
+
 // ProcessCallback 处理上传结果回调
 func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response {
 	// 创建文件系统
@@ -168,9 +185,36 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response
 	// 验证与回调会话中是否一致
 	actualPath := strings.TrimPrefix(callbackSession.SavePath, "/")
 	if callbackSession.Size != info.Size || info.GetSourcePath() != actualPath {
-		// TODO 删除文件信息
+		fs.Handler.(onedrive.Driver).Client.Delete(context.Background(), []string{info.GetSourcePath()})
 		return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
 	}
 	service.Meta = info
 	return ProcessCallback(service, c)
 }
+
+// PreProcess 对COS客户端回调进行预处理
+func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response {
+	// 创建文件系统
+	fs, err := filesystem.NewFileSystemFromCallback(c)
+	if err != nil {
+		return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
+	}
+	defer fs.Recycle()
+
+	// 获取回调会话
+	callbackSessionRaw, _ := c.Get("callbackSession")
+	callbackSession := callbackSessionRaw.(*serializer.UploadSession)
+
+	// 获取文件信息
+	info, err := fs.Handler.(cos.Driver).Meta(context.Background(), callbackSession.SavePath)
+	if err != nil {
+		return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
+	}
+
+	// 验证实际文件信息与回调会话中是否一致
+	if callbackSession.Size != info.Size || callbackSession.Key != info.CallbackKey {
+		return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err)
+	}
+
+	return ProcessCallback(service, c)
+}