diff --git a/go.mod b/go.mod index b2154ccae1c00254bff8b5bb1184fed85b5cd9d3..2b38ee47cb4ca62882505ee599f636b2c96d70ac 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible + github.com/aws/aws-sdk-go v1.31.5 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 github.com/fatih/color v1.7.0 @@ -25,7 +26,7 @@ require ( github.com/mattn/go-colorable v0.1.4 // indirect github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - github.com/pkg/errors v0.8.0 + github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.2.0 github.com/qingwg/payjs v0.0.0-20190928033402-c53dbe16b371 github.com/qiniu/api.v7/v7 v7.4.0 @@ -35,7 +36,7 @@ require ( github.com/smartwalle/alipay/v3 v3.0.13 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/speps/go-hashids v2.0.0+incompatible - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac github.com/upyun/go-sdk v2.1.0+incompatible diff --git a/go.sum b/go.sum index 8393d9d36a9e51df1dbf6f5de4a679e17de5ff14..00f2c4de395dde82deb9c72391788404d419a974 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4= github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0= +github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -75,6 +77,8 @@ github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rm github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -128,6 +132,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -186,6 +192,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= @@ -231,6 +239,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE= github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ= @@ -268,6 +277,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/middleware/auth.go b/middleware/auth.go index e016b0bc92e45c4bd49c5bee909fbbbdd5478b9a..33924f1ef73400e72e430878038375aec53bb0f5 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -5,7 +5,10 @@ import ( "context" "crypto/md5" "fmt" - "github.com/HFO4/cloudreve/models" + "io/ioutil" + "net/http" + + model "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/auth" "github.com/HFO4/cloudreve/pkg/cache" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" @@ -16,8 +19,6 @@ import ( "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" "github.com/qiniu/api.v7/v7/auth/qbox" - "io/ioutil" - "net/http" ) // SignRequired 验证请求签名 @@ -311,6 +312,21 @@ func COSCallbackAuth() gin.HandlerFunc { } } +// S3CallbackAuth Amazon S3回调签名验证 +func S3CallbackAuth() gin.HandlerFunc { + return func(c *gin.Context) { + // 验证key并查找用户 + resp, _ := uploadCallbackCheck(c) + if resp.Code != 0 { + c.JSON(401, serializer.QiniuCallbackFailed{Error: resp.Msg}) + c.Abort() + return + } + + c.Next() + } +} + // IsAdmin 必须为管理员用户组 func IsAdmin() gin.HandlerFunc { return func(c *gin.Context) { diff --git a/models/policy.go b/models/policy.go index a4d4181091b9a2a33b6e2080ba9b344747c715af..e9aec07fa29c4cd5ff568392a63b57cfbdcaaca7 100644 --- a/models/policy.go +++ b/models/policy.go @@ -3,15 +3,16 @@ package model import ( "encoding/gob" "encoding/json" - "github.com/HFO4/cloudreve/pkg/cache" - "github.com/HFO4/cloudreve/pkg/util" - "github.com/jinzhu/gorm" "net/url" "path" "path/filepath" "strconv" "strings" "time" + + "github.com/HFO4/cloudreve/pkg/cache" + "github.com/HFO4/cloudreve/pkg/util" + "github.com/jinzhu/gorm" ) // Policy 存储策略 @@ -48,6 +49,9 @@ type PolicyOption struct { // OdRedirect Onedrive重定向地址 OdRedirect string `json:"od_redirect,omitempty"` + + // Region 区域代码 + Region string `json:"region"` } var thumbSuffix = map[string][]string{ @@ -56,6 +60,7 @@ var thumbSuffix = map[string][]string{ "oss": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"}, "cos": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"}, "upyun": {".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"}, + "s3": {}, "remote": {}, "onedrive": {"*"}, } diff --git a/pkg/filesystem/driver/s3/handler.go b/pkg/filesystem/driver/s3/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..9283bacee4d840fae7dcacddd0126cb1f2e8285c --- /dev/null +++ b/pkg/filesystem/driver/s3/handler.go @@ -0,0 +1,477 @@ +package s3 + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "strings" + "sync" + "time" + + model "github.com/HFO4/cloudreve/models" + "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" + "github.com/HFO4/cloudreve/pkg/filesystem/response" + "github.com/HFO4/cloudreve/pkg/request" + "github.com/HFO4/cloudreve/pkg/serializer" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +// Driver 适配器模板 +type Driver struct { + Policy *model.Policy + sess *session.Session + svc *s3.S3 +} + +// UploadPolicy S3上传策略 +type UploadPolicy struct { + Expiration string `json:"expiration"` + Conditions []interface{} `json:"conditions"` +} + +//MetaData 文件信息 +type MetaData struct { + Size uint64 + Etag string +} + +// InitS3Client 初始化S3会话 +func (handler *Driver) InitS3Client() error { + if handler.Policy == nil { + return errors.New("存储策略为空") + } + + if handler.svc == nil { + // 初始化会话 + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""), + Endpoint: &handler.Policy.Server, + Region: &handler.Policy.OptionsSerialized.Region, + S3ForcePathStyle: aws.Bool(false), + }) + if err != nil { + return err + } + handler.sess = sess + handler.svc = s3.New(sess) + } + return nil +} + +// List 列出给定路径下的文件 +func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { + + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return nil, err + } + + // 初始化列目录参数 + base = strings.TrimPrefix(base, "/") + if base != "" { + base += "/" + } + + opt := &s3.ListObjectsInput{ + Bucket: &handler.Policy.BucketName, + Prefix: &base, + EncodingType: aws.String(""), + MaxKeys: aws.Int64(1000), + } + + // 是否为递归列出 + if !recursive { + opt.Delimiter = aws.String("/") + } + + var ( + objects []*s3.Object + commons []*s3.CommonPrefix + ) + + for { + res, err := handler.svc.ListObjectsWithContext(ctx, opt) + if err != nil { + return nil, err + } + objects = append(objects, res.Contents...) + commons = append(commons, res.CommonPrefixes...) + + // 如果本次未列取完，则继续使用marker获取结果 + if *res.IsTruncated { + opt.Marker = res.NextMarker + } else { + break + } + } + + // 处理列取结果 + res := make([]response.Object, 0, len(objects)+len(commons)) + + // 处理目录 + for _, object := range commons { + rel, err := filepath.Rel(*opt.Prefix, *object.Prefix) + if err != nil { + continue + } + res = append(res, response.Object{ + Name: path.Base(*object.Prefix), + RelativePath: filepath.ToSlash(rel), + Size: 0, + IsDir: true, + LastModify: time.Now(), + }) + } + // 处理文件 + for _, object := range objects { + rel, err := filepath.Rel(*opt.Prefix, *object.Key) + if err != nil { + continue + } + res = append(res, response.Object{ + Name: path.Base(*object.Key), + Source: *object.Key, + RelativePath: filepath.ToSlash(rel), + Size: uint64(*object.Size), + IsDir: false, + LastModify: time.Now(), + }) + } + + return res, nil + +} + +// Get 获取文件 +func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { + + // 获取文件源地址 + downloadURL, err := handler.Source( + ctx, + path, + url.URL{}, + int64(model.GetIntSetting("preview_timeout", 60)), + false, + 0, + ) + if err != nil { + return nil, err + } + + // 获取文件数据流 + client := request.HTTPClient{} + resp, err := client.Request( + "GET", + downloadURL, + nil, + request.WithContext(ctx), + request.WithHeader( + http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}}, + ), + request.WithTimeout(time.Duration(0)), + ).CheckHTTPResponse(200).GetRSCloser() + if err != nil { + return nil, err + } + + resp.SetFirstFakeChunk() + + // 尝试自主获取文件大小 + if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { + resp.SetContentLength(int64(file.Size)) + } + + return resp, nil +} + +// Put 将文件流保存到指定目录 +func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { + + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return err + } + + uploader := s3manager.NewUploader(handler.sess) + + _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: &handler.Policy.BucketName, + Key: &dst, + Body: file, + }) + + if err != nil { + return err + } + + return nil +} + +// Delete 删除一个或多个文件， +// 返回未删除的文件，及遇到的最后一个错误 +func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) { + + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return files, err + } + + var ( + failed = make([]string, 0, len(files)) + lastErr error + currentIndex = 0 + indexLock sync.Mutex + failedLock sync.Mutex + wg sync.WaitGroup + routineNum = 4 + ) + wg.Add(routineNum) + + // S3不支持批量操作，这里开四个协程并行操作 + for i := 0; i < routineNum; i++ { + go func() { + for { + // 取得待删除文件 + indexLock.Lock() + if currentIndex >= len(files) { + // 所有文件处理完成 + wg.Done() + indexLock.Unlock() + return + } + path := files[currentIndex] + currentIndex++ + indexLock.Unlock() + + // 发送异步删除请求 + _, err := handler.svc.DeleteObject( + &s3.DeleteObjectInput{ + Bucket: &handler.Policy.BucketName, + Key: &path, + }) + + // 处理错误 + if err != nil { + failedLock.Lock() + lastErr = err + failed = append(failed, path) + failedLock.Unlock() + } + } + }() + } + + wg.Wait() + return failed, lastErr + +} + +// Thumb 获取文件缩略图 +func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) { + return nil, errors.New("未实现") +} + +// Source 获取外链URL +func (handler Driver) Source( + ctx context.Context, + path string, + baseURL url.URL, + ttl int64, + isDownload bool, + speed int, +) (string, error) { + + // 尝试从上下文获取文件名 + fileName := "" + if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { + fileName = file.Name + } + + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return "", err + } + + req, _ := handler.svc.GetObjectRequest( + &s3.GetObjectInput{ + Bucket: &handler.Policy.BucketName, + Key: &path, + ResponseContentDisposition: aws.String("attachment; filename=\"" + url.PathEscape(fileName) + "\""), + }) + + if ttl == 0 { + ttl = 3600 + } + + signedURL, _ := req.Presign(time.Duration(ttl) * time.Second) + + // 将最终生成的签名URL域名换成用户自定义的加速域名（如果有） + finalURL, err := url.Parse(signedURL) + if err != nil { + return "", err + } + + // 公有空间替换掉Key及不支持的头 + if !handler.Policy.IsPrivate { + finalURL.RawQuery = "" + } + + if handler.Policy.BaseURL != "" { + cdnURL, err := url.Parse(handler.Policy.BaseURL) + if err != nil { + return "", err + } + finalURL.Host = cdnURL.Host + finalURL.Scheme = cdnURL.Scheme + } + + return finalURL.String(), nil +} + +// Token 获取上传策略和认证Token +func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) { + + // 读取上下文中生成的存储路径和文件大小 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + // 生成回调地址 + siteURL := model.GetSiteURL() + apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + key) + apiURL := siteURL.ResolveReference(apiBaseURI) + + // 上传策略 + putPolicy := UploadPolicy{ + Expiration: time.Now().UTC().Add(time.Duration(TTL) * time.Second).Format(time.RFC3339), + Conditions: []interface{}{ + map[string]string{"bucket": handler.Policy.BucketName}, + []string{"starts-with", "$key", savePath}, + []string{"starts-with", "$success_action_redirect", apiURL.String()}, + []string{"starts-with", "$name", ""}, + []string{"starts-with", "$Content-Type", ""}, + map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"}, + }, + } + + if handler.Policy.MaxSize > 0 { + putPolicy.Conditions = append(putPolicy.Conditions, + []interface{}{"content-length-range", 0, handler.Policy.MaxSize}) + } + + // 生成上传凭证 + return handler.getUploadCredential(ctx, putPolicy, apiURL) +} + +// Meta 获取文件信息 +func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) { + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return nil, err + } + + res, err := handler.svc.GetObject( + &s3.GetObjectInput{ + Bucket: &handler.Policy.BucketName, + Key: &path, + }) + + if err != nil { + return nil, err + } + + return &MetaData{ + Size: uint64(*res.ContentLength), + Etag: *res.ETag, + }, nil + +} + +func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL) (serializer.UploadCredential, error) { + + // 读取上下文中生成的存储路径和文件大小 + savePath, ok := ctx.Value(fsctx.SavePathCtx).(string) + if !ok { + return serializer.UploadCredential{}, errors.New("无法获取存储路径") + } + + longDate := time.Now().UTC().Format("20060102T150405Z") + shortDate := time.Now().UTC().Format("20060102") + + credential := handler.Policy.AccessKey + "/" + shortDate + "/" + handler.Policy.OptionsSerialized.Region + "/s3/aws4_request" + policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-credential": credential}) + policy.Conditions = append(policy.Conditions, map[string]string{"x-amz-date": longDate}) + + // 编码上传策略 + policyJSON, err := json.Marshal(policy) + if err != nil { + return serializer.UploadCredential{}, err + } + policyEncoded := base64.StdEncoding.EncodeToString(policyJSON) + + //签名 + signature := getHMAC([]byte("AWS4"+handler.Policy.SecretKey), []byte(shortDate)) + signature = getHMAC(signature, []byte(handler.Policy.OptionsSerialized.Region)) + signature = getHMAC(signature, []byte("s3")) + signature = getHMAC(signature, []byte("aws4_request")) + signature = getHMAC(signature, []byte(policyEncoded)) + + return serializer.UploadCredential{ + Policy: policyEncoded, + Callback: callback.String(), + Token: hex.EncodeToString(signature), + AccessKey: credential, + Path: savePath, + KeyTime: longDate, + }, nil +} + +func getHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// CORS 创建跨域策略 +func (handler Driver) CORS() error { + // 初始化客户端 + if err := handler.InitS3Client(); err != nil { + return err + } + + rule := s3.CORSRule{ + AllowedMethods: aws.StringSlice([]string{ + "GET", + "POST", + "PUT", + "DELETE", + "HEAD", + }), + AllowedOrigins: aws.StringSlice([]string{"*"}), + AllowedHeaders: aws.StringSlice([]string{"*"}), + MaxAgeSeconds: aws.Int64(3600), + } + + _, err := handler.svc.PutBucketCors(&s3.PutBucketCorsInput{ + Bucket: &handler.Policy.BucketName, + CORSConfiguration: &s3.CORSConfiguration{ + CORSRules: []*s3.CORSRule{&rule}, + }, + }) + + return err +} diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 3e1fac526e2b621f347fd06f8b9efb92ff9d03cc..030f6cfc2d27a1eca213ce43efb0e76f61dcc220 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -3,7 +3,12 @@ package filesystem import ( "context" "errors" - "github.com/HFO4/cloudreve/models" + "io" + "net/http" + "net/url" + "sync" + + model "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/auth" "github.com/HFO4/cloudreve/pkg/conf" "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" @@ -12,16 +17,13 @@ import ( "github.com/HFO4/cloudreve/pkg/filesystem/driver/oss" "github.com/HFO4/cloudreve/pkg/filesystem/driver/qiniu" "github.com/HFO4/cloudreve/pkg/filesystem/driver/remote" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/s3" "github.com/HFO4/cloudreve/pkg/filesystem/driver/upyun" "github.com/HFO4/cloudreve/pkg/filesystem/response" "github.com/HFO4/cloudreve/pkg/request" "github.com/HFO4/cloudreve/pkg/serializer" "github.com/gin-gonic/gin" cossdk "github.com/tencentyun/cos-go-sdk-v5" - "io" - "net/http" - "net/url" - "sync" ) // FSPool 文件系统资源池 @@ -219,6 +221,11 @@ func (fs *FileSystem) DispatchHandler() error { HTTPClient: request.HTTPClient{}, } return nil + case "s3": + fs.Handler = s3.Driver{ + Policy: currentPolicy, + } + return nil default: return ErrUnknownPolicyType } diff --git a/routers/controllers/callback.go b/routers/controllers/callback.go index 340d453dcd3b7d80b5b5eb698f2250568e7c665a..16d6e79f49ca22e38c580439796d4e41f55f8cdc 100644 --- a/routers/controllers/callback.go +++ b/routers/controllers/callback.go @@ -1,12 +1,13 @@ package controllers import ( + "net/url" + "strconv" + "github.com/HFO4/cloudreve/pkg/serializer" "github.com/HFO4/cloudreve/pkg/util" "github.com/HFO4/cloudreve/service/callback" "github.com/gin-gonic/gin" - "net/url" - "strconv" ) // RemoteCallback 远程上传回调 @@ -106,3 +107,14 @@ func COSCallback(c *gin.Context) { c.JSON(200, ErrorResponse(err)) } } + +// S3Callback S3上传完成客户端回调 +func S3Callback(c *gin.Context) { + var callbackBody callback.S3Callback + if err := c.ShouldBindQuery(&callbackBody); err == nil { + res := callbackBody.PreProcess(c) + c.JSON(200, res) + } else { + c.JSON(200, ErrorResponse(err)) + } +} diff --git a/routers/router.go b/routers/router.go index 083a007c5077fa7f9f60c11835efe745b5d8fee2..71a3d57b7313a322b714379a28d665de285744f6 100644 --- a/routers/router.go +++ b/routers/router.go @@ -221,6 +221,12 @@ func InitMasterRouter() *gin.Engine { middleware.COSCallbackAuth(), controllers.COSCallback, ) + // AWS S3策略上传回调 + callback.GET( + "s3/:key", + middleware.S3CallbackAuth(), + controllers.S3Callback, + ) } // 分享相关 diff --git a/service/admin/policy.go b/service/admin/policy.go index f89fbe633456d3f8e33e42a5c05166fb14218c5a..c066d7f8ffef3bac38a8465acd4c06ea9d20caab 100644 --- a/service/admin/policy.go +++ b/service/admin/policy.go @@ -5,6 +5,13 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + model "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/auth" "github.com/HFO4/cloudreve/pkg/cache" @@ -12,17 +19,12 @@ import ( "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" "github.com/HFO4/cloudreve/pkg/filesystem/driver/oss" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/s3" "github.com/HFO4/cloudreve/pkg/request" "github.com/HFO4/cloudreve/pkg/serializer" "github.com/HFO4/cloudreve/pkg/util" "github.com/gin-gonic/gin" cossdk "github.com/tencentyun/cos-go-sdk-v5" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" - "time" ) // PathTestService 本地路径测试服务 @@ -170,6 +172,13 @@ func (service *PolicyService) AddCORS() serializer.Response { if err := handler.CORS(); err != nil { return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) } + case "s3": + handler := s3.Driver{ + Policy: &policy, + } + if err := handler.CORS(); err != nil { + return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err) + } default: return serializer.ParamErr("不支持此策略", nil) } diff --git a/service/callback/upload.go b/service/callback/upload.go index d7a24a4e034195a35acb940a8b20ba08f3fe9eca..e9894d559c4a6969c206d70ccf969d4785bfc9df 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -3,15 +3,17 @@ package callback import ( "context" "fmt" + "strings" + "github.com/HFO4/cloudreve/pkg/filesystem" "github.com/HFO4/cloudreve/pkg/filesystem/driver/cos" "github.com/HFO4/cloudreve/pkg/filesystem/driver/local" "github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive" + "github.com/HFO4/cloudreve/pkg/filesystem/driver/s3" "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" "github.com/HFO4/cloudreve/pkg/serializer" "github.com/HFO4/cloudreve/pkg/util" "github.com/gin-gonic/gin" - "strings" ) // CallbackProcessService 上传请求回调正文接口 @@ -59,6 +61,13 @@ type COSCallback struct { Etag string `form:"etag"` } +// S3Callback S3 客户端回调正文 +type S3Callback struct { + Bucket string `form:"bucket"` + Etag string `form:"etag"` + Key string `form:"key"` +} + // GetBody 返回回调正文 func (service UpyunCallbackService) GetBody(session *serializer.UploadSession) serializer.UploadCallback { res := serializer.UploadCallback{ @@ -107,6 +116,16 @@ func (service COSCallback) GetBody(session *serializer.UploadSession) serializer } } +// GetBody 返回回调正文 +func (service S3Callback) GetBody(session *serializer.UploadSession) serializer.UploadCallback { + return serializer.UploadCallback{ + Name: session.Name, + SourceName: session.SavePath, + PicInfo: "", + Size: session.Size, + } +} + // ProcessCallback 处理上传结果回调 func ProcessCallback(service CallbackProcessService, c *gin.Context) serializer.Response { // 创建文件系统 @@ -222,3 +241,30 @@ func (service *COSCallback) PreProcess(c *gin.Context) serializer.Response { return ProcessCallback(service, c) } + +// PreProcess 对S3客户端回调进行预处理 +func (service *S3Callback) PreProcess(c *gin.Context) serializer.Response { + // 创建文件系统 + fs, err := filesystem.NewFileSystemFromCallback(c) + if err != nil { + return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err) + } + defer fs.Recycle() + + // 获取回调会话 + callbackSessionRaw, _ := c.Get("callbackSession") + callbackSession := callbackSessionRaw.(*serializer.UploadSession) + + // 获取文件信息 + info, err := fs.Handler.(s3.Driver).Meta(context.Background(), callbackSession.SavePath) + if err != nil { + return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) + } + + // 验证实际文件信息与回调会话中是否一致 + if callbackSession.Size != info.Size || service.Etag != info.Etag { + return serializer.Err(serializer.CodeUploadFailed, "文件信息不一致", err) + } + + return ProcessCallback(service, c) +}