Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
FinClip
ligase
提交
7596c19f
ligase
项目概览
FinClip
/
ligase
通知
36
Star
8
Fork
1
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
ligase
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7596c19f
编写于
9月 21, 2017
作者:
M
Mark Haines
提交者:
GitHub
9月 21, 2017
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add context to the mediaapi database (#250)
上级
a7773d3d
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
242 addition
and
59 deletion
+242
-59
src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go
...x-org/dendrite/mediaapi/storage/media_repository_table.go
+11
-5
src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go
...ithub.com/matrix-org/dendrite/mediaapi/storage/storage.go
+27
-10
src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go
...m/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go
+20
-7
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go
...m/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go
+13
-2
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go
...rix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go
+42
-7
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go
...rix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go
+42
-7
src/github.com/matrix-org/dendrite/mediaapi/writers/download.go
...thub.com/matrix-org/dendrite/mediaapi/writers/download.go
+57
-14
src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go
...github.com/matrix-org/dendrite/mediaapi/writers/upload.go
+30
-7
未找到文件。
src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
storage
import
(
"context"
"database/sql"
"time"
...
...
@@ -74,9 +75,12 @@ func (s *mediaStatements) prepare(db *sql.DB) (err error) {
}
.
prepare
(
db
)
}
func
(
s
*
mediaStatements
)
insertMedia
(
mediaMetadata
*
types
.
MediaMetadata
)
error
{
func
(
s
*
mediaStatements
)
insertMedia
(
ctx
context
.
Context
,
mediaMetadata
*
types
.
MediaMetadata
,
)
error
{
mediaMetadata
.
CreationTimestamp
=
types
.
UnixMs
(
time
.
Now
()
.
UnixNano
()
/
1000000
)
_
,
err
:=
s
.
insertMediaStmt
.
Exec
(
_
,
err
:=
s
.
insertMediaStmt
.
ExecContext
(
ctx
,
mediaMetadata
.
MediaID
,
mediaMetadata
.
Origin
,
mediaMetadata
.
ContentType
,
...
...
@@ -89,13 +93,15 @@ func (s *mediaStatements) insertMedia(mediaMetadata *types.MediaMetadata) error
return
err
}
func
(
s
*
mediaStatements
)
selectMedia
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
)
(
*
types
.
MediaMetadata
,
error
)
{
func
(
s
*
mediaStatements
)
selectMedia
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
)
(
*
types
.
MediaMetadata
,
error
)
{
mediaMetadata
:=
types
.
MediaMetadata
{
MediaID
:
mediaID
,
Origin
:
mediaOrigin
,
}
err
:=
s
.
selectMediaStmt
.
QueryRow
(
mediaMetadata
.
MediaID
,
mediaMetadata
.
Origin
,
err
:=
s
.
selectMediaStmt
.
QueryRow
Context
(
ctx
,
mediaMetadata
.
MediaID
,
mediaMetadata
.
Origin
,
)
.
Scan
(
&
mediaMetadata
.
ContentType
,
&
mediaMetadata
.
FileSizeBytes
,
...
...
src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
storage
import
(
"context"
"database/sql"
// Import the postgres database driver.
...
...
@@ -44,15 +45,19 @@ func Open(dataSourceName string) (*Database, error) {
// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
// Returns an error if the combination of MediaID and Origin are not unique in the table.
func
(
d
*
Database
)
StoreMediaMetadata
(
mediaMetadata
*
types
.
MediaMetadata
)
error
{
return
d
.
statements
.
media
.
insertMedia
(
mediaMetadata
)
func
(
d
*
Database
)
StoreMediaMetadata
(
ctx
context
.
Context
,
mediaMetadata
*
types
.
MediaMetadata
,
)
error
{
return
d
.
statements
.
media
.
insertMedia
(
ctx
,
mediaMetadata
)
}
// GetMediaMetadata returns metadata about media stored on this server.
// The media could have been uploaded to this server or fetched from another server and cached here.
// Returns nil metadata if there is no metadata associated with this media.
func
(
d
*
Database
)
GetMediaMetadata
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
)
(
*
types
.
MediaMetadata
,
error
)
{
mediaMetadata
,
err
:=
d
.
statements
.
media
.
selectMedia
(
mediaID
,
mediaOrigin
)
func
(
d
*
Database
)
GetMediaMetadata
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
)
(
*
types
.
MediaMetadata
,
error
)
{
mediaMetadata
,
err
:=
d
.
statements
.
media
.
selectMedia
(
ctx
,
mediaID
,
mediaOrigin
)
if
err
!=
nil
&&
err
==
sql
.
ErrNoRows
{
return
nil
,
nil
}
...
...
@@ -61,15 +66,25 @@ func (d *Database) GetMediaMetadata(mediaID types.MediaID, mediaOrigin gomatrixs
// StoreThumbnail inserts the metadata about the thumbnail into the database.
// Returns an error if the combination of MediaID and Origin are not unique in the table.
func
(
d
*
Database
)
StoreThumbnail
(
thumbnailMetadata
*
types
.
ThumbnailMetadata
)
error
{
return
d
.
statements
.
thumbnail
.
insertThumbnail
(
thumbnailMetadata
)
func
(
d
*
Database
)
StoreThumbnail
(
ctx
context
.
Context
,
thumbnailMetadata
*
types
.
ThumbnailMetadata
,
)
error
{
return
d
.
statements
.
thumbnail
.
insertThumbnail
(
ctx
,
thumbnailMetadata
)
}
// GetThumbnail returns metadata about a specific thumbnail.
// The media could have been uploaded to this server or fetched from another server and cached here.
// Returns nil metadata if there is no metadata associated with this thumbnail.
func
(
d
*
Database
)
GetThumbnail
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
width
,
height
int
,
resizeMethod
string
)
(
*
types
.
ThumbnailMetadata
,
error
)
{
thumbnailMetadata
,
err
:=
d
.
statements
.
thumbnail
.
selectThumbnail
(
mediaID
,
mediaOrigin
,
width
,
height
,
resizeMethod
)
func
(
d
*
Database
)
GetThumbnail
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
width
,
height
int
,
resizeMethod
string
,
)
(
*
types
.
ThumbnailMetadata
,
error
)
{
thumbnailMetadata
,
err
:=
d
.
statements
.
thumbnail
.
selectThumbnail
(
ctx
,
mediaID
,
mediaOrigin
,
width
,
height
,
resizeMethod
,
)
if
err
!=
nil
&&
err
==
sql
.
ErrNoRows
{
return
nil
,
nil
}
...
...
@@ -79,8 +94,10 @@ func (d *Database) GetThumbnail(mediaID types.MediaID, mediaOrigin gomatrixserve
// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
// The media could have been uploaded to this server or fetched from another server and cached here.
// Returns nil metadata if there are no thumbnails associated with this media.
func
(
d
*
Database
)
GetThumbnails
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
)
([]
*
types
.
ThumbnailMetadata
,
error
)
{
thumbnails
,
err
:=
d
.
statements
.
thumbnail
.
selectThumbnails
(
mediaID
,
mediaOrigin
)
func
(
d
*
Database
)
GetThumbnails
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
)
([]
*
types
.
ThumbnailMetadata
,
error
)
{
thumbnails
,
err
:=
d
.
statements
.
thumbnail
.
selectThumbnails
(
ctx
,
mediaID
,
mediaOrigin
)
if
err
!=
nil
&&
err
==
sql
.
ErrNoRows
{
return
nil
,
nil
}
...
...
src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
storage
import
(
"context"
"database/sql"
"time"
...
...
@@ -82,9 +83,12 @@ func (s *thumbnailStatements) prepare(db *sql.DB) (err error) {
}
.
prepare
(
db
)
}
func
(
s
*
thumbnailStatements
)
insertThumbnail
(
thumbnailMetadata
*
types
.
ThumbnailMetadata
)
error
{
func
(
s
*
thumbnailStatements
)
insertThumbnail
(
ctx
context
.
Context
,
thumbnailMetadata
*
types
.
ThumbnailMetadata
,
)
error
{
thumbnailMetadata
.
MediaMetadata
.
CreationTimestamp
=
types
.
UnixMs
(
time
.
Now
()
.
UnixNano
()
/
1000000
)
_
,
err
:=
s
.
insertThumbnailStmt
.
Exec
(
_
,
err
:=
s
.
insertThumbnailStmt
.
ExecContext
(
ctx
,
thumbnailMetadata
.
MediaMetadata
.
MediaID
,
thumbnailMetadata
.
MediaMetadata
.
Origin
,
thumbnailMetadata
.
MediaMetadata
.
ContentType
,
...
...
@@ -97,7 +101,13 @@ func (s *thumbnailStatements) insertThumbnail(thumbnailMetadata *types.Thumbnail
return
err
}
func
(
s
*
thumbnailStatements
)
selectThumbnail
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
width
,
height
int
,
resizeMethod
string
)
(
*
types
.
ThumbnailMetadata
,
error
)
{
func
(
s
*
thumbnailStatements
)
selectThumbnail
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
width
,
height
int
,
resizeMethod
string
,
)
(
*
types
.
ThumbnailMetadata
,
error
)
{
thumbnailMetadata
:=
types
.
ThumbnailMetadata
{
MediaMetadata
:
&
types
.
MediaMetadata
{
MediaID
:
mediaID
,
...
...
@@ -109,7 +119,8 @@ func (s *thumbnailStatements) selectThumbnail(mediaID types.MediaID, mediaOrigin
ResizeMethod
:
resizeMethod
,
},
}
err
:=
s
.
selectThumbnailStmt
.
QueryRow
(
err
:=
s
.
selectThumbnailStmt
.
QueryRowContext
(
ctx
,
thumbnailMetadata
.
MediaMetadata
.
MediaID
,
thumbnailMetadata
.
MediaMetadata
.
Origin
,
thumbnailMetadata
.
ThumbnailSize
.
Width
,
...
...
@@ -123,9 +134,11 @@ func (s *thumbnailStatements) selectThumbnail(mediaID types.MediaID, mediaOrigin
return
&
thumbnailMetadata
,
err
}
func
(
s
*
thumbnailStatements
)
selectThumbnails
(
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
)
([]
*
types
.
ThumbnailMetadata
,
error
)
{
rows
,
err
:=
s
.
selectThumbnailsStmt
.
Query
(
mediaID
,
mediaOrigin
,
func
(
s
*
thumbnailStatements
)
selectThumbnails
(
ctx
context
.
Context
,
mediaID
types
.
MediaID
,
mediaOrigin
gomatrixserverlib
.
ServerName
,
)
([]
*
types
.
ThumbnailMetadata
,
error
)
{
rows
,
err
:=
s
.
selectThumbnailsStmt
.
QueryContext
(
ctx
,
mediaID
,
mediaOrigin
,
)
if
err
!=
nil
{
return
nil
,
err
...
...
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
thumbnailer
import
(
"context"
"fmt"
"math"
"os"
...
...
@@ -130,8 +131,18 @@ func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.Active
delete
(
activeThumbnailGeneration
.
PathToResult
,
string
(
dst
))
}
func
isThumbnailExists
(
dst
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
bool
,
error
)
{
thumbnailMetadata
,
err
:=
db
.
GetThumbnail
(
mediaMetadata
.
MediaID
,
mediaMetadata
.
Origin
,
config
.
Width
,
config
.
Height
,
config
.
ResizeMethod
)
func
isThumbnailExists
(
ctx
context
.
Context
,
dst
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
bool
,
error
)
{
thumbnailMetadata
,
err
:=
db
.
GetThumbnail
(
ctx
,
mediaMetadata
.
MediaID
,
mediaMetadata
.
Origin
,
config
.
Width
,
config
.
Height
,
config
.
ResizeMethod
,
)
if
err
!=
nil
{
logger
.
Error
(
"Failed to query database for thumbnail."
)
return
false
,
err
...
...
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go
浏览文件 @
7596c19f
...
...
@@ -17,6 +17,7 @@
package
thumbnailer
import
(
"context"
"os"
"time"
...
...
@@ -28,7 +29,16 @@ import (
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func
GenerateThumbnails
(
src
types
.
Path
,
configs
[]
config
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
GenerateThumbnails
(
ctx
context
.
Context
,
src
types
.
Path
,
configs
[]
config
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
buffer
,
err
:=
bimg
.
Read
(
string
(
src
))
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithField
(
"src"
,
src
)
.
Error
(
"Failed to read src file"
)
...
...
@@ -37,7 +47,10 @@ func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMet
img
:=
bimg
.
NewImage
(
buffer
)
for
_
,
config
:=
range
configs
{
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy
,
err
=
createThumbnail
(
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
)
busy
,
err
=
createThumbnail
(
ctx
,
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
,
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithField
(
"src"
,
src
)
.
Error
(
"Failed to generate thumbnails"
)
return
false
,
err
...
...
@@ -50,7 +63,16 @@ func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMet
}
// GenerateThumbnail generates the configured thumbnail size for the source file
func
GenerateThumbnail
(
src
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
GenerateThumbnail
(
ctx
context
.
Context
,
src
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
buffer
,
err
:=
bimg
.
Read
(
string
(
src
))
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
...
...
@@ -60,7 +82,10 @@ func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata
}
img
:=
bimg
.
NewImage
(
buffer
)
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy
,
err
=
createThumbnail
(
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
)
busy
,
err
=
createThumbnail
(
ctx
,
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
,
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
"src"
:
src
,
...
...
@@ -75,7 +100,17 @@ func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata
// createThumbnail checks if the thumbnail exists, and if not, generates it
// Thumbnail generation is only done once for each non-existing thumbnail.
func
createThumbnail
(
src
types
.
Path
,
img
*
bimg
.
Image
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
createThumbnail
(
ctx
context
.
Context
,
src
types
.
Path
,
img
*
bimg
.
Image
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
logger
=
logger
.
WithFields
(
log
.
Fields
{
"Width"
:
config
.
Width
,
"Height"
:
config
.
Height
,
...
...
@@ -111,7 +146,7 @@ func createThumbnail(src types.Path, img *bimg.Image, config types.ThumbnailSize
}()
}
exists
,
err
:=
isThumbnailExists
(
dst
,
config
,
mediaMetadata
,
db
,
logger
)
exists
,
err
:=
isThumbnailExists
(
ctx
,
dst
,
config
,
mediaMetadata
,
db
,
logger
)
if
err
!=
nil
||
exists
{
return
false
,
err
}
...
...
@@ -147,7 +182,7 @@ func createThumbnail(src types.Path, img *bimg.Image, config types.ThumbnailSize
},
}
err
=
db
.
StoreThumbnail
(
thumbnailMetadata
)
err
=
db
.
StoreThumbnail
(
ctx
,
thumbnailMetadata
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
"ActualWidth"
:
width
,
...
...
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go
浏览文件 @
7596c19f
...
...
@@ -17,6 +17,7 @@
package
thumbnailer
import
(
"context"
"image"
"image/draw"
// Imported for gif codec
...
...
@@ -35,7 +36,16 @@ import (
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func
GenerateThumbnails
(
src
types
.
Path
,
configs
[]
config
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
GenerateThumbnails
(
ctx
context
.
Context
,
src
types
.
Path
,
configs
[]
config
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
img
,
err
:=
readFile
(
string
(
src
))
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithField
(
"src"
,
src
)
.
Error
(
"Failed to read src file"
)
...
...
@@ -43,7 +53,10 @@ func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMet
}
for
_
,
config
:=
range
configs
{
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy
,
err
=
createThumbnail
(
src
,
img
,
types
.
ThumbnailSize
(
config
),
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
)
busy
,
err
=
createThumbnail
(
ctx
,
src
,
img
,
types
.
ThumbnailSize
(
config
),
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
,
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithField
(
"src"
,
src
)
.
Error
(
"Failed to generate thumbnails"
)
return
false
,
err
...
...
@@ -56,7 +69,16 @@ func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMet
}
// GenerateThumbnail generates the configured thumbnail size for the source file
func
GenerateThumbnail
(
src
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
GenerateThumbnail
(
ctx
context
.
Context
,
src
types
.
Path
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
img
,
err
:=
readFile
(
string
(
src
))
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
...
...
@@ -65,7 +87,10 @@ func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata
return
false
,
err
}
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy
,
err
=
createThumbnail
(
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
)
busy
,
err
=
createThumbnail
(
ctx
,
src
,
img
,
config
,
mediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
logger
,
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
"src"
:
src
,
...
...
@@ -107,7 +132,17 @@ func writeFile(img image.Image, dst string) (err error) {
// createThumbnail checks if the thumbnail exists, and if not, generates it
// Thumbnail generation is only done once for each non-existing thumbnail.
func
createThumbnail
(
src
types
.
Path
,
img
image
.
Image
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
)
(
busy
bool
,
errorReturn
error
)
{
func
createThumbnail
(
ctx
context
.
Context
,
src
types
.
Path
,
img
image
.
Image
,
config
types
.
ThumbnailSize
,
mediaMetadata
*
types
.
MediaMetadata
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
db
*
storage
.
Database
,
logger
*
log
.
Entry
,
)
(
busy
bool
,
errorReturn
error
)
{
logger
=
logger
.
WithFields
(
log
.
Fields
{
"Width"
:
config
.
Width
,
"Height"
:
config
.
Height
,
...
...
@@ -143,7 +178,7 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
}()
}
exists
,
err
:=
isThumbnailExists
(
dst
,
config
,
mediaMetadata
,
db
,
logger
)
exists
,
err
:=
isThumbnailExists
(
ctx
,
dst
,
config
,
mediaMetadata
,
db
,
logger
)
if
err
!=
nil
||
exists
{
return
false
,
err
}
...
...
@@ -179,7 +214,7 @@ func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize
},
}
err
=
db
.
StoreThumbnail
(
thumbnailMetadata
)
err
=
db
.
StoreThumbnail
(
ctx
,
thumbnailMetadata
)
if
err
!=
nil
{
logger
.
WithError
(
err
)
.
WithFields
(
log
.
Fields
{
"ActualWidth"
:
width
,
...
...
src/github.com/matrix-org/dendrite/mediaapi/writers/download.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
writers
import
(
"context"
"encoding/json"
"fmt"
"io"
...
...
@@ -118,7 +119,9 @@ func Download(
return
}
metadata
,
err
:=
dReq
.
doDownload
(
w
,
cfg
,
db
,
activeRemoteRequests
,
activeThumbnailGeneration
)
metadata
,
err
:=
dReq
.
doDownload
(
req
.
Context
(),
w
,
cfg
,
db
,
activeRemoteRequests
,
activeThumbnailGeneration
,
)
if
err
!=
nil
{
// TODO: Handle the fact we might have started writing the response
dReq
.
jsonErrorResponse
(
w
,
util
.
ErrorResponse
(
err
))
...
...
@@ -192,6 +195,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
}
func
(
r
*
downloadRequest
)
doDownload
(
ctx
context
.
Context
,
w
http
.
ResponseWriter
,
cfg
*
config
.
Dendrite
,
db
*
storage
.
Database
,
...
...
@@ -199,7 +203,9 @@ func (r *downloadRequest) doDownload(
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
)
(
*
types
.
MediaMetadata
,
error
)
{
// check if we have a record of the media in our database
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
)
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
ctx
,
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
)
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"error querying the database"
)
}
...
...
@@ -209,7 +215,9 @@ func (r *downloadRequest) doDownload(
return
nil
,
nil
}
// If we do not have a record and the origin is remote, we need to fetch it and respond with that file
resErr
:=
r
.
getRemoteFile
(
cfg
,
db
,
activeRemoteRequests
,
activeThumbnailGeneration
)
resErr
:=
r
.
getRemoteFile
(
ctx
,
cfg
,
db
,
activeRemoteRequests
,
activeThumbnailGeneration
,
)
if
resErr
!=
nil
{
return
nil
,
resErr
}
...
...
@@ -217,12 +225,17 @@ func (r *downloadRequest) doDownload(
// If we have a record, we can respond from the local file
r
.
MediaMetadata
=
mediaMetadata
}
return
r
.
respondFromLocalFile
(
w
,
cfg
.
Media
.
AbsBasePath
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
,
db
,
cfg
.
Media
.
DynamicThumbnails
,
cfg
.
Media
.
ThumbnailSizes
)
return
r
.
respondFromLocalFile
(
ctx
,
w
,
cfg
.
Media
.
AbsBasePath
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
,
db
,
cfg
.
Media
.
DynamicThumbnails
,
cfg
.
Media
.
ThumbnailSizes
,
)
}
// respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter
// If no file was found then returns nil, nil
func
(
r
*
downloadRequest
)
respondFromLocalFile
(
ctx
context
.
Context
,
w
http
.
ResponseWriter
,
absBasePath
config
.
Path
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
...
...
@@ -256,7 +269,10 @@ func (r *downloadRequest) respondFromLocalFile(
var
responseFile
*
os
.
File
var
responseMetadata
*
types
.
MediaMetadata
if
r
.
IsThumbnailRequest
{
thumbFile
,
thumbMetadata
,
resErr
:=
r
.
getThumbnailFile
(
types
.
Path
(
filePath
),
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
dynamicThumbnails
,
thumbnailSizes
)
thumbFile
,
thumbMetadata
,
resErr
:=
r
.
getThumbnailFile
(
ctx
,
types
.
Path
(
filePath
),
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
dynamicThumbnails
,
thumbnailSizes
,
)
if
thumbFile
!=
nil
{
defer
thumbFile
.
Close
()
// nolint: errcheck
}
...
...
@@ -306,6 +322,7 @@ func (r *downloadRequest) respondFromLocalFile(
// Note: Thumbnail generation may be ongoing asynchronously.
// If no thumbnail was found then returns nil, nil, nil
func
(
r
*
downloadRequest
)
getThumbnailFile
(
ctx
context
.
Context
,
filePath
types
.
Path
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
...
...
@@ -317,7 +334,10 @@ func (r *downloadRequest) getThumbnailFile(
var
err
error
if
dynamicThumbnails
{
thumbnail
,
err
=
r
.
generateThumbnail
(
filePath
,
r
.
ThumbnailSize
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
)
thumbnail
,
err
=
r
.
generateThumbnail
(
ctx
,
filePath
,
r
.
ThumbnailSize
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
)
if
err
!=
nil
{
return
nil
,
nil
,
err
}
...
...
@@ -326,7 +346,9 @@ func (r *downloadRequest) getThumbnailFile(
// to trying to use a pre-generated thumbnail
if
thumbnail
==
nil
{
var
thumbnails
[]
*
types
.
ThumbnailMetadata
thumbnails
,
err
=
db
.
GetThumbnails
(
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
)
thumbnails
,
err
=
db
.
GetThumbnails
(
ctx
,
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
)
if
err
!=
nil
{
return
nil
,
nil
,
errors
.
Wrap
(
err
,
"error looking up thumbnails"
)
}
...
...
@@ -343,7 +365,10 @@ func (r *downloadRequest) getThumbnailFile(
"Height"
:
thumbnailSize
.
Height
,
"ResizeMethod"
:
thumbnailSize
.
ResizeMethod
,
})
.
Info
(
"Pre-generating thumbnail for immediate response."
)
thumbnail
,
err
=
r
.
generateThumbnail
(
filePath
,
*
thumbnailSize
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
)
thumbnail
,
err
=
r
.
generateThumbnail
(
ctx
,
filePath
,
*
thumbnailSize
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
)
if
err
!=
nil
{
return
nil
,
nil
,
err
}
...
...
@@ -378,6 +403,7 @@ func (r *downloadRequest) getThumbnailFile(
}
func
(
r
*
downloadRequest
)
generateThumbnail
(
ctx
context
.
Context
,
filePath
types
.
Path
,
thumbnailSize
types
.
ThumbnailSize
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
...
...
@@ -389,7 +415,10 @@ func (r *downloadRequest) generateThumbnail(
"Height"
:
thumbnailSize
.
Height
,
"ResizeMethod"
:
thumbnailSize
.
ResizeMethod
,
})
busy
,
err
:=
thumbnailer
.
GenerateThumbnail
(
filePath
,
thumbnailSize
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
)
busy
,
err
:=
thumbnailer
.
GenerateThumbnail
(
ctx
,
filePath
,
thumbnailSize
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
,
)
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"error creating thumbnail"
)
}
...
...
@@ -397,7 +426,10 @@ func (r *downloadRequest) generateThumbnail(
return
nil
,
nil
}
var
thumbnail
*
types
.
ThumbnailMetadata
thumbnail
,
err
=
db
.
GetThumbnail
(
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
thumbnailSize
.
Width
,
thumbnailSize
.
Height
,
thumbnailSize
.
ResizeMethod
)
thumbnail
,
err
=
db
.
GetThumbnail
(
ctx
,
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
thumbnailSize
.
Width
,
thumbnailSize
.
Height
,
thumbnailSize
.
ResizeMethod
,
)
if
err
!=
nil
{
return
nil
,
errors
.
Wrap
(
err
,
"error looking up thumbnail"
)
}
...
...
@@ -409,6 +441,7 @@ func (r *downloadRequest) generateThumbnail(
// regardless of how many download requests are received.
// Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines.
func
(
r
*
downloadRequest
)
getRemoteFile
(
ctx
context
.
Context
,
cfg
*
config
.
Dendrite
,
db
*
storage
.
Database
,
activeRemoteRequests
*
types
.
ActiveRemoteRequests
,
...
...
@@ -434,14 +467,20 @@ func (r *downloadRequest) getRemoteFile(
}()
// check if we have a record of the media in our database
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
)
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
ctx
,
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
)
if
err
!=
nil
{
return
errors
.
Wrap
(
err
,
"error querying the database."
)
}
if
mediaMetadata
==
nil
{
// If we do not have a record, we need to fetch the remote file first and then respond from the local file
err
:=
r
.
fetchRemoteFileAndStoreMetadata
(
cfg
.
Media
.
AbsBasePath
,
*
cfg
.
Media
.
MaxFileSizeBytes
,
db
,
cfg
.
Media
.
ThumbnailSizes
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
)
err
:=
r
.
fetchRemoteFileAndStoreMetadata
(
ctx
,
cfg
.
Media
.
AbsBasePath
,
*
cfg
.
Media
.
MaxFileSizeBytes
,
db
,
cfg
.
Media
.
ThumbnailSizes
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
,
)
if
err
!=
nil
{
return
errors
.
Wrap
(
err
,
"error querying the database."
)
}
...
...
@@ -501,6 +540,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act
// fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
func
(
r
*
downloadRequest
)
fetchRemoteFileAndStoreMetadata
(
ctx
context
.
Context
,
absBasePath
config
.
Path
,
maxFileSizeBytes
config
.
FileSizeBytes
,
db
*
storage
.
Database
,
...
...
@@ -521,7 +561,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(
})
.
Info
(
"Storing file metadata to media repository database"
)
// FIXME: timeout db request
if
err
:=
db
.
StoreMediaMetadata
(
r
.
MediaMetadata
);
err
!=
nil
{
if
err
:=
db
.
StoreMediaMetadata
(
ctx
,
r
.
MediaMetadata
);
err
!=
nil
{
// If the file is a duplicate (has the same hash as an existing file) then
// there is valid metadata in the database for that file. As such we only
// remove the file if it is not a duplicate.
...
...
@@ -535,7 +575,10 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(
}
go
func
()
{
busy
,
err
:=
thumbnailer
.
GenerateThumbnails
(
finalPath
,
thumbnailSizes
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
)
busy
,
err
:=
thumbnailer
.
GenerateThumbnails
(
context
.
Background
(),
finalPath
,
thumbnailSizes
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
,
)
if
err
!=
nil
{
r
.
Logger
.
WithError
(
err
)
.
Warn
(
"Error generating thumbnails"
)
}
...
...
src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go
浏览文件 @
7596c19f
...
...
@@ -15,6 +15,7 @@
package
writers
import
(
"context"
"fmt"
"io"
"net/http"
...
...
@@ -57,7 +58,7 @@ func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activ
return
*
resErr
}
if
resErr
=
r
.
doUpload
(
req
.
Body
,
cfg
,
db
,
activeThumbnailGeneration
);
resErr
!=
nil
{
if
resErr
=
r
.
doUpload
(
req
.
Context
(),
req
.
Body
,
cfg
,
db
,
activeThumbnailGeneration
);
resErr
!=
nil
{
return
*
resErr
}
...
...
@@ -97,7 +98,13 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe
return
r
,
nil
}
func
(
r
*
uploadRequest
)
doUpload
(
reqReader
io
.
Reader
,
cfg
*
config
.
Dendrite
,
db
*
storage
.
Database
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
)
*
util
.
JSONResponse
{
func
(
r
*
uploadRequest
)
doUpload
(
ctx
context
.
Context
,
reqReader
io
.
Reader
,
cfg
*
config
.
Dendrite
,
db
*
storage
.
Database
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
)
*
util
.
JSONResponse
{
r
.
Logger
.
WithFields
(
log
.
Fields
{
"UploadName"
:
r
.
MediaMetadata
.
UploadName
,
"FileSizeBytes"
:
r
.
MediaMetadata
.
FileSizeBytes
,
...
...
@@ -134,7 +141,9 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *
})
.
Info
(
"File uploaded"
)
// check if we already have a record of the media in our database and if so, we can remove the temporary directory
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
)
mediaMetadata
,
err
:=
db
.
GetMediaMetadata
(
ctx
,
r
.
MediaMetadata
.
MediaID
,
r
.
MediaMetadata
.
Origin
,
)
if
err
!=
nil
{
r
.
Logger
.
WithError
(
err
)
.
Error
(
"Error querying the database."
)
resErr
:=
jsonerror
.
InternalServerError
()
...
...
@@ -152,7 +161,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *
}
}
if
resErr
:=
r
.
storeFileAndMetadata
(
tmpDir
,
cfg
.
Media
.
AbsBasePath
,
db
,
cfg
.
Media
.
ThumbnailSizes
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
);
resErr
!=
nil
{
if
resErr
:=
r
.
storeFileAndMetadata
(
ctx
,
tmpDir
,
cfg
.
Media
.
AbsBasePath
,
db
,
cfg
.
Media
.
ThumbnailSizes
,
activeThumbnailGeneration
,
cfg
.
Media
.
MaxThumbnailGenerators
,
);
resErr
!=
nil
{
return
resErr
}
...
...
@@ -208,7 +220,15 @@ func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JS
// The order of operations is important as it avoids metadata entering the database before the file
// is ready, and if we fail to move the file, it never gets added to the database.
// Returns a util.JSONResponse error and cleans up directories in case of error.
func
(
r
*
uploadRequest
)
storeFileAndMetadata
(
tmpDir
types
.
Path
,
absBasePath
config
.
Path
,
db
*
storage
.
Database
,
thumbnailSizes
[]
config
.
ThumbnailSize
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
)
*
util
.
JSONResponse
{
func
(
r
*
uploadRequest
)
storeFileAndMetadata
(
ctx
context
.
Context
,
tmpDir
types
.
Path
,
absBasePath
config
.
Path
,
db
*
storage
.
Database
,
thumbnailSizes
[]
config
.
ThumbnailSize
,
activeThumbnailGeneration
*
types
.
ActiveThumbnailGeneration
,
maxThumbnailGenerators
int
,
)
*
util
.
JSONResponse
{
finalPath
,
duplicate
,
err
:=
fileutils
.
MoveFileWithHashCheck
(
tmpDir
,
r
.
MediaMetadata
,
absBasePath
,
r
.
Logger
)
if
err
!=
nil
{
r
.
Logger
.
WithError
(
err
)
.
Error
(
"Failed to move file."
)
...
...
@@ -221,7 +241,7 @@ func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath conf
r
.
Logger
.
WithField
(
"dst"
,
finalPath
)
.
Info
(
"File was stored previously - discarding duplicate"
)
}
if
err
=
db
.
StoreMediaMetadata
(
r
.
MediaMetadata
);
err
!=
nil
{
if
err
=
db
.
StoreMediaMetadata
(
ctx
,
r
.
MediaMetadata
);
err
!=
nil
{
r
.
Logger
.
WithError
(
err
)
.
Warn
(
"Failed to store metadata"
)
// If the file is a duplicate (has the same hash as an existing file) then
// there is valid metadata in the database for that file. As such we only
...
...
@@ -236,7 +256,10 @@ func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath conf
}
go
func
()
{
busy
,
err
:=
thumbnailer
.
GenerateThumbnails
(
finalPath
,
thumbnailSizes
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
)
busy
,
err
:=
thumbnailer
.
GenerateThumbnails
(
context
.
Background
(),
finalPath
,
thumbnailSizes
,
r
.
MediaMetadata
,
activeThumbnailGeneration
,
maxThumbnailGenerators
,
db
,
r
.
Logger
,
)
if
err
!=
nil
{
r
.
Logger
.
WithError
(
err
)
.
Warn
(
"Error generating thumbnails"
)
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录