Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
李少辉-开发者
gitlab-foss
提交
2f40fb45
G
gitlab-foss
项目概览
李少辉-开发者
/
gitlab-foss
通知
15
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
gitlab-foss
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2f40fb45
编写于
6月 13, 2018
作者:
M
Michael Kozono
提交者:
Stan Hu
6月 13, 2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add support for verifying remote uploads, artifacts, and LFS objects in check rake tasks
上级
cc1b0354
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
147 addition
and
41 deletion
+147
-41
changelogs/unreleased/mk-rake-task-verify-remote-files.yml
changelogs/unreleased/mk-rake-task-verify-remote-files.yml
+5
-0
doc/administration/raketasks/check.md
doc/administration/raketasks/check.md
+4
-3
lib/gitlab/verify/batch_verifier.rb
lib/gitlab/verify/batch_verifier.rb
+49
-10
lib/gitlab/verify/job_artifacts.rb
lib/gitlab/verify/job_artifacts.rb
+9
-1
lib/gitlab/verify/lfs_objects.rb
lib/gitlab/verify/lfs_objects.rb
+10
-2
lib/gitlab/verify/rake_task.rb
lib/gitlab/verify/rake_task.rb
+1
-1
lib/gitlab/verify/uploads.rb
lib/gitlab/verify/uploads.rb
+10
-2
spec/lib/gitlab/verify/job_artifacts_spec.rb
spec/lib/gitlab/verify/job_artifacts_spec.rb
+26
-3
spec/lib/gitlab/verify/lfs_objects_spec.rb
spec/lib/gitlab/verify/lfs_objects_spec.rb
+16
-9
spec/lib/gitlab/verify/uploads_spec.rb
spec/lib/gitlab/verify/uploads_spec.rb
+17
-10
未找到文件。
changelogs/unreleased/mk-rake-task-verify-remote-files.yml
0 → 100644
浏览文件 @
2f40fb45
---
title
:
Add support for verifying remote uploads, artifacts, and LFS objects in check rake tasks
merge_request
:
19501
author
:
type
:
added
doc/administration/raketasks/check.md
浏览文件 @
2f40fb45
...
...
@@ -78,9 +78,10 @@ Example output:
## Uploaded Files Integrity
Various types of file can be uploaded to a GitLab installation by users.
Checksums are generated and stored in the database upon upload, and integrity
checks using those checksums can be run. These checks also detect missing files.
Various types of files can be uploaded to a GitLab installation by users.
These integrity checks can detect missing files. Additionally, for locally
stored files, checksums are generated and stored in the database upon upload,
and these checks will verify them against current files.
Currently, integrity checks are supported for the following types of file:
...
...
lib/gitlab/verify/batch_verifier.rb
浏览文件 @
2f40fb45
...
...
@@ -7,13 +7,15 @@ module Gitlab
@batch_size
=
batch_size
@start
=
start
@finish
=
finish
fix_google_api_logger
end
# Yields a Range of IDs and a Hash of failed verifications (object => error)
def
run_batches
(
&
blk
)
relation
.
in_batches
(
of:
batch_size
,
start:
start
,
finish:
finish
)
do
|
relation
|
# rubocop: disable Cop/InBatches
range
=
relation
.
first
.
id
..
relation
.
last
.
id
failures
=
run_batch
(
relation
)
all_relation
.
in_batches
(
of:
batch_size
,
start:
start
,
finish:
finish
)
do
|
batch
|
# rubocop: disable Cop/InBatches
range
=
batch
.
first
.
id
..
batch
.
last
.
id
failures
=
run_batch
_for
(
batch
)
yield
(
range
,
failures
)
end
...
...
@@ -29,24 +31,56 @@ module Gitlab
private
def
run_batch
(
relation
)
relation
.
map
{
|
upload
|
verify
(
upload
)
}.
compact
.
to_h
def
run_batch
_for
(
batch
)
batch
.
map
{
|
upload
|
verify
(
upload
)
}.
compact
.
to_h
end
def
verify
(
object
)
local?
(
object
)
?
verify_local
(
object
)
:
verify_remote
(
object
)
rescue
=>
err
failure
(
object
,
err
.
inspect
)
end
def
verify_local
(
object
)
expected
=
expected_checksum
(
object
)
actual
=
actual_checksum
(
object
)
raise
'Checksum missing'
unless
expected
.
present?
raise
'Checksum mismatch'
unless
expected
==
actual
return
failure
(
object
,
'Checksum missing'
)
unless
expected
.
present?
return
failure
(
object
,
'Checksum mismatch'
)
unless
expected
==
actual
success
end
# We don't calculate checksum for remote objects, so just check existence
def
verify_remote
(
object
)
return
failure
(
object
,
'Remote object does not exist'
)
unless
remote_object_exists?
(
object
)
success
end
def
success
nil
rescue
=>
err
[
object
,
err
]
end
def
failure
(
object
,
message
)
[
object
,
message
]
end
# It's already set to Logger::INFO, but acts as if it is set to
# Logger::DEBUG, and this fixes it...
def
fix_google_api_logger
if
Object
.
const_defined?
(
'Google::Apis'
)
Google
::
Apis
.
logger
.
level
=
Logger
::
INFO
end
end
# This should return an ActiveRecord::Relation suitable for calling #in_batches on
def
relation
def
all_relation
raise
NotImplementedError
.
new
end
# Should return true if the object is stored locally
def
local?
(
_object
)
raise
NotImplementedError
.
new
end
...
...
@@ -59,6 +93,11 @@ module Gitlab
def
actual_checksum
(
_object
)
raise
NotImplementedError
.
new
end
# Be sure to perform a hard check of the remote object (don't just check DB value)
def
remote_object_exists?
(
object
)
raise
NotImplementedError
.
new
end
end
end
end
lib/gitlab/verify/job_artifacts.rb
浏览文件 @
2f40fb45
...
...
@@ -11,10 +11,14 @@ module Gitlab
private
def
relation
def
all_
relation
::
Ci
::
JobArtifact
.
all
end
def
local?
(
artifact
)
artifact
.
local_store?
end
def
expected_checksum
(
artifact
)
artifact
.
file_sha256
end
...
...
@@ -22,6 +26,10 @@ module Gitlab
def
actual_checksum
(
artifact
)
Digest
::
SHA256
.
file
(
artifact
.
file
.
path
).
hexdigest
end
def
remote_object_exists?
(
artifact
)
artifact
.
file
.
file
.
exists?
end
end
end
end
lib/gitlab/verify/lfs_objects.rb
浏览文件 @
2f40fb45
...
...
@@ -11,8 +11,12 @@ module Gitlab
private
def
relation
LfsObject
.
with_files_stored_locally
def
all_relation
LfsObject
.
all
end
def
local?
(
lfs_object
)
lfs_object
.
local_store?
end
def
expected_checksum
(
lfs_object
)
...
...
@@ -22,6 +26,10 @@ module Gitlab
def
actual_checksum
(
lfs_object
)
LfsObject
.
calculate_oid
(
lfs_object
.
file
.
path
)
end
def
remote_object_exists?
(
lfs_object
)
lfs_object
.
file
.
file
.
exists?
end
end
end
end
lib/gitlab/verify/rake_task.rb
浏览文件 @
2f40fb45
...
...
@@ -45,7 +45,7 @@ module Gitlab
return
unless
verbose?
failures
.
each
do
|
object
,
error
|
say
" -
#{
verifier
.
describe
(
object
)
}
:
#{
error
.
inspect
}
"
.
color
(
:red
)
say
" -
#{
verifier
.
describe
(
object
)
}
:
#{
error
}
"
.
color
(
:red
)
end
end
end
...
...
lib/gitlab/verify/uploads.rb
浏览文件 @
2f40fb45
...
...
@@ -11,8 +11,12 @@ module Gitlab
private
def
relation
Upload
.
with_files_stored_locally
def
all_relation
Upload
.
all
end
def
local?
(
upload
)
upload
.
local?
end
def
expected_checksum
(
upload
)
...
...
@@ -22,6 +26,10 @@ module Gitlab
def
actual_checksum
(
upload
)
Upload
.
hexdigest
(
upload
.
absolute_path
)
end
def
remote_object_exists?
(
upload
)
upload
.
build_uploader
.
file
.
exists?
end
end
end
end
spec/lib/gitlab/verify/job_artifacts_spec.rb
浏览文件 @
2f40fb45
...
...
@@ -21,15 +21,38 @@ describe Gitlab::Verify::JobArtifacts do
FileUtils
.
rm_f
(
artifact
.
file
.
path
)
expect
(
failures
.
keys
).
to
contain_exactly
(
artifact
)
expect
(
failure
).
to
be_a
(
Errno
::
ENOENT
)
expect
(
failure
.
to_s
).
to
include
(
artifact
.
file
.
path
)
expect
(
failure
).
to
include
(
'No such file or directory'
)
expect
(
failure
).
to
include
(
artifact
.
file
.
path
)
end
it
'fails artifacts with a mismatched checksum'
do
File
.
truncate
(
artifact
.
file
.
path
,
0
)
expect
(
failures
.
keys
).
to
contain_exactly
(
artifact
)
expect
(
failure
.
to_s
).
to
include
(
'Checksum mismatch'
)
expect
(
failure
).
to
include
(
'Checksum mismatch'
)
end
context
'with remote files'
do
let
(
:file
)
{
double
(
:file
)
}
before
do
stub_artifacts_object_storage
artifact
.
update!
(
file_store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
CarrierWave
::
Storage
::
Fog
::
File
).
to
receive
(
:new
).
and_return
(
file
)
end
it
'passes artifacts in object storage that exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
true
)
expect
(
failures
).
to
eq
({})
end
it
'fails artifacts in object storage that do not exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
false
)
expect
(
failures
.
keys
).
to
contain_exactly
(
artifact
)
expect
(
failure
).
to
include
(
'Remote object does not exist'
)
end
end
end
end
spec/lib/gitlab/verify/lfs_objects_spec.rb
浏览文件 @
2f40fb45
...
...
@@ -21,30 +21,37 @@ describe Gitlab::Verify::LfsObjects do
FileUtils
.
rm_f
(
lfs_object
.
file
.
path
)
expect
(
failures
.
keys
).
to
contain_exactly
(
lfs_object
)
expect
(
failure
).
to
be_a
(
Errno
::
ENOENT
)
expect
(
failure
.
to_s
).
to
include
(
lfs_object
.
file
.
path
)
expect
(
failure
).
to
include
(
'No such file or directory'
)
expect
(
failure
).
to
include
(
lfs_object
.
file
.
path
)
end
it
'fails LFS objects with a mismatched oid'
do
File
.
truncate
(
lfs_object
.
file
.
path
,
0
)
expect
(
failures
.
keys
).
to
contain_exactly
(
lfs_object
)
expect
(
failure
.
to_s
).
to
include
(
'Checksum mismatch'
)
expect
(
failure
).
to
include
(
'Checksum mismatch'
)
end
context
'with remote files'
do
let
(
:file
)
{
double
(
:file
)
}
before
do
stub_lfs_object_storage
lfs_object
.
update!
(
file_store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
CarrierWave
::
Storage
::
Fog
::
File
).
to
receive
(
:new
).
and_return
(
file
)
end
it
'skips LFS objects in object storage'
do
local_failure
=
create
(
:lfs_object
)
create
(
:lfs_object
,
:object_storage
)
it
'passes LFS objects in object storage that exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
true
)
expect
(
failures
).
to
eq
({})
end
failures
=
{}
described_class
.
new
(
batch_size:
10
).
run_batches
{
|
_
,
failed
|
failures
.
merge!
(
failed
)
}
it
'fails LFS objects in object storage that do not exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
false
)
expect
(
failures
.
keys
).
to
contain_exactly
(
local_failure
)
expect
(
failures
.
keys
).
to
contain_exactly
(
lfs_object
)
expect
(
failure
).
to
include
(
'Remote object does not exist'
)
end
end
end
...
...
spec/lib/gitlab/verify/uploads_spec.rb
浏览文件 @
2f40fb45
...
...
@@ -23,37 +23,44 @@ describe Gitlab::Verify::Uploads do
FileUtils
.
rm_f
(
upload
.
absolute_path
)
expect
(
failures
.
keys
).
to
contain_exactly
(
upload
)
expect
(
failure
).
to
be_a
(
Errno
::
ENOENT
)
expect
(
failure
.
to_s
).
to
include
(
upload
.
absolute_path
)
expect
(
failure
).
to
include
(
'No such file or directory'
)
expect
(
failure
).
to
include
(
upload
.
absolute_path
)
end
it
'fails uploads with a mismatched checksum'
do
upload
.
update!
(
checksum:
'something incorrect'
)
expect
(
failures
.
keys
).
to
contain_exactly
(
upload
)
expect
(
failure
.
to_s
).
to
include
(
'Checksum mismatch'
)
expect
(
failure
).
to
include
(
'Checksum mismatch'
)
end
it
'fails uploads with a missing precalculated checksum'
do
upload
.
update!
(
checksum:
''
)
expect
(
failures
.
keys
).
to
contain_exactly
(
upload
)
expect
(
failure
.
to_s
).
to
include
(
'Checksum missing'
)
expect
(
failure
).
to
include
(
'Checksum missing'
)
end
context
'with remote files'
do
let
(
:file
)
{
double
(
:file
)
}
before
do
stub_uploads_object_storage
(
AvatarUploader
)
upload
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
CarrierWave
::
Storage
::
Fog
::
File
).
to
receive
(
:new
).
and_return
(
file
)
end
it
'skips uploads in object storage'
do
local_failure
=
create
(
:upload
)
create
(
:upload
,
:object_storage
)
it
'passes uploads in object storage that exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
true
)
expect
(
failures
).
to
eq
({})
end
failures
=
{}
described_class
.
new
(
batch_size:
10
).
run_batches
{
|
_
,
failed
|
failures
.
merge!
(
failed
)
}
it
'fails uploads in object storage that do not exist'
do
expect
(
file
).
to
receive
(
:exists?
).
and_return
(
false
)
expect
(
failures
.
keys
).
to
contain_exactly
(
local_failure
)
expect
(
failures
.
keys
).
to
contain_exactly
(
upload
)
expect
(
failure
).
to
include
(
'Remote object does not exist'
)
end
end
end
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录