Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
李少辉-开发者
gitlab-foss
提交
0d00d02e
G
gitlab-foss
项目概览
李少辉-开发者
/
gitlab-foss
通知
15
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
gitlab-foss
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
0d00d02e
编写于
6月 03, 2018
作者:
S
Shinya Maeda
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Directly refer application code from migration code
上级
b626fcc0
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
22 addition
and
85 deletion
+22
-85
app/models/ci/build.rb
app/models/ci/build.rb
+5
-0
db/post_migrate/20180529152628_archive_legacy_traces.rb
db/post_migrate/20180529152628_archive_legacy_traces.rb
+4
-15
lib/gitlab/background_migration/archive_legacy_traces.rb
lib/gitlab/background_migration/archive_legacy_traces.rb
+11
-66
lib/tasks/gitlab/traces.rake
lib/tasks/gitlab/traces.rake
+1
-3
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb
...gitlab/background_migration/archive_legacy_traces_spec.rb
+1
-1
未找到文件。
app/models/ci/build.rb
浏览文件 @
0d00d02e
...
...
@@ -55,6 +55,11 @@ module Ci
where
(
'(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)'
,
''
,
Ci
::
JobArtifact
.
select
(
1
).
where
(
'ci_builds.id = ci_job_artifacts.job_id'
).
archive
)
end
scope
:without_archived_trace
,
->
()
do
where
(
'NOT EXISTS (?)'
,
Ci
::
JobArtifact
.
select
(
1
).
where
(
'ci_builds.id = ci_job_artifacts.job_id'
).
trace
)
end
scope
:with_artifacts_stored_locally
,
->
{
with_artifacts_archive
.
where
(
artifacts_file_store:
[
nil
,
LegacyArtifactUploader
::
Store
::
LOCAL
])
}
scope
:with_artifacts_not_expired
,
->
()
{
with_artifacts_archive
.
where
(
'artifacts_expire_at IS NULL OR artifacts_expire_at > ?'
,
Time
.
now
)
}
scope
:with_expired_artifacts
,
->
()
{
with_artifacts_archive
.
where
(
'artifacts_expire_at < ?'
,
Time
.
now
)
}
...
...
db/post_migrate/20180529152628_archive_legacy_traces.rb
浏览文件 @
0d00d02e
...
...
@@ -2,7 +2,7 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
BATCH_SIZE
=
10_
000
BATCH_SIZE
=
5
000
BACKGROUND_MIGRATION_CLASS
=
'ArchiveLegacyTraces'
disable_ddl_transaction!
...
...
@@ -14,25 +14,14 @@ class ArchiveLegacyTraces < ActiveRecord::Migration
scope
:finished
,
->
{
where
(
status:
[
:success
,
:failed
,
:canceled
])
}
scope
:without_new_traces
,
->
()
do
where
(
'NOT EXISTS (?)'
,
::
ArchiveLegacyTraces
::
JobArtifact
.
select
(
1
).
trace
.
where
(
'ci_builds.id = ci_job_artifacts.job_id'
))
scope
:without_archived_trace
,
->
do
where
(
'NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)'
)
end
end
class
JobArtifact
<
ActiveRecord
::
Base
self
.
table_name
=
'ci_job_artifacts'
enum
file_type:
{
archive:
1
,
metadata:
2
,
trace:
3
}
end
def
up
queue_background_migration_jobs_by_range_at_intervals
(
::
ArchiveLegacyTraces
::
Build
.
finished
.
without_
new_traces
,
::
ArchiveLegacyTraces
::
Build
.
finished
.
without_
archived_trace
,
BACKGROUND_MIGRATION_CLASS
,
5
.
minutes
,
batch_size:
BATCH_SIZE
)
...
...
lib/gitlab/background_migration/archive_legacy_traces.rb
浏览文件 @
0d00d02e
...
...
@@ -5,73 +5,18 @@
module
Gitlab
module
BackgroundMigration
class
ArchiveLegacyTraces
class
Build
<
ActiveRecord
::
Base
include
::
HasStatus
self
.
table_name
=
'ci_builds'
self
.
inheritance_column
=
:_type_disabled
# Disable STI
belongs_to
:project
,
foreign_key: :project_id
,
class_name:
'ArchiveLegacyTraces::Project'
has_one
:job_artifacts_trace
,
->
()
{
where
(
file_type:
ArchiveLegacyTraces
::
JobArtifact
.
file_types
[
:trace
])
},
class_name:
'ArchiveLegacyTraces::JobArtifact'
,
foreign_key: :job_id
has_many
:trace_chunks
,
foreign_key: :build_id
,
class_name:
'ArchiveLegacyTraces::BuildTraceChunk'
scope
:finished
,
->
{
where
(
status:
[
:success
,
:failed
,
:canceled
])
}
scope
:without_new_traces
,
->
()
do
finished
.
where
(
'NOT EXISTS (?)'
,
BackgroundMigration
::
ArchiveLegacyTraces
::
JobArtifact
.
select
(
1
).
trace
.
where
(
'ci_builds.id = ci_job_artifacts.job_id'
))
end
def
trace
::
Gitlab
::
Ci
::
Trace
.
new
(
self
)
end
def
trace
=
(
data
)
raise
NotImplementedError
end
def
old_trace
read_attribute
(
:trace
)
end
def
erase_old_trace!
update_column
(
:trace
,
nil
)
end
end
class
JobArtifact
<
ActiveRecord
::
Base
self
.
table_name
=
'ci_job_artifacts'
belongs_to
:build
belongs_to
:project
mount_uploader
:file
,
JobArtifactUploader
enum
file_type:
{
archive:
1
,
metadata:
2
,
trace:
3
}
end
class
BuildTraceChunk
<
ActiveRecord
::
Base
self
.
table_name
=
'ci_build_trace_chunks'
belongs_to
:build
end
class
Project
<
ActiveRecord
::
Base
self
.
table_name
=
'projects'
has_many
:builds
,
foreign_key: :project_id
,
class_name:
'ArchiveLegacyTraces::Build'
end
def
perform
(
start_id
,
stop_id
)
BackgroundMigration
::
ArchiveLegacyTraces
::
Build
.
finished
.
without_new_traces
.
where
(
id:
(
start_id
..
stop_id
)).
find_each
do
|
build
|
build
.
trace
.
archive!
# This background migrations directly refer ::Ci::Build model which is defined in application code.
# In general, migration code should be isolated as much as possible in order to be idempotent.
# However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly
# and we don't change the `archive!` logic until 11.1
::
Ci
::
Build
.
finished
.
without_archived_trace
.
where
(
id:
start_id
..
stop_id
).
find_each
do
|
build
|
begin
build
.
trace
.
archive!
rescue
=>
e
Rails
.
logger
.
error
"Failed to archive live trace. id:
#{
build
.
id
}
message:
#{
e
.
message
}
"
end
end
end
end
...
...
lib/tasks/gitlab/traces.rake
浏览文件 @
0d00d02e
...
...
@@ -8,9 +8,7 @@ namespace :gitlab do
logger
=
Logger
.
new
(
STDOUT
)
logger
.
info
(
'Archiving legacy traces'
)
Ci
::
Build
.
finished
.
where
(
'NOT EXISTS (?)'
,
Ci
::
JobArtifact
.
select
(
1
).
trace
.
where
(
'ci_builds.id = ci_job_artifacts.job_id'
))
Ci
::
Build
.
finished
.
without_archived_trace
.
order
(
id: :asc
)
.
find_in_batches
(
batch_size:
1000
)
do
|
jobs
|
job_ids
=
jobs
.
map
{
|
job
|
[
job
.
id
]
}
...
...
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb
浏览文件 @
0d00d02e
...
...
@@ -14,7 +14,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2
@legacy_trace_dir
=
File
.
join
(
Settings
.
gitlab_ci
.
builds_path
,
build
.
created_at
.
utc
.
strftime
(
"%Y_%m"
),
build
.
project_id
.
to_s
)
FileUtils
.
mkdir_p
(
@legacy_trace_dir
)
@legacy_trace_path
=
File
.
join
(
@legacy_trace_dir
,
"
#{
build
.
id
}
.log"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录