Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
李少辉-开发者
gitlab-foss
提交
4f6999fa
G
gitlab-foss
项目概览
李少辉-开发者
/
gitlab-foss
通知
15
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
gitlab-foss
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4f6999fa
编写于
12月 04, 2018
作者:
O
Oswaldo Ferreira
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Populate MR metrics with events table information (migration)
上级
f94388f6
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
247 addition
and
1 deletion
+247
-1
changelogs/unreleased/osw-update-mr-metrics-with-events-data.yml
...ogs/unreleased/osw-update-mr-metrics-with-events-data.yml
+5
-0
db/post_migrate/20181204154019_populate_mr_metrics_with_events_data.rb
...te/20181204154019_populate_mr_metrics_with_events_data.rb
+38
-0
db/schema.rb
db/schema.rb
+1
-1
lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb
...pulate_merge_request_metrics_with_events_data_improved.rb
+99
-0
spec/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved_spec.rb
...e_merge_request_metrics_with_events_data_improved_spec.rb
+57
-0
spec/migrations/populate_mr_metrics_with_events_data_spec.rb
spec/migrations/populate_mr_metrics_with_events_data_spec.rb
+47
-0
未找到文件。
changelogs/unreleased/osw-update-mr-metrics-with-events-data.yml
0 → 100644
浏览文件 @
4f6999fa
---
title
:
Populate MR metrics with events table information (migration)
merge_request
:
23564
author
:
type
:
performance
db/post_migrate/20181204154019_populate_mr_metrics_with_events_data.rb
0 → 100644
浏览文件 @
4f6999fa
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class
PopulateMrMetricsWithEventsData
<
ActiveRecord
::
Migration
[
4.2
]
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
BATCH_SIZE
=
10_000
MIGRATION
=
'PopulateMergeRequestMetricsWithEventsDataImproved'
PREVIOUS_MIGRATION
=
'PopulateMergeRequestMetricsWithEventsData'
disable_ddl_transaction!
def
up
# Perform any ongoing background migration that might still be running from
# previous try (see https://gitlab.com/gitlab-org/gitlab-ce/issues/47676).
Gitlab
::
BackgroundMigration
.
steal
(
PREVIOUS_MIGRATION
)
say
'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
# It will update around 4_000_000 records in batches of 10_000 merge
# requests (running between 5 minutes) and should take around 53 hours to complete.
# Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples
# per minute. So this should give us enough space.
#
# More information about the updates in `PopulateMergeRequestMetricsWithEventsDataImproved` class.
#
MergeRequest
.
all
.
each_batch
(
of:
BATCH_SIZE
)
do
|
relation
,
index
|
range
=
relation
.
pluck
(
'MIN(id)'
,
'MAX(id)'
).
first
BackgroundMigrationWorker
.
perform_in
(
index
*
8
.
minutes
,
MIGRATION
,
range
)
end
end
def
down
end
end
db/schema.rb
浏览文件 @
4f6999fa
...
@@ -10,7 +10,7 @@
...
@@ -10,7 +10,7 @@
#
#
# It's strongly recommended that you check this file into your version control system.
# It's strongly recommended that you check this file into your version control system.
ActiveRecord
::
Schema
.
define
(
version:
2018120
3002526
)
do
ActiveRecord
::
Schema
.
define
(
version:
2018120
4154019
)
do
# These are extensions that must be enabled in order to support this database
# These are extensions that must be enabled in order to support this database
enable_extension
"plpgsql"
enable_extension
"plpgsql"
...
...
lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb
0 → 100644
浏览文件 @
4f6999fa
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module
Gitlab
module
BackgroundMigration
class
PopulateMergeRequestMetricsWithEventsDataImproved
CLOSED_EVENT_ACTION
=
3
MERGED_EVENT_ACTION
=
7
def
perform
(
min_merge_request_id
,
max_merge_request_id
)
insert_metrics_for_range
(
min_merge_request_id
,
max_merge_request_id
)
update_metrics_with_events_data
(
min_merge_request_id
,
max_merge_request_id
)
end
# Inserts merge_request_metrics records for merge_requests without it for
# a given merge request batch.
def
insert_metrics_for_range
(
min
,
max
)
metrics_not_exists_clause
=
<<-
SQL
.
strip_heredoc
NOT EXISTS (SELECT 1 FROM merge_request_metrics
WHERE merge_request_metrics.merge_request_id = merge_requests.id)
SQL
MergeRequest
.
where
(
metrics_not_exists_clause
).
where
(
id:
min
..
max
).
each_batch
do
|
batch
|
select_sql
=
batch
.
select
(
:id
,
:created_at
,
:updated_at
).
to_sql
execute
(
"INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at)
#{
select_sql
}
"
)
end
end
def
update_metrics_with_events_data
(
min
,
max
)
if
Gitlab
::
Database
.
postgresql?
psql_update_metrics_with_events_data
(
min
,
max
)
else
mysql_update_metrics_with_events_data
(
min
,
max
)
end
end
def
psql_update_metrics_with_events_data
(
min
,
max
)
update_sql
=
<<-
SQL
.
strip_heredoc
UPDATE merge_request_metrics
SET (latest_closed_at,
latest_closed_by_id) =
( SELECT updated_at,
author_id
FROM events
WHERE target_id = merge_request_id
AND target_type = 'MergeRequest'
AND action =
#{
CLOSED_EVENT_ACTION
}
ORDER BY id DESC
LIMIT 1 ),
merged_by_id =
( SELECT author_id
FROM events
WHERE target_id = merge_request_id
AND target_type = 'MergeRequest'
AND action =
#{
MERGED_EVENT_ACTION
}
ORDER BY id DESC
LIMIT 1 )
WHERE merge_request_id BETWEEN
#{
min
}
AND
#{
max
}
SQL
execute
(
update_sql
)
end
def
mysql_update_metrics_with_events_data
(
min
,
max
)
closed_updated_at_subquery
=
mysql_events_select
(
:updated_at
,
CLOSED_EVENT_ACTION
)
closed_author_id_subquery
=
mysql_events_select
(
:author_id
,
CLOSED_EVENT_ACTION
)
merged_author_id_subquery
=
mysql_events_select
(
:author_id
,
MERGED_EVENT_ACTION
)
update_sql
=
<<-
SQL
.
strip_heredoc
UPDATE merge_request_metrics
SET latest_closed_at = (
#{
closed_updated_at_subquery
}
),
latest_closed_by_id = (
#{
closed_author_id_subquery
}
),
merged_by_id = (
#{
merged_author_id_subquery
}
)
WHERE merge_request_id BETWEEN
#{
min
}
AND
#{
max
}
SQL
execute
(
update_sql
)
end
def
mysql_events_select
(
column
,
action
)
<<-
SQL
.
strip_heredoc
SELECT
#{
column
}
FROM events
WHERE target_id = merge_request_id
AND target_type = 'MergeRequest'
AND action =
#{
action
}
ORDER BY id DESC
LIMIT 1
SQL
end
def
execute
(
sql
)
@connection
||=
ActiveRecord
::
Base
.
connection
@connection
.
execute
(
sql
)
end
end
end
end
spec/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved_spec.rb
0 → 100644
浏览文件 @
4f6999fa
# frozen_string_literal: true
require
'rails_helper'
describe
Gitlab
::
BackgroundMigration
::
PopulateMergeRequestMetricsWithEventsDataImproved
,
:migration
,
schema:
20181204154019
do
let
(
:namespaces
)
{
table
(
:namespaces
)
}
let
(
:projects
)
{
table
(
:projects
)
}
let
(
:users
)
{
table
(
:users
)
}
let
(
:events
)
{
table
(
:events
)
}
let
(
:user
)
{
users
.
create!
(
email:
'test@example.com'
,
projects_limit:
100
,
username:
'test'
)
}
let
(
:namespace
)
{
namespaces
.
create
(
name:
'gitlab'
,
path:
'gitlab-org'
)
}
let
(
:project
)
{
projects
.
create
(
namespace_id:
namespace
.
id
,
name:
'foo'
)
}
let
(
:merge_requests
)
{
table
(
:merge_requests
)
}
def
create_merge_request
(
id
,
params
=
{})
params
.
merge!
(
id:
id
,
target_project_id:
project
.
id
,
target_branch:
'master'
,
source_project_id:
project
.
id
,
source_branch:
'mr name'
,
title:
"mr name
#{
id
}
"
)
merge_requests
.
create
(
params
)
end
def
create_merge_request_event
(
id
,
params
=
{})
params
.
merge!
(
id:
id
,
project_id:
project
.
id
,
author_id:
user
.
id
,
target_type:
'MergeRequest'
)
events
.
create
(
params
)
end
describe
'#perform'
do
it
'creates and updates closed and merged events'
do
timestamp
=
Time
.
new
(
'2018-01-01 12:00:00'
).
utc
create_merge_request
(
1
)
create_merge_request_event
(
1
,
target_id:
1
,
action:
3
,
updated_at:
timestamp
)
create_merge_request_event
(
2
,
target_id:
1
,
action:
3
,
updated_at:
timestamp
+
10
.
seconds
)
create_merge_request_event
(
3
,
target_id:
1
,
action:
7
,
updated_at:
timestamp
)
create_merge_request_event
(
4
,
target_id:
1
,
action:
7
,
updated_at:
timestamp
+
10
.
seconds
)
subject
.
perform
(
1
,
1
)
merge_request
=
MergeRequest
.
first
expect
(
merge_request
.
metrics
).
to
have_attributes
(
latest_closed_by_id:
user
.
id
,
latest_closed_at:
timestamp
+
10
.
seconds
,
merged_by_id:
user
.
id
)
end
end
end
spec/migrations/populate_mr_metrics_with_events_data_spec.rb
0 → 100644
浏览文件 @
4f6999fa
# frozen_string_literal: true
require
'spec_helper'
require
Rails
.
root
.
join
(
'db'
,
'post_migrate'
,
'20181204154019_populate_mr_metrics_with_events_data.rb'
)
describe
PopulateMrMetricsWithEventsData
,
:migration
,
:sidekiq
do
let
(
:namespaces
)
{
table
(
:namespaces
)
}
let
(
:projects
)
{
table
(
:projects
)
}
let
(
:namespace
)
{
namespaces
.
create
(
name:
'gitlab'
,
path:
'gitlab-org'
)
}
let
(
:project
)
{
projects
.
create
(
namespace_id:
namespace
.
id
,
name:
'foo'
)
}
let
(
:merge_requests
)
{
table
(
:merge_requests
)
}
def
create_merge_request
(
id
)
params
=
{
id:
id
,
target_project_id:
project
.
id
,
target_branch:
'master'
,
source_project_id:
project
.
id
,
source_branch:
'mr name'
,
title:
"mr name
#{
id
}
"
}
merge_requests
.
create!
(
params
)
end
it
'correctly schedules background migrations'
do
create_merge_request
(
1
)
create_merge_request
(
2
)
create_merge_request
(
3
)
stub_const
(
"
#{
described_class
.
name
}
::BATCH_SIZE"
,
2
)
Sidekiq
::
Testing
.
fake!
do
Timecop
.
freeze
do
migrate!
expect
(
described_class
::
MIGRATION
)
.
to
be_scheduled_delayed_migration
(
8
.
minutes
,
1
,
2
)
expect
(
described_class
::
MIGRATION
)
.
to
be_scheduled_delayed_migration
(
16
.
minutes
,
3
,
3
)
expect
(
BackgroundMigrationWorker
.
jobs
.
size
).
to
eq
(
2
)
end
end
end
end
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录