Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
喜羊羊3508
Dak
提交
7d548bdf
D
Dak
项目概览
喜羊羊3508
/
Dak
12 个月 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
Dak
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7d548bdf
编写于
6月 08, 2015
作者:
N
Niels Thykier
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Rewrite auto-decruft to group removals
Signed-off-by:
N
Niels Thykier
<
niels@thykier.net
>
上级
ff1199ac
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
170 addition
and
103 deletion
+170
-103
dak/auto_decruft.py
dak/auto_decruft.py
+170
-103
未找到文件。
dak/auto_decruft.py
浏览文件 @
7d548bdf
...
...
@@ -35,6 +35,8 @@ Check for obsolete binary packages
import
sys
import
apt_pkg
from
itertools
import
chain
,
product
from
collections
import
defaultdict
from
daklib.config
import
Config
from
daklib.dbconn
import
*
...
...
@@ -44,6 +46,7 @@ from daklib.rm import remove, ReverseDependencyChecker
################################################################################
def
usage
(
exit_code
=
0
):
print
"""Usage: dak cruft-report
Check for obsolete or duplicated packages.
...
...
@@ -55,8 +58,87 @@ Check for obsolete or duplicated packages.
################################################################################
def
remove_sourceless_cruft
(
suite_name
,
suite_id
,
session
,
dryrun
,
debug
):
"""Remove binaries without a source
def
compute_sourceless_groups
(
suite_id
,
session
):
"""Find binaries without a source
@type suite_id: int
@param suite_id: The id of the suite donated by suite_name
@type session: SQLA Session
@param session: The database session in use
"""""
rows
=
query_without_source
(
suite_id
,
session
)
message
=
'[auto-cruft] no longer built from source, no reverse dependencies'
arch_all_id_tuple
=
tuple
([
get_architecture
(
'all'
,
session
=
session
)])
arch_all_list
=
[
"all"
]
for
row
in
rows
:
package
=
row
[
0
]
group_info
=
{
"name"
:
"sourceless:%s"
%
package
,
"packages"
:
tuple
([
package
]),
"architectures"
:
arch_all_list
,
"architecture_ids"
:
arch_all_id_tuple
,
"message"
:
message
,
"removal_request"
:
{
package
:
arch_all_list
,
},
}
yield
group_info
def
compute_nbs_groups
(
suite_id
,
suite_name
,
session
):
"""Find binaries no longer built
@type suite_id: int
@param suite_id: The id of the suite donated by suite_name
@type suite_name: string
@param suite_name: The name of the suite to remove from
@type session: SQLA Session
@param session: The database session in use
"""""
rows
=
queryNBS
(
suite_id
,
session
)
arch2ids
=
dict
((
a
.
arch_string
,
a
.
arch_id
)
for
a
in
get_suite_architectures
(
suite_name
))
for
row
in
rows
:
(
pkg_list
,
arch_list
,
source
,
_
)
=
row
message
=
'[auto-cruft] NBS (no longer built by %s, no reverse dependencies)'
%
source
removal_request
=
dict
((
pkg
,
arch_list
)
for
pkg
in
pkg_list
)
group_info
=
{
"name"
:
"NBS:%s"
%
source
,
"packages"
:
tuple
(
sorted
(
pkg_list
)),
"architectures"
:
sorted
(
arch_list
,
cmp
=
utils
.
arch_compare_sw
),
"architecture_ids"
:
tuple
(
arch2ids
[
arch
]
for
arch
in
arch_list
),
"message"
:
message
,
"removal_request"
:
removal_request
,
}
yield
group_info
def
remove_groups
(
groups
,
suite_id
,
suite_name
,
session
):
for
group
in
groups
:
message
=
group
[
"message"
]
params
=
{
"architecture_ids"
:
group
[
"architecture_ids"
],
"packages"
:
group
[
"packages"
],
"suite_id"
:
suite_id
}
q
=
session
.
execute
(
"""
SELECT b.package, b.version, a.arch_string, b.id
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN architecture a ON b.architecture = a.id
JOIN suite su ON ba.suite = su.id
WHERE a.id IN :architecture_ids AND b.package IN :packages AND su.id = :suite_id
"""
,
params
)
remove
(
session
,
message
,
[
suite_name
],
list
(
q
),
partial
=
True
,
whoami
=
"DAK's auto-decrufter"
)
def
auto_decruft_suite
(
suite_name
,
suite_id
,
session
,
dryrun
,
debug
):
"""Run the auto-decrufter on a given suite
@type suite_name: string
@param suite_name: The name of the suite to remove from
...
...
@@ -72,27 +154,51 @@ def remove_sourceless_cruft(suite_name, suite_id, session, dryrun, debug):
@type debug: bool
@param debug: If True, print some extra information
"""""
global
Options
rows
=
query_without_source
(
suite_id
,
session
)
arch_all_id
=
get_architecture
(
'all'
,
session
=
session
)
discarded_removal
=
set
()
message
=
'[auto-cruft] no longer built from source, no reverse dependencies'
all_packages
=
dict
((
row
[
0
],
None
)
for
row
in
rows
)
if
not
all_packages
:
"""
all_architectures
=
[
a
.
arch_string
for
a
in
get_suite_architectures
(
suite_name
)]
pkg_arch2groups
=
defaultdict
(
set
)
group_order
=
[]
groups
=
{}
full_removal_request
=
[]
group_generator
=
chain
(
compute_sourceless_groups
(
suite_id
,
session
),
compute_nbs_groups
(
suite_id
,
suite_name
,
session
)
)
for
group
in
group_generator
:
group_name
=
group
[
"name"
]
pkgs
=
group
[
"packages"
]
affected_archs
=
group
[
"architectures"
]
removal_request
=
group
[
"removal_request"
]
# If we remove an arch:all package, then the breakage can occur on any
# of the architectures.
if
"all"
in
affected_archs
:
affected_archs
=
all_architectures
for
pkg_arch
in
product
(
pkgs
,
affected_archs
):
pkg_arch2groups
[
pkg_arch
].
add
(
group_name
)
groups
[
group_name
]
=
group
group_order
.
append
(
group_name
)
full_removal_request
.
extend
(
removal_request
.
iteritems
())
if
not
groups
:
if
debug
:
print
"N: Found no candidates"
return
if
debug
:
print
"N: Considering to remove %s"
%
str
(
sorted
(
all_packages
.
iterkeys
()))
print
"N: Considering to remove the following packages:"
for
group_name
in
sorted
(
groups
):
group_info
=
groups
[
group_name
]
pkgs
=
group_info
[
"packages"
]
archs
=
group_info
[
"architectures"
]
print
"N: * %s: %s [%s]"
%
(
group_name
,
", "
.
join
(
pkgs
),
" "
.
join
(
archs
))
if
debug
:
print
"N: Compiling ReverseDependencyChecker (RDC) - please hold ..."
rdc
=
ReverseDependencyChecker
(
session
,
suite_name
)
if
debug
:
print
"N: Computing initial breakage..."
breakage
=
rdc
.
check_reverse_depends
(
all_packages
)
breakage
=
rdc
.
check_reverse_depends
(
full_removal_request
)
while
breakage
:
by_breakers
=
[(
len
(
breakage
[
x
]),
x
,
breakage
[
x
])
for
x
in
breakage
]
by_breakers
.
sort
(
reverse
=
True
)
...
...
@@ -104,112 +210,74 @@ def remove_sourceless_cruft(suite_name, suite_id, session, dryrun, debug):
broken_str
=
", "
.
join
(
"%s/%s"
%
b
for
b
in
sorted
(
broken
))
print
"N: * %s => %s"
%
(
bname
,
broken_str
)
_
,
worst_package_arch
,
worst_breakage
=
by_breakers
.
pop
(
0
)
averted_breakage
=
set
(
worst_breakage
)
del
all_packages
[
worst_package_arch
[
0
]]
discarded_removal
.
add
(
worst_package_arch
[
0
])
if
debug
:
print
"N: - skip removal of %s (due to %s)"
%
(
worst_package_arch
[
0
],
sorted
(
averted_breakage
))
averted_breakage
=
set
()
for
_
,
package_arch
,
breakage
in
by_breakers
:
package
=
package_arch
[
0
]
if
breakage
<=
averted_breakage
:
# We already avoided this break
continue
if
package
in
discarded_removal
:
averted_breakage
|=
breakage
continue
guilty_groups
=
pkg_arch2groups
[
package_arch
]
if
not
guilty_groups
:
utils
.
fubar
(
"Cannot figure what group provided %s"
%
str
(
package_arch
))
if
debug
:
print
"N: - skip removal of %s (due to %s)"
%
(
package
,
str
(
sorted
(
breakage
-
averted_breakage
)))
discarded_removal
.
add
(
package
)
# Only output it, if it truly a new group being discarded
# - a group can reach this part multiple times, if it breaks things on
# more than one architecture. This being rather common in fact.
already_discard
=
True
if
any
(
group_name
for
group_name
in
guilty_groups
if
group_name
in
groups
):
already_discard
=
False
if
not
already_discard
:
avoided
=
sorted
(
breakage
-
averted_breakage
)
print
"N: - skipping removal of %s (breakage: %s)"
%
(
", "
.
join
(
sorted
(
guilty_groups
)),
str
(
avoided
))
averted_breakage
|=
breakage
del
all_packages
[
package
]
for
group_name
in
guilty_groups
:
if
group_name
in
groups
:
del
groups
[
group_name
]
if
not
all_package
s
:
if
not
group
s
:
if
debug
:
print
"N: Nothing left to remove"
return
if
debug
:
print
"N: Now considering to remove %s"
%
str
(
sorted
(
all_packages
.
iterkeys
()))
breakage
=
rdc
.
check_reverse_depends
(
all_packages
)
print
"N: Now considering to remove: %s"
%
str
(
", "
.
join
(
sorted
(
groups
.
iterkeys
())))
# Rebuild the removal request with the remaining groups and off
# we go to (not) break the world once more time
full_removal_request
=
[]
for
group_info
in
groups
.
itervalues
():
full_removal_request
.
extend
(
group_info
[
"removal_request"
].
iteritems
())
breakage
=
rdc
.
check_reverse_depends
(
full_removal_request
)
if
debug
:
print
"N: Removal looks good"
if
dryrun
:
# Embed the -R just in case someone wants to run it manually later
print
'Would do: dak rm -m "%s" -s %s -a all -p -R -b %s'
%
\
(
message
,
suite_name
,
" "
.
join
(
sorted
(
all_packages
)))
else
:
params
=
{
arch_all_id
:
arch_all_id
,
all_packages
:
tuple
(
all_packages
),
suite_id
:
suite_id
}
q
=
session
.
execute
(
"""
SELECT b.package, b.version, a.arch_string, b.id
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN architecture a ON b.architecture = a.id
JOIN suite su ON ba.suite = su.id
WHERE a.id = :arch_all_id AND b.package IN :all_packages AND su.id = :suite_id
"""
,
params
)
remove
(
session
,
message
,
[
suite_name
],
list
(
q
),
partial
=
True
,
whoami
=
"DAK's auto-decrufter"
)
def
removeNBS
(
suite_name
,
suite_id
,
session
,
dryrun
):
"""Remove binaries no longer built
@type suite_name: string
@param suite_name: The name of the suite to remove from
@type suite_id: int
@param suite_id: The id of the suite donated by suite_name
@type session: SQLA Session
@param session: The database session in use
@type dryrun: bool
@param dryrun: If True, just print the actions rather than actually doing them
"""""
global
Options
rows
=
queryNBS
(
suite_id
,
session
)
arch2ids
=
{}
for
row
in
rows
:
(
pkg_list
,
arch_list
,
source
,
_
)
=
row
if
utils
.
check_reverse_depends
(
pkg_list
,
suite_name
,
arch_list
,
session
,
cruft
=
True
,
quiet
=
True
):
continue
arch_string
=
','
.
join
(
arch_list
)
message
=
'[auto-cruft] NBS (no longer built by %s, no reverse dependencies)'
%
source
print
"Would remove the equivalent of:"
for
group_name
in
group_order
:
if
group_name
not
in
groups
:
continue
group_info
=
groups
[
group_name
]
pkgs
=
group_info
[
"packages"
]
archs
=
group_info
[
"architectures"
]
message
=
group_info
[
"message"
]
if
dryrun
:
# Embed the -R just in case someone wants to run it manually later
pkg_string
=
' '
.
join
(
pkg_list
)
print
'Would do: dak rm -m "%s" -s %s -a %s -p -R -b %s'
%
\
(
message
,
suite_name
,
arch_string
,
pkg_string
)
else
:
for
architecture
in
arch_list
:
if
architecture
in
arch2ids
:
arch2ids
[
architecture
]
=
utils
.
get_architecture
(
architecture
,
session
=
session
)
arch_ids
=
tuple
(
arch2ids
[
architecture
]
for
architecture
in
arch_list
)
params
=
{
suite_id
:
suite_id
,
arch_ids
:
arch2ids
,
pkg_list
:
tuple
(
pkg_list
),
}
q
=
session
.
execute
(
"""
SELECT b.package, b.version, a.arch_string, b.id
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN architecture a ON b.architecture = a.id
JOIN suite su ON ba.suite = su.id
WHERE a.id IN :arch_ids AND b.package IN :pkg_db_set AND su.id = :suite_id
"""
,
params
)
remove
(
session
,
message
,
[
suite_name
],
list
(
q
),
partial
=
True
,
whoami
=
"DAK's auto-decrufter"
)
print
' dak rm -m "{message}" -s {suite} -a {architectures} -p -R -b {packages}'
.
format
(
message
=
message
,
suite
=
suite_name
,
architectures
=
","
.
join
(
archs
),
packages
=
" "
.
join
(
pkgs
),
)
print
print
"Note: The removals may be interdependent. A non-breaking result may require the execution of all"
print
"of the removals"
else
:
remove_groups
(
groups
.
itervalues
(),
suite_id
,
suite_name
,
session
)
################################################################################
...
...
@@ -249,8 +317,7 @@ def main ():
suite_id
=
suite
.
suite_id
suite_name
=
suite
.
suite_name
.
lower
()
remove_sourceless_cruft
(
suite_name
,
suite_id
,
session
,
dryrun
,
debug
)
#removeNBS(suite_name, suite_id, session, dryrun)
auto_decruft_suite
(
suite_name
,
suite_id
,
session
,
dryrun
,
debug
)
################################################################################
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录