Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
喜羊羊3508
Dak
提交
778a4394
D
Dak
项目概览
喜羊羊3508
/
Dak
12 个月 前同步成功
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
Dak
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
778a4394
编写于
11月 04, 2009
作者:
M
Mark Hymers
浏览文件
操作
浏览文件
下载
差异文件
Merge commit 'mhy/master'
上级
81594833
2d5e7e78
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
342 addition
and
3 deletion
+342
-3
dak/dakdb/update24.py
dak/dakdb/update24.py
+62
-0
dak/manage_build_queues.py
dak/manage_build_queues.py
+102
-0
dak/update_db.py
dak/update_db.py
+1
-1
daklib/dbconn.py
daklib/dbconn.py
+177
-2
未找到文件。
dak/dakdb/update24.py
0 → 100755
浏览文件 @
778a4394
#!/usr/bin/env python
"""
Add some meta info to queues
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2009 Mark Hymers <mhy@debian.org>
@license: GNU General Public License version 2 or later
"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import
psycopg2
def
do_update
(
self
):
print
"Add meta info columns to queues."
try
:
c
=
self
.
db
.
cursor
()
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN generate_metadata BOOL DEFAULT FALSE NOT NULL"
)
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN origin TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN label TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN releasedescription TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN signingkey TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE policy_queue ADD COLUMN stay_of_execution INT4 NOT NULL DEFAULT 86400 CHECK (stay_of_execution >= 0)"
)
c
.
execute
(
"""ALTER TABLE policy_queue
ADD CONSTRAINT policy_queue_meta_sanity_check
CHECK ( (generate_metadata IS FALSE)
OR (origin IS NOT NULL AND label IS NOT NULL AND releasedescription IS NOT NULL) )"""
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN generate_metadata BOOL DEFAULT FALSE NOT NULL"
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN origin TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN label TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN releasedescription TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN signingkey TEXT DEFAULT NULL"
)
c
.
execute
(
"ALTER TABLE build_queue ADD COLUMN stay_of_execution INT4 NOT NULL DEFAULT 86400 CHECK (stay_of_execution >= 0)"
)
c
.
execute
(
"""ALTER TABLE build_queue
ADD CONSTRAINT build_queue_meta_sanity_check
CHECK ( (generate_metadata IS FALSE)
OR (origin IS NOT NULL AND label IS NOT NULL AND releasedescription IS NOT NULL) )"""
)
print
"Committing"
c
.
execute
(
"UPDATE config SET value = '24' WHERE name = 'db_revision'"
)
self
.
db
.
commit
()
except
psycopg2
.
InternalError
,
msg
:
self
.
db
.
rollback
()
raise
DBUpdateError
,
"Database error, rollback issued. Error message : %s"
%
(
str
(
msg
))
dak/manage_build_queues.py
0 → 100755
浏览文件 @
778a4394
#!/usr/bin/env python
"""Manage build queues"""
# Copyright (C) 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
# Copyright (C) 2009 Mark Hymers <mhy@debian.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
################################################################################
import
os
,
os
.
path
,
stat
,
sys
from
datetime
import
datetime
import
apt_pkg
from
daklib
import
daklog
from
daklib.dbconn
import
*
from
daklib.config
import
Config
################################################################################
Options
=
None
Logger
=
None
################################################################################
def
usage
(
exit_code
=
0
):
print
"""Usage: dak manage-build-queues [OPTIONS] buildqueue1 buildqueue2
Manage the contents of one or more build queues
-a, --all run on all known build queues
-n, --no-action don't do anything
-v, --verbose explain what is being done
-h, --help show this help and exit"""
sys
.
exit
(
exit_code
)
################################################################################
def
main
():
global
Options
,
Logger
cnf
=
Config
()
for
i
in
[
"Help"
,
"No-Action"
,
"Verbose"
"All"
]:
if
not
cnf
.
has_key
(
"Manage-Build-Queues::Options::%s"
%
(
i
)):
cnf
[
"Manage-Build-Queues::Options::%s"
%
(
i
)]
=
""
Arguments
=
[(
'h'
,
"help"
,
"Manage-Build-Queues::Options::Help"
),
(
'n'
,
"no-action"
,
"Manage-Build-Queues::Options::No-Action"
),
(
'a'
,
"all"
,
"Manage-Build-Queues::Options::All"
),
(
'v'
,
"verbose"
,
"Manage-Build-Queues::Options::Verbose"
)]
queue_names
=
apt_pkg
.
ParseCommandLine
(
cnf
.
Cnf
,
Arguments
,
sys
.
argv
)
Options
=
cnf
.
SubTree
(
"Manage-Build-Queues::Options"
)
if
Options
[
"Help"
]:
usage
()
Logger
=
daklog
.
Logger
(
cnf
,
'manage-build-queues'
,
Options
[
'No-Action'
])
starttime
=
datetime
.
now
()
session
=
DBConn
().
session
()
if
Options
[
"All"
]:
if
len
(
queue_names
)
!=
0
:
print
"E: Cannot use both -a and a queue_name"
sys
.
exit
(
1
)
queues
=
session
.
query
(
BuildQueue
).
all
()
else
:
queues
=
[]
for
q
in
queue_name
:
queue
=
get_build_queue
(
q
.
lower
(),
session
)
if
queue
:
queues
.
append
(
queue
)
else
:
Logger
.
log
([
'cannot find queue %s'
%
q
])
# For each given queue, look up object and call manage_queue
for
q
in
queues
:
Logger
.
log
([
'cleaning queue %s using datetime %s'
%
(
q
.
queue_name
,
starttime
)])
q
.
clean_and_update
(
starttime
,
dryrun
=
Options
[
"No-Action"
])
Logger
.
close
()
#######################################################################################
if
__name__
==
'__main__'
:
main
()
dak/update_db.py
浏览文件 @
778a4394
...
...
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################
Cnf
=
None
required_database_schema
=
2
3
required_database_schema
=
2
4
################################################################################
...
...
daklib/dbconn.py
浏览文件 @
778a4394
...
...
@@ -37,7 +37,9 @@ import os
import
re
import
psycopg2
import
traceback
from
datetime
import
datetime
from
datetime
import
datetime
,
timedelta
from
errno
import
ENOENT
from
tempfile
import
mkstemp
,
mkdtemp
from
inspect
import
getargspec
...
...
@@ -429,6 +431,45 @@ __all__.append('BinaryACLMap')
################################################################################
MINIMAL_APT_CONF
=
"""
Dir
{
ArchiveDir "%(archivepath)s";
OverrideDir "/srv/ftp.debian.org/scripts/override/";
CacheDir "/srv/ftp.debian.org/database/";
};
Default
{
Packages::Compress ". bzip2 gzip";
Sources::Compress ". bzip2 gzip";
DeLinkLimit 0;
FileMode 0664;
}
bindirectory "incoming"
{
Packages "Packages";
Contents " ";
BinOverride "override.sid.all3";
BinCacheDB "packages-accepted.db";
FileList "%(filelist)s";
PathPrefix "";
Packages::Extensions ".deb .udeb";
};
bindirectory "incoming/"
{
Sources "Sources";
BinOverride "override.sid.all3";
SrcOverride "override.sid.all3.src";
FileList "%(filelist)s";
};
"""
class
BuildQueue
(
object
):
def
__init__
(
self
,
*
args
,
**
kwargs
):
pass
...
...
@@ -436,6 +477,135 @@ class BuildQueue(object):
def
__repr__
(
self
):
return
'<BuildQueue %s>'
%
self
.
queue_name
def
write_metadata
(
self
,
starttime
,
force
=
False
):
# Do we write out metafiles?
if
not
(
force
or
self
.
generate_metadata
):
return
session
=
DBConn
().
session
().
object_session
(
self
)
fl_fd
=
fl_name
=
ac_fd
=
ac_name
=
None
tempdir
=
None
arches
=
" "
.
join
([
a
.
arch_string
for
a
in
session
.
query
(
Architecture
).
all
()
if
a
.
arch_string
!=
'source'
])
startdir
=
os
.
getcwd
()
try
:
# Grab files we want to include
newer
=
session
.
query
(
BuildQueueFile
).
filter_by
(
build_queue_id
=
self
.
queue_id
).
filter
(
BuildQueueFile
.
lastused
+
timedelta
(
seconds
=
self
.
stay_of_execution
)
>
starttime
).
all
()
# Write file list with newer files
(
fl_fd
,
fl_name
)
=
mkstemp
()
for
n
in
newer
:
os
.
write
(
fl_fd
,
'%s
\n
'
%
n
.
fullpath
)
os
.
close
(
fl_fd
)
# Write minimal apt.conf
# TODO: Remove hardcoding from template
(
ac_fd
,
ac_name
)
=
mkstemp
()
os
.
write
(
ac_fd
,
MINIMAL_APT_CONF
%
{
'archivepath'
:
self
.
path
,
'filelist'
:
fl_name
})
os
.
close
()
# Run apt-ftparchive generate
os
.
chdir
(
os
.
path
.
dirname
(
ac_name
))
os
.
system
(
'apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s'
%
os
.
path
.
basename
(
ac_name
))
# Run apt-ftparchive release
# TODO: Eww - fix this
bname
=
os
.
path
.
basename
(
self
.
path
)
os
.
chdir
(
self
.
path
)
os
.
chdir
(
'..'
)
os
.
system
(
"""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="${archs}" release %s > Release"""
,
[
self
.
origin
,
self
.
label
,
self
.
releasedescription
,
arches
,
bname
])
# Sign if necessary
if
self
.
signingkey
:
cnf
=
Config
()
keyring
=
"--secret-keyring
\"
%s
\"
"
%
cnf
[
"Dinstall::SigningKeyring"
]
if
cnf
.
has_key
(
"Dinstall::SigningPubKeyring"
):
keyring
+=
" --keyring
\"
%s
\"
"
%
cnf
[
"Dinstall::SigningPubKeyring"
]
os
.
system
(
"gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey))
# Move the files if we got this far
os.rename('Release', os.path.join(bname, 'Release'))
if self.signingkey:
os.rename('Release.gpg', os.path.join(bname, 'Release.gpg'))
# Clean up any left behind files
finally:
os.chdir(startdir)
if fl_fd:
try:
os.close(fl_fd)
except OSError:
pass
if fl_name:
try:
os.unlink(fl_name)
except OSError:
pass
if ac_fd:
try:
os.close(ac_fd)
except OSError:
pass
if ac_name:
try:
os.unlink(ac_name)
except OSError:
pass
def clean_and_update(self, starttime, dryrun=False):
"""WARNING: This routine commits for you"""
session = DBConn().session().object_session(self)
if self.generate_metadata:
self.write_metadata(starttime)
# Grab files older than our execution time
older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all()
for o in older:
killdb = False
try:
if dryrun:
print "
I
:
Would
have
removed
%
s
from
the
queue
"
else:
print "
I
:
Removing
%
s
from
the
queue
"
os.unlink(o.fullpath)
killdb = True
except OSError, e:
# If it wasn't there, don't worry
if e.errno == ENOENT:
killdb = True
else:
# TODO: Replace with proper logging call
print "
E
:
Could
not
remove
%
s
" % o.fullpath
if killdb:
session.delete(o)
session.commit()
for f in os.listdir(self.path):
if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release'):
continue
try:
r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one()
except NoResultFound:
fp = os.path.join(self.path, f)
if dryrun:
print "
I
:
Would
remove
unused
link
%
s
" % fp
else:
print "
I
:
Removing
unused
link
%
s
" % fp
try:
os.unlink(fp)
except OSError:
print "
E
:
Failed
to
unlink
unreferenced
file
%
s
" % r.fullpath
def add_file_from_pool(self, poolfile):
"""Copies a file into the pool. Assumes that the PoolFile object is
attached to the same SQLAlchemy session as the Queue object is.
...
...
@@ -516,7 +686,12 @@ class BuildQueueFile(object):
pass
def __repr__(self):
return
'<BuildQueueFile %s (%s)>'
%
(
self
.
filename
,
self
.
queue_id
)
return '<BuildQueueFile %s (%s)>' % (self.filename, self.build_queue_id)
@property
def fullpath(self):
return os.path.join(self.buildqueue.path, self.filename)
__all__.append('BuildQueueFile')
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录