!18 add missing scripts for obs

Merge pull request !18 from TommyLike/feature/add-missing-scripts

obs/tf/startup/README.md

@@ -125,3 +125,9 @@ Worker8:     172.16.1.12
 Worker9:     172.16.1.14
 Worker10:    172.16.1.157
 ```
+
+## download full fedora packages
+please see the document in folder `download_fedora_packages`
+
+## prepare gitee projects
+please see the document in folder `download_package_codes`

obs/tf/startup/backend.sh

@@ -87,7 +87,9 @@ sed -i "s/when you touch hostname or port/when you touch hostname\n\$ipaccess->{
 
 sed -i "s/our \$srcserver = \"http:\/\/\$hostname:5352\";/our \$srcserver = \"http:\/\/source.openeuler.org:5352\";/g" /usr/lib/obs/server/BSConfig.pm
 sed -i "s/our \$reposerver = \"http:\/\/\$hostname:5252\";/our \$reposerver = \"http:\/\/backend.openeuler.org:5252\";/g" /usr/lib/obs/server/BSConfig.pm
-sed -i "s/our \$serviceserver = \"http:\/\/\$hostname:5152\";/our \$serviceserver = \"http:\/\/backend.openeuler.org:5152\";/g" /usr/lib/obs/server/BSConfig.pm
+sed -i "s/our \$serviceserver = \"http:\/\/\$hostname:5152\";/our \$serviceserver = \"http:\/\/source.openeuler.org:5152\";/g" /usr/lib/obs/server/BSConfig.pm
+# user used by obs source service
+sed -i "s/our \$bsserviceuser = 'obsservicerun';/our \$bsserviceuser = 'obsrun';/g" /usr/lib/obs/server/BSConfig.pm
 
 sed -i "s/\$HOSTNAME/backend.openeuler.org/g" /etc/slp.reg.d/obs.repo_server.reg
 sed -i "s/\$HOSTNAME/source.openeuler.org/g" /etc/slp.reg.d/obs.source_server.reg
@@ -140,7 +142,6 @@ echo ${repo_id} -en > /srv/obs/projects/_repoid
 echo "Restarting backend service"
 # restart the frontend service
 systemctl enable obsrepserver.service
-systemctl enable obsservice.service
 systemctl enable obsdodup.service
 systemctl enable obssignd.service
 systemctl enable obsscheduler.service
@@ -150,7 +151,6 @@ systemctl enable obssigner.service
 systemctl enable obswarden.service
 
 systemctl start obsrepserver.service
-systemctl start obsservice.service
 systemctl start obsdodup.service
 systemctl start obssignd.service
 systemctl start obsscheduler.service

obs/tf/startup/build-pkg-rpm

+#
+# RPM specific functions.
+#
+################################################################
+#
+# Copyright (c) 1995-2014 SUSE Linux Products GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program (see the file COPYING); if not, write to the
+# Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+#
+################################################################
+
+
+pkg_initdb_rpm() {
+    echo "initializing rpm db..."
+    mkdir -p $BUILD_ROOT/var/lib/rpm
+    # rpm v5 does not have initdb
+    if ! test -e $BUILD_ROOT/usr/lib/rpm/cpuinfo.yaml ; then
+	if test -x $BUILD_ROOT/usr/bin/rpmdb ; then
+	    chroot $BUILD_ROOT /usr/bin/rpmdb --initdb || cleanup_and_exit 1
+	else
+	    chroot $BUILD_ROOT rpm --initdb || cleanup_and_exit 1
+	fi
+    fi
+    # hack: add nofsync to db config to speed up install
+    mkdir -p $BUILD_ROOT/root
+    DBI_OTHER=`chroot $BUILD_ROOT rpm --eval '%{?__dbi_other}'`
+    echo "%__dbi_other $DBI_OTHER nofsync" > $BUILD_ROOT/.rpmmacros
+    echo "%__dbi_other $DBI_OTHER nofsync" > $BUILD_ROOT/root/.rpmmacros
+}
+
+pkg_prepare_rpm() {
+    rpm_set_checkopts
+    rpm_init_cumulate
+}
+
+pkg_erase_rpm() {
+    chroot $BUILD_ROOT rpm --nodeps -e $PKG 2>&1 | {
+	local retry
+	while read line; do
+	    case "$line" in
+		r*failed:\ No\ such\ file\ or\ directory) ;;
+		error:\ failed\ to\ stat\ *:\ No\ such\ file\ or\ directory) ;;
+		error:\ *scriptlet\ failed*)
+		    echo "$line"
+		    retry=1
+		;;
+		*) echo "$line" ;;
+	    esac
+	done
+	if test -n "$retry" ; then
+	    echo "re-try deleting $PKG using --noscripts"
+	    chroot $BUILD_ROOT rpm --nodeps --noscripts -e $PKG || true
+	fi
+    }
+}
+
+rpm_set_checkopts() {
+    RPMCHECKOPTS=
+    # on Fedora 10 rpmbuild is in a separate package so we need something else to
+    # detect rpm4
+    test -x $BUILD_ROOT/usr/bin/rpmquery && RPMCHECKOPTS="--nodigest --nosignature"
+}
+
+rpm_init_cumulate() {
+    cumulate=-1
+    CUMULATED_LIST=()
+    CUMULATED_PIDS=()
+    CUMULATED_HMD5=()
+    DO_CUMULATE=
+    typeset -ri suse_version=$(chroot $BUILD_ROOT rpm --eval '%{?suse_version}' 2>/dev/null)
+    if ((suse_version > 1220)) ; then 
+	DO_CUMULATE=true
+    fi
+}
+
+pkg_verify_installed_rpm() {
+    chroot $BUILD_ROOT rpm --verify $PKG 2>&1 | tee $TMPFILE
+    if grep ^missing $TMPFILE > /dev/null ; then
+	return 1
+    fi
+    return 0
+}
+
+pkg_cumulate_rpm() {
+    test "$DO_CUMULATE" = true || return 1
+    # work around for cross-build installs, we must not overwrite the running rpm
+    if test "$PKG" = rpm ; then
+	for i in $BUILD_ROOT/.init_b_cache/preinstalls/rpm-x86-* ; do
+	    test -e "$i" && return 1
+	done
+    fi
+    let cumulate++
+    CUMULATED_LIST[$cumulate]=".init_b_cache/$PKG.rpm"
+    CUMULATED_PIDS[$cumulate]="$PKGID"
+    CUMULATED_HMD5[$cumulate]="$PKG_HDRMD5"
+    return 0
+}
+
+pkg_install_rpm() {
+    export ADDITIONAL_PARAMS=
+    if test "$USE_FORCE" = true ; then
+	export ADDITIONAL_PARAMS="$ADDITIONAL_PARAMS --force"
+    fi
+    # work around for cross-build installs, we must not overwrite the running rpm
+    if test "$PKG" = rpm ; then
+	for i in $BUILD_ROOT/.init_b_cache/preinstalls/rpm-x86-* ; do
+	    test -e "$i" && ADDITIONAL_PARAMS="$ADDITIONAL_PARAMS --justdb"
+	done
+    fi
+    ( cd $BUILD_ROOT && chroot $BUILD_ROOT rpm --ignorearch --nodeps -U --oldpackage --ignoresize $RPMCHECKOPTS \
+		$ADDITIONAL_PARAMS .init_b_cache/$PKG.rpm 2>&1 || chroot $BUILD_ROOT rpm --ignorearch --nodeps -U --oldpackage --ignoresize $RPMCHECKOPTS $ADDITIONAL_PARAMS .init_b_cache/$PKG.rpm 2>&1 || chroot $BUILD_ROOT rpm --ignorearch --nodeps -U --oldpackage --ignoresize $RPMCHECKOPTS $ADDITIONAL_PARAMS .init_b_cache/$PKG.rpm 2>&1 || chroot $BUILD_ROOT rpm --ignorearch --nodeps -U --oldpackage --ignoresize $RPMCHECKOPTS $ADDITIONAL_PARAMS .init_b_cache/$PKG.rpm 2>&1 || chroot $BUILD_ROOT rpm --ignorearch --nodeps -U --oldpackage --ignoresize $RPMCHECKOPTS $ADDITIONAL_PARAMS .init_b_cache/$PKG.rpm ||\
+	  touch $BUILD_ROOT/exit ) | \
+	      grep -v "^warning:.*saved as.*rpmorig$"
+}
+
+pkg_finalize_rpm() {
+    if test -n "${CUMULATED_LIST[*]}" ; then
+	echo "now installing cumulated packages"
+	for ((num=0; num<=cumulate; num++)) ; do
+	    echo ${CUMULATED_LIST[$num]}
+	    PKG=${CUMULATED_LIST[$num]##*/}
+	    test "$BUILD_ROOT/.init_b_cache/rpms/$PKG" -ef "$BUILD_ROOT/${CUMULATED_LIST[$num]}" && continue
+	    rm -f $BUILD_ROOT/${CUMULATED_LIST[$num]}
+	    cp $BUILD_ROOT/.init_b_cache/rpms/$PKG $BUILD_ROOT/${CUMULATED_LIST[$num]} || cleanup_and_exit 1
+	done > $BUILD_ROOT/.init_b_cache/manifest
+	( cd $BUILD_ROOT && chroot $BUILD_ROOT rpm --ignorearch --nodeps -Uh --oldpackage --ignoresize --verbose $RPMCHECKOPTS \
+		$ADDITIONAL_PARAMS .init_b_cache/manifest 2>&1 || touch $BUILD_ROOT/exit )
+	for ((num=0; num<=cumulate; num++)) ; do
+	    rm -f $BUILD_ROOT/${CUMULATED_LIST[$num]}
+	done
+	rm -f $BUILD_ROOT/.init_b_cache/manifest
+	check_exit
+	for ((num=0; num<=cumulate; num++)) ; do
+	    PKG=${CUMULATED_LIST[$num]##*/}
+	    echo "${CUMULATED_PIDS[$num]}" > $BUILD_ROOT/installed-pkg/${PKG%.rpm}
+	    test -n "${CUMULATED_HMD5[$num]}" || continue
+	    echo "${CUMULATED_HMD5[$num]} ${CUMULATED_PIDS[$num]}" > $BUILD_ROOT/.preinstall_image/${PKG%.rpm}
+	done
+    fi
+}
+
+pkg_preinstall_rpm() {
+    PAYLOADDECOMPRESS=cat
+    case `rpm -qp --nodigest --nosignature --qf "%{PAYLOADCOMPRESSOR}\n" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.rpm"` in
+	lzma) rpm --showrc | egrep 'PayloadIsLzma|_lzma' > /dev/null || PAYLOADDECOMPRESS="lzma -d" ;;
+	xz) rpm --showrc | egrep 'PayloadIsXz|_xz' > /dev/null || PAYLOADDECOMPRESS="xz -d" ;;
+    esac
+    if test "$PAYLOADDECOMPRESS" = "lzma -d" ; then
+	if ! lzma </dev/null >/dev/null 2>&1 ; then
+	    test -f "$BUILD_DIR/lzmadec.sh" && PAYLOADDECOMPRESS="bash $BUILD_DIR/lzmadec.sh"
+	fi
+    fi
+    if test "$PAYLOADDECOMPRESS" = "xz -d" ; then
+	if ! xz </dev/null >/dev/null 2>&1 ; then
+	    test -f "$BUILD_DIR/xzdec.sh" && PAYLOADDECOMPRESS="bash $BUILD_DIR/xzdec.sh"
+	fi
+    fi
+    if test "$PAYLOADDECOMPRESS" = cat ; then
+	rpm2cpio "$BUILD_ROOT/.init_b_cache/rpms/$PKG.rpm" | $CPIO
+    else
+	rpm2cpio "$BUILD_ROOT/.init_b_cache/rpms/$PKG.rpm" | $PAYLOADDECOMPRESS | $CPIO
+    fi
+    if test -e ".init_b_cache/scripts/$PKG.run" ; then
+	rpm -qp --nodigest --nosignature --qf "%{PREIN}" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.rpm" > ".init_b_cache/scripts/$PKG.pre"
+	rpm -qp --nodigest --nosignature --qf "%{POSTIN}" "$BUILD_ROOT/.init_b_cache/rpms/$PKG.rpm" > ".init_b_cache/scripts/$PKG.post"
+	echo -n '(none)' > .init_b_cache/scripts/.none
+	cmp -s ".init_b_cache/scripts/$PKG.pre" .init_b_cache/scripts/.none && rm -f ".init_b_cache/scripts/$PKG.pre"
+	cmp -s ".init_b_cache/scripts/$PKG.post" .init_b_cache/scripts/.none && rm -f ".init_b_cache/scripts/$PKG.post"
+	rm -f .init_b_cache/scripts/.none
+    fi
+    # hack for rpm erasures
+    if test -d "$BUILD_ROOT/installed-pkg" ; then
+	# call for rpm-4.x and not rpm-devel
+        test -z "${PKG##rpm-[0-9]*}" && chroot $BUILD_ROOT rpm --rebuilddb
+        # also exec for exchanged rpm !  naming is rpm-x86-<target>-<ver>
+        test -z "${PKG##rpm-x86-*[0-9]*}" && chroot $BUILD_ROOT rpm --rebuilddb
+    fi
+}
+
+pkg_runscripts_rpm() {
+    if test -e "$BUILD_ROOT/.init_b_cache/scripts/$PKG.pre" ; then
+	echo "running $PKG preinstall script"
+	(cd $BUILD_ROOT && chroot $BUILD_ROOT sh ".init_b_cache/scripts/$PKG.pre" 0)
+	rm -f "$BUILD_ROOT/.init_b_cache/scripts/$PKG.pre"
+    fi
+    if test -e "$BUILD_ROOT/.init_b_cache/scripts/$PKG.post" ; then
+	echo "running $PKG postinstall script"
+	(cd $BUILD_ROOT && chroot $BUILD_ROOT sh ".init_b_cache/scripts/$PKG.post" 1)
+	rm -f "$BUILD_ROOT/.init_b_cache/scripts/$PKG.post"
+    fi
+}

obs/tf/startup/download_fedora_packages/README.md

+# NOTE
+Please put both 'package_lists' and 'prepare.py' files into the destination folder
+you want to download before executing command, and then generate the execute sh file
+via command:
+```$xslt
+python prepare.py
+```
+this py file will generate a new executable file 'download.sh', since it will take a
+long time to finish you can execute the `nohup` command:
+```$xslt
+chmod +x download.sh
+nohup ./download.sh > result.log 2>&1 &
+```  
+when all packages has been downloaded, we need refresh the project via command (assume we
+create the project named openEuler:BaseOS)
+```$xslt
+cd /srv/obs/build/openEuler:BaseOS/standard_aarch64/aarch64
+chown -R obsrun:obsrun .
+obs_admin --rescan-repository openEuler:BaseOS standard_aarch64 aarch64
+```

obs/team_usage/obs_meta/obs/prepare.sh.py → obs/tf/startup/download_fedora_packages/prepare.py

 # Usage: move this file into the destination folder you used for downloading.
 import os.path
 
-RPMFILE = "rpmlist"
+RPMFILE = "package_lists"
 try:
     results = []
     # Read to plain object
@@ -15,15 +15,17 @@ try:
                     full_url = "https://dl.fedoraproject.org/pub/" \
                                "fedora/linux/releases/29/Everything/" \
                                "aarch64/os/Packages/"+package_name[0]+"/"+package_name
-                    results.append("echo 'starting to download {0}: {1}'".format(str(count), package_name))
                     results.append("curl -o {0} {1}".format(package_name, full_url))
                     count = count+1
             content = rpmlist.readline()
 
-    with open("results.sh", "w") as result_file:
+    with open("download.sh", "w") as result_file:
         result_file.write("#!/bin/bash\n")
         result_file.write("echo 'starting to download full packages : {0}'\n".format(len(results)))
+        count = 1
         for item in results:
+            results.append("echo 'starting to download {0}/{1}'".format(count, len(results)))
             result_file.write("%s\n" % item)
+            count = count+1
 except Exception as e:
     print(e)

obs/tf/startup/download_package_codes/README.md

+# NOTE
+prepare.py file used to generate the command script which will be used to clone the whole projects,
+please clone the source projects in the folder `/srv/cache/obs/tar_scm/repo/next/openEuler`
+```$xslt
+python prepare.py
+chmod +x download.sh
+cp download.sh /srv/cache/obs/tar_scm/repo/next/openEuler/download.sh
+cd /srv/cache/obs/tar_scm/repo/next/openEuler
+nohup ./download.sh > result.log 2>&1 & 
+```

obs/tf/startup/download_package_codes/prepare.py

+import xmlplain
+
+try:
+    # Read to plain object
+    with open("projects_raw.xml") as inf:
+        root = xmlplain.xml_to_obj(inf, strip_space=True, fold_dict=True)
+
+    projects = []
+    for project in root['manifest']:
+        projects.append(
+            "git clone https://gitee.com/openeuler/{0}.git".format(project['project']['@path'].split("/")[-1]))
+
+    # Output plain YAML
+    with open("download.sh", "w") as outf:
+        outf.write("#!/bin/bash\n")
+        for line in projects:
+            outf.write("echo 'git clone project: {0}'\n".format(line))
+            outf.write("{0}\n".format(line))
+
+
+except Exception as e:
+    print(e)

obs/tf/startup/service/extract_file

+#!/bin/bash
+
+# A simple script to checkout or update a svn or git repo as source service
+
+# defaults
+MYARCHIVE=""
+MYFILES=""
+OUTFILE="."
+
+while test $# -gt 0; do
+  case $1 in
+    *-archive)
+      MYARCHIVE="${2##*/}"
+      shift
+    ;;
+    *-file|*-files)
+      MYFILES="$MYFILES ${2}"
+      shift
+    ;;
+    *-outfilename)
+      OUTFILE="${2}"
+      shift
+    ;;
+    *-outdir)
+      MYOUTDIR="$2"
+      shift
+    ;;
+    *)
+      echo Unknown parameter $1.
+      echo 'Usage: extract_file --archive $ARCHIVE --file $FILE --outdir $OUT'
+      exit 1
+    ;;
+  esac
+  shift
+done
+
+if [ -z "$MYARCHIVE" ]; then
+  echo "ERROR: no archive specified!"
+  exit 1
+fi
+if [ -z "$MYFILES" ]; then
+  echo "ERROR: no checkout URL is given via --file parameter!"
+  exit 1
+fi
+if [ -z "$MYOUTDIR" ]; then
+  echo "ERROR: no output directory is given via --outdir parameter!"
+  exit 1
+fi
+
+existing_archive="$PWD/$(echo $MYARCHIVE)"
+cd "$MYOUTDIR"
+
+if [ -e "$existing_archive" ]; then
+  if [ "${existing_archive%.tar.gz}" != "$existing_archive" ]; then
+    tar xfz "$existing_archive" --wildcards $MYFILES || exit 1
+  elif [ "${existing_archive%.tar.bz2}" != "$existing_archive" ]; then
+    tar xfj "$existing_archive" --wildcards $MYFILES || exit 1
+  elif [ "${existing_archive%.tar.xz}" != "$existing_archive" ]; then
+    tar xfJ "$existing_archive" --wildcards $MYFILES || exit 1
+  elif [ "${existing_archive%.tar}" != "$existing_archive" ]; then
+    tar xf "$existing_archive" --wildcards $MYFILES || exit 1
+  elif [ "${existing_archive%.zip}" != "$existing_archive" ]; then
+    unzip "$existing_archive" $MYFILES || exit 1
+  else
+    echo "ERROR: unknown archive format $existing_archive"
+    exit 1
+  fi
+  for i in $MYFILES; do 
+    mv "$i" "$OUTFILE"
+  done
+else
+  echo "ERROR: archive not found: $existing_archive"
+  exit 1
+fi
+
+exit 0

obs/tf/startup/service/extract_file.service

+<service name="extract_file">
+  <summary>Extract files from archive</summary>
+  <description>This service can be used to extract a file from an tar or zip archive. This can be used to used a spec file directly from a tar.</description>
+  <parameter name="archive">
+    <description>Archive to be used</description>
+    <required/>
+  </parameter>
+  <parameter name="files">
+    <description>Files to extract. You may use this parameter multiple times</description>
+    <required/>
+  </parameter>
+  <parameter name="outfilename">
+    <description>Rename file into outfilename</description>
+  </parameter>
+</service>
+

obs/tf/startup/service/logger.conf

+#logger.conf
+###############################################
+[loggers]
+keys=root,publishedhook_depends_update
+[logger_root]
+level=DEBUG
+handlers=hand01
+[logger_publishedhook_depends_update]
+level=DEBUG
+handlers=hand02
+qualname=publishedhook_depends_update
+###############################################
+[handlers]
+keys=hand01,hand02
+[handler_hand01]
+class=StreamHandler
+level=INFO
+formatter=form01
+args=(sys.stderr,)
+[handler_hand02]
+class=handlers.RotatingFileHandler
+level=DEBUG
+formatter=form01
+args=('/srv/obs/log/publishedhook_depends_update.log', 'a', 10*1024*1024, 5)
+###############################################
+[formatters]
+keys=form01,form02
+[formatter_form01]
+format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s : %(message)s
+#datefmt=%a, %d %b %Y %H:%M:%S
+[formatter_form02]
+format=%(name)-12s: %(levelname)-8s %(message)s
+datefmt=

obs/tf/startup/service/publishedhook_depends_update.py

+#!/usr/bin/env python2
+# -*- encoding=utf-8 -*-
+
+import os,sys
+from subprocess  import *
+import re
+import shutil
+import traceback
+import hashlib
+import time
+from xml.dom import minidom
+from xml.etree import ElementTree as ET
+import logging
+import logging.config
+
+source_host = '/srv/obs'
+publishedhook_workspace = '/usr/lib/obs/source_md5'
+
+logging.config.fileConfig("/usr/lib/obs/service/logger.conf")
+logger = logging.getLogger("publishedhook_depends_update")
+
+def shell_cmd(s_cmd_line, inmsg=None):
+    p = Popen(s_cmd_line, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
+    if inmsg:
+        p.stdin.write(inmsg)
+    out, err = p.communicate()
+    return p.returncode, out, err
+
+def parse_args(argv):
+    if len(argv) == 0:
+        logger.error("Usage:EulerOS:Update/standard_aarch64 /srv/obs/repos/EulerOS:/Update/standard_aarch64 aarch64/wget-1.14-15.1.h2.aarch64.rpm")
+        return False
+
+    for line in argv:
+        logger.info( "%s" % line )
+
+    publish_rpmlist = []
+    archs=[]
+    project_build_path = '/srv/obs/build/%s' % argv[0]
+    (prj,repo)=argv[0].split('/')
+    base_project = '/srv/obs/build/%s_baseline' % prj
+
+    if not os.path.exists( base_project ):
+        logger.info( "Found none %s_baseline project. skipped ..." % prj )
+        return True
+
+    if len(argv) > 2:
+        for line in argv[2:]:
+            (arch_name,pkgname)=line.split('/')
+            if arch_name not in ['src','noarch']:
+                if not os.path.exists( os.path.join(project_build_path, arch_name) ):
+                    cmd = "find %s -name %s | sed -n '1p'" % (project_build_path, pkgname)
+                    ret, out, err = shell_cmd(cmd)
+                    if ret != 0:
+                        logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+                        return False
+                    else:
+                        logger.info("%s" % out)
+                        arch_name = out.split('/')[6]
+                if arch_name not in archs:
+                    archs.append(arch_name)
+            if not re.match( r'.*src\.rpm', pkgname ):
+                publish_rpmlist.append( pkgname )
+
+    if len(archs) == 0:
+        archs = filter( lambda x: os.path.isdir( os.path.join(project_build_path,x) ) , os.listdir(project_build_path) )
+    for xxarch in archs:
+        rpmlist = []
+        for rpm in publish_rpmlist:
+            if rpm.split('.')[-2] in [ 'noarch', xxarch ]:
+                rpmlist.append(rpm)
+        project = (prj,repo,xxarch,rpmlist)
+        projects.append(project)
+
+    return True
+
+def scan_update_change(project,pkg_dict,prj_base_dict):
+    publish_info = []
+    ls = os.linesep
+    xml_head = '<?xml version="1.0" encoding="utf-8"?>'
+    project_build_path = '/srv/obs/build/%s/%s/%s' % (project[0], project[1], project[2])
+
+    publish_info.append( xml_head )
+    publish_info.append('<packages>')
+    packages = filter( lambda x: not re.match(r'^:.*',x), os.listdir(project_build_path) )
+
+    pkg_rev_path = '%s/projects/%s.pkg'  % ( source_host, project[0]  )
+    pkg_src_path = '%s/trees/%s' % (source_host, project[0] )
+    pkg_hash_path = '%s/trees/%s'  % (source_host, project[0] )
+    for package in packages:
+        cmd = "sed -n '$p' %s/%s.rev" % ( pkg_rev_path, package )
+        ret, out1, err = shell_cmd(cmd)
+        if ret != 0:
+            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out1, err) )
+            return False
+        cmd = "sed -n '1p' %s/%s/%s-MD5SUMS" % ( pkg_src_path, package, out1.split('|')[2] )
+        ret, out2, err = shell_cmd(cmd)
+        if ret != 0:
+            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out2, err) )
+            return False
+        if out2.find('/SERVICE') != -1:
+            srcfile = '%s/%s/%s-MD5SUMS' % (pkg_hash_path, package, out2.split('  ')[0])
+            if os.path.exists(srcfile):
+                cmd = "cat %s | grep '_service:'" % srcfile
+            else:
+                cmd = "echo 'E1001'" 
+        else:
+            cmd = "cat %s/%s/%s-MD5SUMS" % ( pkg_src_path, package, out1.split('|')[2] )
+        ret, out3, err = shell_cmd(cmd)
+        if ret != 0:
+            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out3, err) )
+            return False
+        if re.match(r'E100.*', out3):
+            logger.info( '%s pending %s skip ...' % (package,out3) )
+            pkg_hash = prj_base_dict[package][0]
+            rpms = prj_base_dict[package][1]
+        else:
+            digest = hashlib.md5()
+            digest.update(out3)
+            rpms = filter( lambda x: re.match(r'.*\.rpm',x), os.listdir( os.path.join(project_build_path, package) ) )
+            rpms.sort()
+            digest.update( ' '.join(rpms) )
+            pkg_hash = digest.hexdigest()
+        rpmsStr = ' '.join(rpms)
+        update_info = '<package name="%s" hash="%s" binary="%s"></package>' % (package, pkg_hash, rpmsStr)
+        pkg_dict[package] = (pkg_hash,rpms)
+        publish_info.append( update_info )
+
+    cmd = "sed -n '$p' %s/_project.rev | awk -F'|' '{print $3}';sed -n '$p' %s/_project.mrev | awk -F'|' '{print $3}'" % ( pkg_rev_path, pkg_rev_path )
+    ret, out, err = shell_cmd(cmd)
+    if ret != 0:
+        logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+        return False
+    pdigest = hashlib.md5()
+    pdigest.update(out)
+    pkg_hash = pdigest.hexdigest()
+    update_info = '<package name="_project" hash="%s" binary=""></package>' % ( pkg_hash )
+    pkg_dict['_project']  = (pkg_hash,'')
+    publish_info.append( update_info )
+    publish_info.append('</packages>')
+
+    update_info_dir = '%s/%s/%s/%s' % (publishedhook_workspace, project[0], project[1], project[2])
+    if not os.path.exists( update_info_dir ):
+        os.makedirs( update_info_dir )
+    update_info_file = '%s/update.xml' % (update_info_dir)
+    fobj_update_info_file = open(update_info_file, 'w')
+    fobj_update_info_file.writelines( ['%s%s' % ( update_info, ls ) for update_info in publish_info ] )
+    fobj_update_info_file.close()
+
+    return True
+
+def check_base(project):
+    check_flag = 0
+    build_path='/srv/obs/build/%s_baseline/%s/%s/:full' %(project[0],project[1],project[2])
+    repo_ppath='/srv/obs/build/%s/%s/%s/:repo' %(project[0],project[1],project[2])
+
+    build_rpms = os.listdir( build_path )
+    repos_rpms = filter( lambda x: not re.match(r'.*src\.rpm',x), os.listdir( repo_ppath ) )
+
+    for rpm in build_rpms:
+        if rpm not in repos_rpms:
+           build_file=os.path.join(build_path,rpm)
+           os.remove(build_file) 
+           check_flag = 1
+           logger.info( "check: should delete %s" % rpm )
+
+    for rpm in repos_rpms:
+        if rpm not in build_rpms:
+            repo_file=os.path.join(repo_ppath,rpm)
+            shutil.copy(repo_file,build_path)
+            check_flag = 1
+            logger.info( "check: shoud add %s" % rpm )
+
+    if check_flag == 1:
+        return False
+    else:
+        return True
+
+def depends_update(project, prj_update_dict, prj_base_dict):
+    diff_add_rpm = []
+    diff_delete_rpm = []
+    for key,value in prj_update_dict.items():
+        if prj_base_dict.has_key(key):
+            if value[0].decode("utf-8") == prj_base_dict[key][0]:
+                continue
+            else:
+                ##Update add
+                for rpm in value[1]:
+                    if rpm not in diff_add_rpm:
+                        if not re.match( r'.*src\.rpm', rpm ) and re.match( r'.*\.rpm', rpm ):
+                            diff_add_rpm.append(rpm)
+                            logger.info( "source change add  %s" % rpm )
+                #Update del
+                for rpm in prj_base_dict[key][1]:
+                    if rpm not in diff_delete_rpm:
+                        if not re.match( r'.*src\.rpm', rpm ) and re.match( r'.*\.rpm', rpm ):
+                            diff_delete_rpm.append(rpm)
+                            logger.info( "source change delete  %s" % rpm )
+        #New add
+        else:
+            for rpm in value[1]:
+                if rpm not in diff_add_rpm:
+                    if not re.match( r'.*src\.rpm', rpm ) and re.match( r'.*\.rpm', rpm ):
+                        diff_add_rpm.append(rpm)
+                        logger.info( "new package add  %s" % rpm )
+
+    for key, value in prj_base_dict.items():
+        if prj_update_dict.has_key(key):
+                continue
+        #New del
+        else:
+            for rpm in value[1]:
+                if rpm not in diff_delete_rpm:
+                    if not re.match( r'.*src\.rpm', rpm ) and re.match( r'.*\.rpm', rpm ):
+                        diff_delete_rpm.append(rpm)
+                        logger.info( "deleted package delete  %s" % rpm )
+
+    #global config change
+    if prj_base_dict.has_key('_project') and prj_base_dict['_project'][0] != prj_update_dict['_project'][0]:
+        logger.info( "%s project prjconfig or meta change" % project[0] )
+        for rpm in project[3]:
+            if rpm not in diff_add_rpm:
+                diff_add_rpm.append(rpm)
+                logger.info( "project config change update %s" % rpm )
+
+    baseline_path = '/srv/obs/build/%s_baseline/%s/%s/:full' % ( project[0], project[1], project[2] )
+    if not os.path.exists( baseline_path ):
+        os.makedirs( baseline_path )
+
+    #delete first
+    for rpm in diff_delete_rpm:
+        rpm_path = os.path.join( baseline_path, rpm )
+        if os.path.exists( rpm_path ) and re.match( r'.*\.rpm', rpm_path ):
+            os.remove( rpm_path )
+    for rpm in diff_add_rpm:
+        rpm_path = '/srv/obs/repos/%s/%s/%s/%s' % ( project[0].replace(':',':/'), project[1], rpm.split('.')[-2], rpm )
+        base_rpm = os.path.join(baseline_path,rpm)
+        if os.path.exists( base_rpm ) and re.match( r'.*\.rpm', base_rpm):
+            os.remove( base_rpm )
+        shutil.copy( rpm_path, baseline_path )
+
+    update_baseline_info_dir = '%s/%s_baseline/%s/%s' %( publishedhook_workspace, project[0], project[1], project[2] )
+    update_baseline_info_file = '%s/%s_baseline/%s/%s/update.xml' %( publishedhook_workspace, project[0], project[1], project[2] )
+    update_info_file = '%s/%s/%s/%s/update.xml' %( publishedhook_workspace, project[0], project[1], project[2] )
+    if not os.path.exists( update_baseline_info_dir ):
+        os.makedirs( update_baseline_info_dir )
+    shutil.copy( update_info_file, update_baseline_info_dir )
+
+    return True
+
+def rescan_repository(projects):
+    for project in projects:
+        cmd = '/usr/lib/obs/server/bs_admin --rescan-repository %s_baseline %s %s' % ( project[0], project[1], project[2] )
+        logger.info( "%s" % cmd)
+        ret, out, err = shell_cmd(cmd)
+        if ret != 0:
+            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+            return False
+
+    return True
+
+def parse_base_change( project, pkg_dict ):
+    xml_file = '%s/%s_baseline/%s/%s/update.xml' %( publishedhook_workspace, project[0], project[1], project[2] )
+
+    if not os.path.exists(xml_file):
+        return True
+    if os.path.isfile(xml_file) == False:
+        logger.error(" %s is not a file" % xml_file )
+        return False
+    try:
+        xmldoc = minidom.parse(xml_file)
+    except:
+        logger.error("Can't parse Xml File of %s." % (xml_file) )
+        return False
+    package = xmldoc.getElementsByTagName('package')   
+    for i in range(0, len(package)):
+        pkg_dict[package[i].attributes['name'].value] = (package[i].attributes['hash'].value,package[i].attributes['binary'].value.split(' '))
+
+    return True
+
+def update_baseline(projects):
+    for project in projects:
+        prj_update_dict = {}
+        prj_base_dict = {}
+        logger.info(" %s %s %s" % (project[0], project[1], project[2]))
+        if not parse_base_change(project, prj_base_dict):
+            logger.error("parse_base_change error")
+            return False
+        if not scan_update_change(project, prj_update_dict, prj_base_dict):
+            logger.error("scan_update_change error")
+            return False
+        if not depends_update(project, prj_update_dict, prj_base_dict):
+            logger.error("depends_update error")
+            return False
+        if not check_base(project):
+            logger.error("check_base error")
+            return False
+
+    return True
+
+if __name__ == '__main__':
+    projects = []
+    try:
+        if not parse_args(sys.argv[1:]):
+            raise Exception("parse_args failed")
+    except Exception as ie:
+        logger.info( "%s" % ie )
+        ret, out, err = shell_cmd(notice_cmd)
+        if ret != 0:
+            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+            sys.exit(1)
+    else:
+        try:
+            if not update_baseline(projects):
+                raise Exception("update_baseline failed")
+        except Exception as ie:
+            traceback.print_exc()
+            logger.info( "%s" % ie )
+            ret, out, err = shell_cmd(notice_cmd)
+            if ret != 0:
+                logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+                sys.exit(1)
+        finally:
+            if not rescan_repository(projects):
+                if not rescan_repository(projects):
+                    if not rescan_repository(projects):
+                        ret, out, err = shell_cmd(notice_cmd)
+                        if ret != 0:
+                            logger.error( "run: %s : failed,the out is %s,the err is %s" % (cmd, out, err) )
+                            sys.exit(1)

obs/tf/startup/service/recompress

+#!/bin/bash
+
+# A simple script to checkout or update a svn or git repo as source service
+#  
+# (C) 2010 by Adrian Schröter <adrian@suse.de>
+#  
+# This program is free software; you can redistribute it and/or  
+# modify it under the terms of the GNU General Public License  
+# as published by the Free Software Foundation; either version 2  
+# of the License, or (at your option) any later version.  
+# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.  
+
+# defaults
+MYCOMPRESSION=""
+FILES=""
+
+while test $# -gt 0; do
+  case $1 in
+    *-compression)
+      MYCOMPRESSION="$2"
+      shift
+    ;;
+    *-file)
+      FILES="$FILES ${2##*/}"
+      shift
+    ;;
+    *-outdir)
+      MYOUTDIR="$2"
+      shift
+    ;;
+    *)
+      echo Unknown parameter $1.
+      echo 'Usage: recompress --compression $COMPRESSION --file $FILE --outdir $OUT'
+      exit 1
+    ;;
+  esac
+  shift
+done
+
+if [ -z "$MYCOMPRESSION" ]; then
+  MYCOMPRESSION="bz2"
+fi
+if [ -z "$FILES" ]; then
+  echo "ERROR: no inputs files are given via --file parameter!"
+  exit 1
+fi
+if [ -z "$MYOUTDIR" ]; then
+  echo "ERROR: no output directory is given via --outdir parameter!"
+  exit 1
+fi
+
+for i in $FILES; do
+  FILE=`ls -1 "$i" || ls -1 "_service:*:$i"`
+  if [ ! -f "$FILE" ]; then
+    echo "Unknown file $i"
+    exit 1
+  fi
+  UNCOMPRESS="cat"
+  BASENAME="$FILE"
+  if [ "${FILE%.gz}" != "$FILE" ]; then
+    UNCOMPRESS="gunzip -c"
+    BASENAME="${FILE%.gz}"
+  elif [ "${FILE%.tgz}" != "$FILE" ]; then
+    UNCOMPRESS="gunzip -c"
+    BASENAME="${FILE%.tgz}.tar"
+  elif [ "${FILE%.bz2}" != "$FILE" ]; then
+    UNCOMPRESS="bunzip2 -c"
+    BASENAME="${FILE%.bz2}"
+  elif [ "${FILE%.xz}" != "$FILE" ]; then
+    UNCOMPRESS="xz -dc"
+    BASENAME="${FILE%.xz}"
+  fi
+
+  if [ "$MYCOMPRESSION" == "gz" ]; then
+    COMPRESS="gzip -c -n --rsyncable -"
+    NEWFILE="${BASENAME#_service:}.gz"
+  elif [ "$MYCOMPRESSION" == "bz2" ]; then
+    COMPRESS="bzip2 -c -"
+    NEWFILE="${BASENAME#_service:}.bz2"
+  elif [ "$MYCOMPRESSION" == "xz" ]; then
+    COMPRESS="xz -c -"
+    NEWFILE="${BASENAME#_service:}.xz"
+  elif [ "$MYCOMPRESSION" == "none" ]; then
+    COMPRESS="cat -"
+    NEWFILE="${BASENAME#_service:}"
+  else
+    echo "ERROR: Unknown compression"
+    exit 1
+  fi
+
+  # recompress
+  mkdir uncompress_dir
+  cd uncompress_dir
+  mv ../$FILE uncompress.tar
+  tar -xf uncompress.tar
+  rm -rf uncompress.tar
+  #tar --mtime="2017-01-01 00:00:00" --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0,ctime:=0,mtime:=0 -cf ../$FILE `ls -A`
+  tar --mtime="2017-01-01 00:00:00" --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0,ctime:=0 -cf ../$FILE `ls -A`
+  cd ../
+  rm -rf uncompress_dir
+  # do the real work
+  $UNCOMPRESS -- "$FILE" | $COMPRESS > "$MYOUTDIR/$NEWFILE" || exit 1
+
+  # Check if the (compressed) target file already exists in the directory where
+  # the service is invoked and drop the newly generated one. Avoids overwriting
+  # otherwise identical files which only have different timestamps. Note that
+  # zdiff and co all fail to do that properly...
+  if [ -f $NEWFILE ] ; then
+    DIFF_TMPDIR=$(mktemp -d)
+    SRC_DIR="$PWD"
+    cd $DIFF_TMPDIR
+    mkdir new old
+    $(cd new ; tar -xf "$MYOUTDIR/$NEWFILE" 2> /dev/null || mv -- "$MYOUTDIR/$NEWFILE" .)
+    $(cd old ; tar -xf "$SRC_DIR/$NEWFILE" 2> /dev/null || mv -- "$SRC_DIR/$NEWFILE" .)
+    if diff -r new old > /dev/null ; then
+      echo "Identical target file $NEWFILE already exists, skipping.."
+      rm -r -- "$MYOUTDIR/$NEWFILE"
+    else
+      echo "Compressed $FILE to $NEWFILE"
+    fi
+    cd $SRC_DIR
+    rm -rf -- "$DIFF_TMPDIR"
+  else
+    echo "Compressed $FILE to $NEWFILE"
+  fi
+  
+  # we can remove service files, no need to store them twice
+  rm -f -- "$FILE"
+done
+
+exit 0

obs/tf/startup/service/tar_scm.service

+<service name="tar_scm">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/service/tar_scm_kernel.service

+<service name="tar_scm_kernel">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/service/tar_scm_kernel_repo.service

+<service name="tar_scm">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/service/tar_scm_kernels_repo.service

+<service name="tar_scm">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/service/tar_scm_repo.service

+<service name="tar_scm">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/service/tar_scm_repo_docker.service

+<service name="tar_scm">
+  <summary>Create a tarball from SCM repository</summary>
+  <description>This service uses a SCM client to checkout or update from a given repository.  Supported are svn, git, hg and bzr.</description>
+  <parameter name="scm">
+    <description>Specify SCM to use.</description>
+    <allowedvalue>svn</allowedvalue>
+    <allowedvalue>git</allowedvalue>
+    <allowedvalue>hg</allowedvalue>
+    <allowedvalue>bzr</allowedvalue>
+    <required/>
+  </parameter>
+  <parameter name="url">
+    <description>Specify URL to checkout.</description>
+    <required/>
+  </parameter>
+  <parameter name="subdir">
+    <description>Package just a subdirectory.</description>
+  </parameter>
+  <parameter name="version">
+    <description>Specify version to be used in tarball.  Defaults to automatically detected value formatted by versionformat parameter.</description>
+  </parameter>
+  <parameter name="versionformat">
+    <description>
+      Auto-generate version from checked out source using this format
+      string.  This parameter is used if the 'version' parameter is not
+      specified.
+
+      For git, the value is passed to git log --date=short --pretty=format:...
+      for the topmost commit, and the output from git is cleaned up to
+      remove some unhelpful characters.  Here are some useful examples of
+      strings which are expanded, see the git-log documentation for more.
+
+      %ct               Commit time as a UNIX timestamp, e.g. 1384855776.
+                        This is the default.
+
+      %at               Author time as a UNIX timestamp, e.g. 1384855776.
+
+      %cd               Commit date in YYYYMMDD format, e.g. 20131119
+
+      %ad               Author date in YYYYMMDD format, e.g. 20131119
+
+      %h                Abbreviated hash, e.g. cc62c54
+
+      @PARENT_TAG@      The first tag that is reachable, e.g. v0.2.3
+
+      @TAG_OFFSET@      The commit count since @PARENT_TAG@, e.g. 9
+
+      For hg, the value is passed to hg log --template=....  See the
+      hg documentation for more information.  The default is '{rev}'
+
+      For bzr and svn, '%r' is expanded to the revision, and is the default.
+    </description>
+  </parameter>
+  <parameter name="versionprefix">
+    <description>Specify a base version as prefix.</description>
+  </parameter>
+  <parameter name="revision">
+    <description>
+       Specify revision of source to check out.
+
+       When using git, revision may refer to any of the following:
+
+       * explicit SHA1: a1b2c3d4....
+         - the SHA1 must be reachable from a default clone/fetch
+           (generally, must be reachable from some branch or tag on the
+           remote).
+         - set by: git checkout ${SHA1}
+
+       * short branch name: "master", "devel" etc.
+         - set by: git checkout ${branch}
+                   git pull
+
+       * explicit ref: refs/heads/master, refs/tags/v1.2.3,
+         refs/changes/49/11249/1
+         - set by: git fetch ${url} ${revision}:${revision}
+                   git checkout ${revision}
+    </description>
+  </parameter>
+  <parameter name="filename">
+    <description>Specify name of package, which is used together with version to determine tarball name.</description>
+  </parameter>
+  <parameter name="exclude">
+    <description>Specify regexp to exclude when creating the tarball.</description>
+  </parameter>
+  <parameter name="include">
+    <description>Specify subset of files/subdirectories to pack in the tarball.</description>
+  </parameter>
+  <parameter name="package-meta">
+    <description>Package the metadata of SCM to allow the user or OBS to update after un-tar.</description>
+    <allowedvalue>yes</allowedvalue>
+  </parameter>
+  <parameter name="history-depth">
+    <description>Obsolete parameter which will be ignored.</description>
+  </parameter>
+  <parameter name="submodules">
+    <description>Specify whether to include git submodules.  Default is 'enable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesgenerate">
+    <description>Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision).  Default is 'disable'.</description>
+    <allowedvalue>enable</allowedvalue>
+    <allowedvalue>disable</allowedvalue>
+  </parameter>
+  <parameter name="changesauthor">
+    <description>Specify author of the changes file entry to be written.  Defaults to first email entry in ~/.oscrc, or "opensuse-packaging@opensuse.org" if there is no .oscrc found.</description>
+  </parameter>
+</service>

obs/tf/startup/source_server.sh

@@ -84,7 +84,8 @@ sed -i "s/when you touch hostname or port/when you touch hostname\n\$ipaccess->{
 
 sed -i "s/our \$srcserver = \"http:\/\/\$hostname:5352\";/our \$srcserver = \"http:\/\/source.openeuler.org:5352\";/g" /usr/lib/obs/server/BSConfig.pm
 sed -i "s/our \$reposerver = \"http:\/\/\$hostname:5252\";/our \$reposerver = \"http:\/\/backend.openeuler.org:5252\";/g" /usr/lib/obs/server/BSConfig.pm
-sed -i "s/our \$serviceserver = \"http:\/\/\$hostname:5152\";/our \$serviceserver = \"http:\/\/backend.openeuler.org:5152\";/g" /usr/lib/obs/server/BSConfig.pm
+sed -i "s/our \$serviceserver = \"http:\/\/\$hostname:5152\";/our \$serviceserver = \"http:\/\/source.openeuler.org:5152\";/g" /usr/lib/obs/server/BSConfig.pm
+sed -i "s/our \$bsserviceuser = 'obsservicerun';/our \$bsserviceuser = 'obsrun';/g" /usr/lib/obs/server/BSConfig.pm
 
 sed -i "s/\$HOSTNAME/backend.openeuler.org/g" /etc/slp.reg.d/obs.repo_server.reg
 sed -i "s/\$HOSTNAME/source.openeuler.org/g" /etc/slp.reg.d/obs.source_server.reg
@@ -123,13 +124,26 @@ fi
 cd /root/.config/osc
 curl -o oscrc https://openeuler.obs.cn-south-1.myhuaweicloud.com:443/infrastructure/oscrc
 
+# update cache folder for scm service
+echo CACHEDIRECTORY="/srv/cache/obs/tar_scm" > /etc/obs/services/tar_scm
+
+# copy service files into
+cp ../service/* /usr/lib/obs/service/
+cp ../build-pkg-rpm /usr/lib/build/build-pkg-rpm
+mkdir -p /usr/lib/obs/source_md5
+chmod 777 /usr/lib/obs/source_md5
+mkdir -p /srv/cache/obs/tar_scm/{incoming,repo,repourl}
+mkdir -p /srv/cache/obs/tar_scm/repo/euleros-version
+
 echo "Restarting source service"
 # restart the frontend service
 systemctl enable obsstoragesetup.service
 systemctl enable obssrcserver.service
 systemctl enable obsdeltastore.service
 systemctl enable obsservicedispatch.service
+systemctl enable obsservice.service
 
+systemctl start obsservice.service
 systemctl start obsstoragesetup.service
 systemctl start obssrcserver.service
 systemctl start obsdeltastore.service