From a57864f03ec811902011d8646279b184d6fb3075 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Mon, 3 Feb 2020 15:01:57 +0800 Subject: [PATCH] Support rsync server in repo server --- .../dockerfiles/Dockerfile.nginx_uswgi_flask | 9 -- cd/repo/dockerfiles/app/main.py | 53 --------- cd/repo/dockerfiles/entrypoint.sh | 3 - cd/repo/dockerfiles/repo_tools.py | 103 ------------------ cd/repo/dockerfiles/rsyncd/Dockerfile | 16 +++ .../deployment_with_rsync_server.yaml | 48 +++----- 6 files changed, 34 insertions(+), 198 deletions(-) delete mode 100644 cd/repo/dockerfiles/Dockerfile.nginx_uswgi_flask delete mode 100644 cd/repo/dockerfiles/app/main.py delete mode 100755 cd/repo/dockerfiles/entrypoint.sh delete mode 100755 cd/repo/dockerfiles/repo_tools.py diff --git a/cd/repo/dockerfiles/Dockerfile.nginx_uswgi_flask b/cd/repo/dockerfiles/Dockerfile.nginx_uswgi_flask deleted file mode 100644 index ad95033..0000000 --- a/cd/repo/dockerfiles/Dockerfile.nginx_uswgi_flask +++ /dev/null @@ -1,9 +0,0 @@ -FROM tiangolo/uwsgi-nginx-flask:python3.7 -MAINTAINER tommylike - -RUN pip install Flask-BasicAuth && \ - curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \ - chmod +x kubectl && \ - mv kubectl /usr/bin/ - -COPY ./app /app diff --git a/cd/repo/dockerfiles/app/main.py b/cd/repo/dockerfiles/app/main.py deleted file mode 100644 index ac9a14f..0000000 --- a/cd/repo/dockerfiles/app/main.py +++ /dev/null @@ -1,53 +0,0 @@ -import subprocess -import os -import sys -import json -import base64 -import os.path -import datetime -from flask import Flask -from flask import request -from flask_basicauth import BasicAuth - - -# Validate the required job yaml file - -JOB_YAML = "/etc/repo-update/update-repo-job.yaml" -if not os.path.isfile(JOB_YAML): - print("job yaml file not exist, exiting...") - sys.exit(1) - -app = Flask(__name__) -app.config['BASIC_AUTH_USERNAME'] = os.environ.get('BASIC_AUTH_USERNAME') -app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('BASIC_AUTH_PASSWORD') -basic_auth = BasicAuth(app) - - -@app.route('/republish', methods=['POST']) -@basic_auth.required -def republish(): - if request.json is None or not "projects" in request.json or request.json["projects"] is None: - return "invalid request body, please specify the projects to republish", 400 - if not isinstance(request.json["projects"], list) or len(request.json["projects"]) == 0: - return "invalid request body, please specify the projects via array", 400 - print("[{0}]: starting to republish repo with projects {1}".format(datetime.datetime.now(), request.json)) - exist = subprocess.run(["kubectl get job/update-repo-job -n {0}".format(os.environ.get('K8S_NAMESPACE'))], shell=True) - if exist.returncode == 0: - #This would be a little arbitary, but we delete it if it's existed for simple logic - subprocess.run(["kubectl delete job/update-repo-job -n {0} --wait".format(os.environ.get('K8S_NAMESPACE'))], shell=True) - # NOTE: update yaml and then apply - content = base64.b64encode(json.dumps(request.json).encode('utf-8')).decode('utf-8') - print("start to create update job with content {0}".format(content)) - result = subprocess.run(["cat {0} | sed -e \"s/PROJECT_VARIABLE/{1}/g\" | kubectl apply -f -".format(JOB_YAML, content)], shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - if result.returncode == 0: - return "successfully triggered", 200 - else: - return "failed to trigger update job error: \n {0}".format(result.stdout), 400 - -@app.route('/', methods=['GET']) -def index(): - return "http is not enabled for repo service, please use https instead", 200 - - -if __name__ == '__main__': - app.run() \ No newline at end of file diff --git a/cd/repo/dockerfiles/entrypoint.sh b/cd/repo/dockerfiles/entrypoint.sh deleted file mode 100755 index ab8b102..0000000 --- a/cd/repo/dockerfiles/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -python3 /root/repo_tools.py $@ diff --git a/cd/repo/dockerfiles/repo_tools.py b/cd/repo/dockerfiles/repo_tools.py deleted file mode 100755 index 5893876..0000000 --- a/cd/repo/dockerfiles/repo_tools.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: UTF-8 -*- -import argparse -import datetime -import json -import base64 -import os -import pytz -import sys -import subprocess - -# Used to prepare the key&cert file used by nginx service -def prepare_repo(arg_instance): - print("starting to prepare for nginx server") - local_folder = "/etc/nginx/ssl/" - if not os.path.isdir(local_folder): - os.mkdir(local_folder) - print("starting to download key file {0}".format(arg_instance.key_file)) - ret = subprocess.run(["curl -o {0} {1}".format(os.path.join(local_folder, "privkey.pem"), arg_instance.key_file)], shell=True) - if ret.returncode != 0: - print("failed to get key file for nginx service: {0}", ret.stdout) - print("starting to download cert file {0}".format(arg_instance.cert_file)) - ret = subprocess.run(["curl -o {0} {1}".format(os.path.join(local_folder, "fullchain.pem"), arg_instance.cert_file)], shell=True) - if ret.returncode != 0: - print("failed to get cert file for nginx service: {0}", ret.stdout) - -# The acceptable json content would be like: -# { -# "projects": [ -# { -# "localpath": "xxxx", -# "http_url": "" -# }, -# { -# "localpath": "xxxx", -# "http_url": "" -# } -# ] -# } -# 'update_repo' will download every rpm packages (overwrite if it exists) specified in 'http_url' root folder via wget tool and the rpm will be arranged in the format of -# . -# ├── packages -# | └────── AAA.rpm -# | └────── BBB.rpm -# | └────── repodata -def update_repo(arg_instance, working_dir): - content = base64.b64decode(arg_instance.repo_json.encode('utf-8')).decode('utf-8') - repo = json.loads(content) - if "projects" not in repo or repo["projects"] is None or not isinstance(repo["projects"], list): - print("unacceptable json content when trying to update repo {0}".format(arg_instance.repo_json)) - sys.exit(1) - for project in repo["projects"]: - if "localpath" not in project or "http_url" not in project: - print("project {0} format unacceptable, skipping".format(project)) - continue - handle_single_repo_update(project, working_dir) - -def handle_single_repo_update(project, working_dir): - #check and create the base repo folder - base_repo_folder = os.path.join(working_dir, project["localpath"]) - if not os.path.isdir(base_repo_folder): - os.makedirs(base_repo_folder) - # download rpms - package_folder= os.path.join(base_repo_folder, "packages") - if not os.path.isdir(package_folder): - os.mkdir(package_folder) - print("starting to sync rpms from {0} into folder {1}".format(project['http_url'], project['localpath'])) - ret = subprocess.run(["cd {0} && wget -N -r -nd -np -k -L -p -A '*.rpm' --tries=5 {1}".format(package_folder, project['http_url'])], shell=True) - if ret.returncode != 0: - print("failed to download rpms from http url {0}, stdout {1}".format(project["http_url"], ret.stdout)) - sys.exit(1) - #create or update repo - print("starting to sync repodata folder {0}".format(os.path.join(package_folder, 'repodata'))) - if os.path.isdir(os.path.join(package_folder, 'repodata')): - ret = subprocess.run(["cd {0} && createrepo --update .".format(package_folder)], shell=True) - else: - ret = subprocess.run(["cd {0} && createrepo .".format(package_folder)], shell=True) - if ret.returncode != 0: - print("failed to update repo with command result: {0}".format(ret.stdout)) - sys.exit(1) - #Add timestamp file - subprocess.run(["echo {0} > {1}".format(datetime.datetime.now(pytz.timezone('Hongkong')), os.path.join(package_folder, 'release_time.txt'))], shell=True) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='repo action collection.') - parser.add_argument('action', type=str, metavar='ACTION', help='specify the action to perform, now only "prepare" or "update" are supported') - parser.add_argument('--key-file', type=str, nargs='?', help='key file used in nginx for tls') - parser.add_argument('--cert-file', type=str, nargs='?', help='cert file used in nginx for tls') - parser.add_argument('--repo-json', type=str, nargs='?', default="{}", help='repo data used to update the official repo(s)') - args = parser.parse_args() - print("starting to perform action via command: {0}".format(args)) - - if str(args.action) == "prepare": - prepare_repo(args) - elif str(args.action) == "update": - working_dir = os.environ.get("WORKING_DIR", "") - if working_dir == "": - print("Must specify 'WORKING_DIR' when perform repo update action") - sys.exit(1) - update_repo(args, working_dir) - else: - print("unsupported actions {0}, please specify 'prepare' or 'update'.".format(args.action)) - sys.exit(1) - sys.exit(0) diff --git a/cd/repo/dockerfiles/rsyncd/Dockerfile b/cd/repo/dockerfiles/rsyncd/Dockerfile index 47a0148..cbcbe0c 100644 --- a/cd/repo/dockerfiles/rsyncd/Dockerfile +++ b/cd/repo/dockerfiles/rsyncd/Dockerfile @@ -3,9 +3,12 @@ MAINTAINER tommylikehu@gmail.com EXPOSE 873 +# install required packages RUN apt-get update && \ apt-get -y install rsync && \ apt-get -y install net-tools && \ + apt-get -y install createrepo && \ + apt-get -y install vim && \ apt-get -y install openssh-server EXPOSE 22 @@ -15,5 +18,18 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile +#Update apt sources +RUN source_file=/etc/apt/sources.list \ + && echo "deb http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse" > ${source_file} \ + && echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse" >> ${source_file} \ + && echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse" >> ${source_file} \ + && echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse" >> ${source_file} \ + && echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse" >> ${source_file} \ + && echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse" >> ${source_file} \ + && echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse" >> ${source_file} \ + && echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse" >> ${source_file} \ + && echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse" >> ${source_file} \ + && echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse" >> ${source_file} + COPY entrypoint.sh /usr/local/bin/ CMD ["entrypoint.sh"] \ No newline at end of file diff --git a/cd/repo/openeuler.org/deployment_with_rsync_server.yaml b/cd/repo/openeuler.org/deployment_with_rsync_server.yaml index 08e3ac9..ee42ba8 100644 --- a/cd/repo/openeuler.org/deployment_with_rsync_server.yaml +++ b/cd/repo/openeuler.org/deployment_with_rsync_server.yaml @@ -1,4 +1,3 @@ -# Source: repo-chart/templates/config.yaml --- apiVersion: v1 kind: ConfigMap @@ -7,7 +6,9 @@ metadata: namespace: repo2 data: rsyncd.secrets: | - root:openeuler@!234 + root:openeuler@1234 + ssh.pub: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQChSk+/FF79F8ut0hpNuYQ4uhAgUSY4hzRIPvQ2uVTukP0B0A99NYPfDNICp7gLa6e7yzaj+bvCpgWZoZCNAWTBWkq+zHTgDDHrLtUE4zPC1guzuR+Gz3yeIzSt0iAzi9uG2p+qh7jUfl8QAwNOs3wosNenoZj7NmgsCF9M1o85msimRc8Roxnn5caao1RtdNkDHDqhw5QiS9doSUjoxT+esD0CI7RHAyMgMCfSlXXl/phpdSU2hVJSFXsHVBTiymkuMQe8Ylmls+OEmCe8Cy7lIqE/Q+56l62Pxv4UJpOWs9T/SrOEr5vtEPGghZzgo5ViewzAs3dGMaODSO25XhrIiRZ7hjBK9tjLOX7ZXfAsb4DpJljq2aPCrlEaGJsHc2laixKIOKogDbqFffM3eXwgEAxPUevX/mYnuyhAVrsRUl8HXIQnTuVIEeYVdmn2MA8I/y6MPWdN5VbLR5gOiNLOuSVhCq3sQLpTZ9CONF+zq+1layCoaGMIZw5JwYgWCQgCGNclbx8eMVY3+J3slH6VjzE/05Eys18HtpuAMCivBLGgpBGeCVdpQqkHodTn5ZjNZEBAgzHypiOgR/txMHhkTi4+1ZvTmSRWs9hggv4/IcDQFaj1f2JVds6lxGvJyOnoy9k1VUC0q4N6sptpJ/n+ElCJ4UaHE48La3Mu79R6+Q== openeuler_hosts rsyncd.conf: | log file = /dev/stdout use chroot = yes @@ -137,11 +138,11 @@ spec: name: openeuler-data-volume resources: requests: - cpu: 8000m + cpu: 4000m memory: 8000Mi - name: rsync-server - image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.4 - imagePullPolicy: "IfNotPresent" + image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.5 + imagePullPolicy: "Always" volumeMounts: - mountPath: /etc/rsyncd.conf name: repo-nginx-configmap-volume @@ -151,35 +152,24 @@ spec: subPath: rsyncd.secrets - mountPath: /repo/openeuler name: openeuler-data-volume + - mountPath: /root/.ssh/authorized_keys.ro + name: repo-nginx-configmap-volume + subPath: ssh.pub resources: requests: cpu: 4000m - memory: 8000Mi - command: - - /bin/sh - - -c - - | - cp /etc/rsyncd.secrets.ro /etc/rsyncd.secrets; - chmod 0400 /etc/rsyncd.secrets; - exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf; - - name: rsync-client - image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.4 - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /etc/rsyncd.conf - name: repo-nginx-configmap-volume - subPath: rsyncd.conf - - mountPath: /etc/rsyncd.secrets - name: repo-nginx-configmap-volume - subPath: rsyncd.secrets - - mountPath: /repo/openeuler - name: openeuler-data-volume + memory: 6000Mi command: - /bin/sh - -c - | + cp /etc/rsyncd.secrets.ro /etc/rsyncd.secrets + chmod 0400 /etc/rsyncd.secrets + cp /root/.ssh/authorized_keys.ro /root/.ssh/authorized_keys + chmod 0400 /root/.ssh/authorized_keys + chown root:root /root/.ssh/authorized_keys /usr/sbin/sshd & - tail -f /dev/null; + exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf; volumes: - name: repo-nginx-configmap-volume configMap: @@ -190,10 +180,8 @@ spec: - name: website-secrets-volume secret: secretName: website-secrets - defaultMode: 400 --- -# Source: repo-chart/templates/service.yaml apiVersion: v1 kind: Service metadata: @@ -201,7 +189,7 @@ metadata: namespace: repo2 annotations: kubernetes.io/elb.class: union - kubernetes.io/elb.id: + kubernetes.io/elb.id: b0fa0739-f69c-4abd-bcbf-840c8dd1b44e kubernetes.io/elb.lb-algorithm: ROUND_ROBIN spec: externalTrafficPolicy: Cluster @@ -218,4 +206,4 @@ spec: selector: app: repo-nginx-pod type: LoadBalancer - loadBalancerIP: + loadBalancerIP: 114.116.245.239 -- GitLab