Skip to content

I
infrastructure

openeuler / infrastructure

通知 1
Star 0
Fork 0
I
infrastructure

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 a57864f0 编写于 作者: T TommyLike
浏览文件
操作

Support rsync server in repo server

上级 5c333f51
隐藏空白更改
内联 并排
Showing with 34 addition and 198 deletion
cd/repo/dockerfiles/Dockerfile.nginx_uswgi_flask 已删除 100644 → 0
浏览文件 @ 5c333f51
FROM tiangolo/uwsgi-nginx-flask:python3.7
MAINTAINER tommylike <tommylikehu@gmail.com>
RUN pip install Flask-BasicAuth && \
curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \
chmod +x kubectl && \
mv kubectl /usr/bin/
COPY ./app /app
cd/repo/dockerfiles/app/main.py 已删除 100644 → 0
浏览文件 @ 5c333f51
import subprocess
import os
import sys
import json
import base64
import os.path
import datetime
from flask import Flask
from flask import request
from flask_basicauth import BasicAuth
# Validate the required job yaml file
JOB_YAML = "/etc/repo-update/update-repo-job.yaml"
if not os.path.isfile(JOB_YAML):
print("job yaml file not exist, exiting...")
sys.exit(1)
app = Flask(__name__)
app.config['BASIC_AUTH_USERNAME'] = os.environ.get('BASIC_AUTH_USERNAME')
app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('BASIC_AUTH_PASSWORD')
basic_auth = BasicAuth(app)
@app.route('/republish', methods=['POST'])
@basic_auth.required
def republish():
if request.json is None or not "projects" in request.json or request.json["projects"] is None:
return "invalid request body, please specify the projects to republish", 400
if not isinstance(request.json["projects"], list) or len(request.json["projects"]) == 0:
return "invalid request body, please specify the projects via array", 400
print("[{0}]: starting to republish repo with projects {1}".format(datetime.datetime.now(), request.json))
exist = subprocess.run(["kubectl get job/update-repo-job -n {0}".format(os.environ.get('K8S_NAMESPACE'))], shell=True)
if exist.returncode == 0:
#This would be a little arbitary, but we delete it if it's existed for simple logic
subprocess.run(["kubectl delete job/update-repo-job -n {0} --wait".format(os.environ.get('K8S_NAMESPACE'))], shell=True)
# NOTE: update yaml and then apply
content = base64.b64encode(json.dumps(request.json).encode('utf-8')).decode('utf-8')
print("start to create update job with content {0}".format(content))
result = subprocess.run(["cat {0} | sed -e \"s/PROJECT_VARIABLE/{1}/g\" | kubectl apply -f -".format(JOB_YAML, content)], shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
if result.returncode == 0:
return "successfully triggered", 200
else:
return "failed to trigger update job error: \n {0}".format(result.stdout), 400
@app.route('/', methods=['GET'])
def index():
return "http is not enabled for repo service, please use https instead", 200
if __name__ == '__main__':
app.run()
\ No newline at end of file
cd/repo/dockerfiles/entrypoint.sh 已删除 100755 → 0
浏览文件 @ 5c333f51
#!/usr/bin/env bash
python3 /root/repo_tools.py $@
cd/repo/dockerfiles/repo_tools.py 已删除 100755 → 0
浏览文件 @ 5c333f51
# -*- coding: UTF-8 -*-
import argparse
import datetime
import json
import base64
import os
import pytz
import sys
import subprocess
# Used to prepare the key&cert file used by nginx service
def prepare_repo(arg_instance):
print("starting to prepare for nginx server")
local_folder = "/etc/nginx/ssl/"
if not os.path.isdir(local_folder):
os.mkdir(local_folder)
print("starting to download key file {0}".format(arg_instance.key_file))
ret = subprocess.run(["curl -o {0} {1}".format(os.path.join(local_folder, "privkey.pem"), arg_instance.key_file)], shell=True)
if ret.returncode != 0:
print("failed to get key file for nginx service: {0}", ret.stdout)
print("starting to download cert file {0}".format(arg_instance.cert_file))
ret = subprocess.run(["curl -o {0} {1}".format(os.path.join(local_folder, "fullchain.pem"), arg_instance.cert_file)], shell=True)
if ret.returncode != 0:
print("failed to get cert file for nginx service: {0}", ret.stdout)
# The acceptable json content would be like:
# {
# "projects": [
# {
# "localpath": "xxxx",
# "http_url": ""
# },
# {
# "localpath": "xxxx",
# "http_url": ""
# }
# ]
# }
# 'update_repo' will download every rpm packages (overwrite if it exists) specified in 'http_url' root folder via wget tool and the rpm will be arranged in the format of
# .
# ├── packages
# | └────── AAA.rpm
# | └────── BBB.rpm
# | └────── repodata
def update_repo(arg_instance, working_dir):
content = base64.b64decode(arg_instance.repo_json.encode('utf-8')).decode('utf-8')
repo = json.loads(content)
if "projects" not in repo or repo["projects"] is None or not isinstance(repo["projects"], list):
print("unacceptable json content when trying to update repo {0}".format(arg_instance.repo_json))
sys.exit(1)
for project in repo["projects"]:
if "localpath" not in project or "http_url" not in project:
print("project {0} format unacceptable, skipping".format(project))
continue
handle_single_repo_update(project, working_dir)
def handle_single_repo_update(project, working_dir):
#check and create the base repo folder
base_repo_folder = os.path.join(working_dir, project["localpath"])
if not os.path.isdir(base_repo_folder):
os.makedirs(base_repo_folder)
# download rpms
package_folder= os.path.join(base_repo_folder, "packages")
if not os.path.isdir(package_folder):
os.mkdir(package_folder)
print("starting to sync rpms from {0} into folder {1}".format(project['http_url'], project['localpath']))
ret = subprocess.run(["cd {0} && wget -N -r -nd -np -k -L -p -A '*.rpm' --tries=5 {1}".format(package_folder, project['http_url'])], shell=True)
if ret.returncode != 0:
print("failed to download rpms from http url {0}, stdout {1}".format(project["http_url"], ret.stdout))
sys.exit(1)
#create or update repo
print("starting to sync repodata folder {0}".format(os.path.join(package_folder, 'repodata')))
if os.path.isdir(os.path.join(package_folder, 'repodata')):
ret = subprocess.run(["cd {0} && createrepo --update .".format(package_folder)], shell=True)
else:
ret = subprocess.run(["cd {0} && createrepo .".format(package_folder)], shell=True)
if ret.returncode != 0:
print("failed to update repo with command result: {0}".format(ret.stdout))
sys.exit(1)
#Add timestamp file
subprocess.run(["echo {0} > {1}".format(datetime.datetime.now(pytz.timezone('Hongkong')), os.path.join(package_folder, 'release_time.txt'))], shell=True)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='repo action collection.')
parser.add_argument('action', type=str, metavar='ACTION', help='specify the action to perform, now only "prepare" or "update" are supported')
parser.add_argument('--key-file', type=str, nargs='?', help='key file used in nginx for tls')
parser.add_argument('--cert-file', type=str, nargs='?', help='cert file used in nginx for tls')
parser.add_argument('--repo-json', type=str, nargs='?', default="{}", help='repo data used to update the official repo(s)')
args = parser.parse_args()
print("starting to perform action via command: {0}".format(args))
if str(args.action) == "prepare":
prepare_repo(args)
elif str(args.action) == "update":
working_dir = os.environ.get("WORKING_DIR", "")
if working_dir == "":
print("Must specify 'WORKING_DIR' when perform repo update action")
sys.exit(1)
update_repo(args, working_dir)
else:
print("unsupported actions {0}, please specify 'prepare' or 'update'.".format(args.action))
sys.exit(1)
sys.exit(0)
cd/repo/dockerfiles/rsyncd/Dockerfile
浏览文件 @ a57864f0
......@@ -3,9 +3,12 @@ MAINTAINER tommylikehu@gmail.com
EXPOSE 873
# install required packages
RUN apt-get update && \
apt-get -y install rsync && \
apt-get -y install net-tools && \
apt-get -y install createrepo && \
apt-get -y install vim && \
apt-get -y install openssh-server
EXPOSE 22
......@@ -15,5 +18,18 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so
ENV NOTVISIBLE "in users profile"
RUN echo "export VISIBLE=now" >> /etc/profile
#Update apt sources
RUN source_file=/etc/apt/sources.list \
&& echo "deb http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse" > ${source_file} \
&& echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse" >> ${source_file} \
&& echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse" >> ${source_file} \
&& echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse" >> ${source_file} \
&& echo "deb http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse" >> ${source_file} \
&& echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse" >> ${source_file} \
&& echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse" >> ${source_file} \
&& echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse" >> ${source_file} \
&& echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse" >> ${source_file} \
&& echo "deb-src http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse" >> ${source_file}
COPY entrypoint.sh /usr/local/bin/
CMD ["entrypoint.sh"]
\ No newline at end of file
cd/repo/openeuler.org/deployment_with_rsync_server.yaml
浏览文件 @ a57864f0
# Source: repo-chart/templates/config.yaml
---
apiVersion: v1
kind: ConfigMap
......@@ -7,7 +6,9 @@ metadata:
namespace: repo2
data:
rsyncd.secrets: |
root:openeuler@!234
root:openeuler@1234
ssh.pub: |
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQChSk+/FF79F8ut0hpNuYQ4uhAgUSY4hzRIPvQ2uVTukP0B0A99NYPfDNICp7gLa6e7yzaj+bvCpgWZoZCNAWTBWkq+zHTgDDHrLtUE4zPC1guzuR+Gz3yeIzSt0iAzi9uG2p+qh7jUfl8QAwNOs3wosNenoZj7NmgsCF9M1o85msimRc8Roxnn5caao1RtdNkDHDqhw5QiS9doSUjoxT+esD0CI7RHAyMgMCfSlXXl/phpdSU2hVJSFXsHVBTiymkuMQe8Ylmls+OEmCe8Cy7lIqE/Q+56l62Pxv4UJpOWs9T/SrOEr5vtEPGghZzgo5ViewzAs3dGMaODSO25XhrIiRZ7hjBK9tjLOX7ZXfAsb4DpJljq2aPCrlEaGJsHc2laixKIOKogDbqFffM3eXwgEAxPUevX/mYnuyhAVrsRUl8HXIQnTuVIEeYVdmn2MA8I/y6MPWdN5VbLR5gOiNLOuSVhCq3sQLpTZ9CONF+zq+1layCoaGMIZw5JwYgWCQgCGNclbx8eMVY3+J3slH6VjzE/05Eys18HtpuAMCivBLGgpBGeCVdpQqkHodTn5ZjNZEBAgzHypiOgR/txMHhkTi4+1ZvTmSRWs9hggv4/IcDQFaj1f2JVds6lxGvJyOnoy9k1VUC0q4N6sptpJ/n+ElCJ4UaHE48La3Mu79R6+Q== openeuler_hosts
rsyncd.conf: |
log file = /dev/stdout
use chroot = yes
......@@ -137,11 +138,11 @@ spec:
name: openeuler-data-volume
resources:
requests:
cpu: 8000m
cpu: 4000m
memory: 8000Mi
- name: rsync-server
image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.4
imagePullPolicy: "IfNotPresent"
image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.5
imagePullPolicy: "Always"
volumeMounts:
- mountPath: /etc/rsyncd.conf
name: repo-nginx-configmap-volume
......@@ -151,35 +152,24 @@ spec:
subPath: rsyncd.secrets
- mountPath: /repo/openeuler
name: openeuler-data-volume
- mountPath: /root/.ssh/authorized_keys.ro
name: repo-nginx-configmap-volume
subPath: ssh.pub
resources:
requests:
cpu: 4000m
memory: 8000Mi
command:
- /bin/sh
- -c
- |
cp /etc/rsyncd.secrets.ro /etc/rsyncd.secrets;
chmod 0400 /etc/rsyncd.secrets;
exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf;
- name: rsync-client
image: swr.cn-north-4.myhuaweicloud.com/openeuler/rsyncd:0.0.4
imagePullPolicy: "IfNotPresent"
volumeMounts:
- mountPath: /etc/rsyncd.conf
name: repo-nginx-configmap-volume
subPath: rsyncd.conf
- mountPath: /etc/rsyncd.secrets
name: repo-nginx-configmap-volume
subPath: rsyncd.secrets
- mountPath: /repo/openeuler
name: openeuler-data-volume
memory: 6000Mi
command:
- /bin/sh
- -c
- |
cp /etc/rsyncd.secrets.ro /etc/rsyncd.secrets
chmod 0400 /etc/rsyncd.secrets
cp /root/.ssh/authorized_keys.ro /root/.ssh/authorized_keys
chmod 0400 /root/.ssh/authorized_keys
chown root:root /root/.ssh/authorized_keys
/usr/sbin/sshd &
tail -f /dev/null;
exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf;
volumes:
- name: repo-nginx-configmap-volume
configMap:
......@@ -190,10 +180,8 @@ spec:
- name: website-secrets-volume
secret:
secretName: website-secrets
defaultMode: 400
---
# Source: repo-chart/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
......@@ -201,7 +189,7 @@ metadata:
namespace: repo2
annotations:
kubernetes.io/elb.class: union
kubernetes.io/elb.id:
kubernetes.io/elb.id: b0fa0739-f69c-4abd-bcbf-840c8dd1b44e
kubernetes.io/elb.lb-algorithm: ROUND_ROBIN
spec:
externalTrafficPolicy: Cluster
......@@ -218,4 +206,4 @@ spec:
selector:
app: repo-nginx-pod
type: LoadBalancer
loadBalancerIP:
loadBalancerIP: 114.116.245.239
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或