diff --git a/CHANGELOG.md b/CHANGELOG.md index ddd61395716f5314c0b2bcdb1a47417b14584939..1b0d473090844ee973cf1dc128832eeb48bad136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub () - Project task subsets () +- Kubernetes templates and guide for their deployment () - [WiderFace](http://shuoyang1213.me/WIDERFACE/) format support () - [VGGFace2](https://github.com/ox-vgg/vgg_face2) format support () diff --git a/kubernetes-templates/01_namespace.yml b/kubernetes-templates/01_namespace.yml new file mode 100644 index 0000000000000000000000000000000000000000..fa0aaa12f714944bb8ed64b0e75967bc1371f718 --- /dev/null +++ b/kubernetes-templates/01_namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cvat diff --git a/kubernetes-templates/02_cvat_backend_storage.yml b/kubernetes-templates/02_cvat_backend_storage.yml new file mode 100644 index 0000000000000000000000000000000000000000..6d0f51b35006b7543f400e97a7d16aa83ced8036 --- /dev/null +++ b/kubernetes-templates/02_cvat_backend_storage.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cvat-backend-data + namespace: cvat +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 20Gi diff --git a/kubernetes-templates/02_database_secrets.yml b/kubernetes-templates/02_database_secrets.yml new file mode 100644 index 0000000000000000000000000000000000000000..4c811f007284de42e6cf2118aeefe918847da13c --- /dev/null +++ b/kubernetes-templates/02_database_secrets.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: cvat-postgres-secret + namespace: cvat + labels: + app: cvat-app + tier: db +stringData: + POSTGRES_DB: cvat + POSTGRES_USER: root + POSTGRES_PASSWORD: POSTGRES_ADMIN_PW diff --git a/kubernetes-templates/02_database_storage.yml b/kubernetes-templates/02_database_storage.yml new file mode 100644 index 0000000000000000000000000000000000000000..c331d85b74299ed4e9ca4b8a4edbade241c01fd6 --- /dev/null +++ b/kubernetes-templates/02_database_storage.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cvat-postgres-data + namespace: cvat +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 20Gi diff --git a/kubernetes-templates/03_database_deployment.yml b/kubernetes-templates/03_database_deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..84bdf3fd62e3c029d178cd5cdd1156e7255d912e --- /dev/null +++ b/kubernetes-templates/03_database_deployment.yml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-postgres + namespace: cvat + labels: + app: cvat-app + tier: db +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: db + template: + metadata: + labels: + app: cvat-app + tier: db + spec: + containers: + - name: cvat-postgres + image: postgres:10.3-alpine + imagePullPolicy: "IfNotPresent" + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - sh + - -c + - su - postgres -c "pg_isready --host=$POD_IP" + initialDelaySeconds: 15 + timeoutSeconds: 2 + resources: {} + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgredb + subPath: postgres + volumes: + - name: postgredb + persistentVolumeClaim: + claimName: cvat-postgres-data diff --git a/kubernetes-templates/03_redis_deployment.yml b/kubernetes-templates/03_redis_deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..eafb52836d0b47afcb509b439587e8f322aaf63b --- /dev/null +++ b/kubernetes-templates/03_redis_deployment.yml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-redis + namespace: cvat + labels: + app: cvat-app + tier: redis-app +spec: + replicas: 1 + selector: + matchLabels: + app: cvat-app + tier: redis-app + template: + metadata: + labels: + app: cvat-app + tier: redis-app + spec: + containers: + - image: redis:4.0.5-alpine + name: cvat-redis + imagePullPolicy: Always + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" diff --git a/kubernetes-templates/04_cvat_backend_deployment.yml b/kubernetes-templates/04_cvat_backend_deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..7b1ea7a0f0589db6e75df69e386e53f25d0b94c4 --- /dev/null +++ b/kubernetes-templates/04_cvat_backend_deployment.yml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-backend + namespace: cvat + labels: + app: cvat-app + tier: backend +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: backend + template: + metadata: + labels: + app: cvat-app + tier: backend + spec: + containers: + - name: cvat-backend-app-container + image: openvino/cvat_server:v1.2.0 + imagePullPolicy: Always + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: DJANGO_MODWSGI_EXTRA_ARGS + value: "" + - name: UI_PORT + value: "80" + - name: UI_HOST + value: "cvat-frontend-service" + - name: ALLOWED_HOSTS + value: "*" + - name: CVAT_REDIS_HOST + value: "cvat-redis-service" + - name: CVAT_POSTGRES_HOST + value: "cvat-postgres-service" + - name: CVAT_POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: CVAT_POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: CVAT_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models + initContainers: + - name: user-data-permission-fix + image: busybox + command: ["/bin/chmod", "-R", "777", "/home/django"] + volumeMounts: + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models + volumes: + - name: cvat-backend-data + persistentVolumeClaim: + claimName: cvat-backend-data + imagePullSecrets: + - name: gitlab-registry diff --git a/kubernetes-templates/04_cvat_frontend_deployment.yml b/kubernetes-templates/04_cvat_frontend_deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..362ebdd5a29c3487ab96d1b4928e20ea451565c1 --- /dev/null +++ b/kubernetes-templates/04_cvat_frontend_deployment.yml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-frontend + namespace: cvat + labels: + app: cvat-app + tier: frontend +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: frontend + template: + metadata: + labels: + app: cvat-app + tier: frontend + spec: + containers: + - name: cvat-frontend-app-container + image: openvino/cvat_ui:v1.2.0 + imagePullPolicy: Always + ports: + - containerPort: 80 + resources: {} diff --git a/kubernetes-templates/04_database_service.yml b/kubernetes-templates/04_database_service.yml new file mode 100644 index 0000000000000000000000000000000000000000..bb1d04f4065a9c672242e3e653e45ee08c9a74b6 --- /dev/null +++ b/kubernetes-templates/04_database_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-postgres-service + namespace: cvat + labels: + app: cvat-app + tier: db +spec: + type: ClusterIP + selector: + app: cvat-app + tier: db + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP + name: http diff --git a/kubernetes-templates/04_redis_service.yml b/kubernetes-templates/04_redis_service.yml new file mode 100644 index 0000000000000000000000000000000000000000..436b99b2757740f7f50be03f6e1b6fbcf9f5db06 --- /dev/null +++ b/kubernetes-templates/04_redis_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-redis-service + namespace: cvat + labels: + app: cvat-app + tier: redis-app +spec: + type: ClusterIP + selector: + app: cvat-app + tier: redis-app + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_backend_service.yml b/kubernetes-templates/05_cvat_backend_service.yml new file mode 100644 index 0000000000000000000000000000000000000000..d6c4659cb4f92b78e3dd525e0147248a2c6e88cf --- /dev/null +++ b/kubernetes-templates/05_cvat_backend_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-backend-service + namespace: cvat + labels: + app: cvat-app + tier: backend +spec: + type: ClusterIP + selector: + app: cvat-app + tier: backend + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_frontend_service.yml b/kubernetes-templates/05_cvat_frontend_service.yml new file mode 100644 index 0000000000000000000000000000000000000000..0c97278c60e3efa030faca7a8f4ecb6473fb0762 --- /dev/null +++ b/kubernetes-templates/05_cvat_frontend_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-frontend-service + namespace: cvat + labels: + app: cvat-app + tier: frontend +spec: + type: ClusterIP + selector: + app: cvat-app + tier: frontend + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_proxy_configmap.yml b/kubernetes-templates/05_cvat_proxy_configmap.yml new file mode 100644 index 0000000000000000000000000000000000000000..a7ca810626ee9b92d73040c6420eb7838349bb88 --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_configmap.yml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cvat-nginx-conf + namespace: cvat +data: + nginx.conf: | + worker_processes 2; + + error_log /dev/stdout info; + + events { + worker_connections 1024; + } + + http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + # For long domain names (e.g. AWS hosts) + server_names_hash_bucket_size 128; + + include /etc/nginx/cvat.d/*.conf; + client_max_body_size 0; + } + cvat.conf: | + server { + listen 80; + server_name _ default; + return 404; + } + + server { + listen 80; + server_name {MY_SERVER_URL_COM}; + + proxy_pass_header X-CSRFToken; + proxy_set_header Host $http_host; + proxy_pass_header Set-Cookie; + + location ~* /api/.*|git/.*|tensorflow/.*|auto_annotation/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.*|dextr/.*|reid/.* { + proxy_pass http://cvat-backend-service:8080; + } + + # workaround for match location by arguments + location = / { + error_page 418 = @annotation_ui; + + if ( $query_string ~ "^id=\d+.*" ) { return 418; } + proxy_pass http://cvat-frontend-service:80; + } + + location / { + proxy_pass http://cvat-frontend-service:80; + } + + # old annotation ui, will be removed in the future. + location @annotation_ui { + proxy_pass http://cvat-backend-service:8080; + } + } + mime.types: | + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg svgz; + image/webp webp; + + application/font-woff woff; + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; + } diff --git a/kubernetes-templates/05_cvat_proxy_deployment.yml b/kubernetes-templates/05_cvat_proxy_deployment.yml new file mode 100644 index 0000000000000000000000000000000000000000..456bec2ea901693ecc2a18c46736608ce044c4c0 --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_deployment.yml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-nginx + namespace: cvat + labels: + app: cvat-app + tier: proxy +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: proxy + template: + metadata: + labels: + app: cvat-app + tier: proxy + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /etc/nginx + readOnly: true + name: cvat-nginx-conf + - mountPath: /var/log/nginx + name: log + volumes: + - name: cvat-nginx-conf + configMap: + name: cvat-nginx-conf + items: + - key: nginx.conf + path: nginx.conf + - key: mime.types + path: mime.types + - key: cvat.conf + path: cvat.d/cvat.conf + - name: log + emptyDir: {} diff --git a/kubernetes-templates/05_cvat_proxy_service.yml b/kubernetes-templates/05_cvat_proxy_service.yml new file mode 100644 index 0000000000000000000000000000000000000000..18229d3b3dd795e312bef87f4e3afc9fa0a95927 --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-proxy-service + namespace: cvat + labels: + app: cvat-app + tier: proxy +spec: + type: NodePort + selector: + app: cvat-app + tier: proxy + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http diff --git a/kubernetes-templates/README.md b/kubernetes-templates/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ce0c6a06d4dc2a7d6df3be8f014d44bc41fa23a7 --- /dev/null +++ b/kubernetes-templates/README.md @@ -0,0 +1,85 @@ +# Deploying cvat in a kubernetes cluster + +This guide will focus on how to deploy cvat in an kubernetes environment. +It was tested on Kubernetes v1.19.3 but should work for >=v1.9, eventhough it is untested. + +## Building the container - optional +Since prebuild container images are now available [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and +[cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) this steps becomes optional. + +If you would like to build your one image the following steps need to be followd. +1. Build the cvat backend and frontend images and push them to a registry that you can pull from within the cluster. +1. Replace the `openvino/...` image source in + `04_cvat_backend_deployment.yml` and `04_cvat_frontend_deployment.yml` with your newly build image. + +```bash +export CI_REGISTRY_IMAGE="your.private.registry" + +echo "Building backend" +docker build --cache-from $CI_REGISTRY_IMAGE/backend:release-1.1.0 \ + --build-arg TF_ANNOTATION=no --build-arg AUTO_SEGMENTATION=no \ + --build-arg WITH_TESTS=no --build-arg TZ="Etc/UTC" --build-arg OPENVINO_TOOLKIT=no \ + --build-arg USER=django --build-arg DJANGO_CONFIGURATION=production \ + --build-arg TZ="Etc/UTC" . +docker push $CI_REGISTRY_IMAGE/backend:release-1.1.0 + +echo "Building frontend" +docker build --file Dockerfile.ui \ + --tag $CI_REGISTRY_IMAGE/frontend:release-1.1.0 - . +docker push $CI_REGISTRY_IMAGE/frontend:release-1.1.0 +``` + +## Adjusting the kubernetes templates + +1. Replacing the domain dummy with your real domain name `cvat.my.cool.domain.com`. + Replace `{MY_SERVER_URL_COM}` in `kubernetes-templates/04_cvat_frontend_deployment.yml` + and `kubernetes-templates/05_cvat_proxy_configmap.yml`. +1. Insert your choosen database password the `kubernetes-templates/02_database_secrets.yml` + +## Deploying to the cluster +Deploy everything to your cluster with `kubectl apply -f kubernetes-templates/` + +### Expose the deployment +The service `cvat-proxy-service` is the accesspoint to the deployment. +In order to expose this resource an ingress might be handy [kubernetes ingress documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/). + +For debugging puposes it is usefull to forward this service to a port on your localhost. +In the following example `8080` will be used for this purpose [localhost:8080](http://localhost:8080). + +```bash +kubectl port-forward service/cvat-proxy-service -n cvat 8080:80 +``` + +**Hint:** +If you are developing locally it might be usefull to replace `{MY_SERVER_URL_COM}` with `localhost`, +such that `/etc/hosts` does not need to override the DNS. + +## Create the django super user + +``` +kubectl get pods --namespace cvat +kubectl --namespace cvat exec -it cvat-backend-78c954f84f-qxb8b -- /bin/bash +python3 ~/manage.py createsuperuser +``` + +## Debugging hints +Due to different kubernetes versions or other deployment environments + +### Incorect storage class +Depending on the selected kubernetes environment certain storage classes might not be available. +The selected "standard" class is available with in all maijor kubernetes platforms (GKE, EKS, ...), +but not in some local development environemnts such as miniKube. +This is the case, if `kubectl describe pod -n cvat cvat-backend` shows that the volume claim is pending. +To fix this, `class: standard` needs to be adjusted in `02_cvat_backend_storage.yaml` and `02_database_storage.yml`. + +### Creating the django super user fails +Depending on your kuberenets version you creating the super user might not be possible with in one line. +Therefore you need to get bash access within the consol and call the manage script manually. +```bash +kubectl --namespace cvat exec -it cvat-backend-7c954d5cf6-xfdcm bash +python3 ~/manage.py createsuperuser +``` + +### Running out of storage +By default the backend is reserving 20GB of storage if this is not enough, +you will need to ajust the `02_cvat_backend_storage.yml` persistant volume claim to increase it.