Adding Kuberenetes templates and deployment guide (#1962)

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and
   [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub (<https://github.com/openvinotoolkit/cvat/pull/2766>)
 - Project task subsets (<https://github.com/openvinotoolkit/cvat/pull/2774>)
+- Kubernetes templates and guide for their deployment (<https://github.com/openvinotoolkit/cvat/pull/1962>)
 - [WiderFace](http://shuoyang1213.me/WIDERFACE/) format support (<https://github.com/openvinotoolkit/cvat/pull/2864>)
 - [VGGFace2](https://github.com/ox-vgg/vgg_face2) format support (<https://github.com/openvinotoolkit/cvat/pull/2865>)
 

kubernetes-templates/01_namespace.yml

+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cvat

kubernetes-templates/02_cvat_backend_storage.yml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cvat-backend-data
+  namespace: cvat
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: standard
+  resources:
+    requests:
+      storage: 20Gi

kubernetes-templates/02_database_secrets.yml

+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: cvat-postgres-secret
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: db
+stringData:
+  POSTGRES_DB: cvat
+  POSTGRES_USER: root
+  POSTGRES_PASSWORD: POSTGRES_ADMIN_PW

kubernetes-templates/02_database_storage.yml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cvat-postgres-data
+  namespace: cvat
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: standard
+  resources:
+    requests:
+      storage: 20Gi

kubernetes-templates/03_database_deployment.yml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cvat-postgres
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: db
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: cvat-app
+      tier: db
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: db
+    spec:
+      containers:
+        - name: cvat-postgres
+          image: postgres:10.3-alpine
+          imagePullPolicy: "IfNotPresent"
+          env:
+          - name: POSTGRES_DB
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_DB
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_USER
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_PASSWORD
+          ports:
+          - containerPort: 5432
+          readinessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - su - postgres -c "pg_isready --host=$POD_IP"
+            initialDelaySeconds: 15
+            timeoutSeconds: 2
+          resources: {}
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgredb
+              subPath: postgres
+      volumes:
+        - name: postgredb
+          persistentVolumeClaim:
+            claimName: cvat-postgres-data

kubernetes-templates/03_redis_deployment.yml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cvat-redis
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: redis-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cvat-app
+      tier: redis-app
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: redis-app
+    spec:
+      containers:
+      - image: redis:4.0.5-alpine
+        name: cvat-redis
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 6379
+        resources:
+          limits:
+            cpu: "0.1"

kubernetes-templates/04_cvat_backend_deployment.yml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cvat-backend
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: backend
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: cvat-app
+      tier: backend
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: backend
+    spec:
+      containers:
+        - name: cvat-backend-app-container
+          image: openvino/cvat_server:v1.2.0
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 10m
+              memory: 100Mi
+          env:
+          - name: DJANGO_MODWSGI_EXTRA_ARGS
+            value: ""
+          - name: UI_PORT
+            value: "80"
+          - name: UI_HOST
+            value: "cvat-frontend-service"
+          - name: ALLOWED_HOSTS
+            value: "*"
+          - name: CVAT_REDIS_HOST
+            value: "cvat-redis-service"
+          - name: CVAT_POSTGRES_HOST
+            value: "cvat-postgres-service"
+          - name: CVAT_POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_USER
+          - name: CVAT_POSTGRES_DBNAME
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_DB
+          - name: CVAT_POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: cvat-postgres-secret
+                key: POSTGRES_PASSWORD
+          ports:
+          - containerPort: 8080
+          volumeMounts:
+          - mountPath: /home/django/data
+            name: cvat-backend-data
+            subPath: data
+          - mountPath: /home/django/keys
+            name: cvat-backend-data
+            subPath: keys
+          - mountPath: /home/django/logs
+            name: cvat-backend-data
+            subPath: logs
+          - mountPath: /home/django/models
+            name: cvat-backend-data
+            subPath: models
+      initContainers:
+        - name: user-data-permission-fix
+          image: busybox
+          command: ["/bin/chmod", "-R", "777", "/home/django"]
+          volumeMounts:
+          - mountPath: /home/django/data
+            name: cvat-backend-data
+            subPath: data
+          - mountPath: /home/django/keys
+            name: cvat-backend-data
+            subPath: keys
+          - mountPath: /home/django/logs
+            name: cvat-backend-data
+            subPath: logs
+          - mountPath: /home/django/models
+            name: cvat-backend-data
+            subPath: models
+      volumes:
+        - name: cvat-backend-data
+          persistentVolumeClaim:
+            claimName: cvat-backend-data
+      imagePullSecrets:
+        - name: gitlab-registry

kubernetes-templates/04_cvat_frontend_deployment.yml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cvat-frontend
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: frontend
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: cvat-app
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: frontend
+    spec:
+      containers:
+        - name: cvat-frontend-app-container
+          image: openvino/cvat_ui:v1.2.0
+          imagePullPolicy: Always
+          ports:
+          - containerPort: 80
+          resources: {}

kubernetes-templates/04_database_service.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: cvat-postgres-service
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: db
+spec:
+  type: ClusterIP
+  selector:
+      app: cvat-app
+      tier: db
+  ports:
+    - port: 5432
+      targetPort: 5432
+      protocol: TCP
+      name: http

kubernetes-templates/04_redis_service.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: cvat-redis-service
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: redis-app
+spec:
+  type: ClusterIP
+  selector:
+      app: cvat-app
+      tier: redis-app
+  ports:
+    - port: 6379
+      targetPort: 6379
+      protocol: TCP
+      name: http

kubernetes-templates/05_cvat_backend_service.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: cvat-backend-service
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: backend
+spec:
+  type: ClusterIP
+  selector:
+      app: cvat-app
+      tier: backend
+  ports:
+    - port: 8080
+      targetPort: 8080
+      protocol: TCP
+      name: http

kubernetes-templates/05_cvat_frontend_service.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: cvat-frontend-service
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: frontend
+spec:
+  type: ClusterIP
+  selector:
+      app: cvat-app
+      tier: frontend
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP
+      name: http

kubernetes-templates/05_cvat_proxy_configmap.yml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cvat-nginx-conf
+  namespace: cvat
+data:
+  nginx.conf: |
+    worker_processes 2;
+
+    error_log /dev/stdout info;
+
+    events {
+        worker_connections  1024;
+    }
+
+    http {
+        include       mime.types;
+        default_type  application/octet-stream;
+        sendfile      on;
+        keepalive_timeout  65;
+        # For long domain names (e.g. AWS hosts)
+        server_names_hash_bucket_size 128;
+
+        include /etc/nginx/cvat.d/*.conf;
+        client_max_body_size 0;
+    }
+  cvat.conf: |
+    server {
+        listen       80;
+        server_name  _ default;
+        return       404;
+    }
+
+    server {
+        listen       80;
+        server_name  {MY_SERVER_URL_COM};
+
+        proxy_pass_header       X-CSRFToken;
+        proxy_set_header        Host $http_host;
+        proxy_pass_header       Set-Cookie;
+
+        location ~* /api/.*|git/.*|tensorflow/.*|auto_annotation/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.*|dextr/.*|reid/.*  {
+            proxy_pass              http://cvat-backend-service:8080;
+        }
+
+        # workaround for match location by arguments
+        location = / {
+            error_page 418 = @annotation_ui;
+
+            if ( $query_string ~ "^id=\d+.*" ) { return 418; }
+            proxy_pass              http://cvat-frontend-service:80;
+        }
+
+        location / {
+            proxy_pass              http://cvat-frontend-service:80;
+        }
+
+        # old annotation ui, will be removed in the future.
+        location @annotation_ui {
+            proxy_pass              http://cvat-backend-service:8080;
+        }
+    }
+  mime.types: |
+    types {
+      text/html                             html htm shtml;
+      text/css                              css;
+      text/xml                              xml;
+      image/gif                             gif;
+      image/jpeg                            jpeg jpg;
+      application/javascript                js;
+      application/atom+xml                  atom;
+      application/rss+xml                   rss;
+
+      text/mathml                           mml;
+      text/plain                            txt;
+      text/vnd.sun.j2me.app-descriptor      jad;
+      text/vnd.wap.wml                      wml;
+      text/x-component                      htc;
+
+      image/png                             png;
+      image/tiff                            tif tiff;
+      image/vnd.wap.wbmp                    wbmp;
+      image/x-icon                          ico;
+      image/x-jng                           jng;
+      image/x-ms-bmp                        bmp;
+      image/svg+xml                         svg svgz;
+      image/webp                            webp;
+
+      application/font-woff                 woff;
+      application/java-archive              jar war ear;
+      application/json                      json;
+      application/mac-binhex40              hqx;
+      application/msword                    doc;
+      application/pdf                       pdf;
+      application/postscript                ps eps ai;
+      application/rtf                       rtf;
+      application/vnd.apple.mpegurl         m3u8;
+      application/vnd.ms-excel              xls;
+      application/vnd.ms-fontobject         eot;
+      application/vnd.ms-powerpoint         ppt;
+      application/vnd.wap.wmlc              wmlc;
+      application/vnd.google-earth.kml+xml  kml;
+      application/vnd.google-earth.kmz      kmz;
+      application/x-7z-compressed           7z;
+      application/x-cocoa                   cco;
+      application/x-java-archive-diff       jardiff;
+      application/x-java-jnlp-file          jnlp;
+      application/x-makeself                run;
+      application/x-perl                    pl pm;
+      application/x-pilot                   prc pdb;
+      application/x-rar-compressed          rar;
+      application/x-redhat-package-manager  rpm;
+      application/x-sea                     sea;
+      application/x-shockwave-flash         swf;
+      application/x-stuffit                 sit;
+      application/x-tcl                     tcl tk;
+      application/x-x509-ca-cert            der pem crt;
+      application/x-xpinstall               xpi;
+      application/xhtml+xml                 xhtml;
+      application/xspf+xml                  xspf;
+      application/zip                       zip;
+
+      application/octet-stream              bin exe dll;
+      application/octet-stream              deb;
+      application/octet-stream              dmg;
+      application/octet-stream              iso img;
+      application/octet-stream              msi msp msm;
+
+      application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;
+      application/vnd.openxmlformats-officedocument.spreadsheetml.sheet          xlsx;
+      application/vnd.openxmlformats-officedocument.presentationml.presentation  pptx;
+
+      audio/midi                            mid midi kar;
+      audio/mpeg                            mp3;
+      audio/ogg                             ogg;
+      audio/x-m4a                           m4a;
+      audio/x-realaudio                     ra;
+
+      video/3gpp                            3gpp 3gp;
+      video/mp2t                            ts;
+      video/mp4                             mp4;
+      video/mpeg                            mpeg mpg;
+      video/quicktime                       mov;
+      video/webm                            webm;
+      video/x-flv                           flv;
+      video/x-m4v                           m4v;
+      video/x-mng                           mng;
+      video/x-ms-asf                        asx asf;
+      video/x-ms-wmv                        wmv;
+      video/x-msvideo                       avi;
+    }

kubernetes-templates/05_cvat_proxy_deployment.yml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cvat-nginx
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: proxy
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: cvat-app
+      tier: proxy
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: proxy
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+        volumeMounts:
+        - mountPath: /etc/nginx
+          readOnly: true
+          name: cvat-nginx-conf
+        - mountPath: /var/log/nginx
+          name: log
+      volumes:
+      - name: cvat-nginx-conf
+        configMap:
+          name: cvat-nginx-conf
+          items:
+            - key: nginx.conf
+              path: nginx.conf
+            - key: mime.types
+              path: mime.types
+            - key: cvat.conf
+              path: cvat.d/cvat.conf
+      - name: log
+        emptyDir: {}

kubernetes-templates/05_cvat_proxy_service.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: cvat-proxy-service
+  namespace: cvat
+  labels:
+    app: cvat-app
+    tier: proxy
+spec:
+  type: NodePort
+  selector:
+      app: cvat-app
+      tier: proxy
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP
+      name: http

kubernetes-templates/README.md

+# Deploying cvat in a kubernetes cluster
+
+This guide will focus on how to deploy cvat in an kubernetes environment.
+It was tested on Kubernetes v1.19.3 but should work for >=v1.9, eventhough it is untested.
+
+## Building the container - optional
+Since prebuild container images are now available [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and
+[cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) this steps becomes optional.
+
+If you would like to build your one image the following steps need to be followd.
+1. Build the cvat backend and frontend images and push them to a registry that you can pull from within the cluster.
+1. Replace the `openvino/...` image source in
+    `04_cvat_backend_deployment.yml` and `04_cvat_frontend_deployment.yml` with your newly build image.
+
+```bash
+export CI_REGISTRY_IMAGE="your.private.registry"
+
+echo "Building backend"
+docker build --cache-from $CI_REGISTRY_IMAGE/backend:release-1.1.0 \
+  --build-arg TF_ANNOTATION=no --build-arg AUTO_SEGMENTATION=no \
+  --build-arg WITH_TESTS=no --build-arg TZ="Etc/UTC" --build-arg OPENVINO_TOOLKIT=no \
+  --build-arg USER=django --build-arg DJANGO_CONFIGURATION=production \
+  --build-arg TZ="Etc/UTC" .
+docker push $CI_REGISTRY_IMAGE/backend:release-1.1.0
+
+echo "Building frontend"
+docker build --file Dockerfile.ui \
+  --tag $CI_REGISTRY_IMAGE/frontend:release-1.1.0 - .
+docker push $CI_REGISTRY_IMAGE/frontend:release-1.1.0
+```
+
+## Adjusting the kubernetes templates
+
+1. Replacing the domain dummy with your real domain name `cvat.my.cool.domain.com`.
+    Replace `{MY_SERVER_URL_COM}` in `kubernetes-templates/04_cvat_frontend_deployment.yml`
+    and `kubernetes-templates/05_cvat_proxy_configmap.yml`.
+1. Insert your choosen database password the `kubernetes-templates/02_database_secrets.yml`
+
+## Deploying to the cluster
+Deploy everything to your cluster with `kubectl apply -f kubernetes-templates/`
+
+### Expose the deployment
+The service `cvat-proxy-service` is the accesspoint to the deployment.
+In order to expose this resource an ingress might be handy [kubernetes ingress documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/).
+
+For debugging puposes it is usefull to forward this service to a port on your localhost.
+In the following example `8080` will be used for this purpose [localhost:8080](http://localhost:8080).
+
+```bash
+kubectl port-forward service/cvat-proxy-service -n cvat 8080:80
+```
+
+**Hint:**
+If you are developing locally it might be usefull to replace `{MY_SERVER_URL_COM}` with `localhost`,
+such that `/etc/hosts` does not need to override the DNS.
+
+## Create the django super user
+
+```
+kubectl get pods --namespace cvat
+kubectl --namespace cvat exec -it cvat-backend-78c954f84f-qxb8b -- /bin/bash
+python3 ~/manage.py createsuperuser
+```
+
+## Debugging hints
+Due to different kubernetes versions or other deployment environments
+
+### Incorect storage class
+Depending on the selected kubernetes environment certain storage classes might not be available.
+The selected "standard" class is available with in all maijor kubernetes platforms (GKE, EKS, ...),
+but not in some local development environemnts such as miniKube.
+This is the case, if `kubectl describe pod -n cvat cvat-backend` shows that the volume claim is pending.
+To fix this, `class: standard` needs to be adjusted in `02_cvat_backend_storage.yaml` and `02_database_storage.yml`.
+
+### Creating the django super user fails
+Depending on your kuberenets version you creating the super user might not be possible with in one line.
+Therefore you need to get bash access within the consol and call the manage script manually.
+```bash
+kubectl --namespace cvat exec -it cvat-backend-7c954d5cf6-xfdcm bash
+python3 ~/manage.py createsuperuser
+```
+
+### Running out of storage
+By default the backend is reserving 20GB of storage if this is not enough,
+you will need to ajust the `02_cvat_backend_storage.yml` persistant volume claim to increase it.