未验证 提交 6055be49 编写于 作者: T Tadej Svetina 提交者: GitHub

Replace Nginx proxy with Traefik (#3409)

* Replace Nginx with Traefik

* Comment Traefik dashboard commands and ports

* Fix cvat service port

* Simplify traefik dashboard instructions

* Add license to docker-compose files

* Update all mentions of CVAT_HOST in the docs

* Add link to Traefik documentation on router rules

* Return base CVAT port to 8080

* Fix spelling in documentation

* Fix port indentaion in docker-compose file

* Fix Traefik dashboard config

* Update changelog

* Adapt serverless dockerfile

* Update analytics dockerfile

* Update analytics docker compose file

* Fix linting issues

* fixed linter issues
Co-authored-by: NAndrey Zhavoronkov <andrey.zhavoronkov@intel.com>
上级 f4382fec
......@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Update to Django 3.1.12 (<https://github.com/openvinotoolkit/cvat/pull/3378>)
- Updated visibility for removable points in AI tools (<https://github.com/openvinotoolkit/cvat/pull/3417>)
- Updated UI handling for IOG serverless function (<https://github.com/openvinotoolkit/cvat/pull/3417>)
- Changed Nginx proxy to Traefik in `docker-compose.yml` (<https://github.com/openvinotoolkit/cvat/pull/3409>)
- Simplify the process of deploying CVAT with HTTPS (<https://github.com/openvinotoolkit/cvat/pull/3409>)
### Deprecated
......
version: '3.3'
services:
cvat_elasticsearch:
elasticsearch:
container_name: cvat_elasticsearch
image: cvat_elasticsearch
networks:
default:
aliases:
- elasticsearch
- cvat
build:
context: ./components/analytics/elasticsearch
args:
......@@ -15,18 +13,16 @@ services:
- cvat_events:/usr/share/elasticsearch/data
restart: always
cvat_kibana:
kibana:
container_name: cvat_kibana
image: cvat_kibana
networks:
default:
aliases:
- kibana
- cvat
build:
context: ./components/analytics/kibana
args:
ELK_VERSION: 6.4.0
depends_on: ['cvat_elasticsearch']
depends_on: ['elasticsearch']
restart: always
cvat_kibana_setup:
......@@ -35,6 +31,8 @@ services:
volumes: ['./components/analytics/kibana:/home/django/kibana:ro']
depends_on: ['cvat']
working_dir: '/home/django'
networks:
- cvat
entrypoint:
[
'bash',
......@@ -56,13 +54,11 @@ services:
environment:
no_proxy: elasticsearch,kibana,${no_proxy}
cvat_logstash:
logstash:
container_name: cvat_logstash
image: cvat_logstash
networks:
default:
aliases:
- logstash
- cvat
build:
context: ./components/analytics/logstash
args:
......@@ -73,7 +69,7 @@ services:
LOGSTASH_OUTPUT_HOST: elasticsearch:9200
LOGSTASH_OUTPUT_USER:
LOGSTASH_OUTPUT_PASS:
depends_on: ['cvat_elasticsearch']
depends_on: ['elasticsearch']
restart: always
cvat:
......
version: '3.3'
services:
serverless:
nuclio:
container_name: nuclio
image: quay.io/nuclio/dashboard:1.5.16-amd64
restart: always
networks:
default:
aliases:
- nuclio
- cvat
volumes:
- /tmp:/tmp
- /var/run/docker.sock:/var/run/docker.sock
......
server {
listen 80;
server_name _ default;
return 404;
}
server {
listen 80;
server_name ${CVAT_HOST};
proxy_pass_header X-CSRFToken;
proxy_set_header Host $http_host;
proxy_pass_header Set-Cookie;
location ~* /api/.*|git/.*|opencv/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? {
proxy_pass http://cvat:8080;
}
location / {
proxy_pass http://cvat_ui;
}
}
worker_processes 2;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
# For long domain names (e.g. AWS hosts)
server_names_hash_bucket_size 128;
include /etc/nginx/conf.d/*.conf;
client_max_body_size 0;
}
# Copyright (C) 2018-2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
version: '3.3'
services:
cvat:
labels:
- traefik.http.routers.cvat.entrypoints=websecure
- traefik.http.routers.cvat.tls.certresolver=lets-encrypt
cvat_ui:
labels:
- traefik.http.routers.cvat-ui.entrypoints=websecure
- traefik.http.routers.cvat-ui.tls.certresolver=lets-encrypt
traefik:
image: traefik:v2.4
container_name: traefik
command:
- "--providers.docker.exposedByDefault=false"
- "--providers.docker.network=cvat"
- "--entryPoints.web.address=:80"
- "--entryPoints.web.http.redirections.entryPoint.to=websecure"
- "--entryPoints.web.http.redirections.entryPoint.scheme=https"
- "--entryPoints.websecure.address=:443"
- "--certificatesResolvers.lets-encrypt.acme.email=${ACME_EMAIL:?Please set the ACME_EMAIL env variable}"
- "--certificatesResolvers.lets-encrypt.acme.tlsChallenge=true"
- "--certificatesResolvers.lets-encrypt.acme.storage=/letsencrypt/acme.json"
# Uncomment to get Traefik dashboard
# - "--entryPoints.dashboard.address=:8090"
# - "--api.dashboard=true"
ports:
- 80:80
- 443:443
volumes:
- cvat_letsencrypt:/letsencrypt
volumes:
cvat_letsencrypt:
#
# Copyright (C) 2018-2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
version: '3.3'
services:
cvat_db:
container_name: cvat_db
image: postgres:10-alpine
networks:
default:
aliases:
- db
restart: always
environment:
POSTGRES_USER: root
......@@ -20,15 +15,15 @@ services:
POSTGRES_HOST_AUTH_METHOD: trust
volumes:
- cvat_db:/var/lib/postgresql/data
networks:
- cvat
cvat_redis:
container_name: cvat_redis
image: redis:4.0-alpine
networks:
default:
aliases:
- redis
restart: always
networks:
- cvat
cvat:
container_name: cvat
......@@ -43,47 +38,61 @@ services:
CVAT_REDIS_HOST: 'cvat_redis'
CVAT_POSTGRES_HOST: 'cvat_db'
ADAPTIVE_AUTO_ANNOTATION: 'false'
labels:
- traefik.enable=true
- traefik.http.services.cvat.loadbalancer.server.port=8080
- traefik.http.routers.cvat.rule=Host(`${CVAT_HOST:-localhost}`) &&
PathPrefix(`/api/`, `/git/`, `/opencv/`, `/analytics/`, `/static/`, `/admin`, `/documentation/`, `/django-rq`)
- traefik.http.routers.cvat.entrypoints=web
volumes:
- cvat_data:/home/django/data
- cvat_keys:/home/django/keys
- cvat_logs:/home/django/logs
networks:
- cvat
cvat_ui:
container_name: cvat_ui
image: openvino/cvat_ui
restart: always
networks:
default:
aliases:
- ui
depends_on:
- cvat
cvat_proxy:
container_name: cvat_proxy
image: nginx:stable-alpine
restart: always
depends_on:
labels:
- traefik.enable=true
- traefik.http.services.cvat-ui.loadbalancer.server.port=80
- traefik.http.routers.cvat-ui.rule=Host(`${CVAT_HOST:-localhost}`)
- traefik.http.routers.cvat-ui.entrypoints=web
networks:
- cvat
- cvat_ui
environment:
CVAT_HOST: localhost
traefik:
image: traefik:v2.4
container_name: traefik
command:
- "--providers.docker.exposedByDefault=false"
- "--providers.docker.network=cvat"
- "--entryPoints.web.address=:8080"
# Uncomment to get Traefik dashboard
# - "--entryPoints.dashboard.address=:8090"
# - "--api.dashboard=true"
# labels:
# - traefik.enable=true
# - traefik.http.routers.dashboard.entrypoints=dashboard
# - traefik.http.routers.dashboard.service=api@internal
# - traefik.http.routers.dashboard.rule=Host(`${CVAT_HOST:-localhost}`)
ports:
- '8080:80'
- 8080:8080
- 8090:8090
volumes:
- ./cvat_proxy/nginx.conf:/etc/nginx/nginx.conf:ro
- ./cvat_proxy/conf.d/cvat.conf.template:/etc/nginx/conf.d/cvat.conf.template:ro
command: /bin/sh -c "envsubst '$$CVAT_HOST' < /etc/nginx/conf.d/cvat.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
networks:
default:
ipam:
driver: default
config:
- subnet: 172.28.0.0/24
- /var/run/docker.sock:/var/run/docker.sock:ro
networks:
- cvat
volumes:
cvat_db:
cvat_data:
cvat_keys:
cvat_logs:
networks:
cvat:
\ No newline at end of file
......@@ -21,14 +21,11 @@ There are two ways of deploying the CVAT.
[installation instructions](/docs/administration/basics/installation/).
The additional step is to add a [security group and rule to allow incoming connections](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html).
For any of above, don't forget to add exposed AWS public IP address or hostname to `docker-compose.override.yml`:
For any of above, don't forget to set the `CVAT_HOST` environemnt variable to the exposed
AWS public IP address or hostname:
```
version: "2.3"
services:
cvat_proxy:
environment:
CVAT_HOST: your-instance.amazonaws.com
export CVAT_HOST=your-instance.amazonaws.com
```
In case of problems with using hostname, you can also use the public IPV4 instead of hostname.
......@@ -37,5 +34,4 @@ the public IPV4 and hostname changes with every stop and reboot.
To address this efficiently, avoid using spot instances that cannot be stopped,
since copying the EBS to an AMI and restarting it throws problems.
On the other hand, when a regular instance is stopped and restarted,
the new hostname/IPV4 can be used in the `CVAT_HOST` variable in the `docker-compose.override.yml`
and the build can happen instantly with CVAT tasks being available through the new IPV4.
the new hostname/IPV4 can be used to set the `CVAT_HOST` environment variable.
......@@ -281,6 +281,40 @@ which starts containers and add JSON such as the following:
These environment variables are set automatically within any container.
Please see the [Docker documentation](https://docs.docker.com/network/proxy/) for more details.
### Using the Traefik dashboard
If you are customizing the docker compose files and you come upon some unexpected issues, using the Traefik
dashboard might be very useful to see if the problem is with Traefik configuration, or with some of the services.
You can enable the Traefik dashboard by uncommenting the following lines from `docker-compose.yml`
```
services:
traefik:
# Uncomment to get Traefik dashboard
# - "--entryPoints.dashboard.address=:8090"
# - "--api.dashboard=true"
# labels:
# - traefik.enable=true
# - traefik.http.routers.dashboard.entrypoints=dashboard
# - traefik.http.routers.dashboard.service=api@internal
# - traefik.http.routers.dashboard.rule=Host(`${CVAT_HOST:-localhost}`)
```
and if you are using `docker-compose.https.yml`, also uncomment these lines
```
services:
traefik:
command:
# Uncomment to get Traefik dashboard
# - "--entryPoints.dashboard.address=:8090"
# - "--api.dashboard=true"
```
Note that this "insecure" dashboard is not recommended in production (and if your instance is publicly available);
if you want to keep the dashboard in production you should read Traefik's
[documentation](https://doc.traefik.io/traefik/operations/dashboard/) on how to properly secure it.
### Additional components
- [Analytics: management and monitoring of data annotation team](/docs/administration/advanced/analytics/)
......@@ -304,24 +338,14 @@ created by `up`.
docker-compose down
```
### Advanced settings
### Use your own domain
If you want to access your instance of CVAT outside of your localhost you should
specify the `CVAT_HOST` environment variable. The best way to do that is to create
[docker-compose.override.yml](https://docs.docker.com/compose/extends/) and put
all your extra settings here.
```yml
version: '3.3'
If you want to access your instance of CVAT outside of your localhost (on another domain),
you should specify the `CVAT_HOST` environment variable, like this:
services:
cvat_proxy:
environment:
CVAT_HOST: .example.com
```
Please don't forget include this file to docker-compose commands using the `-f`
option (in some cases it can be omitted).
export CVAT_HOST=<YOUR_DOMAIN>
```
### Share path
......@@ -379,249 +403,27 @@ for details.
### Deploy CVAT on the Scaleway public cloud
Please follow [this tutorial](https://blog.scaleway.com/smart-data-annotation-for-your-computer-vision-projects-cvat-on-scaleway/)
Please follow
[this tutorial](https://blog.scaleway.com/smart-data-annotation-for-your-computer-vision-projects-cvat-on-scaleway/)
to install and set up remote access to CVAT on a Scaleway cloud instance with data in a mounted object storage bucket.
### Deploy secure CVAT instance with HTTPS
Certificates (issued by let's encrypt) to cloud instance.
#### Prerequisites
We assume that:
- you have a virtual instance (machine) in the cloud provider with docker installed;
- there is no root permissions required if user is in docker group;
- there is no services listen 80 and 443 tcp ports on virtual instance.
There are multiple approaches. Our approach suggests:
- easy setup automatic certificate updates;
- leave certificates in safe place on docker host (protect from `docker-compose down` cleanup);
- no unnecessary certificate files copying between container and host.
#### Roadmap
We will go through the following sequence of steps to get CVAT over HTTPS:
- Install [acme.sh](https://github.com/acmesh-official/acme.sh) on the virtual instance (docker host).
- Configure Nginx site template `HOME/cvat/cvat_proxy/conf.d/cvat.conf.template` used in `cvat_proxy` container.
- Deploy CVAT services in the most common way with docker-compose utilizes default HTTP scheme.
- Create the https certificates with `acme.sh` client.
- Reconfigure Nginx to serve over HTTPS.
- Make sure that certificates will be able to automatically update via cron job.
#### Step-by-step instructions
##### 1. Make the proxy listen on 80 and 443 ports
Prepare nginx for the ACME challenge via webroot method
Using Traefik, you can automatically obtain TLS certificate for your domain from Let's Encrypt,
enabling you to use HTTPS protocol to access your website.
Let's assume the server domain name is `CVAT.example.com`.
Clone repo and point you shell in cvat repository directory, usually `cd $HOME/cvat`:
Install and create the required directories for letsencrypt webroot operation and acme folder passthrough.
```bash
# on the docker host
# this will create ~/.acme.sh directory
curl https://get.acme.sh | sh
# create a subdirs for acme-challenge webroot manually
mkdir -p $HOME/cvat/letsencrypt-webroot/.well-known/acme-challenge
```
Create `docker-compose.override.yml` in repo root like follows:
> modify CVAT_HOST with your own domain name
> (nginx tests the request’s header field “Host” to determine which server the request should be routed to)
```yaml
version: '3.3'
services:
cvat_proxy:
environment:
CVAT_HOST: CVAT.example.com
ports:
- '80:80'
- '443:443'
volumes:
- ./letsencrypt-webroot:/var/tmp/letsencrypt-webroot
- /etc/ssl/private:/etc/ssl/private
cvat:
environment:
ALLOWED_HOSTS: '*'
```
Update a CVAT site proxy template `$HOME/cvat/cvat_proxy/conf.d/cvat.conf.template` on docker(system) host.
Site config updates from this template each time `cvat_proxy` container start.
Add a location to server with `server_name ${CVAT_HOST};` ahead others:
To enable this, first set the the `CVAT_HOST` (the domain of your website) and `ACME_EMAIL`
(contact email for Let's Encrypt) environment variables:
```
location ^~ /.well-known/acme-challenge/ {
default_type "text/plain";
root /var/tmp/letsencrypt-webroot;
}
export CVAT_HOST=<YOUR_DOMAIN>
export ACME_EMAIL=<YOUR_EMAIL>
```
Make the changes where necessary, e.g. base.py or somewhere else.
Build the containers with new configurations updated in `docker-compose.override.yml`
E.g. including `analytics` module:
Then, use the `docker-compose.https.yml` file to override the base `docker-compose.yml` file:
```
docker-compose -f docker-compose.yml -f components/analytics/docker-compose.analytics.yml -f docker-compose.override.yml up -d --build
docker-compose -f docker-compose.yml -f docker-compose.https.yml up -d
```
Your server should be available (and unsecured) at `http://CVAT.example.com`
Something went wrong ? The most common cause is a containers and images cache which were built earlier.
This will enable serving `http://CVAT.example.com/.well-known/acme-challenge/`
route from `/var/tmp/letsencrypt-webroot` directory on the container's filesystem
which is bind mounted from docker host `$HOME/cvat/letsencrypt-webroot`.
That volume needed for issue and renewing certificates only.
Another volume `/etc/ssl/private` should be used within web server according to [acme.sh](https://github.com/acmesh-official/acme.sh#3-install-the-cert-to-apachenginx-etc) documentation
At this point your deployment is up and running, ready for run acme-challenge for issue a new certificate
##### 2. Issue a certificate and run HTTPS versions with `acme.sh` helper
###### Create certificate files using an ACME challenge on docker host
**Prepare certificates**
Point you shell in cvat repository directory, usually `cd $HOME/cvat` on docker host.
Let’s Encrypt provides rate limits to ensure fair usage by as many people as possible.
They recommend utilize their staging environment instead of the production API during testing.
So first try to get a test certificate.
```
~/.acme.sh/acme.sh --issue --staging -d CVAT.example.com -w $HOME/cvat/letsencrypt-webroot --debug
```
> Debug note: nginx server logs for cvat_proxy are not saved in container. You shall see it at docker host by with: `docker logs cvat_proxy`.
If certificates is issued a successful we can test a renew:
```
~/.acme.sh/acme.sh --renew --force --staging -d CVAT.example.com -w $HOME/cvat/letsencrypt-webroot --debug
```
**Remove test certificate, if success**
```
~/.acme.sh/acme.sh --remove -d CVAT.example.com --debug
rm -r /root/.acme.sh/CVAT.example.com
```
**Issue a production certificate**
```
~/.acme.sh/acme.sh --issue -d CVAT.example.com -w $HOME/cvat/letsencrypt-webroot --debug
```
**Install production certificate and a user cron job (`crontab -e`) for update it**
This will copy necessary certificate files to a permanent directory for serve.
According to acme.sh [documentation](https://github.com/acmesh-official/acme.sh#3-install-the-cert-to-apachenginx-etc)
Additionally, we must create a directory for our domain.
Acme supports a valid install configuration options in domain config file
E.g. `~/.acme.sh/CVAT.example.com/lsoft-cvat.cvisionlab.com.conf`.
```
mkdir /etc/ssl/private/CVAT.example.com
acme.sh --install-cert -d CVAT.example.com \
--cert-file /etc/ssl/private/CVAT.example.com/site.cer \
--key-file /etc/ssl/private/CVAT.example.com/site.key \
--fullchain-file /etc/ssl/private/CVAT.example.com/fullchain.cer \
--reloadcmd "/usr/bin/docker restart cvat_proxy"
```
Down the cvat_proxy container for setup https with issued certificate.
```bash
docker stop cvat_proxy
```
**Reconfigure nginx for use certificates**
Bring the configuration file `$HOME/cvat/cvat_proxy/conf.d/cvat.conf.template` to the following form:
- add location with redirect `return 301` from http to https port;
- change main cvat server to listen on 443 port;
- add ssl certificates options.
Final configuration file should look like:
> for a more accurate proxy configuration according to upstream,
> do not neglect the verification with
> this configuration [file](https://github.com/openvinotoolkit/cvat/blob/v1.2.0/cvat_proxy/conf.d/cvat.conf.template).
```
server {
listen 80;
server_name _ default;
return 404;
}
server {
listen 80;
server_name ${CVAT_HOST};
location ^~ /.well-known/acme-challenge/ {
default_type "text/plain";
root /var/tmp/letsencrypt-webroot;
}
location / {
return 301 https://$server_name$request_uri;
}
}
server {
listen 443 ssl;
server_name ${CVAT_HOST};
ssl_certificate /etc/ssl/private/${CVAT_HOST}/site.cer;
ssl_certificate_key /etc/ssl/private/${CVAT_HOST}/site.key;
ssl_trusted_certificate /etc/ssl/private/${CVAT_HOST}/fullchain.cer;
# security options
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_stapling on;
ssl_session_timeout 24h;
ssl_session_cache shared:SSL:2m;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!3DES';
proxy_pass_header X-CSRFToken;
proxy_set_header Host $http_host;
proxy_pass_header Set-Cookie;
location ~* /api/.*|git/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? {
proxy_pass http://cvat:8080;
}
location / {
proxy_pass http://cvat_ui;
}
}
```
Start cvat_proxy container with https enabled.
```bash
docker start cvat_proxy
```
Then, the CVAT instance will be available at your domain on ports 443 (HTTPS) and 80 (HTTP, redirects to 443).
......@@ -46,21 +46,25 @@ You should free up disk space or change the threshold, to do so check: [Elastics
## How to change default CVAT hostname or port
The best way to do that is to create docker-compose.override.yml and override the host and port settings here.
To change the hostname, simply set the `CVAT_HOST` environemnt variable
version: "3.3"
```
export CVAT_HOST=<YOUR_HOSTNAME>
```
```yaml
If you want to change the port, change the `entryPoints.web.address` part of `traefik` image command in `docker-compose.yml`
```
services:
cvat_proxy:
environment:
CVAT_HOST: example.com
ports:
- '80:80'
traefik:
command:
- "--providers.docker.exposedByDefault=false"
- "--providers.docker.network=test"
- "--entryPoints.web.address=:<YOUR_PORT>"
```
Please don't forget to include this file in docker-compose commands
using the `-f` option (in some cases it can be omitted).
Note that changing the port does not make sense if you are using HTTPS - port 443 is conventionally
used for HTTPS connections, and is needed for Let's Encrypt [TLS challenge](https://doc.traefik.io/traefik/https/acme/#tlschallenge).
## How to configure connected share folder on Windows
......@@ -130,13 +134,21 @@ You should build CVAT images with ['Analytics' component](https://github.com/ope
You can upload annotation for a multi-job task from the Dasboard view or the Task view.
Uploading of annotation from the Annotation view only affects the current job.
## How to specify multiple hostnames for CVAT_HOST
## How to specify multiple hostnames
To do this, you will need to edit `traefik.http.<router>.cvat.rule` docker label for both the
`cvat` and `cvat_ui` services, like so
(see [the documentation](https://doc.traefik.io/traefik/routing/routers/#rule) on Traefik rules for more details):
```yaml
services:
cvat_proxy:
environment:
CVAT_HOST: example1.com example2.com
cvat:
labels:
- traefik.http.routers.cvat.rule=(Host(`example1.com`) || Host(`example2.com`)) &&
PathPrefix(`/api/`, `/git/`, `/opencv/`, `/analytics/`, `/static/`, `/admin`, `/documentation/`, `/django-rq`)
cvat_ui:
labels:
- traefik.http.routers.cvat-ui.rule=Host(`example1.com`) || Host(`example2.com`)
```
## How to create a task with multiple jobs
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册