Extend SDK layer 1 docs (#5011)

.github/workflows/full.yml

@@ -64,12 +64,13 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Create image directory
+      - name: Create artifact directories
         run: |
           mkdir /tmp/cvat_server
           mkdir /tmp/cvat_ui
+          mkdir /tmp/cvat_sdk
 
-      - name: CVAT server. Build
+      - name: CVAT server. Build and push
         uses: docker/build-push-action@v3
         with:
           cache-from: type=local,src=/tmp/cvat_cache_server
@@ -78,7 +79,7 @@ jobs:
           tags: cvat/server
           outputs: type=docker,dest=/tmp/cvat_server/image.tar
 
-      - name: CVAT UI. Build
+      - name: CVAT UI. Build and push
         uses: docker/build-push-action@v3
         with:
           cache-from: type=local,src=/tmp/cvat_cache_ui
@@ -87,6 +88,18 @@ jobs:
           tags: cvat/ui
           outputs: type=docker,dest=/tmp/cvat_ui/image.tar
 
+      - name: CVAT SDK. Build
+        run: |
+          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
+            --entrypoint /bin/bash -u root cvat/server \
+            -c 'python manage.py spectacular --file /transfer/schema.yml'
+          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          cd cvat-sdk/
+          gen/generate.sh
+          cd ..
+
+          cp -r cvat-sdk/* /tmp/cvat_sdk/
+
       - name: Upload CVAT server artifact
         uses: actions/upload-artifact@v3
         with:
@@ -99,6 +112,12 @@ jobs:
           name: cvat_ui
           path: /tmp/cvat_ui/image.tar
 
+      - name: Upload CVAT SDK artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: cvat_sdk
+          path: /tmp/cvat_sdk/
+
   rest_api:
     needs: build
     runs-on: ubuntu-latest
@@ -158,6 +177,12 @@ jobs:
           name: cvat_ui
           path: /tmp/cvat_ui/
 
+      - name: Download CVAT SDK package
+        uses: actions/download-artifact@v3
+        with:
+          name: cvat_sdk
+          path: /tmp/cvat_sdk/
+
       - name: Load Docker images
         run: |
           docker load --input /tmp/cvat_server/image.tar
@@ -168,15 +193,7 @@ jobs:
 
       - name: Running REST API and SDK tests
         run: |
-          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
-            --entrypoint /bin/bash -u root cvat/server \
-            -c 'python manage.py spectacular --file /transfer/schema.yml'
-          pip3 install --user -r cvat-sdk/gen/requirements.txt
-          cd cvat-sdk/
-          gen/generate.sh
-          cd ..
-
-          pip3 install --user cvat-sdk/
+          pip3 install --user /tmp/cvat_sdk/
           pip3 install --user cvat-cli/
           pip3 install --user -r tests/python/requirements.txt
           pytest tests/python -s -v

.github/workflows/github_pages.yml

-name: Github pages
-
-on:
-  push:
-    branches:
-      - develop
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-          fetch-depth: 0
-
-      - name: Setup Hugo
-        uses: peaceiris/actions-hugo@v2
-        with:
-          hugo-version: '0.83.1'
-          extended: true
-
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16.x'
-
-      - name: Install npm packages
-        working-directory: ./site
-        run: |
-          npm ci
-
-      - name: Build docs
-        run: |
-          pip install -r site/requirements.txt
-          python site/build_docs.py
-
-      - name: Deploy
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./public
-          force_orphan: true

.github/workflows/main.yml

@@ -65,10 +65,11 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Create image directory
+      - name: Create artifact directories
         run: |
           mkdir /tmp/cvat_server
           mkdir /tmp/cvat_ui
+          mkdir /tmp/cvat_sdk
 
       - name: CVAT server. Build and push
         uses: docker/build-push-action@v3
@@ -88,6 +89,18 @@ jobs:
           tags: cvat/ui
           outputs: type=docker,dest=/tmp/cvat_ui/image.tar
 
+      - name: CVAT SDK. Build
+        run: |
+          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
+            --entrypoint /bin/bash -u root cvat/server \
+            -c 'python manage.py spectacular --file /transfer/schema.yml'
+          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          cd cvat-sdk/
+          gen/generate.sh
+          cd ..
+
+          cp -r cvat-sdk/* /tmp/cvat_sdk/
+
       - name: Upload CVAT server artifact
         uses: actions/upload-artifact@v3
         with:
@@ -100,7 +113,13 @@ jobs:
           name: cvat_ui
           path: /tmp/cvat_ui/image.tar
 
-  rest_api:
+      - name: Upload CVAT SDK artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: cvat_sdk
+          path: /tmp/cvat_sdk/
+
+  rest_api_testing:
     needs: build
     runs-on: ubuntu-latest
     steps:
@@ -125,6 +144,12 @@ jobs:
           name: cvat_ui
           path: /tmp/cvat_ui/
 
+      - name: Download CVAT SDK package
+        uses: actions/download-artifact@v3
+        with:
+          name: cvat_sdk
+          path: /tmp/cvat_sdk/
+
       - name: Load Docker images
         run: |
           docker load --input /tmp/cvat_server/image.tar
@@ -135,22 +160,14 @@ jobs:
 
       - name: Running REST API tests
         run: |
-          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
-            --entrypoint /bin/bash -u root cvat/server \
-            -c 'python manage.py spectacular --file /transfer/schema.yml'
-          pip3 install --user -r cvat-sdk/gen/requirements.txt
-          cd cvat-sdk/
-          gen/generate.sh
-          cd ..
-
-          pip3 install --user cvat-sdk/
+          pip3 install --user /tmp/cvat_sdk/
           pip3 install --user -r tests/python/requirements.txt
           pytest tests/python/rest_api -k 'GET' -s
 
       - name: Creating a log file from cvat containers
         if: failure()
         env:
-          LOGS_DIR: "${{ github.workspace }}/rest_api"
+          LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
         run: |
             mkdir $LOGS_DIR
             docker logs test_cvat_server_1 > $LOGS_DIR/cvat.log
@@ -161,7 +178,7 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: container_logs
-          path: "${{ github.workspace }}/rest_api"
+          path: "${{ github.workspace }}/rest_api_testing"
 
   unit_testing:
     needs: build
@@ -320,9 +337,64 @@ jobs:
           name: cypress_screenshots_${{ matrix.specs }}
           path: ${{ github.workspace }}/tests/cypress/screenshots
 
+  generate_github_pages:
+    if: github.ref == 'refs/heads/develop'
+    needs: [rest_api_testing, unit_testing, e2e_testing]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Download CVAT server images
+        uses: actions/download-artifact@v3
+        with:
+          name: cvat_server
+          path: /tmp/cvat_server/
+
+      - name: Download CVAT server images
+        uses: actions/download-artifact@v3
+        with:
+          name: cvat_sdk
+          path: /tmp/cvat_sdk/
+
+      - name: Load Docker images
+        run: |
+          docker load --input /tmp/cvat_server/image.tar
+
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@v2
+        with:
+          hugo-version: '0.83.1'
+          extended: true
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.x'
+
+      - name: Install npm packages
+        working-directory: ./site
+        run: |
+          npm ci
+
+      - name: Build docs
+        run: |
+          pip install -r site/requirements.txt
+          python site/process_sdk_docs.py --input-dir /tmp/cvat_sdk/docs/ --site-root site/
+          python site/build_docs.py
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./public
+          force_orphan: true
+
   publish_dev_images:
     if: github.ref == 'refs/heads/develop'
-    needs: [rest_api, unit_testing, e2e_testing]
+    needs: [rest_api_testing, unit_testing, e2e_testing, generate_github_pages]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

.vscode/launch.json

@@ -252,6 +252,21 @@
             "cwd": "${workspaceFolder}",
             "console": "integratedTerminal"
         },
+        {
+            "name": "sdk docs: Postprocess generated docs",
+            "type": "python",
+            "request": "launch",
+            "justMyCode": false,
+            "stopOnEntry": false,
+            "python": "${command:python.interpreterPath}",
+            "program": "${workspaceFolder}/site/process_sdk_docs.py",
+            "args": [
+                "--input-dir", "${workspaceFolder}/cvat-sdk/docs/",
+                "--site-root", "${workspaceFolder}/site/",
+            ],
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal"
+        },
         {
             "name": "server: Generate REST API Schema",
             "type": "python",

cvat-sdk/gen/design.md

-# API design decisions
-
-Generated API is modified from what `openapi-generator` does by default.
-Changes are mostly focused on better user experience - including better
-usage patterns and simpler/faster ways to achieve results.
-
-## Changes
-
-- Added type annotations for return types and class members
-  This change required us to implement a custom post-processing script,
-  which converts generated types into correct type annotations. The types
-  generated by default are supposed to work with the API implementation
-  (parameter validation and parsing), but they are not applicable as
-  type annotations (they have incorrect syntax). Custom post-processing
-  allowed us to make these types correct type annotations.
-  Other possible solutions:
-  - There is a `python-experimental` API generator, which may solve
-    some issues, but it is unstable and requires python 3.9. Our API
-    works with 3.7, which is the lowest supported version now.
-  - Custom templates - partially works, but only in limited cases
-    (model fields). It's very hard to maintain the template code and
-    logic for this. Only `if` checks and `for` loops are available in
-    mustache templates, which is not enough for annotation generation.
-
-- Separate APIs are embedded into the general `APIClient` class
-  Now we have:
-  ```python
-  with ApiClient(config) as api_client:
-    result1 = api_client.foo_api.operation1()
-    result2 = api_client.bar_api.operation2()
-  ```
-
-  This showed to be more convenient than default:
-  ```python
-  with ApiClient(config) as api_client:
-    foo_api = FooApi(api_client)
-    result1 = foo_api.operation1()
-    result2 = foo_api.operation2()
-
-    bar_api = BarApi(api_client)
-    result3 = bar_api.operation3()
-    result4 = bar_api.operation4()
-  ```
-
-  This also required custom post-processing. Operation Ids are
-  [supposed to be unique](https://swagger.io/specification/#operation-object)
-  in the OpenAPI / Swagger specification. Therefore, we can't generate such
-  schema on the server, nor we can't expect it to be supported in the
-  API generator.
-
-- Operations have IDs like `<api>/<method>_<object>`
-  This also showed to be more readable and more natural than DRF-spectacular's
-  default `<api>/<object>_<method>`.
-
-- Server operations have different types for input and output values
-  While it can be expected that an endopint with POST/PUT methods available
-  (like `create` or `partial_update`) has the same type for input and output
-  (because it looks natural), it also leads to the situation, in which there
-  are lots of read-/write-only fields, and it becomes hard for understanding.
-  This clear type separation is supposed to make it simpler for users.

cvat-sdk/gen/generate.sh

@@ -13,7 +13,7 @@ LIB_NAME="cvat_sdk"
 LAYER1_LIB_NAME="${LIB_NAME}/api_client"
 DST_DIR="."
 TEMPLATE_DIR="gen"
-PYTHON_POST_PROCESS_FILE="${TEMPLATE_DIR}/postprocess.py"
+POST_PROCESS_SCRIPT="${TEMPLATE_DIR}/postprocess.py"
 
 mkdir -p "${DST_DIR}/"
 rm -f -r "${DST_DIR}/docs" "${DST_DIR}/${LAYER1_LIB_NAME}" "requirements/"
@@ -34,5 +34,19 @@ cp -r "${TEMPLATE_DIR}/templates/requirements" "${DST_DIR}/"
 cp -r "${TEMPLATE_DIR}/templates/MANIFEST.in" "${DST_DIR}/"
 mv "${DST_DIR}/requirements.txt" "${DST_DIR}/requirements/api_client.txt"
 
-# Do custom postprocessing
-"${PYTHON_POST_PROCESS_FILE}" --schema "schema/schema.yml" --input-path "${DST_DIR}/${LIB_NAME}"
+# Do custom postprocessing for code files
+"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/${LIB_NAME}"
+
+# Do custom postprocessing for docs files
+"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/docs" --file-ext '.md'
+"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/README.md"
+
+API_DOCS_DIR="${DST_DIR}/docs/apis/"
+MODEL_DOCS_DIR="${DST_DIR}/docs/models/"
+mkdir "${API_DOCS_DIR}"
+mkdir "${MODEL_DOCS_DIR}"
+mv "${DST_DIR}/docs/"*Api.md "${API_DOCS_DIR}"
+mv "${DST_DIR}/docs/"*.md "${MODEL_DOCS_DIR}"
+mv "${DST_DIR}/README.md" "${DST_DIR}/docs/"
+
+cp "${TEMPLATE_DIR}/templates/README.md.template" "${DST_DIR}/README.md"

cvat-sdk/gen/postprocess.py

 #!/usr/bin/env python
 
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
+import argparse
 import os.path as osp
 import re
 import sys
-from argparse import ArgumentParser
 from glob import glob
 
 from inflection import underscore
@@ -29,7 +29,7 @@ def collect_operations(schema):
     return operations
 
 
-class Processor:
+class Replacer:
     REPLACEMENT_TOKEN = r"%%%"
     ARGS_TOKEN = r"!!!"
 
@@ -77,7 +77,7 @@ class Processor:
 
     def _process_file(self, contents: str):
         processor_pattern = re.compile(
-            f"{self.REPLACEMENT_TOKEN}(.*?){self.ARGS_TOKEN}(.*){self.REPLACEMENT_TOKEN}"
+            f"{self.REPLACEMENT_TOKEN}(.*?){self.ARGS_TOKEN}(.*?){self.REPLACEMENT_TOKEN}"
         )
 
         matches = list(processor_pattern.finditer(contents))
@@ -102,8 +102,8 @@ class Processor:
         with open(src_path, "w") as f:
             f.write(contents)
 
-    def process_dir(self, dir_path: str):
-        for filename in glob(dir_path + "/**/*.py", recursive=True):
+    def process_dir(self, dir_path: str, *, file_ext: str = ".py"):
+        for filename in glob(dir_path + f"/**/*{file_ext}", recursive=True):
             try:
                 self.process_file(filename)
             except Exception as e:
@@ -117,26 +117,36 @@ def parse_schema(path):
 
 
 def parse_args(args=None):
-    parser = ArgumentParser(
+    parser = argparse.ArgumentParser(
         add_help=True,
-        description="""
-        Processes generator output files in a custom way, saves results inplace.
-
-        Replacement token: '%(repl_token)s'.
-        Args separator token: '%(args_token)s'.
-        Replaces the following patterns in python files:
-            '%(repl_token)sREPLACER%(args_token)sARG1%(args_token)sARG2...%(repl_token)s'
-            ->
-            REPLACER(ARG1, ARG2, ...) value
-
+        formatter_class=argparse.RawTextHelpFormatter,
+        description="""\
+Processes generator output files in a custom way, saves results inplace.
+
+Replacement token: '%(repl_token)s'.
+Arg separator token: '%(args_token)s'.
+Replaces the following patterns in files:
+    '%(repl_token)sREPLACER%(args_token)sARG1%(args_token)sARG2...%(repl_token)s'
+    ->
+    REPLACER(ARG1, ARG2, ...) value
+
+Available REPLACERs:
+    %(replacers)s
         """
         % {
-            "repl_token": Processor.REPLACEMENT_TOKEN,
-            "args_token": Processor.ARGS_TOKEN,
+            "repl_token": Replacer.REPLACEMENT_TOKEN,
+            "args_token": Replacer.ARGS_TOKEN,
+            "replacers": "\n    ".join(Replacer.allowed_actions),
         },
     )
-    parser.add_argument("--schema", required=True)
-    parser.add_argument("--input-path", required=True)
+    parser.add_argument("--schema", required=True, help="Path to server schema yaml")
+    parser.add_argument("--input-path", required=True, help="Path to target file or directory")
+    parser.add_argument(
+        "--file-ext",
+        default=".py",
+        help="If working on a directory, look for "
+        "files with the specified extension (default: %(default)s)",
+    )
 
     return parser.parse_args(args)
 
@@ -145,10 +155,10 @@ def main(args=None):
     args = parse_args(args)
 
     schema = parse_schema(args.schema)
-    processor = Processor(schema=schema)
+    processor = Replacer(schema=schema)
 
     if osp.isdir(args.input_path):
-        processor.process_dir(args.input_path)
+        processor.process_dir(args.input_path, file_ext=args.file_ext)
     elif osp.isfile(args.input_path):
         processor.process_file(args.input_path)
 

cvat-sdk/gen/templates/README.md.template

+# SDK for [Computer Vision Annotation Tool (CVAT)](https://github.com/cvat-ai/cvat)
+
+This package provides a Python client library for CVAT server. It can be useful for
+workflow automation and writing custom CVAT server clients.
+
+The SDK API includes 2 layers:
+- Server API wrappers (`ApiClient`). Located in at `cvat_sdk.api_client`
+- High-level tools (`Core`). Located at `cvat_sdk.core`
+
+Package documentation is available [here](https://opencv.github.io/cvat/docs/api_sdk/sdk).
+
+## Installation & Usage
+
+To install a prebuilt package, run the following command in the terminal:
+
+```bash
+pip install cvat-sdk
+```
+
+To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/api_sdk/sdk/developer_guide).
+
+After installation you can import the package:
+
+```python
+import cvat_sdk
+```

cvat-sdk/gen/templates/openapi-generator/README.mustache

@@ -23,7 +23,7 @@ To install a prebuilt package, run the following command in the terminal:
 pip install cvat-sdk
 ```
 
-To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/integration/sdk/developer_guide).
+To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/api_sdk/sdk/developer_guide).
 
 After installation you can import the package:
 

cvat-sdk/gen/templates/openapi-generator/README_common.mustache

@@ -155,16 +155,16 @@ with make_client(host="{{{basePath}}}") as client:
     task.remove()
 ```
 
-## Documentation for API Endpoints
+## Available API Endpoints
 
-All URIs are relative to *{{basePath}}*
+All URIs are relative to _{{basePath}}_
 
 Class | Method | HTTP request | Description
 ------------ | ------------- | ------------- | -------------
-{{#apiInfo}}{{#apis}}{{#operations}}{{#operation}}*{{classname}}* | **{{operationId}}** | **{{httpMethod}}** {{path}} | {{summary}}
+{{#apiInfo}}{{#apis}}{{#operations}}{{#operation}}_{{classname}}_ | [**{{>operation_name}}**](apis/{{classname}}#{{>operation_name}}) | **{{httpMethod}}** {{path}} | {{summary}}
 {{/operation}}{{/operations}}{{/apis}}{{/apiInfo}}
 
-## Documentation For Models
+## Available Models
 
 {{#models}}{{#model}} - {{{classname}}}
 {{/model}}{{/models}}

cvat-sdk/gen/templates/openapi-generator/api_doc.mustache

+# {{classname}}
+
+{{#description}}{{.}}
+{{/description}}
+
+All URIs are relative to _{{basePath}}_
+
+Method | HTTP request | Description
+------------- | ------------- | -------------
+{{#operations}}{{#operation}}[**{{>operation_name}}**]({{classname}}#{{>operation_name}}) | **{{httpMethod}}** {{path}} | {{summary}}
+{{/operation}}{{/operations}}
+
+{{#operations}}
+{{#operation}}
+## **{{>operation_name}}**
+> {{#returnType}}{{{.}}} {{/returnType}}{{>operation_name}}({{#requiredParams}}{{^defaultValue}}{{paramName}}{{^-last}}, {{/-last}}{{/defaultValue}}{{/requiredParams}})
+
+{{{summary}}}{{#notes}}
+
+{{{.}}}{{/notes}}
+
+### Example
+{{> api_doc_example }}
+
+### Parameters
+{{^allParams}}This endpoint does not need any parameter.{{/allParams}}{{#allParams}}{{#-last}}
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------{{/-last}}{{/allParams}}
+{{#requiredParams}}{{^defaultValue}} **{{paramName}}** | {{^baseType}}**{{dataType}}**{{/baseType}}{{#baseType}}[**{{dataType}}**](../models/{{baseType}}){{/baseType}}| {{description}} |
+{{/defaultValue}}{{/requiredParams}}{{#requiredParams}}{{#defaultValue}} **{{paramName}}** | {{^baseType}}**{{dataType}}**{{/baseType}}{{#baseType}}[**{{dataType}}**](../models/{{baseType}}){{/baseType}}| {{description}} | defaults to {{{.}}}
+{{/defaultValue}}{{/requiredParams}}{{#optionalParams}} **{{paramName}}** | {{^baseType}}**{{dataType}}**{{/baseType}}{{#baseType}}[**{{dataType}}**](../models/{{baseType}}){{/baseType}}| {{description}} | [optional]{{#defaultValue}} if omitted the server will use the default value of {{{.}}}{{/defaultValue}}
+{{/optionalParams}}
+
+### Return type
+
+{{#returnType}}{{#returnTypeIsPrimitive}}**{{{returnType}}}**{{/returnTypeIsPrimitive}}{{^returnTypeIsPrimitive}}[**{{{returnType}}}**](../models/{{returnBaseType}}){{/returnTypeIsPrimitive}}{{/returnType}}{{^returnType}}None (empty response body){{/returnType}}
+
+### Authorization
+
+{{^authMethods}}No authorization required{{/authMethods}}{{#authMethods}}{{{name}}}{{^-last}}, {{/-last}}{{/authMethods}}
+
+### HTTP request headers
+
+ - **Content-Type**: {{#consumes}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/consumes}}{{^consumes}}Not defined{{/consumes}}
+ - **Accept**: {{#produces}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/produces}}{{^produces}}Not defined{{/produces}}
+
+{{#responses.0}}
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+{{#responses}}
+**{{code}}** | {{message}} | {{#headers}} * {{baseName}} - {{description}} <br> {{/headers}}{{^headers.0}} - {{/headers.0}} |
+{{/responses}}
+{{/responses.0}}
+
+{{/operation}}
+{{/operations}}

cvat-sdk/gen/templates/openapi-generator/api_doc_example.mustache

+```python
+import time
+from {{{packageName}}} import Configuration, ApiClient, exceptions
+{{#imports}}
+{{.}}
+{{/imports}}
+from pprint import pprint
+
+# Set up an API client
+# Read Configuration class docs for more info about parameters and authentication methods
+configuration = Configuration(
+    host = "{{{basePath}}}",{{#hasAuthMethods}}
+{{#authMethods}}
+{{#isBasic}}
+{{#isBasicBasic}}
+    username = 'YOUR_USERNAME',
+    password = 'YOUR_PASSWORD',
+{{/isBasicBasic}}
+{{/isBasic}}
+{{/authMethods}}
+{{/hasAuthMethods}}
+)
+
+with ApiClient(configuration) as api_client:
+{{#requiredParams}}
+{{^defaultValue}}
+    {{paramName}} = {{{example}}} # {{{dataType}}} | {{{description}}}
+{{/defaultValue}}
+{{/requiredParams}}
+{{#optionalParams}}
+    {{paramName}} = {{{example}}} # {{{dataType}}} | {{{description}}}{{^required}} (optional){{/required}}{{#defaultValue}} if omitted the server will use the default value of {{{.}}}{{/defaultValue}}
+{{/optionalParams}}
+
+    try:
+        {{#returnType}}(data, response) = {{/returnType}}%%%make_api_name!!!{{classname}}%%%.{{{operationId}}}({{#requiredParams}}
+            {{^defaultValue}}{{paramName}},{{/defaultValue}}{{/requiredParams}}{{#optionalParams}}
+            {{paramName}}={{paramName}},{{#-last}}
+        {{/-last}}{{/optionalParams}})
+{{#returnType}}
+        pprint(data)
+{{/returnType}}
+    except exceptions.ApiException as e:
+        print("Exception when calling {{classname}}.{{operationId}}: %s\n" % e)
+```

cvat-sdk/gen/templates/openapi-generator/model_doc.mustache

+{{#models}}{{#model}}# {{classname}}
+
+{{#description}}{{&description}}
+{{/description}}
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+{{#isEnum}}
+**value** | {{^arrayModelType}}**{{dataType}}**{{/arrayModelType}} | {{description}} | {{#defaultValue}}{{#hasRequired}} if omitted the server will use the default value of {{/hasRequired}}{{^hasRequired}}defaults to {{/hasRequired}}{{{.}}}{{/defaultValue}}{{#allowableValues}}{{#defaultValue}}, {{/defaultValue}} must be one of [{{#enumVars}}{{{value}}}, {{/enumVars}}]{{/allowableValues}}
+{{/isEnum}}
+{{#isAlias}}
+**value** | {{^arrayModelType}}**{{dataType}}**{{/arrayModelType}} | {{description}} | {{#defaultValue}}{{#hasRequired}} if omitted the server will use the default value of {{/hasRequired}}{{^hasRequired}}defaults to {{/hasRequired}}{{{.}}}{{/defaultValue}}
+{{/isAlias}}
+{{#isArray}}
+**value** | {{^arrayModelType}}**{{dataType}}**{{/arrayModelType}}{{#arrayModelType}}[**{{dataType}}**]({{arrayModelType}}){{/arrayModelType}} | {{description}} | {{#defaultValue}}{{#hasRequired}} if omitted the server will use the default value of {{/hasRequired}}{{^hasRequired}}defaults to {{/hasRequired}}{{{.}}}{{/defaultValue}}
+{{/isArray}}
+{{#requiredVars}}
+{{^defaultValue}}
+**{{name}}** | {{^complexType}}**{{dataType}}**{{/complexType}}{{#complexType}}[**{{dataType}}**]({{complexType}}){{/complexType}} | {{description}} | {{#isReadOnly}}[readonly] {{/isReadOnly}}
+{{/defaultValue}}
+{{/requiredVars}}
+{{#requiredVars}}
+{{#defaultValue}}
+**{{name}}** | {{^complexType}}**{{dataType}}**{{/complexType}}{{#complexType}}[**{{dataType}}**]({{complexType}}){{/complexType}} | {{description}} | {{^required}}[optional] {{/required}}{{#isReadOnly}}[readonly] {{/isReadOnly}}{{#defaultValue}}defaults to {{{.}}}{{/defaultValue}}
+{{/defaultValue}}
+{{/requiredVars}}
+{{#optionalVars}}
+**{{name}}** | {{^complexType}}**{{dataType}}**{{/complexType}}{{#complexType}}[**{{dataType}}**]({{complexType}}){{/complexType}} | {{description}} | [optional] {{#isReadOnly}}[readonly] {{/isReadOnly}}{{#defaultValue}} if omitted the server will use the default value of {{{.}}}{{/defaultValue}}
+{{/optionalVars}}
+
+{{/model}}{{/models}}

cvat/apps/engine/schema.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import textwrap
 from typing import Type
 from rest_framework import serializers
 from drf_spectacular.extensions import OpenApiSerializerExtension
@@ -78,6 +79,8 @@ class WriteOnceSerializerExtension(OpenApiSerializerExtension):
     """
     Enables support for cvat.apps.engine.serializers.WriteOnceMixin in drf-spectacular.
     Doesn't block other extensions on the target serializer.
+
+    Removes the WriteOnceMixin class docstring from derived class descriptions.
     """
 
     match_subclasses = True
@@ -94,13 +97,20 @@ class WriteOnceSerializerExtension(OpenApiSerializerExtension):
         return False
 
     def map_serializer(self, auto_schema, direction):
-        return auto_schema._map_serializer(
+        from cvat.apps.engine.serializers import WriteOnceMixin
+
+        schema = auto_schema._map_serializer(
             _copy_serializer(self.target, context={
                 'view': auto_schema.view,
                 self._PROCESSED_INDICATOR_NAME: True
             }),
             direction, bypass_extensions=False)
 
+        if schema.get('description') == textwrap.dedent(WriteOnceMixin.__doc__).strip():
+            del schema['description']
+
+        return schema
+
 class OpenApiTypeProxySerializerExtension(PolymorphicProxySerializerExtension):
     """
     Provides support for OpenApiTypes in the PolymorphicProxySerializer list

site/content/en/docs/api_sdk/sdk/_index.md

@@ -11,8 +11,8 @@ SDK is a Python library. It provides you access to Python function and objects,
 simplify server interaction and provide additional functionality like data validation.
 
 SDK API includes 2 layers:
-- Low-level API with REST API wrappers. Located in at `cvat_sdk.api_client`. [Read more](/api_sdk/sdk/lowlevel-api)
-- High-level API. Located at `cvat_sdk.core`. [Read more](/api_sdk/sdk/highlevel-api)
+- Low-level API with REST API wrappers. Located in at `cvat_sdk.api_client`. [Read more](../sdk/lowlevel-api)
+- High-level API. Located at `cvat_sdk.core`. [Read more](../sdk/highlevel-api)
 
 Roughly, low-level API provides single-request operations, while the high-level one allows you
 to use composite, multi-request operations and have local counterparts for server objects.

site/content/en/docs/api_sdk/sdk/developer-guide.md

@@ -7,7 +7,7 @@ description: ''
 
 ## Overview
 
-This package contains manually written and generated files. We store only sources in
+This package contains manually written and autogenerated files. We store only sources in
 the repository. To get the full package, one need to generate missing package files.
 
 ## Package file layout
@@ -23,7 +23,7 @@ the repository. To get the full package, one need to generate missing package fi
 
 If you have a local custom version of the server, run the following command in the terminal.
 You need to be able to execute django server. Server installation instructions are available
-[here](/contributing/development-environment).
+[here](/docs/contributing/development-environment).
 ```bash
 mkdir -p cvat-sdk/schema/ && python manage.py spectacular --file cvat-sdk/schema/schema.yml
 ```
@@ -41,6 +41,10 @@ you can also get schema from `<yourserver>/api/docs`:
 
 ![Download server schema button image](/images/download_server_schema.png)
 
+The official server schema for `app.cvat.ai` is available [here](https://app.cvat.ai/api/docs/).
+
+You can read more about server schema [here](/docs/api_sdk/api#api-schema).
+
 2. Install generator dependencies:
 ```bash
 pip install -r gen/requirements.txt
@@ -78,3 +82,72 @@ To execute, run:
 ```bash
 pytest tests/python/rest_api tests/python/sdk
 ```
+
+## SDK API design decisions
+
+The generated `ApiClient` code is modified from what `openapi-generator` does by default.
+Changes are mostly focused on better user experience - including better
+usage patterns and simpler/faster ways to achieve results.
+
+### Modifications
+
+- Added Python type annotations for return types and class members.
+  This change required us to implement a custom post-processing script,
+  which converts generated types into correct type annotations. The types
+  generated by default are supposed to work with the API implementation
+  (parameter validation and parsing), but they are not applicable as
+  type annotations (they have incorrect syntax). Custom post-processing
+  allowed us to make these types correct type annotations.
+  Other possible solutions:
+  - There is the `python-experimental` API generator, which may solve
+    some issues, but it is unstable and requires python 3.9. Our API
+    works with 3.7, which is the lowest supported version now.
+  - Custom templates - partially works, but only in limited cases
+    (model fields). It's very hard to maintain the template code and
+    logic for this. Only `if` checks and `for` loops are available in
+    mustache templates, which is not enough for annotation generation.
+
+- Separate APIs are embedded into the general `APIClient` class.
+  Now we have:
+  ```python
+  with ApiClient(config) as api_client:
+    result1 = api_client.foo_api.operation1()
+    result2 = api_client.bar_api.operation2()
+  ```
+
+  This showed to be more convenient than the default:
+  ```python
+  with ApiClient(config) as api_client:
+    foo_api = FooApi(api_client)
+    result1 = foo_api.operation1()
+    result2 = foo_api.operation2()
+
+    bar_api = BarApi(api_client)
+    result3 = bar_api.operation3()
+    result4 = bar_api.operation4()
+  ```
+
+  This also required custom post-processing. Operation Ids are
+  [supposed to be unique](https://swagger.io/specification/#operation-object)
+  in the OpenAPI / Swagger specification. Therefore, we can't generate such
+  schema on the server, nor we can't expect it to be supported in the
+  API generator.
+
+- Operations have IDs like `<api>/<method>_<object>`.
+  This also showed to be more readable and more natural than DRF-spectacular's
+  default `<api>/<object>_<method>`.
+
+- Server operations have different types for input and output values.
+  While it can be expected that an endopint with POST/PUT methods available
+  (like `create` or `partial_update`) has the same type for input and output
+  (because it looks natural), it also leads to the situation, in which there
+  are lots of read-/write-only fields, and it becomes hard for understanding.
+  This clear type separation is supposed to make it simpler for users.
+
+- Added cookie management in the `ApiClient` class.
+
+- Added interface classes for models to simplify class member usage and lookup.
+
+- Dicts can be passed into API methods and model constructors instead of models.
+  They are automatically parsed as models. In the original implementation, the user
+  is required to pass a `Configuration` object each time, which is clumsy and adds little sense.

site/content/en/docs/api_sdk/sdk/reference/_index.md

+---
+title: "SDK API Reference"
+linkTitle: "API Reference"
+weight: 1
+description: ''
+---

site/content/en/docs/api_sdk/sdk/reference/apis/.gitignore

+# The files are autogenerated here
+*.md

site/content/en/docs/api_sdk/sdk/reference/models/.gitignore

+# The files are autogenerated here
+*.md
+!_index.md

site/content/en/docs/api_sdk/sdk/reference/models/_index.md

+---
+title: 'Models'
+linkTitle: 'Models'
+weight: 1
+description: ''
+---

site/process_sdk_docs.py

+#!/usr/bin/env python
+
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os
+import os.path as osp
+import re
+import shutil
+import sys
+import textwrap
+from glob import iglob
+from typing import Callable, List
+
+from inflection import underscore
+
+
+class Processor:
+    _reference_files: List[str]
+
+    def __init__(self, *, input_dir: str, site_root: str) -> None:
+        self._input_dir = input_dir
+        self._site_root = site_root
+
+        self._content_dir = osp.join(self._site_root, "content")
+        self._sdk_reference_dir = osp.join(
+            self._content_dir, "en/docs/api_sdk/sdk/reference"
+        )
+        self._templates_dir = osp.join(self._site_root, "templates")
+
+    @staticmethod
+    def _copy_files(src_dir: str, glob_pattern: str, dst_dir: str) -> List[str]:
+        copied_files = []
+
+        for src_path in iglob(osp.join(src_dir, glob_pattern), recursive=True):
+            src_filename = osp.relpath(src_path, src_dir)
+            dst_path = osp.join(dst_dir, src_filename)
+            # assume dst dir exists
+            shutil.copy(src_path, dst_path, follow_symlinks=True)
+
+            copied_files.append(dst_path)
+
+        return copied_files
+
+    def _copy_pages(self):
+        self._reference_files = self._copy_files(
+            self._input_dir, "*/**/*.md", self._sdk_reference_dir
+        )
+
+    def _add_page_headers(self):
+        """
+        Adds headers required by hugo to docs pages
+        """
+
+        HEADER_SEPARATOR = "---"
+
+        for p in self._reference_files:
+            with open(p) as f:
+                contents = f.read()
+
+            assert not contents.startswith(HEADER_SEPARATOR), p
+
+            lines = contents.splitlines()
+
+            assert lines[0].startswith("#")
+            classname = lines[0][1:].strip()
+
+            header = textwrap.dedent(
+                """\
+                %(header_separator)s
+                title: '%(classname)s class reference'
+                linkTitle: '%(classname)s'
+                weight: 10
+                description: ''
+                %(header_separator)s
+            """
+                % {"header_separator": HEADER_SEPARATOR, "classname": classname}
+            )
+
+            contents = header + "\n".join(lines[1:])
+
+            with open(p, "w") as f:
+                f.write(contents)
+
+    def _extract_apis_summary(self, readme_path: str) -> str:
+        with open(readme_path) as f:
+            readme_contents = f.read()
+
+        apis_summary = re.search(
+            r"## Available API Endpoints(.*)## Available Models",
+            readme_contents,
+            flags=re.DOTALL,
+        )[1]
+        assert len(apis_summary) > 0
+
+        return apis_summary
+
+    def _move_api_summary(self):
+        """
+        Moves API summary section from README to apis/_index
+        """
+
+        SUMMARY_REPLACE_TOKEN = "{{REPLACEME:apis_summary}}"  # nosec
+
+        apis_summary = self._extract_apis_summary(
+            osp.join(self._input_dir, "README.md")
+        )
+
+        apis_index_filename = osp.join(
+            osp.relpath(self._sdk_reference_dir, self._content_dir), "apis/_index.md"
+        )
+        apis_index_path = osp.join(
+            self._templates_dir, apis_index_filename + ".template"
+        )
+        with open(apis_index_path) as f:
+            contents = f.read()
+
+        contents = contents.replace(SUMMARY_REPLACE_TOKEN, apis_summary)
+
+        with open(osp.join(self._content_dir, apis_index_filename), "w") as f:
+            f.write(contents)
+
+    def _fix_page_links_and_references(self):
+        """
+        Replaces reference page links from full lowercase (which is generated by hugo from the
+        orignal camelcase and creates broken links) ('authapi') to the minus-case ('auth-api'),
+        which is more readable and works.
+        Adds an extra parent directory part to links ('../') as hugo requires, even for neighbor
+        files.
+        """
+
+        mapping = {}
+
+        for src_path in self._reference_files:
+            src_filename = osp.relpath(src_path, self._sdk_reference_dir)
+            dst_filename = underscore(src_filename).replace("_", "-")
+            dst_path = osp.join(self._sdk_reference_dir, dst_filename)
+            os.rename(src_path, dst_path)
+            mapping[src_filename] = dst_filename
+
+        self._reference_files = [
+            osp.join(self._sdk_reference_dir, p) for p in mapping.values()
+        ]
+
+        for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
+            with open(p) as f:
+                contents = f.read()
+
+            for src_filename, dst_filename in mapping.items():
+                src_dir, src_filename = osp.split(osp.splitext(src_filename)[0])
+                dst_filename = osp.basename(osp.splitext(dst_filename)[0])
+                contents = re.sub(
+                    rf"(\[.*?\]\()((?:\.\./)?(?:{src_dir}/)?){src_filename}((?:#[^\)]*?)?\))",
+                    rf"\1../\2{dst_filename}\3",
+                    contents,
+                )
+
+            with open(p, "w") as f:
+                f.write(contents)
+
+    def _process_non_code_blocks(
+        self, text: str, handlers: List[Callable[[str], str]]
+    ) -> str:
+        """
+        Allows to process Markdown documents with passed callbacks. Callbacks are only
+        executed outside code blocks.
+        """
+
+        used_quotes = ""
+        block_start_pos = 0
+        inside_code_block = False
+        while block_start_pos < len(text):
+            pattern = re.compile(used_quotes or "```|`")
+            next_code_block_quote = pattern.search(text, pos=block_start_pos)
+            if next_code_block_quote is not None:
+                if not used_quotes:
+                    inside_code_block = False
+                    block_end_pos = next_code_block_quote.start(0)
+                    used_quotes = next_code_block_quote.group(0)
+                else:
+                    inside_code_block = True
+                    block_end_pos = next_code_block_quote.end(0)
+                    used_quotes = None
+            else:
+                block_end_pos = len(text)
+
+            if not inside_code_block:
+                block = text[block_start_pos:block_end_pos]
+
+                for handler in handlers:
+                    block = handler(block)
+
+                text = text[:block_start_pos] + block + text[block_end_pos:]
+                block_end_pos = block_start_pos + len(block) + len(used_quotes)
+
+            block_start_pos = block_end_pos
+
+        return text
+
+    def _escape_free_square_brackets(self, text: str) -> str:
+        return re.sub(r"\[([^\[\]]*?)\]([^\(])", r"\[\1\]\2", text)
+
+    def _add_angle_brackets_to_free_links(self, text: str) -> str:
+        # Adapted from https://stackoverflow.com/a/31952097
+        URL_REGEX = (
+            # Scheme (HTTP, HTTPS):
+            r"(?:https?:\/\/)"
+            r"(?:"
+            # www:
+            r"(?:www\.)?"
+            # Host and domain (including ccSLD):
+            r"(?:(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.)+)"
+            # TLD:
+            r"(?:[a-zA-Z]{2,6})"
+            # IP Address:
+            r"|(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
+            r")"
+            # Port:
+            r"(?::\d{1,5})?"
+            # Query path:
+            r"(?:(?:\/\S+)*|\/)"
+        )
+
+        text = re.sub(
+            r"(\A|[\.\s])(" + URL_REGEX + r")([\.\s]|\Z)",
+            r"\1<\2>\3",
+            text,
+            flags=re.MULTILINE,
+        )
+
+        return text
+
+    def _fix_parsing_problems(self):
+        """
+        Adds angle brackets to freestanding links, as the linter requires. Such links can appear
+        from the generated model and api descriptions.
+        Adds escapes to freestanding square brackets to make parsing correct.
+        """
+
+        for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
+            with open(p) as f:
+                contents = f.read()
+
+            contents = self._process_non_code_blocks(
+                contents,
+                [
+                    self._add_angle_brackets_to_free_links,
+                    self._escape_free_square_brackets,
+                ],
+            )
+
+            with open(p, "w") as f:
+                f.write(contents)
+
+    def run(self):
+        assert osp.isdir(self._input_dir), self._input_dir
+        assert osp.isdir(self._site_root), self._site_root
+        assert osp.isdir(self._sdk_reference_dir), self._sdk_reference_dir
+        assert osp.isdir(self._templates_dir), self._templates_dir
+
+        self._copy_pages()
+        self._move_api_summary()
+        self._add_page_headers()
+        self._fix_page_links_and_references()
+        self._fix_parsing_problems()
+
+
+def parse_args(args=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input-dir",
+        type=osp.abspath,
+        default="cvat-sdk/docs/",
+        help="Path to the cvat-sdk/docs/ directory",
+    )
+    parser.add_argument(
+        "--site-root",
+        type=osp.abspath,
+        default="site/",
+    )
+
+    return parser.parse_args(args)
+
+
+def main(args=None):
+    args = parse_args(args)
+    processor = Processor(input_dir=args.input_dir, site_root=args.site_root)
+    processor.run()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

site/requirements.txt

+black>=22.1.0
 gitpython
+inflection >= 0.5.1
+isort>=5.10.1
 packaging
-toml
\ No newline at end of file
+toml

site/templates/en/docs/api_sdk/sdk/reference/apis/_index.md.template

+---
+title: 'APIs'
+linkTitle: 'APIs'
+weight: 1
+description: ''
+---
+
+{{REPLACEME:apis_summary}}