Various improvements to the SDK generation process (#5678)

.github/workflows/full.yml

@@ -78,9 +78,7 @@ jobs:
             --entrypoint /bin/bash -u root cvat/server \
             -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
-          cd cvat-sdk/
-          gen/generate.sh
-          cd ..
+          ./cvat-sdk/gen/generate.sh
 
           cp -r cvat-sdk/* /tmp/cvat_sdk/
 

.github/workflows/helm.yml

@@ -72,9 +72,7 @@ jobs:
         kubectl exec $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') -- /bin/bash -c "python manage.py spectacular --file /tmp/schema.yml"
         kubectl cp $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/tmp/schema.yml cvat-sdk/schema/schema.yml
         pip3 install --user -r cvat-sdk/gen/requirements.txt
-        cd cvat-sdk/
-        gen/generate.sh
-        cd ..
+        ./cvat-sdk/gen/generate.sh
 
     - name: Install test requirements
       run: |

.github/workflows/main.yml

@@ -79,9 +79,7 @@ jobs:
             --entrypoint /bin/bash -u root cvat/server \
             -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
-          cd cvat-sdk/
-          gen/generate.sh
-          cd ..
+          ./cvat-sdk/gen/generate.sh
 
           cp -r cvat-sdk/* /tmp/cvat_sdk/
 

.github/workflows/schedule.yml

@@ -203,9 +203,7 @@ jobs:
             --entrypoint /bin/bash -u root cvat/server \
             -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
-          cd cvat-sdk/
-          gen/generate.sh
-          cd ..
+          ./cvat-sdk/gen/generate.sh
 
           pip3 install --user 'cvat-sdk/[pytorch]'
           pip3 install --user cvat-cli/

cvat-sdk/.gitignore

@@ -72,8 +72,6 @@ schema/
 # Generated code
 /cvat_sdk/api_client/
 /cvat_sdk/version.py
-/requirements/
 /docs/
+/requirements/api_client.txt
 /setup.py
-/README.md
-/MANIFEST.in
\ No newline at end of file

cvat-sdk/.openapi-generator-ignore

@@ -23,6 +23,7 @@
 #!docs/README.md
 
 # For safety
+/MANIFEST.in
 /cvat_sdk/__init__.py
 /config
 /gen
@@ -31,6 +32,7 @@
 /types.py
 
 # Don't generate these files
+/README.md
 /git_push.sh
 /setup.cfg
 /test-requirements.txt

cvat-sdk/.remarkignore

 cvat-sdk/docs/
-cvat-sdk/README.md

cvat-sdk/gen/generate.sh

@@ -11,35 +11,35 @@ GENERATOR_VERSION="v6.0.1"
 VERSION="2.3.0"
 LIB_NAME="cvat_sdk"
 LAYER1_LIB_NAME="${LIB_NAME}/api_client"
-DST_DIR="."
-TEMPLATE_DIR="gen"
+DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)"
+TEMPLATE_DIR_NAME="gen"
+TEMPLATE_DIR="$DST_DIR/$TEMPLATE_DIR_NAME"
 POST_PROCESS_SCRIPT="${TEMPLATE_DIR}/postprocess.py"
 
-mkdir -p "${DST_DIR}/"
-rm -f -r "${DST_DIR}/docs" "${DST_DIR}/${LAYER1_LIB_NAME}" "requirements/"
-cp "${TEMPLATE_DIR}/templates/openapi-generator/.openapi-generator-ignore" "${DST_DIR}/"
+rm -f -r "${DST_DIR}/docs" "${DST_DIR}/${LAYER1_LIB_NAME}" \
+    "${DST_DIR}/requirements/api_client.txt"
 
 # Pass template dir here
 # https://github.com/OpenAPITools/openapi-generator/issues/8420
-docker run --rm -v "$PWD":"/local" -u "$(id -u)":"$(id -g)" \
+docker run --rm -v "$DST_DIR:/local" -u "$(id -u)":"$(id -g)" \
     openapitools/openapi-generator-cli:${GENERATOR_VERSION} generate \
-        -t "/local/${TEMPLATE_DIR}/templates/openapi-generator/" \
+        -t "/local/${TEMPLATE_DIR_NAME}/templates/openapi-generator/" \
         -i "/local/schema/schema.yml" \
-        --config "/local/${TEMPLATE_DIR}/generator-config.yml" \
+        --config "/local/${TEMPLATE_DIR_NAME}/generator-config.yml" \
+        -p "packageVersion=$VERSION" \
         -g python \
-        -o "/local/${DST_DIR}/"
+        -o "/local/"
 
-sed -e "s|{{packageVersion}}|${VERSION}|g" "${TEMPLATE_DIR}/templates/version.py.template" > "${DST_DIR}/${LIB_NAME}/version.py"
-cp -r "${TEMPLATE_DIR}/templates/requirements" "${DST_DIR}/"
-cp -r "${TEMPLATE_DIR}/templates/MANIFEST.in" "${DST_DIR}/"
+echo "VERSION = \"$VERSION\"" > "${DST_DIR}/${LIB_NAME}/version.py"
 mv "${DST_DIR}/requirements.txt" "${DST_DIR}/requirements/api_client.txt"
 
 # Do custom postprocessing for code files
-"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/${LIB_NAME}"
+"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \
+    --input-path "${DST_DIR}/${LIB_NAME}"
 
 # Do custom postprocessing for docs files
-"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/docs" --file-ext '.md'
-"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/README.md"
+"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \
+    --input-path "${DST_DIR}/docs" --file-ext '.md'
 
 API_DOCS_DIR="${DST_DIR}/docs/apis/"
 MODEL_DOCS_DIR="${DST_DIR}/docs/models/"
@@ -47,6 +47,3 @@ mkdir "${API_DOCS_DIR}"
 mkdir "${MODEL_DOCS_DIR}"
 mv "${DST_DIR}/docs/"*Api.md "${API_DOCS_DIR}"
 mv "${DST_DIR}/docs/"*.md "${MODEL_DOCS_DIR}"
-mv "${DST_DIR}/README.md" "${DST_DIR}/docs/"
-
-cp "${TEMPLATE_DIR}/templates/README.md.template" "${DST_DIR}/README.md"

cvat-sdk/gen/generator-config.yml

 additionalProperties:
   projectName: "cvat_sdk"
-  packageVersion: "2.0-alpha"
   packageUrl: "https://github.com/cvat-ai/cvat"
   packageName: "cvat_sdk.api_client"
   initRequiredVars: true

cvat-sdk/gen/postprocess.py

@@ -161,6 +161,8 @@ def main(args=None):
         processor.process_dir(args.input_path, file_ext=args.file_ext)
     elif osp.isfile(args.input_path):
         processor.process_file(args.input_path)
+    else:
+        return f"error: input {args.input_path} is neither a file nor a directory"
 
     return 0
 

cvat-sdk/gen/templates/openapi-generator/.openapi-generator-ignore

-# OpenAPI Generator Ignore
-# Generated by openapi-generator https://github.com/openapitools/openapi-generator
-
-# Use this file to prevent files from being overwritten by the generator.
-# The patterns follow closely to .gitignore or .dockerignore.
-
-# As an example, the C# client generator defines ApiClient.cs.
-# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
-#ApiClient.cs
-
-# You can match any string of characters against a directory, file or extension with a single asterisk (*):
-#foo/*/qux
-# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
-
-# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
-#foo/**/qux
-# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
-
-# You can also negate patterns with an exclamation (!).
-# For example, you can ignore all files in a docs folder with the file extension .md:
-#docs/*.md
-# Then explicitly reverse the ignore rule for a single file:
-#!docs/README.md
-
-# For safety
-/cvat_sdk/__init__.py
-/config
-/gen
-/helpers.py
-/utils.py
-/types.py
-
-# Don't generate these files
-/git_push.sh
-/setup.cfg
-/test-requirements.txt
-/tox.ini
-/.gitlab-ci.yml
-/.travis.yml
-/.gitignore

cvat-sdk/gen/templates/openapi-generator/README.mustache

-# {{{projectName}}}
-{{#appDescriptionWithNewLines}}
-{{{.}}}
-{{/appDescriptionWithNewLines}}
-
-This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
-
-- API version: {{appVersion}}
-- Package version: {{packageVersion}}
-{{^hideGenerationTimestamp}}
-- Build date: {{generatedDate}}
-{{/hideGenerationTimestamp}}
-- Build package: {{generatorClass}}
-{{#infoUrl}}
-For more information, please visit [{{{infoUrl}}}]({{{infoUrl}}})
-{{/infoUrl}}
-
-## Installation & Usage
-
-To install a prebuilt package, run the following command in the terminal:
-
-```sh
-pip install cvat-sdk
-```
-
-To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/api_sdk/sdk/developer_guide).
-
-After installation you can import the package:
-
-```python
-import cvat_sdk
-```
-
-## Getting Started
-
-{{> README_common }}

cvat-sdk/gen/templates/openapi-generator/README_common.mustache

-API includes 2 layers:
-- REST API wrappers (`ApiClient`). Located in at `cvat_sdk.api_client`
-- high-level tools (`core`). Located at `cvat_sdk.core`
-
-### `ApiClient` (low level)
-
-This layer is useful if you need to work directly with REST API, but want
-to have data validation and syntax assistance from your code editor. The code
-on this layer is autogenerated.
-
-#### Example
-
-Let's see how a task with local files can be created. We will use the basic auth
-to make things simpler.
-
-```python
-from time import sleep
-from cvat_sdk.api_client import Configuration, ApiClient, models, apis, exceptions
-
-configuration = Configuration(
-    host="{{{basePath}}}",
-    username='YOUR_USERNAME',
-    password='YOUR_PASSWORD',
-)
-
-# Enter a context with an instance of the API client
-with ApiClient(configuration) as api_client:
-    # Parameters can be passed as a plain dict with JSON-serialized data
-    # or as model objects (from cvat_sdk.api_client.models), including
-    # mixed variants.
-    #
-    # In case of dicts, keys must be the same as members of models.I<ModelName>
-    # interfaces and values must be convertible to the corresponding member
-    # value types (e.g. a date or string enum value can be parsed from a string).
-    #
-    # In case of model objects, data must be of the corresponding
-    # models.<ModelName> types.
-    #
-    # Let's use a dict here. It should look like models.ITaskWriteRequest
-    task_spec = {
-        'name': 'example task',
-        "labels": [{
-            "name": "car",
-            "color": "#ff00ff",
-            "attributes": [
-                {
-                    "name": "a",
-                    "mutable": True,
-                    "input_type": "number",
-                    "default_value": "5",
-                    "values": ["4", "5", "6"]
-                }
-            ]
-        }],
-    }
-
-    try:
-        # Apis can be accessed as ApiClient class members
-        # We use different models for input and output data. For input data,
-        # models are typically called like "*Request". Output data models have
-        # no suffix.
-        (task, response) = api_client.tasks_api.create(task_spec)
-    except exceptions.ApiException as e:
-        # We can catch the basic exception type, or a derived type
-        print("Exception when trying to create a task: %s\n" % e)
-
-    # Here we will use models instead of a dict
-    task_data = models.DataRequest(
-        image_quality=75,
-        start_frame=2,
-        stop_frame=5,
-        client_files=[
-            open('image1.jpg', 'rb'),
-            open('image2.jpg', 'rb'),
-        ],
-    )
-
-    # If we pass binary file objects, we need to specify content type.
-    # For this endpoint, we don't have response data
-    (_, response) = api_client.tasks_api.create_data(task.id,
-        data_request=task_data,
-        _content_type="multipart/form-data",
-
-        # we can choose to check the response status manually
-        # and disable the response data parsing
-        _check_status=False, _parse_response=False
-    )
-    assert response.status == 202, response.msg
-
-    # Wait till task data is processed
-    for _ in range(100):
-        (status, _) = api_client.tasks_api.retrieve_status(task.id)
-        if status.state.value in ['Finished', 'Failed']:
-            break
-        sleep(0.1)
-    assert status.state.value == 'Finished', status.message
-
-    # Update the task object and check the task size
-    (task, _) = api_client.tasks_api.retrieve(task.id)
-    assert task.size == 4
-```
-
-### `Core` (high-level)
-
-This layer provides high-level APIs, allowing easier access to server operations.
-API includes *Repositories* and *Entities*. Repositories provide management
-operations for Entitites. Entitites represent separate objects on the server
-(e.g. tasks, jobs etc).
-
-#### Example
-
-```python
-from cvat_sdk import make_client, models
-from cvat_sdk.core.proxies.tasks import ResourceType, Task
-
-with make_client(host="{{{basePath}}}") as client:
-    # Authorize using the basic auth
-    client.login(('YOUR_USERNAME', 'YOUR_PASSWORD'))
-
-    # Models are used the same way as in the layer 1
-    task_spec = {
-        "name": "example task 2",
-        "labels": [
-            {
-                "name": "car",
-                "color": "#ff00ff",
-                "attributes": [
-                    {
-                        "name": "a",
-                        "mutable": True,
-                        "input_type": "number",
-                        "default_value": "5",
-                        "values": ["4", "5", "6"],
-                    }
-                ],
-            }
-        ],
-    }
-
-    # Different repositories can be accessed as the Client class members.
-    # They may provide both simple and complex operations,
-    # such as entity creation, retrieval and removal.
-    task = client.tasks.create_from_data(
-        spec=task_spec,
-        resource_type=ResourceType.LOCAL,
-        resources=['image1.jpg', 'image2.png'],
-    )
-
-    # Task object is already up-to-date with its server counterpart
-    assert task.size == 2
-
-    # An entity needs to be fetch()-ed to reflect the latest changes.
-    # It can be update()-d and remove()-d depending on the entity type.
-    task.update({'name': 'mytask'})
-    task.remove()
-```
-
-## Available API Endpoints
-
-All URIs are relative to _{{basePath}}_
-
-Class | Method | HTTP request | Description
------------- | ------------- | ------------- | -------------
-{{#apiInfo}}{{#apis}}{{#operations}}{{#operation}}_{{classname}}_ | [**{{>operation_name}}**](apis/{{classname}}#{{>operation_name}}) | **{{httpMethod}}** {{path}} | {{summary}}
-{{/operation}}{{/operations}}{{/apis}}{{/apiInfo}}
-
-## Available Models
-
-{{#models}}{{#model}} - {{{classname}}}
-{{/model}}{{/models}}
-
-
-## Author
-
-{{#apiInfo}}{{#apis}}{{#-last}}{{infoEmail}}
-{{/-last}}{{/apis}}{{/apiInfo}}

cvat-sdk/gen/templates/version.py.template

-VERSION = "{{packageVersion}}"