Add analytics (#118)

* ELK stack to analyze logs from client and server
* Short-live container to configure kibana. Import kibana dashboards from export.json on startup.
* Client and server logging is working throw python-logstash
* Minor polishing of events and dashboards.
* Default is discover, new visualizations.
* Make comments more readable inside logger.js.
* Added a path for backups.

.vscode/launch.json

@@ -9,6 +9,7 @@
             "type": "python",
             "request": "launch",
             "stopOnEntry": false,
+            "debugStdLib": true,
             "pythonPath": "${config:python.pythonPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
@@ -23,7 +24,6 @@
                 "DjangoDebugging"
             ],
             "cwd": "${workspaceFolder}",
-            "env": {},
             "envFile": "${workspaceFolder}/.env",
         },
         {
@@ -44,6 +44,7 @@
             "type": "python",
             "request": "launch",
             "stopOnEntry": false,
+            "debugStdLib": true,
             "pythonPath": "${config:python.pythonPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
@@ -65,6 +66,7 @@
             "name": "CVAT RQ - low",
             "type": "python",
             "request": "launch",
+            "debugStdLib": true,
             "stopOnEntry": false,
             "pythonPath": "${config:python.pythonPath}",
             "program": "${workspaceRoot}/manage.py",

README.md

@@ -109,19 +109,13 @@ services:
 ```
 ### Annotation logs
 
-It is possible to proxy annotation logs from client to another server over http. For examlpe you can use Logstash.
-To do that set DJANGO_LOG_SERVER_URL environment variable in cvat section of docker-compose.yml
-file (or add this variable to docker-compose.override.yml).
+It is possible to proxy annotation logs from client to ELK. To do that run the following command below:
 
-```yml
-version: "2.3"
-
-services:
-cvat:
-    environment:
-      DJANGO_LOG_SERVER_URL: https://annotation.example.com:5000
+```bash
+docker-compose -f docker-compose.yml -f analytics/docker-compose.yml up -d --build
 ```
 
+
 ### Share path
 
 You can use a share storage for data uploading during you are creating a task. To do that you can mount it to CVAT docker container. Example of docker-compose.override.yml for this purpose:

analytics/docker-compose.yml

+version: '2.3'
+services:
+  cvat_elasticsearch:
+    container_name: cvat_elasticsearch
+    image: cvat_elasticsearch
+    networks:
+      default:
+        aliases:
+          - elasticsearch
+    build:
+      context: ./analytics/elasticsearch
+      args:
+        ELK_VERSION: 6.4.0
+    restart: always
+
+  cvat_kibana:
+    container_name: cvat_kibana
+    image: cvat_kibana
+    networks:
+      default:
+        aliases:
+          - kibana
+    build:
+      context: ./analytics/kibana
+      args:
+        ELK_VERSION: 6.4.0
+    ports:
+      - "5601:5601"
+    depends_on: ['cvat_elasticsearch']
+    restart: always
+
+  cvat_kibana_setup:
+    container_name: cvat_kibana_setup
+    image: cvat
+    volumes: ['./analytics/kibana:/home/django/kibana:ro']
+    depends_on: ['cvat']
+    working_dir: '/home/django'
+    entrypoint: ['bash', 'wait-for-it.sh', 'elasticsearch:9200', '-t', '0', '--',
+      '/bin/bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--',
+      '/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json']
+    environment:
+      no_proxy: elasticsearch,kibana,${no_proxy}
+
+  cvat_logstash:
+    container_name: cvat_logstash
+    image: cvat_logstash
+    networks:
+      default:
+        aliases:
+          - logstash
+    build:
+      context: ./analytics/logstash
+      args:
+        ELK_VERSION: 6.4.0
+        http_proxy:  ${http_proxy}
+        https_proxy: ${https_proxy}
+    depends_on: ['cvat_elasticsearch']
+    restart: always
+
+  cvat:
+    environment:
+      DJANGO_LOG_SERVER_HOST: "logstash"
+      DJANGO_LOG_SERVER_PORT: 5000
+      no_proxy: logstash,${no_proxy}

analytics/elasticsearch/Dockerfile

+ARG ELK_VERSION
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION}
+COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/

analytics/elasticsearch/elasticsearch.yml

+http.host: 0.0.0.0
+script.painless.regex.enabled: true
+path.repo: ["/usr/share/elasticsearch/backup"]

analytics/kibana/Dockerfile

+ARG ELK_VERSION
+FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION}
+COPY kibana.yml /usr/share/kibana/config/
+
+

analytics/kibana/export.json

+[
+  {
+    "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "Timeline for exceptions",
+      "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8",
+    "_type": "dashboard",
+    "_source": {
+      "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
+      "timeRestore": false,
+      "description": "",
+      "hits": 0,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+      },
+      "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]",
+      "title": "Monitoring",
+      "version": 1
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "List of users",
+      "visState": "{\"title\":\"List of users\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"3-bucket\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucketAgg\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},\"customMetric\":{\"id\":\"3-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{}},\"customLabel\":\"Activity\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working time\",\"customLabel\":\"Working Time (h)\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "Duration of events",
+      "visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8",
+    "_type": "index-pattern",
+    "_source": {
+      "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+      "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}",
+      "title": "cvat*",
+      "timeFieldName": "@timestamp"
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "65918380-c244-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "Number of events",
+      "visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "Activity of users",
+      "visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8",
+    "_type": "search",
+    "_source": {
+      "title": "Table with exceptions",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "task",
+        "type",
+        "userid",
+        "stack"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "b6339c70-c7d4-11e8-a035-258d2bd7d91f",
+    "_type": "visualization",
+    "_source": {
+      "title": "Working calendar",
+      "visState": "{\"title\":\"Working calendar\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 8\":\"rgb(165,0,38)\",\"8 - 15\":\"rgb(249,142,82)\",\"15 - 23\":\"rgb(255,255,190)\",\"23 - 30\":\"rgb(135,203,103)\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8",
+    "_type": "dashboard",
+    "_source": {
+      "title": "Managment",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":21,\"w\":24,\"h\":16,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"gridData\":{\"x\":0,\"y\":21,\"w\":24,\"h\":16,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"b6339c70-c7d4-11e8-a035-258d2bd7d91f\",\"embeddableConfig\":{}}]",
+      "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  },
+  {
+    "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8",
+    "_type": "visualization",
+    "_source": {
+      "title": "Working day",
+      "visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 2
+    }
+  }
+]
\ No newline at end of file

analytics/kibana/kibana.yml

+server.host: 0.0.0.0
+elasticsearch.url: http://elasticsearch:9200
+elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ]
+kibana.defaultAppId: "discover"

analytics/kibana/setup.py

+#/usr/bin/env python
+
+import os
+import argparse
+import requests
+import json
+
+def import_resources(host, port, cfg_file):
+    with open(cfg_file, 'r') as f:
+        for saved_object in json.load(f):
+            _id = saved_object["_id"]
+            _type = saved_object["_type"]
+            _doc = saved_object["_source"]
+            import_saved_object(host, port, _type, _id, _doc)
+
+def import_saved_object(host, port, _type, _id, data):
+    saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format(
+        host, port, _type, _id)
+    request = requests.get(saved_objects_api)
+    if request.status_code == 404:
+        print("Creating {} as {}".format(_type, _id))
+        request = requests.post(saved_objects_api, json={"attributes": data},
+            headers={'kbn-xsrf': 'true'})
+    else:
+        print("Updating {} named {}".format(_type, _id))
+        request = requests.put(saved_objects_api, json={"attributes": data},
+            headers={'kbn-xsrf': 'true'})
+    request.raise_for_status()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='import Kibana 6.x resources',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('export_file', metavar='FILE',
+        help='JSON export file with resources')
+    parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int,
+        help='port of Kibana instance')
+    parser.add_argument('-H', '--host', metavar='HOST', default='kibana',
+        help='host of Kibana instance')
+    args = parser.parse_args()
+    import_resources(args.host, args.port, args.export_file)

analytics/logstash/Dockerfile

+ARG ELK_VERSION
+FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION}
+RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \
+    logstash-filter-prune logstash-output-email
+
+COPY logstash.conf   /usr/share/logstash/pipeline/
+EXPOSE 5000

analytics/logstash/logstash.conf

+input {
+  tcp {
+    port => 5000
+    codec => json
+  }
+}
+
+filter {
+  if [logger_name] =~ /cvat.client/ {
+    # 1. Decode the event from json in 'message' field
+    # 2. Remove unnecessary field from it
+    # 3. Type it as client
+    json {
+      source => "message"
+    }
+
+    date {
+      match => ["timestamp", "UNIX", "UNIX_MS"]
+      remove_field => "timestamp"
+    }
+
+    if [event] == "Send exception" {
+      aggregate {
+        task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}"
+        code => "
+          require 'time'
+
+          map['userid'] ||= event.get('userid');
+          map['application'] ||= event.get('application');
+          map['message'] ||= event.get('message');
+          map['filename'] ||= event.get('filename');
+          map['line'] ||= event.get('line');
+          map['task'] ||= event.get('task');
+
+          map['error_count'] ||= 0;
+          map['error_count'] += 1;
+
+          map['aggregated_message'] ||= '';
+          time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00')
+          map['aggregated_message'] += time.to_s + '\n' +  event.get('stack') + '\n\n\n';"
+
+        timeout => 3600
+        timeout_tags => ['send_email_notification']
+        push_map_as_event_on_timeout => true
+      }
+    }
+
+    prune {
+      blacklist_names => ["level", "host", "logger_name", "message", "path",
+        "port", "stack_info"]
+    }
+
+    mutate {
+      replace => { "type" => "client" }
+    }
+  } else if [logger_name] =~ /cvat.server/ {
+    # 1. Remove unnecessary field from it
+    # 2. Type it as server
+    prune {
+      blacklist_names => ["host", "port"]
+    }
+
+    mutate {
+      replace => { "type" => "server" }
+    }
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+
+  if [type] == "client" {
+    elasticsearch {
+      hosts => ["elasticsearch:9200"]
+      index => "cvat.client"
+    }
+  } else if [type] == "server" {
+    elasticsearch {
+      hosts => ["elasticsearch:9200"]
+      index => "cvat.server"
+    }
+  }
+}

cvat/apps/engine/logging.py

-
 # Copyright (C) 2018 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
 import os
-import inspect
 import logging
 from . import models
 from cvat.settings.base import LOGGING
+from cvat.apps.engine.models import Job, Task
+
+def _get_task(tid):
+    try:
+        return Task.objects.get(pk=tid)
+    except Exception:
+        raise Exception('{} key must be a task identifier'.format(tid))
 
+def _get_job(jid):
+    try:
+        return models.Job.objects.select_related("segment__task").get(id=jid)
+    except Exception:
+        raise Exception('{} key must be a job identifier'.format(jid))
 
 class TaskLoggerStorage:
     def __init__(self):
         self._storage = dict()
-        self._formatter = logging.getLogger('task')
 
     def __getitem__(self, tid):
         if tid not in self._storage:
@@ -21,33 +30,13 @@ class TaskLoggerStorage:
         return self._storage[tid]
 
     def _create_task_logger(self, tid):
-        task = self._get_task(tid)
-        if task is not None:
-            configuration = LOGGING.copy()
-            handler_configuration = configuration['handlers']['file']
-            handler_configuration['filename'] = task.get_log_path()
-            configuration['handlers'] = {
-                'file_{}'.format(tid): handler_configuration
-            }
-            configuration['loggers'] = {
-                'task_{}'.format(tid): {
-                    'handlers': ['file_{}'.format(tid)],
-                    'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
-                }
-            }
-
-            logging.config.dictConfig(configuration)
-            logger = logging.getLogger('task_{}'.format(tid))
-            return logger
-        else:
-            raise Exception('Key must be task indentificator')
-
-    def _get_task(self, tid):
-        try:
-            return models.Task.objects.get(pk=tid)
-        except Exception:
-            return None
+        task = _get_task(tid)
+
+        logger = logging.getLogger('cvat.server.task_{}'.format(tid))
+        server_file = logging.FileHandler(filename=task.get_log_path())
+        logger.addHandler(server_file)
 
+        return logger
 
 class JobLoggerStorage:
     def __init__(self):
@@ -59,17 +48,41 @@ class JobLoggerStorage:
         return self._storage[jid]
 
     def _get_task_logger(self, jid):
-        job = self._get_job(jid)
-        if job is not None:
-            return task_logger[job.segment.task.id]
-        else:
-            raise Exception('Key must be job identificator')
-
-    def _get_job(self, jid):
-        try:
-            return models.Job.objects.select_related("segment__task").get(id=jid)
-        except Exception:
-            return None
+        job = _get_job(jid)
+        return task_logger[job.segment.task.id]
+
+class TaskClientLoggerStorage:
+    def __init__(self):
+        self._storage = dict()
+
+    def __getitem__(self, tid):
+        if tid not in self._storage:
+            self._storage[tid] = self._create_client_logger(tid)
+        return self._storage[tid]
+
+    def _create_client_logger(self, tid):
+        task = _get_task(tid)
+        logger = logging.getLogger('cvat.client.task_{}'.format(tid))
+        client_file = logging.FileHandler(filename=task.get_client_log_path())
+        logger.addHandler(client_file)
+
+        return logger
+
+class JobClientLoggerStorage:
+    def __init__(self):
+        self._storage = dict()
+
+    def __getitem__(self, jid):
+        if jid not in self._storage:
+            self._storage[jid] = self._get_task_logger(jid)
+        return self._storage[jid]
+
+    def _get_task_logger(self, jid):
+        job = _get_job(jid)
+        return task_client_logger[job.segment.task.id]
 
 task_logger = TaskLoggerStorage()
 job_logger = JobLoggerStorage()
+global_logger = logging.getLogger('cvat.server')
+job_client_logger = JobClientLoggerStorage()
+task_client_logger = TaskClientLoggerStorage()
\ No newline at end of file

cvat/apps/engine/static/engine/js/logger.js

@@ -80,7 +80,7 @@ var LoggerHandler = function(applicationName, jobId)
 
         return new Promise( (resolve, reject) => {
             let xhr = new XMLHttpRequest();
-            xhr.open('POST', '/logs/exception/' + this._jobId);
+            xhr.open('POST', '/save/exception/' + this._jobId);
             xhr.setRequestHeader('Content-Type', 'application/json');
             xhr.setRequestHeader("X-CSRFToken", Cookies.get('csrftoken'));
 
@@ -202,25 +202,31 @@ var LoggerHandler = function(applicationName, jobId)
 
 
 /*
-Log message has simple json format - each message is set of "key" : "value" pairs inside curly braces - {"key1" : "string_value", "key2" : number_value, ...}
-Value may be string or number (see json spec)
-required fields for all event types:
+Log message has simple json format - each message is set of "key" : "value"
+pairs inside curly braces - {"key1" : "string_value", "key2" : number_value,
+...} Value may be string or number (see json spec) required fields for all event
+types:
 
     NAME         TYPE           DESCRIPTION
 "event"         string          see EventType enum description of possible values.
-"timestamp"     number          timestamp in UNIX format - the number of seconds or milliseconds that have elapsed since 00:00:00 Thursday, 1 January 1970
+"timestamp"     number          timestamp in UNIX format - the number of seconds
+                                or milliseconds that have elapsed since 00:00:00
+                                Thursday, 1 January 1970
 "application"   string          application name
 "userid"        string          Unique userid
 "task"          string          Unique task id. (Is expected corresponding Jira task id)
 
-"count" is requiered field for "Add object", "Delete object", "Copy track", "Propagate object", "Merge objecrs", "Undo action" and "Redo action"
-events with number value.
+"count" is requiered field for "Add object", "Delete object", "Copy track",
+"Propagate object", "Merge objecrs", "Undo action" and "Redo action" events with
+number value.
 
-Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" : "CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" : "bounding box" }
+Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" :
+"CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" :
+"bounding box" }
 
-Types of supported events.
-Minimum subset of events to generate simple report are Logger.EventType.addObject, Logger.EventType.deleteObject and Logger.EventType.sendTaskInfo.
-Value of "count" property should be a number.
+Types of supported events. Minimum subset of events to generate simple report
+are Logger.EventType.addObject, Logger.EventType.deleteObject and
+Logger.EventType.sendTaskInfo. Value of "count" property should be a number.
 */
 
 var Logger = {
@@ -276,50 +282,67 @@ var Logger = {
     EventType: {
         // dumped as "Paste object". There are no additional required fields.
         pasteObject: 0,
-        // dumped as "Change attribute". There are no additional required fields.
+        // dumped as "Change attribute". There are no additional required
+        // fields.
         changeAttribute: 1,
         // dumped as "Drag object". There are no additional required fields.
         dragObject: 2,
-        // dumped as "Delete object". "count" is required field, value of deleted objects should be positive number.
+        // dumped as "Delete object". "count" is required field, value of
+        // deleted objects should be positive number.
         deleteObject: 3,
         // dumped as "Press shortcut". There are no additional required fields.
         pressShortcut: 4,
         // dumped as "Resize object". There are no additional required fields.
         resizeObject: 5,
-        // dumped as "Send logs". It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Send logs". It's expected that event has "duration" field,
+        // but it isn't necessary.
         sendLogs: 6,
-        // dumped as "Save job". It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Save job". It's expected that event has "duration" field,
+        // but it isn't necessary.
         saveJob: 7,
         // dumped as "Jump frame". There are no additional required fields.
         jumpFrame: 8,
-        // dumped as "Draw object". It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Draw object". It's expected that event has "duration"
+        // field, but it isn't necessary.
         drawObject: 9,
         // dumped as "Change label".
         changeLabel: 10,
-        // dumped as "Send task info". "track count", "frame count", "object count" are required fields. It's expected that event has "current_frame" field.
+        // dumped as "Send task info". "track count", "frame count", "object
+        // count" are required fields. It's expected that event has
+        // "current_frame" field.
         sendTaskInfo: 11,
-        // dumped as "Load job". "track count", "frame count", "object count" are required fields. It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Load job". "track count", "frame count", "object count"
+        // are required fields. It's expected that event has "duration" field,
+        // but it isn't necessary.
         loadJob: 12,
-        // dumped as "Move image". It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Move image". It's expected that event has "duration"
+        // field, but it isn't necessary.
         moveImage: 13,
-        // dumped as "Zoom image". It's expected that event has "duration" field, but it isn't necessary.
+        // dumped as "Zoom image". It's expected that event has "duration"
+        // field, but it isn't necessary.
         zoomImage: 14,
         // dumped as "Lock object". There are no additional required fields.
         lockObject: 15,
-        // dumped as "Merge objects". "count" is required field with positive or negative number value.
+        // dumped as "Merge objects". "count" is required field with positive or
+        // negative number value.
         mergeObjects: 16,
         // dumped as "Copy object". "count" is required field with number value.
         copyObject: 17,
-        // dumped as "Propagate object". "count" is required field with number value.
+        // dumped as "Propagate object". "count" is required field with number
+        // value.
         propagateObject: 18,
-        // dumped as "Undo action". "count" is required field with positive or negative number value.
+        // dumped as "Undo action". "count" is required field with positive or
+        // negative number value.
         undoAction: 19,
-        // dumped as "Redo action". "count" is required field with positive or negative number value.
+        // dumped as "Redo action". "count" is required field with positive or
+        // negative number value.
         redoAction: 20,
-        // dumped as "Send user activity". "working_time" is required field with positive number value.
+        // dumped as "Send user activity". "working_time" is required field with
+        // positive number value.
         sendUserActivity: 21,
-        // dumped as "Send exception". Use to send any exception events to the server.
-        // "message", "filename", "line" are mandatory fields. "stack" and "column" are optional.
+        // dumped as "Send exception". Use to send any exception events to the
+        // server. "message", "filename", "line" are mandatory fields. "stack"
+        // and "column" are optional.
         sendException: 22,
         // dumped as "Change frame". There are no additional required fields.
         changeFrame: 23,
@@ -356,10 +379,12 @@ var Logger = {
 
     /**
      * Logger.addContinuedEvent Use to add log event with duration field.
-     * Duration will be calculated automatically when LogEvent.close() method of returned Object will be called.
-     * Note: in case of LogEvent.close() method will not be callsed event will not be sended to server
+     * Duration will be calculated automatically when LogEvent.close() method of
+     * returned Object will be called. Note: in case of LogEvent.close() method
+     * will not be callsed event will not be sent to server
      * @param {Logger.EventType} type Event Type
-     * @param {Object} values Any event values, for example {count: 1, label: 'vehicle'}
+     * @param {Object} values Any event values, for example {count: 1, label:
+     * 'vehicle'}
      * @return {LogEvent} instance of LogEvent
      * @static
      */
@@ -370,7 +395,8 @@ var Logger = {
 
     /**
      * Logger.shortkeyLogDecorator use for decorating the shortkey handlers.
-     * This decorator just create appropriate log event and close it when decored function will performed.
+     * This decorator just create appropriate log event and close it when
+     * decored function will performed.
      * @param {Function} decoredFunc is function for decorating
      * @return {Function} is decorated decoredFunc
      * @static
@@ -387,7 +413,7 @@ var Logger = {
     },
 
     /**
-     * Logger.sendLogs Try to send exception logs to the server immediatly.
+     * Logger.sendLogs Try to send exception logs to the server immediately.
      * @return {Promise}
      * @param {LogEvent} exceptionEvent
      * @static
@@ -414,7 +440,8 @@ var Logger = {
     },
 
     /**
-     * Logger.setUsername just set username property which will be added to all log messages
+     * Logger.setUsername just set username property which will be added to all
+     * log messages
      * @param {String} username
      * @static
      */
@@ -423,7 +450,8 @@ var Logger = {
         this._logger.setUsername(username);
     },
 
-    /** Logger.updateUserActivityTimer method updates internal timer for working time calculation logic
+    /** Logger.updateUserActivityTimer method updates internal timer for working
+     * time calculation logic
      * @static
      */
     updateUserActivityTimer: function()
@@ -431,11 +459,12 @@ var Logger = {
         this._logger.updateTimer();
     },
 
-    /** Logger.setTimeThreshold set time threshold in ms for EventType.
-     * If time interval betwwen incoming log events less than threshold events will be collapsed.
-     * Note that result event will have timestamp of first event,
-     * In case of time threshold used for continued event duration will be difference between
-     * first and last event timestamps and other fields from last event.
+    /** Logger.setTimeThreshold set time threshold in ms for EventType. If time
+     * interval betwwen incoming log events less than threshold events will be
+     * collapsed. Note that result event will have timestamp of first event, In
+     * case of time threshold used for continued event duration will be
+     * difference between first and last event timestamps and other fields from
+     * last event.
      * @static
      * @param {Logger.EventType} eventType
      * @param {Number} threshold
@@ -445,7 +474,8 @@ var Logger = {
         this._logger.setTimeThreshold(eventType, threshold);
     },
 
-    /** Logger._eventTypeToString private method to transform Logger.EventType to string
+    /** Logger._eventTypeToString private method to transform Logger.EventType
+     * to string
      * @param {Logger.EventType} type Event Type
      * @return {String} string reppresentation of Logger.EventType
      * @static

cvat/apps/engine/task.py

@@ -30,9 +30,7 @@ from pyunpack import Archive
 from distutils.dir_util import copy_tree
 
 from . import models
-from .logging import task_logger, job_logger
-
-global_logger = logging.getLogger(__name__)
+from .logging import task_logger, job_logger, global_logger
 
 ############################# Low Level server API
 

cvat/apps/engine/urls.py

@@ -22,4 +22,5 @@ urlpatterns = [
     path('save/annotation/task/<int:tid>', views.save_annotation_for_task),
     path('get/annotation/job/<int:jid>', views.get_annotation),
     path('get/username', views.get_username),
+    path('save/exception/<int:jid>', views.catch_client_exception)
 ]

cvat/apps/engine/views.py

@@ -5,7 +5,6 @@
 
 import os
 import json
-import logging
 import traceback
 
 from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
@@ -18,13 +17,20 @@ from sendfile import sendfile
 from . import annotation, task, models
 from cvat.settings.base import JS_3RDPARTY
 from cvat.apps.authentication.decorators import login_required
-from cvat.apps.log_proxy.proxy_logger import client_log_proxy
 from requests.exceptions import RequestException
-from .logging import task_logger, job_logger
-
-global_logger = logging.getLogger(__name__)
+import logging
+from .logging import task_logger, job_logger, global_logger, job_client_logger
 
 ############################# High Level server API
+@login_required
+@permission_required('engine.view_task', raise_exception=True)
+def catch_client_exception(request, jid):
+    data = json.loads(request.body.decode('utf-8'))
+    for event in data['exceptions']:
+        job_client_logger[jid].error(json.dumps(event))
+
+    return HttpResponse()
+
 @login_required
 def dispatch_request(request):
     """An entry point to dispatch legacy requests"""
@@ -243,7 +249,8 @@ def save_annotation_for_job(request, jid):
         if 'annotation' in data:
             annotation.save_job(jid, json.loads(data['annotation']))
         if 'logs' in data:
-            client_log_proxy.push_logs(jid, json.loads(data['logs']))
+            for event in json.loads(data['logs']):
+                job_client_logger[jid].info(json.dumps(event))
     except RequestException as e:
         job_logger[jid].error("cannot send annotation logs for job {}".format(jid), exc_info=True)
         return HttpResponseBadRequest(str(e))

cvat/apps/log_proxy/__init__.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-

cvat/apps/log_proxy/admin.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.contrib import admin
-
-# Register your models here.
-

cvat/apps/log_proxy/apps.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.apps import AppConfig
-
-
-class LogProxyConfig(AppConfig):
-    name = 'log_proxy'
-

cvat/apps/log_proxy/migrations/__init__.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-

cvat/apps/log_proxy/models.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.db import models
-
-# Create your models here.
-

cvat/apps/log_proxy/proxy_logger.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.conf import settings
-import os
-import logging
-import requests
-import json
-from urllib.parse import urlparse
-from enum import Enum
-from cvat.apps.engine.models import Job, Task
-
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-class ClientLoggerStorage:
-    def __init__(self):
-        self._storage = dict()
-        self._formatter = logging.Formatter('%(message)s')
-
-    def __getitem__(self, tid):
-        if tid not in self._storage:
-            self._storage[tid] = self._create_client_logger(tid)
-        return self._storage[tid]
-
-    def _create_client_logger(self, tid):
-        task = self._get_task(tid)
-        logger = logging.getLogger(name='client_annotation_logger_{}'.format(tid))
-        logger.setLevel(logging.INFO)
-        handler = logging.FileHandler(filename=task.get_client_log_path())
-        handler.setFormatter(self._formatter)
-        logger.addHandler(handler)
-        return logger
-
-    def _get_task(self, tid):
-        try:
-            return Task.objects.get(pk=tid)
-        except Exception:
-            raise Exception('Key must be task indentificator')
-
-class ClientLogProxy():
-    class _HandlerType(Enum):
-            FILE = 1
-            HTTP = 2
-
-    def __init__(self):
-        self._client_logger = ClientLoggerStorage()
-        def file_log_handler(tid, messages):
-            for event in messages:
-                self._client_logger[tid].info(json.dumps(event))
-
-        self._handlers = {self._HandlerType.FILE: file_log_handler}
-
-        log_server_url = os.environ.get('DJANGO_LOG_SERVER_URL')
-
-        def create_retry_session(retries=3, session=None, backoff_factor=0.3):
-            session = session or requests.Session()
-            retry = Retry(total=retries, backoff_factor=backoff_factor)
-            adapter = HTTPAdapter(max_retries=retry)
-            session.mount('http://', adapter)
-            session.mount('https://', adapter)
-            return session
-
-        if log_server_url:
-            parse_result = urlparse(log_server_url)
-
-            if parse_result.scheme and 'http' not in parse_result.scheme:
-                raise Exception('unsuported annotation log destination')
-
-            def http_log_handler(taskID, messages):
-                r = create_retry_session().post(url=log_server_url, json=messages, verify=False)
-                r.raise_for_status()
-
-            self._handlers[self._HandlerType.HTTP] = http_log_handler
-
-    def push_logs(self, jid, logs):
-        taskID = self._get_task_id(jid)
-
-        for handler in self._handlers.values():
-            handler(taskID, logs)
-
-    def _get_task_id(self, jid):
-        try:
-            job = Job.objects.select_related("segment__task").get(id=jid)
-            return job.segment.task.id
-        except:
-            raise Exception('Key must be job indentificator')
-
-client_log_proxy = ClientLogProxy()

cvat/apps/log_proxy/tests.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.test import TestCase
-
-# Create your tests here.
-

cvat/apps/log_proxy/urls.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.urls import path
-from . import views
-
-urlpatterns = [
-    path('exception/<int:jid>', views.exception_receiver),
-]
-

cvat/apps/log_proxy/views.py

-
-# Copyright (C) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-
-from django.http import HttpResponse, HttpResponseBadRequest
-from django.contrib.auth.decorators import permission_required
-from .proxy_logger import client_log_proxy
-from cvat.apps.authentication.decorators import login_required
-
-
-import json
-
-# Create your views here.
-@login_required()
-@permission_required('engine.view_task', raise_exception=True)
-def exception_receiver(request, jid):
-    data = json.loads(request.body.decode('utf-8'))
-    try:
-        if 'exceptions' in data:
-            client_log_proxy.push_logs(jid, data['exceptions'])
-    except Exception as e:
-        return HttpResponseBadRequest(str(e))
-
-    return HttpResponse()

cvat/requirements/base.txt

@@ -22,3 +22,4 @@ scipy==1.0.1
 sqlparse==0.2.4
 django-sendfile==0.3.11
 dj-pagination==2.3.2
+python-logstash==0.4.6

cvat/settings/base.py

@@ -187,23 +187,40 @@ LOGGING = {
             'class': 'logging.StreamHandler',
             'formatter': 'standard',
         },
-        'file': {
+        'server_file': {
             'class': 'logging.handlers.RotatingFileHandler',
-            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
-            'filename': os.path.join(BASE_DIR, 'logs', 'cvat.log'),
+            'level': 'DEBUG',
+            'filename': os.path.join(BASE_DIR, 'logs', 'cvat_server.log'),
             'formatter': 'standard',
             'maxBytes': 1024*1024*50, # 50 MB
             'backupCount': 5,
+        },
+        'logstash': {
+            'level': 'INFO',
+            'class': 'logstash.TCPLogstashHandler',
+            'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'),
+            'port': os.getenv('DJANGO_LOG_SERVER_PORT', 5000),
+            'version': 1,
+            'message_type': 'django',
         }
     },
     'loggers': {
-        'cvat': {
-            'handlers': ['console', 'file'],
+        'cvat.server': {
+            'handlers': ['console', 'server_file'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
+        },
+
+        'cvat.client': {
+            'handlers': [],
             'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
         }
     },
 }
 
+if os.getenv('DJANGO_LOG_SERVER_HOST'):
+    LOGGING['loggers']['cvat.server']['handlers'] += ['logstash']
+    LOGGING['loggers']['cvat.client']['handlers'] += ['logstash']
+
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.0/howto/static-files/
 

cvat/urls.py

@@ -30,8 +30,7 @@ urlpatterns = [
     path('dashboard/', include('cvat.apps.dashboard.urls')),
     path('django-rq/', include('django_rq.urls')),
     path('auth/', include('cvat.apps.authentication.urls')),
-    path('documentation/', include('cvat.apps.documentation.urls')),
-    path('logs/', include('cvat.apps.log_proxy.urls'))
+    path('documentation/', include('cvat.apps.documentation.urls'))
 ]
 
 if 'yes' == os.environ.get('TF_ANNOTATION', 'no'):

docker-compose.yml

@@ -9,6 +9,10 @@ services:
   cvat_db:
     container_name: cvat_db
     image: postgres:10.3-alpine
+    networks:
+      default:
+        aliases:
+          - db
     restart: always
     environment:
       POSTGRES_USER: root
@@ -19,6 +23,10 @@ services:
   cvat_redis:
     container_name: cvat_redis
     image: redis:4.0.5-alpine
+    networks:
+      default:
+        aliases:
+          - redis
     restart: always
 
   cvat:
@@ -42,7 +50,8 @@ services:
         WITH_TESTS: "no"
     environment:
       DJANGO_MODWSGI_EXTRA_ARGS: ""
-      DJANGO_LOG_SERVER_URL: ""
+      DJANGO_LOG_SERVER_HOST: ""
+      DJANGO_LOG_SERVER_PORT: ""
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys

supervisord.conf

@@ -18,16 +18,19 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker_default]
-command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
+command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic \
+    "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
 numprocs=2
 process_name=rqworker_default_%(process_num)s
 
 [program:rqworker_low]
-command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
+command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
+    "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
 numprocs=1
 
 [program:runserver]
-command=%(ENV_HOME)s/wait-for-it.sh cvat_db:5432 -t 0 -- bash -ic "/usr/bin/python3 ~/manage.py migrate && \
-   exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
+command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \
+    "/usr/bin/python3 ~/manage.py migrate && \
+    exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
     --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \
     %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s"