Merge branch 'regression_detect' into 'master'

support detection of perfomance regression

See merge request !1174

tools/bazel_adb_run.py

@@ -21,13 +21,15 @@
 
 import argparse
 import sys
-
 import sh_commands
 
 from common import *
-
+from dana.dana_util import DanaUtil
+from dana.dana_cli import DanaTrend
 from device import DeviceWrapper, DeviceManager
 
+TABLE_NAME = 'Ops'
+
 
 def unittest_stdout_processor(stdout, device_properties, abi):
     stdout_lines = stdout.split("\n")
@@ -69,7 +71,7 @@ def parse_args():
         type=str,
         default="all",
         help="SoCs (ro.board.platform from getprop) to build, "
-        "comma seperated list or all/random")
+             "comma seperated list or all/random")
     parser.add_argument(
         "--target", type=str, default="//...", help="Bazel target to build")
     parser.add_argument(
@@ -115,10 +117,44 @@ def parse_args():
     return parser.parse_known_args()
 
 
+def report_to_dana(dana_util, item_name, metric_name,
+                   device, soc, abi, value, trend):
+    serie_id = dana_util.create_serie_id_lite(
+        TABLE_NAME, "%s_%s_%s_%s_%s" % (metric_name, device,
+                                        soc, abi, item_name))
+    dana_util.report_benchmark(serie_id=serie_id, value=value, trend=trend)
+
+
+def report_run_statistics(stdouts, device, soc, abi, dana_util):
+    if stdouts is None:
+        print('report_run_statistics failed: stdouts is None.')
+        return
+    for line in stdouts.split('\n'):
+        line = line.strip()
+        parts = line.split()
+        if len(parts) == 5 and parts[0].startswith('MACE'):
+            item_name = str(parts[0])
+            report_to_dana(dana_util, item_name, 'time-ns', device, soc,
+                           abi, float(parts[1]), DanaTrend.SMALLER)
+            report_to_dana(dana_util, item_name, 'iterations', device, soc,
+                           abi, float(parts[2]), DanaTrend.HIGHER)
+            input_mbs = float(parts[3])
+            if input_mbs < sys.float_info.min:
+                input_mbs = sys.float_info.min
+            report_to_dana(dana_util, item_name, 'input-MBS', device, soc,
+                           abi, input_mbs, DanaTrend.HIGHER)
+            gmacps = float(parts[4])
+            if gmacps < sys.float_info.min:
+                gmacps = sys.float_info.min
+            report_to_dana(dana_util, item_name, 'GMACPS', device, soc,
+                           abi, gmacps, DanaTrend.HIGHER)
+
+
 def main(unused_args):
     target = FLAGS.target
     host_bin_path, bin_name = sh_commands.bazel_target_to_bin(target)
     target_abis = FLAGS.target_abis.split(',')
+    dana_util = DanaUtil()
 
     for target_abi in target_abis:
         toolchain = infer_toolchain(target_abi)
@@ -132,7 +168,7 @@ def main(unused_args):
             debug_mode=FLAGS.debug_mode)
         if FLAGS.run_target:
             target_devices = DeviceManager.list_devices(FLAGS.device_yml)
-            if FLAGS.target_socs != TargetSOCTag.all and\
+            if FLAGS.target_socs != TargetSOCTag.all and \
                     FLAGS.target_socs != TargetSOCTag.random:
                 target_socs = set(FLAGS.target_socs.split(','))
                 target_devices = \
@@ -159,6 +195,10 @@ def main(unused_args):
                     address_sanitizer=FLAGS.address_sanitizer,
                     simpleperf=FLAGS.simpleperf)
                 globals()[FLAGS.stdout_processor](stdouts, dev, target_abi)
+                report_run_statistics(stdouts=stdouts,
+                                      device=dev['device_name'],
+                                      soc=dev['target_socs'],
+                                      abi=target_abi, dana_util=dana_util)
 
 
 if __name__ == "__main__":

tools/converter.py

@@ -32,6 +32,8 @@ from utils import config_parser
 import convert
 import encrypt
 
+from dana.dana_util import DanaUtil
+
 ################################
 # set environment
 ################################

tools/dana/dana_cli.py

+# Copyright 2018 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fcntl
+import json
+import os
+import urllib2
+
+DANA_HEADERS = {
+    'User-Agent': 'Mozilla/5.0',
+    'Content-Type': 'application/json',
+}
+
+
+class DanaTrend:
+    SMALLER = 'smaller'
+    HIGHER = 'higher'
+
+
+class DanaCli:
+    def __init__(self, domain, serie_cfg_path):
+        self._domain = domain
+        self._serie_cfg_path = serie_cfg_path
+        self._exist_series = self.load_series_from_cfg()
+
+    def service_available(self):
+        index_url = "http://%s" % self._domain
+        request = urllib2.Request(index_url)
+        response = None
+        try:
+            response = urllib2.urlopen(request).read()
+        except Exception:
+            print("Dana service is not available.")
+        return (response is not None)
+
+    def serie_available(self, serie_id):
+        return (serie_id in self._exist_series)
+
+    # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addbuild
+    def add_build(self, project_id, build_id, build_hash, abbrev_hash,
+                  author_name, author_mail, subject, url=None, override=False):
+        build_data = {
+            'projectId': project_id,
+            'build': {
+                'buildId': build_id,
+                'infos': {
+                    'hash': build_hash,
+                    'abbrevHash': abbrev_hash,
+                    'authorName': author_name,
+                    'authorEmail': author_mail,
+                    'subject': subject,
+                    'url': url
+                }
+            },
+            'override': override
+        }
+        request_url = "http://%s/apis/addBuild" % self._domain
+        succ = self.post_data(request_url, build_data)
+        if succ:
+            print("Add a build id, build_data: %s" % build_data)
+        else:
+            print("Add build id failed. build_data: %s" % build_data)
+        return succ
+
+    # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addserie
+    def add_benchmark_serie(self, project_id, serie_id, range, required,
+                            trend, base_id=None, description=None, infos=None,
+                            override=True):
+        serie_data = {
+            'projectId': project_id,
+            'serieId': serie_id,
+            'analyse': {
+                'benchmark': {
+                    'range': range,
+                    'required': required,
+                    'trend': trend
+                }
+            },
+            'override': override
+        }
+        succ = self.add_serie(serie_data, base_id, description, infos)
+        if succ:
+            print("Add a benchmark serie, serie_data: %s" % serie_data)
+        else:
+            print("Add benchmark serie failed. serie_data: %s" % serie_data)
+        return succ
+
+    # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addserie
+    def add_test_serie(self, project_id, serie_id, base_id=None,
+                       propagate=True, description=None, infos=None,
+                       override=True):
+        serie_data = {
+            'projectId': project_id,
+            'serieId': serie_id,
+            'analyse': {
+                'test': {
+                    'propagate': propagate
+                }
+            },
+            'override': override
+        }
+        succ = self.add_serie(serie_data, base_id, description, infos)
+        if succ:
+            print("Add a test serie, serie_data: %s" % serie_data)
+        else:
+            print("Add test serie failed. serie_data: %s" % serie_data)
+        return succ
+
+    # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addsample
+    # for test, value is true/false
+    # for benchmark, value is zero integer
+    def add_sample(self, project_id, serie_id, build_id,
+                   value, override=False, skip_analysis=False):
+        samples = [{
+            'buildId': build_id,
+            'value': value
+        }]
+        return self.add_samples(project_id, serie_id,
+                                samples, override, skip_analysis)
+
+    # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addsample
+    # samples should be an array includes {'buildId': build_id, 'value': value}
+    def add_samples(self, project_id, serie_id, samples,
+                    override=False, skip_analysis=False):
+        samples_data = {
+            'projectId': project_id,
+            'serieId': serie_id,
+            'samples': samples,
+            'override': override,
+            'skipAnalysis': skip_analysis
+        }
+        request_url = "http://%s/apis/addSample" % self._domain
+        succ = self.post_data(request_url, samples_data)
+        if succ:
+            print("Add samples, samples_data: %s" % samples_data)
+        else:
+            print("Add samples failed. samples_data: %s" % samples_data)
+        return succ
+
+    def add_serie(self, serie_data, base_id=None,
+                  description=None, infos=None):
+        if base_id is not None:
+            serie_data['analyse']['base'] = base_id
+        if description is not None:
+            serie_data['description'] = description
+        if infos is not None:
+            serie_data['infos'] = infos
+        request_url = "http://%s/apis/addSerie" % self._domain
+        succ = self.post_data(request_url, serie_data)
+        if succ:
+            succ = self.write_serie_to_cfg(serie_data['serieId'])
+        return succ
+
+    def post_data(self, request_url, json_data):
+        json_data = json.dumps(json_data).replace('\'', '\"')
+        data = json_data.encode()
+        request = urllib2.Request(request_url,
+                                  headers=DANA_HEADERS,
+                                  data=data)
+        try:
+            response = urllib2.urlopen(request).read()
+        except Exception as e:
+            print("Http error, url=%s\npost_data=%s\n%s" %
+                  (request_url, json_data, e))
+            exit(1)
+        result = response.decode()
+        if len(result) < 30 and "successfull" in result:
+            return True
+        else:
+            return False
+
+    def load_series_from_cfg(self):
+        exist_series = []
+        if os.path.exists(self._serie_cfg_path):
+            with open(self._serie_cfg_path, "r") as f:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                for serie in f.readlines():
+                    exist_series.append(serie.strip())
+        return exist_series
+
+    def write_serie_to_cfg(self, serie_id):
+        if len(serie_id) == 0:
+            print('serie_id is empty.')
+            return False
+        if serie_id in self._exist_series:
+            return True
+
+        with open(self._serie_cfg_path, "a") as f:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+            f.write("%s\n" % serie_id)
+            self._exist_series.append(serie_id)
+        return True

tools/dana/dana_util.py

+# Copyright 2018 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dana.dana_cli import DanaCli
+from dana.dana_cli import DanaTrend
+import os
+import re
+import sh
+import time
+
+DANA_DOMAIN = 'localhost:7000'
+DEFAULT_RANGE = '5%'
+DEFAULT_REQUIRED = 2
+MACE_PROJECT_ID = 'Mace'
+SERIE_CFG_PATH = "%s/.dana/mace_dana.conf" % os.environ['HOME']
+SERVICE_LIFE = 60  # 60s
+
+
+class DanaUtil:
+    def __init__(self, domain=DANA_DOMAIN, serie_cfg_path=SERIE_CFG_PATH):
+        dir_path = os.path.dirname(serie_cfg_path)
+        if not os.path.isdir(dir_path):
+            os.makedirs(dir_path)
+        self._dana_cli = DanaCli(domain, serie_cfg_path)
+        self._valid_time = None
+        self._build_id = None
+        self._service_available = False
+
+    def fast_report_test(self, table_name, base_name, device_name,
+                         soc, target_abi, runtime, value):
+        return self.fast_report_sample(True, table_name, base_name,
+                                       device_name, soc, target_abi,
+                                       runtime, value)
+
+    def fast_report_benchmark(self, table_name, base_name, device_name,
+                              soc, target_abi, runtime, value, trend):
+        return self.fast_report_sample(False, table_name, base_name,
+                                       device_name, soc, target_abi,
+                                       runtime, value, trend)
+
+    def fast_report_sample(self, is_test, table_name, base_name, device_name,
+                           soc, target_abi, runtime, value, trend):
+        if not self.service_available():
+            return True
+        serie_id = self.create_serie_id(table_name, base_name, device_name,
+                                        soc, target_abi, runtime)
+        if is_test:
+            succ = self.report_test(serie_id, value)
+        else:
+            succ = self.report_benchmark(serie_id, value, trend)
+        return succ
+
+    def get_and_create_build_from_git(self, project_id=MACE_PROJECT_ID):
+        commit_id = sh.git('log', '--no-merges', '-n1',
+                           '--pretty=format:%H', _tty_out=False).strip()
+        abbrev_commit_id = sh.git('rev-parse', "%s~1" % commit_id,
+                                  _tty_out=False).strip()
+        author_name = sh.git('log', '--pretty=format:%an',
+                             '-1', commit_id, _tty_out=False).strip()
+        author_mail = sh.git('log', '--pretty=format:%ae',
+                             '-1', commit_id, _tty_out=False).strip()
+        subject = sh.git('log', '--pretty=format:%s',
+                         '-1', commit_id, _tty_out=False).strip()
+        return self.get_and_create_build(commit_id, abbrev_commit_id,
+                                         author_name, author_mail,
+                                         subject, project_id)
+
+    def get_and_create_build(self, commit_id, abbrev_commit_id, author_name,
+                             author_mail, subject, project_id=MACE_PROJECT_ID):
+        if self._build_id is None:
+            build_id = self.create_build_id()
+            succ = self._dana_cli.add_build(project_id=project_id,
+                                            build_id=build_id,
+                                            build_hash=commit_id,
+                                            abbrev_hash=abbrev_commit_id,
+                                            author_name=author_name,
+                                            author_mail=author_mail,
+                                            subject=subject)
+            if succ:
+                self._build_id = build_id
+        return self._build_id
+
+    def create_build_id(self):
+        if False:
+            latest_id = sh.git('log', '-n1', '--pretty=format:%H',
+                               _tty_out=False).strip()
+            timestamp = sh.git('log', '--pretty=format:%ad', '-1',
+                               "--date=format:%Y-%m-%d %H:%M:%S",
+                               latest_id, _tty_out=False).strip()
+            unix_time = time.mktime(
+                time.strptime(timestamp, '%Y-%m-%d %H:%M:%S'))
+            build_id = int(unix_time)
+        else:
+            build_id = int(time.time())
+        return build_id
+
+    def create_serie_id(self, table_name, base_name, device_name,
+                        soc, target_abi, runtime):
+        base_serie_id = "%s_%s_%s_%s_%s" % (base_name, device_name,
+                                            soc, target_abi, runtime)
+        return self.create_serie_id_lite(table_name, base_serie_id)
+
+    def create_serie_id_lite(self, table_name, base_name):
+        serie_id = "%s_%s" % (table_name, base_name)
+        serie_id = re.sub(r'\s+', '', serie_id.strip())
+        return serie_id
+
+    def report_benchmark(self, serie_id, value, trend,
+                         project_id=MACE_PROJECT_ID):
+        if not self._dana_cli.serie_available(serie_id):
+            succ = self._dana_cli.add_benchmark_serie(
+                project_id=project_id, serie_id=serie_id,
+                range=DEFAULT_RANGE, required=DEFAULT_REQUIRED,
+                trend=trend)
+            if not succ:
+                print("Add benchmark serie_id(%s) failed." % serie_id)
+                return False
+        build_id = self.get_and_create_build_from_git()
+        if build_id is None:
+            print("Add build id failed.")
+            return False
+        succ = self._dana_cli.add_sample(project_id=project_id,
+                                         serie_id=serie_id,
+                                         build_id=build_id,
+                                         value=value)
+        return succ
+
+    def report_test(self, serie_id, value,
+                    project_id=MACE_PROJECT_ID):
+        if not self._dana_cli.serie_available(serie_id):
+            succ = self._dana_cli.add_test_serie(project_id=project_id,
+                                                 serie_id=serie_id)
+            if not succ:
+                print("Add test serie_id(%s) failed." % serie_id)
+                return False
+        build_id = self.get_and_create_build_from_git()
+        if build_id is None:
+            print("Add build id failed.")
+            return False
+        succ = self._dana_cli.add_sample(project_id=project_id,
+                                         serie_id=serie_id,
+                                         build_id=build_id,
+                                         value=value)
+        return succ
+
+    def service_available(self):
+        cur_time = int(time.time())
+        if self._valid_time is None or \
+                cur_time - self._valid_time > SERVICE_LIFE:
+            self._service_available = self._dana_cli.service_available()
+            self._valid_time = cur_time
+        return self._service_available
+
+
+if __name__ == '__main__':
+    DanaUtil().fast_report_benchmark(
+        base_name="%s-%s" % ('init', "model_name"),
+        device_name='device_name',
+        soc='target_socs',
+        target_abi='target_abi',
+        runtime='device_type', value=90)

tools/device.py

@@ -25,6 +25,9 @@ import yaml
 
 import common
 from common import *
+from dana.dana_cli import DanaTrend
+from dana.dana_util import DanaUtil
+
 
 import sh_commands
 
@@ -41,6 +44,7 @@ class DeviceWrapper:
                        device_name, target_abis, target_socs, system,
                         address, username
         """
+        self._dana_util = DanaUtil()
         diff = set(device_dict.keys()) - set(YAMLKeyword.__dict__.keys())
         if len(diff) > 0:
             six.print_('Wrong key detected: ')
@@ -783,13 +787,13 @@ class DeviceWrapper:
                                     YAMLKeyword.validation_outputs_data],
                                 log_file=log_file,
                             )
-                        if flags.report and flags.round > 0:
+                        if flags.round > 0:
                             tuned = tuning and device_type == DeviceType.GPU
                             self.report_run_statistics(
                                 target_abi=target_abi,
                                 model_name=model_name,
                                 device_type=device_type,
-                                output_dir=flags.report_dir,
+                                flags=flags,
                                 tuned=tuned)
 
         if model_output_dirs:
@@ -826,7 +830,7 @@ class DeviceWrapper:
                               target_abi,
                               model_name,
                               device_type,
-                              output_dir,
+                              flags,
                               tuned):
         metrics = [0] * 3
         for line in self.stdout.split('\n'):
@@ -837,14 +841,16 @@ class DeviceWrapper:
                 metrics[1] = str(float(parts[3]))
                 metrics[2] = str(float(parts[4]))
                 break
-        report_filename = output_dir + '/report.csv'
-        if not os.path.exists(report_filename):
-            with open(report_filename, 'w') as f:
-                f.write('model_name,device_name,soc,abi,runtime,'
-                        'init(ms),warmup(ms),run_avg(ms),tuned\n')
-
-        data_str = '{model_name},{device_name},{soc},{abi},{device_type},' \
-                   '{init},{warmup},{run_avg},{tuned}\n'.format(
+        if flags.report:
+            report_filename = flags.report_dir + '/report.csv'
+            if not os.path.exists(report_filename):
+                with open(report_filename, 'w') as f:
+                    f.write('model_name,device_name,soc,abi,runtime,'
+                            'init(ms),warmup(ms),run_avg(ms),tuned\n')
+
+            data_str = \
+                '{model_name},{device_name},{soc},{abi},{device_type},' \
+                '{init},{warmup},{run_avg},{tuned}\n'.format(
                     model_name=model_name,
                     device_name=self.device_name,
                     soc=self.target_socs,
@@ -854,8 +860,25 @@ class DeviceWrapper:
                     warmup=metrics[1],
                     run_avg=metrics[2],
                     tuned=tuned)
-        with open(report_filename, 'a') as f:
-            f.write(data_str)
+            with open(report_filename, 'a') as f:
+                f.write(data_str)
+        self.report_to_dana(model_name, target_abi, device_type,
+                            {
+                                'init': metrics[0],
+                                'warmup': metrics[1],
+                                'run-avg': metrics[2]
+                            })
+
+    def report_to_dana(self, model_name, target_abi, device_type, metrics):
+        if not self._dana_util.service_available():
+            return
+        for (key, value) in metrics.items():
+            value = float(value)
+            self._dana_util.fast_report_benchmark(
+                table_name="Models", base_name="%s_%s" % (key, model_name),
+                device_name=self.device_name, soc=self.target_socs,
+                target_abi=target_abi, runtime=device_type,
+                value=value, trend=DanaTrend.SMALLER)
 
     def run(self,
             abi,