diff --git a/tools/bazel_adb_run.py b/tools/bazel_adb_run.py index 92f02f021f83fe9a1eaeab9b1d3a530d1a7e49d5..309ac90f36caff841c4067f6fb0dd3f3ac5383e5 100644 --- a/tools/bazel_adb_run.py +++ b/tools/bazel_adb_run.py @@ -21,13 +21,15 @@ import argparse import sys - import sh_commands from common import * - +from dana.dana_util import DanaUtil +from dana.dana_cli import DanaTrend from device import DeviceWrapper, DeviceManager +TABLE_NAME = 'Ops' + def unittest_stdout_processor(stdout, device_properties, abi): stdout_lines = stdout.split("\n") @@ -69,7 +71,7 @@ def parse_args(): type=str, default="all", help="SoCs (ro.board.platform from getprop) to build, " - "comma seperated list or all/random") + "comma seperated list or all/random") parser.add_argument( "--target", type=str, default="//...", help="Bazel target to build") parser.add_argument( @@ -115,10 +117,44 @@ def parse_args(): return parser.parse_known_args() +def report_to_dana(dana_util, item_name, metric_name, + device, soc, abi, value, trend): + serie_id = dana_util.create_serie_id_lite( + TABLE_NAME, "%s_%s_%s_%s_%s" % (metric_name, device, + soc, abi, item_name)) + dana_util.report_benchmark(serie_id=serie_id, value=value, trend=trend) + + +def report_run_statistics(stdouts, device, soc, abi, dana_util): + if stdouts is None: + print('report_run_statistics failed: stdouts is None.') + return + for line in stdouts.split('\n'): + line = line.strip() + parts = line.split() + if len(parts) == 5 and parts[0].startswith('MACE'): + item_name = str(parts[0]) + report_to_dana(dana_util, item_name, 'time-ns', device, soc, + abi, float(parts[1]), DanaTrend.SMALLER) + report_to_dana(dana_util, item_name, 'iterations', device, soc, + abi, float(parts[2]), DanaTrend.HIGHER) + input_mbs = float(parts[3]) + if input_mbs < sys.float_info.min: + input_mbs = sys.float_info.min + report_to_dana(dana_util, item_name, 'input-MBS', device, soc, + abi, input_mbs, DanaTrend.HIGHER) + gmacps = float(parts[4]) + if gmacps < sys.float_info.min: + gmacps = sys.float_info.min + report_to_dana(dana_util, item_name, 'GMACPS', device, soc, + abi, gmacps, DanaTrend.HIGHER) + + def main(unused_args): target = FLAGS.target host_bin_path, bin_name = sh_commands.bazel_target_to_bin(target) target_abis = FLAGS.target_abis.split(',') + dana_util = DanaUtil() for target_abi in target_abis: toolchain = infer_toolchain(target_abi) @@ -132,7 +168,7 @@ def main(unused_args): debug_mode=FLAGS.debug_mode) if FLAGS.run_target: target_devices = DeviceManager.list_devices(FLAGS.device_yml) - if FLAGS.target_socs != TargetSOCTag.all and\ + if FLAGS.target_socs != TargetSOCTag.all and \ FLAGS.target_socs != TargetSOCTag.random: target_socs = set(FLAGS.target_socs.split(',')) target_devices = \ @@ -159,6 +195,10 @@ def main(unused_args): address_sanitizer=FLAGS.address_sanitizer, simpleperf=FLAGS.simpleperf) globals()[FLAGS.stdout_processor](stdouts, dev, target_abi) + report_run_statistics(stdouts=stdouts, + device=dev['device_name'], + soc=dev['target_socs'], + abi=target_abi, dana_util=dana_util) if __name__ == "__main__": diff --git a/tools/converter.py b/tools/converter.py index a98c304dc7acf1e026d198513e0e1429de3adc2a..1a13ffabac010a7d85b9b51a8ed60f69bf9f98ab 100644 --- a/tools/converter.py +++ b/tools/converter.py @@ -32,6 +32,8 @@ from utils import config_parser import convert import encrypt +from dana.dana_util import DanaUtil + ################################ # set environment ################################ diff --git a/tools/dana/__init__.py b/tools/dana/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/dana/dana_cli.py b/tools/dana/dana_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..07aeb17fd73d83d0018196f75156d0829ea1157c --- /dev/null +++ b/tools/dana/dana_cli.py @@ -0,0 +1,203 @@ +# Copyright 2018 The MACE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fcntl +import json +import os +import urllib2 + +DANA_HEADERS = { + 'User-Agent': 'Mozilla/5.0', + 'Content-Type': 'application/json', +} + + +class DanaTrend: + SMALLER = 'smaller' + HIGHER = 'higher' + + +class DanaCli: + def __init__(self, domain, serie_cfg_path): + self._domain = domain + self._serie_cfg_path = serie_cfg_path + self._exist_series = self.load_series_from_cfg() + + def service_available(self): + index_url = "http://%s" % self._domain + request = urllib2.Request(index_url) + response = None + try: + response = urllib2.urlopen(request).read() + except Exception: + print("Dana service is not available.") + return (response is not None) + + def serie_available(self, serie_id): + return (serie_id in self._exist_series) + + # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addbuild + def add_build(self, project_id, build_id, build_hash, abbrev_hash, + author_name, author_mail, subject, url=None, override=False): + build_data = { + 'projectId': project_id, + 'build': { + 'buildId': build_id, + 'infos': { + 'hash': build_hash, + 'abbrevHash': abbrev_hash, + 'authorName': author_name, + 'authorEmail': author_mail, + 'subject': subject, + 'url': url + } + }, + 'override': override + } + request_url = "http://%s/apis/addBuild" % self._domain + succ = self.post_data(request_url, build_data) + if succ: + print("Add a build id, build_data: %s" % build_data) + else: + print("Add build id failed. build_data: %s" % build_data) + return succ + + # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addserie + def add_benchmark_serie(self, project_id, serie_id, range, required, + trend, base_id=None, description=None, infos=None, + override=True): + serie_data = { + 'projectId': project_id, + 'serieId': serie_id, + 'analyse': { + 'benchmark': { + 'range': range, + 'required': required, + 'trend': trend + } + }, + 'override': override + } + succ = self.add_serie(serie_data, base_id, description, infos) + if succ: + print("Add a benchmark serie, serie_data: %s" % serie_data) + else: + print("Add benchmark serie failed. serie_data: %s" % serie_data) + return succ + + # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addserie + def add_test_serie(self, project_id, serie_id, base_id=None, + propagate=True, description=None, infos=None, + override=True): + serie_data = { + 'projectId': project_id, + 'serieId': serie_id, + 'analyse': { + 'test': { + 'propagate': propagate + } + }, + 'override': override + } + succ = self.add_serie(serie_data, base_id, description, infos) + if succ: + print("Add a test serie, serie_data: %s" % serie_data) + else: + print("Add test serie failed. serie_data: %s" % serie_data) + return succ + + # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addsample + # for test, value is true/false + # for benchmark, value is zero integer + def add_sample(self, project_id, serie_id, build_id, + value, override=False, skip_analysis=False): + samples = [{ + 'buildId': build_id, + 'value': value + }] + return self.add_samples(project_id, serie_id, + samples, override, skip_analysis) + + # ref: https://github.com/google/dana/blob/master/docs/Apis.md#addsample + # samples should be an array includes {'buildId': build_id, 'value': value} + def add_samples(self, project_id, serie_id, samples, + override=False, skip_analysis=False): + samples_data = { + 'projectId': project_id, + 'serieId': serie_id, + 'samples': samples, + 'override': override, + 'skipAnalysis': skip_analysis + } + request_url = "http://%s/apis/addSample" % self._domain + succ = self.post_data(request_url, samples_data) + if succ: + print("Add samples, samples_data: %s" % samples_data) + else: + print("Add samples failed. samples_data: %s" % samples_data) + return succ + + def add_serie(self, serie_data, base_id=None, + description=None, infos=None): + if base_id is not None: + serie_data['analyse']['base'] = base_id + if description is not None: + serie_data['description'] = description + if infos is not None: + serie_data['infos'] = infos + request_url = "http://%s/apis/addSerie" % self._domain + succ = self.post_data(request_url, serie_data) + if succ: + succ = self.write_serie_to_cfg(serie_data['serieId']) + return succ + + def post_data(self, request_url, json_data): + json_data = json.dumps(json_data).replace('\'', '\"') + data = json_data.encode() + request = urllib2.Request(request_url, + headers=DANA_HEADERS, + data=data) + try: + response = urllib2.urlopen(request).read() + except Exception as e: + print("Http error, url=%s\npost_data=%s\n%s" % + (request_url, json_data, e)) + exit(1) + result = response.decode() + if len(result) < 30 and "successfull" in result: + return True + else: + return False + + def load_series_from_cfg(self): + exist_series = [] + if os.path.exists(self._serie_cfg_path): + with open(self._serie_cfg_path, "r") as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + for serie in f.readlines(): + exist_series.append(serie.strip()) + return exist_series + + def write_serie_to_cfg(self, serie_id): + if len(serie_id) == 0: + print('serie_id is empty.') + return False + if serie_id in self._exist_series: + return True + + with open(self._serie_cfg_path, "a") as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + f.write("%s\n" % serie_id) + self._exist_series.append(serie_id) + return True diff --git a/tools/dana/dana_util.py b/tools/dana/dana_util.py new file mode 100644 index 0000000000000000000000000000000000000000..ab4daf00cc87acdfd2d498a6b8fd5599f28a4f44 --- /dev/null +++ b/tools/dana/dana_util.py @@ -0,0 +1,172 @@ +# Copyright 2018 The MACE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dana.dana_cli import DanaCli +from dana.dana_cli import DanaTrend +import os +import re +import sh +import time + +DANA_DOMAIN = 'localhost:7000' +DEFAULT_RANGE = '5%' +DEFAULT_REQUIRED = 2 +MACE_PROJECT_ID = 'Mace' +SERIE_CFG_PATH = "%s/.dana/mace_dana.conf" % os.environ['HOME'] +SERVICE_LIFE = 60 # 60s + + +class DanaUtil: + def __init__(self, domain=DANA_DOMAIN, serie_cfg_path=SERIE_CFG_PATH): + dir_path = os.path.dirname(serie_cfg_path) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + self._dana_cli = DanaCli(domain, serie_cfg_path) + self._valid_time = None + self._build_id = None + self._service_available = False + + def fast_report_test(self, table_name, base_name, device_name, + soc, target_abi, runtime, value): + return self.fast_report_sample(True, table_name, base_name, + device_name, soc, target_abi, + runtime, value) + + def fast_report_benchmark(self, table_name, base_name, device_name, + soc, target_abi, runtime, value, trend): + return self.fast_report_sample(False, table_name, base_name, + device_name, soc, target_abi, + runtime, value, trend) + + def fast_report_sample(self, is_test, table_name, base_name, device_name, + soc, target_abi, runtime, value, trend): + if not self.service_available(): + return True + serie_id = self.create_serie_id(table_name, base_name, device_name, + soc, target_abi, runtime) + if is_test: + succ = self.report_test(serie_id, value) + else: + succ = self.report_benchmark(serie_id, value, trend) + return succ + + def get_and_create_build_from_git(self, project_id=MACE_PROJECT_ID): + commit_id = sh.git('log', '--no-merges', '-n1', + '--pretty=format:%H', _tty_out=False).strip() + abbrev_commit_id = sh.git('rev-parse', "%s~1" % commit_id, + _tty_out=False).strip() + author_name = sh.git('log', '--pretty=format:%an', + '-1', commit_id, _tty_out=False).strip() + author_mail = sh.git('log', '--pretty=format:%ae', + '-1', commit_id, _tty_out=False).strip() + subject = sh.git('log', '--pretty=format:%s', + '-1', commit_id, _tty_out=False).strip() + return self.get_and_create_build(commit_id, abbrev_commit_id, + author_name, author_mail, + subject, project_id) + + def get_and_create_build(self, commit_id, abbrev_commit_id, author_name, + author_mail, subject, project_id=MACE_PROJECT_ID): + if self._build_id is None: + build_id = self.create_build_id() + succ = self._dana_cli.add_build(project_id=project_id, + build_id=build_id, + build_hash=commit_id, + abbrev_hash=abbrev_commit_id, + author_name=author_name, + author_mail=author_mail, + subject=subject) + if succ: + self._build_id = build_id + return self._build_id + + def create_build_id(self): + if False: + latest_id = sh.git('log', '-n1', '--pretty=format:%H', + _tty_out=False).strip() + timestamp = sh.git('log', '--pretty=format:%ad', '-1', + "--date=format:%Y-%m-%d %H:%M:%S", + latest_id, _tty_out=False).strip() + unix_time = time.mktime( + time.strptime(timestamp, '%Y-%m-%d %H:%M:%S')) + build_id = int(unix_time) + else: + build_id = int(time.time()) + return build_id + + def create_serie_id(self, table_name, base_name, device_name, + soc, target_abi, runtime): + base_serie_id = "%s_%s_%s_%s_%s" % (base_name, device_name, + soc, target_abi, runtime) + return self.create_serie_id_lite(table_name, base_serie_id) + + def create_serie_id_lite(self, table_name, base_name): + serie_id = "%s_%s" % (table_name, base_name) + serie_id = re.sub(r'\s+', '', serie_id.strip()) + return serie_id + + def report_benchmark(self, serie_id, value, trend, + project_id=MACE_PROJECT_ID): + if not self._dana_cli.serie_available(serie_id): + succ = self._dana_cli.add_benchmark_serie( + project_id=project_id, serie_id=serie_id, + range=DEFAULT_RANGE, required=DEFAULT_REQUIRED, + trend=trend) + if not succ: + print("Add benchmark serie_id(%s) failed." % serie_id) + return False + build_id = self.get_and_create_build_from_git() + if build_id is None: + print("Add build id failed.") + return False + succ = self._dana_cli.add_sample(project_id=project_id, + serie_id=serie_id, + build_id=build_id, + value=value) + return succ + + def report_test(self, serie_id, value, + project_id=MACE_PROJECT_ID): + if not self._dana_cli.serie_available(serie_id): + succ = self._dana_cli.add_test_serie(project_id=project_id, + serie_id=serie_id) + if not succ: + print("Add test serie_id(%s) failed." % serie_id) + return False + build_id = self.get_and_create_build_from_git() + if build_id is None: + print("Add build id failed.") + return False + succ = self._dana_cli.add_sample(project_id=project_id, + serie_id=serie_id, + build_id=build_id, + value=value) + return succ + + def service_available(self): + cur_time = int(time.time()) + if self._valid_time is None or \ + cur_time - self._valid_time > SERVICE_LIFE: + self._service_available = self._dana_cli.service_available() + self._valid_time = cur_time + return self._service_available + + +if __name__ == '__main__': + DanaUtil().fast_report_benchmark( + base_name="%s-%s" % ('init', "model_name"), + device_name='device_name', + soc='target_socs', + target_abi='target_abi', + runtime='device_type', value=90) diff --git a/tools/device.py b/tools/device.py index 66d8d1e6adc7dcf288f81d03be920f7cf40e5213..d428756af67dcaa89242d1e9d1f2bb6c5c52f24a 100644 --- a/tools/device.py +++ b/tools/device.py @@ -25,6 +25,9 @@ import yaml import common from common import * +from dana.dana_cli import DanaTrend +from dana.dana_util import DanaUtil + import sh_commands @@ -41,6 +44,7 @@ class DeviceWrapper: device_name, target_abis, target_socs, system, address, username """ + self._dana_util = DanaUtil() diff = set(device_dict.keys()) - set(YAMLKeyword.__dict__.keys()) if len(diff) > 0: six.print_('Wrong key detected: ') @@ -783,13 +787,13 @@ class DeviceWrapper: YAMLKeyword.validation_outputs_data], log_file=log_file, ) - if flags.report and flags.round > 0: + if flags.round > 0: tuned = tuning and device_type == DeviceType.GPU self.report_run_statistics( target_abi=target_abi, model_name=model_name, device_type=device_type, - output_dir=flags.report_dir, + flags=flags, tuned=tuned) if model_output_dirs: @@ -826,7 +830,7 @@ class DeviceWrapper: target_abi, model_name, device_type, - output_dir, + flags, tuned): metrics = [0] * 3 for line in self.stdout.split('\n'): @@ -837,14 +841,16 @@ class DeviceWrapper: metrics[1] = str(float(parts[3])) metrics[2] = str(float(parts[4])) break - report_filename = output_dir + '/report.csv' - if not os.path.exists(report_filename): - with open(report_filename, 'w') as f: - f.write('model_name,device_name,soc,abi,runtime,' - 'init(ms),warmup(ms),run_avg(ms),tuned\n') - - data_str = '{model_name},{device_name},{soc},{abi},{device_type},' \ - '{init},{warmup},{run_avg},{tuned}\n'.format( + if flags.report: + report_filename = flags.report_dir + '/report.csv' + if not os.path.exists(report_filename): + with open(report_filename, 'w') as f: + f.write('model_name,device_name,soc,abi,runtime,' + 'init(ms),warmup(ms),run_avg(ms),tuned\n') + + data_str = \ + '{model_name},{device_name},{soc},{abi},{device_type},' \ + '{init},{warmup},{run_avg},{tuned}\n'.format( model_name=model_name, device_name=self.device_name, soc=self.target_socs, @@ -854,8 +860,25 @@ class DeviceWrapper: warmup=metrics[1], run_avg=metrics[2], tuned=tuned) - with open(report_filename, 'a') as f: - f.write(data_str) + with open(report_filename, 'a') as f: + f.write(data_str) + self.report_to_dana(model_name, target_abi, device_type, + { + 'init': metrics[0], + 'warmup': metrics[1], + 'run-avg': metrics[2] + }) + + def report_to_dana(self, model_name, target_abi, device_type, metrics): + if not self._dana_util.service_available(): + return + for (key, value) in metrics.items(): + value = float(value) + self._dana_util.fast_report_benchmark( + table_name="Models", base_name="%s_%s" % (key, model_name), + device_name=self.device_name, soc=self.target_socs, + target_abi=target_abi, runtime=device_type, + value=value, trend=DanaTrend.SMALLER) def run(self, abi,