#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright (c) 2020 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import subprocess from datetime import datetime from distutils.spawn import find_executable from hb_internal.common.utils import exec_command from hb_internal.common.utils import hb_warning from hb_internal.common.config import Config class PreBuild: def __init__(self, config): self._root_path = config.root_path self._out_path = config.out_path self._log_path = config.log_path def set_ccache(self): ccache_local_dir = os.environ.get('CCACHE_LOCAL_DIR') ccache_log_suffix = os.environ.get('CCACHE_LOG_SUFFIX') if not ccache_local_dir: ccache_local_dir = '.ccache' ccache_base = os.environ.get('CCACHE_BASE') # The default value is HOME for local users if not ccache_base: ccache_base = os.environ.get('HOME') ccache_base = os.path.join(ccache_base, ccache_local_dir) if not os.path.exists(ccache_base): os.makedirs(ccache_base) ccache_log_file_name = "ccache.log" if ccache_log_suffix: ccache_log_file_name = "ccache." + ccache_log_suffix + ".log" logfile = os.path.join(ccache_base, ccache_log_file_name) if os.path.exists(logfile): oldfile_name = ccache_log_file_name + ".old" oldfile = os.path.join(ccache_base, oldfile_name) if os.path.exists(oldfile): os.unlink(oldfile) os.rename(logfile, oldfile) ccache_path = find_executable('ccache') if ccache_path is None: hb_warning('Failed to find ccache, ccache disabled.') return os.environ['CCACHE_EXEC'] = ccache_path os.environ['CCACHE_LOGFILE'] = logfile os.environ['USE_CCACHE'] = '1' os.environ['CCACHE_DIR'] = ccache_base os.environ['CCACHE_UMASK'] = '002' os.environ['CCACHE_BASEDIR'] = self._root_path ccache_max_size = os.environ.get('CCACHE_MAXSIZE') if not ccache_max_size: ccache_max_size = '100G' cmd = ['ccache', '-M', ccache_max_size] exec_command(cmd, log_path=self._log_path) def set_pycache(self): pycache_dir = os.environ.get('CCACHE_BASE') # The default value is HOME for local users if not pycache_dir: pycache_dir = os.environ.get('HOME') pycache_dir = os.path.join(pycache_dir, '.pycache') os.environ['PYCACHE_DIR'] = pycache_dir pyd_start_cmd = [ 'python3', '{}/build/scripts/util/pyd.py'.format(self._root_path), '--root', pycache_dir, '--start', ] cmd = ['/bin/bash', '-c', ' '.join(pyd_start_cmd), '&'] subprocess.Popen(cmd) def rename_last_logfile(self): logfile = os.path.join(self._out_path, 'build.log') if os.path.exists(logfile): mtime = os.stat(logfile).st_mtime os.rename(logfile, '{}/build.{}.log'.format(self._out_path, mtime)) def prepare(self, args): actions = [self.set_ccache, self.rename_last_logfile] for action in actions: action() class PostBuild: def __init__(self, config): self._root_path = config.root_path self._out_path = config.out_path self._log_path = config.log_path def clean(self, start_time, disable_post_build_args): if not disable_post_build_args or 'stat_ccache' not in disable_post_build_args: self.stat_ccache() if not disable_post_build_args or 'generate_ninja_trace' not in disable_post_build_args: self.generate_ninja_trace(start_time) if not disable_post_build_args or 'get_warning_list' not in disable_post_build_args: self.get_warning_list() if not disable_post_build_args or 'compute_overlap_rate' not in disable_post_build_args: self.compute_overlap_rate() def package_image(self): image_path = os.path.join(self._out_path, 'packages/phone/images/') if os.path.exists(image_path): packaged_file_path = os.path.join(self._out_path, 'images.tar.gz') cmd = ['tar', '-zcvf', packaged_file_path, image_path] exec_command(cmd, log_path=self._log_path) def stat_pycache(self): cmd = [ 'python3', '{}/build/scripts/util/pyd.py'.format(self._root_path), '--stat' ] exec_command(cmd, log_path=self._log_path) def manage_cache_data(self): cmd = [ 'python3', '{}/build/scripts/util/pyd.py'.format(self._root_path), '--manage' ] exec_command(cmd, log_path=self._log_path) def stop_pyd(self): cmd = [ 'python3', '{}/build/scripts/util/pyd.py'.format(self._root_path), '--stop' ] exec_command(cmd, log_path=self._log_path) def stat_ccache(self): ccache_path = find_executable('ccache') if ccache_path is None: return ccache_log_file_name = "ccache.log" ccache_log_suffix = os.environ.get('CCACHE_LOG_SUFFIX') if ccache_log_suffix: ccache_log_file_name = "ccache." + ccache_log_suffix + ".log" ccache_local_dir = os.environ.get('CCACHE_LOCAL_DIR') if not ccache_local_dir: ccache_local_dir = '.ccache' ccache_base = os.environ.get('CCACHE_BASE') # The default value is HOME for local users if not ccache_base: ccache_base = os.environ.get('HOME') ccache_base = os.path.join(ccache_base, ccache_local_dir) cmd = [ 'python3', '{}/build/scripts/summary_ccache_hitrate.py'.format( self._root_path), '{}/{}'.format(ccache_base, ccache_log_file_name) ] exec_command(cmd, log_path=self._log_path) def get_warning_list(self): cmd = [ 'python3', '{}/build/scripts/get_warnings.py'.format(self._root_path), '--build-log-file', '{}/build.log'.format(self._out_path), '--warning-out-file', '{}/packages/WarningList.txt'.format(self._out_path), ] exec_command(cmd, log_path=self._log_path) def generate_ninja_trace(self, start_time): def get_unixtime(dt): epoch = datetime.utcfromtimestamp(0) unixtime = '%f' % ((dt - epoch).total_seconds() * 10**9) return str(unixtime) cmd = [ 'python3', '{}/build/scripts/ninja2trace.py'.format(self._root_path), '--ninja-log', '{}/.ninja_log'.format(self._out_path), "--trace-file", "{}/build.trace".format(self._out_path), "--ninja-start-time", get_unixtime(start_time), "--duration-file", "{}/sorted_action_duration.txt".format(self._out_path), ] exec_command(cmd, log_path=self._log_path) def compute_overlap_rate(self): conf = Config() subsystem_config_overlay_path = conf.product_path + '/subsystem_config_overlay.json' if os.path.isfile(subsystem_config_overlay_path): cmd = [ 'python3', '{}/build/ohos/statistics/build_overlap_statistics.py'.format( self._root_path), "--build-out-dir", self._out_path, "--subsystem-config-file", "{}/build/subsystem_config.json".format(self._root_path), "--subsystem-config-overlay-file", "{}/subsystem_config_overlay.json".format(conf.product_path), "--root-source-dir", self._root_path ] else: cmd = [ 'python3', '{}/build/ohos/statistics/build_overlap_statistics.py'.format( self._root_path), "--build-out-dir", self._out_path, "--subsystem-config-file", "{}/build/subsystem_config.json".format(self._root_path), "--root-source-dir", self._root_path ] exec_command(cmd, log_path=self._log_path)