!209 auto download clang, gn and ninja

Merge pull request !209 from yangming_ha/master

hb/__init__.py

@@ -42,4 +42,24 @@ CONFIG_STRUCT = {
     "device_path": None,
     "patch_cache": None
 }
-VERSION = '0.4.2'
+
+VERSION = '0.4.3'
+
+BUILD_TOOLS_CFG = {
+    "Linux": {
+        "build_tools_path": 'prebuilts/build-tools/linux-x86/bin',
+        "gn": 'https://repo.huaweicloud.com/harmonyos/compiler/gn/' +\
+              '1717/linux/gn-linux-x86-1717.tar.gz',
+        "ninja": 'https://repo.huaweicloud.com/harmonyos/compiler/ninja/' +\
+                 '1.10.1/linux/ninja-linux-x86-1.10.1.tar.gz',
+        "clang": 'https://repo.huaweicloud.com/harmonyos/compiler/clang/' +\
+                 '10.0.1-62608/linux/llvm.tar.gz'
+    },
+    "Windows": {
+        "build_tools_path": 'prebuilts\\build-tools\\win-x86\\bin',
+        "gn": 'https://repo.huaweicloud.com/harmonyos/compiler/gn/' +\
+              '1744/windows/gn-windows-amd64.zip',
+        "ninja": 'https://repo.huaweicloud.com/harmonyos/compiler/ninja/' +\
+                 '1.9.0/windows/ninja-win.zip'
+    }
+}

hb/common/config.py

@@ -21,10 +21,13 @@ from distutils.spawn import find_executable
 
 from hb import CONFIG_JSON
 from hb import CONFIG_STRUCT
+from hb import BUILD_TOOLS_CFG
 from hb.common.utils import read_json_file
 from hb.common.utils import dump_json_file
 from hb.common.utils import Singleton
 from hb.common.utils import OHOSException
+from hb.common.utils import download_tool
+from hb.common.utils import makedirs
 
 
 class Config(metaclass=Singleton):
@@ -41,6 +44,7 @@ class Config(metaclass=Singleton):
         self._patch_cache = config_content.get('patch_cache', None)
         self._out_path = None
         self.fs_attr = set()
+        self.platform = platform.system()
 
     @property
     def root_path(self):
@@ -157,45 +161,41 @@ class Config(metaclass=Singleton):
 
     @property
     def build_tools_path(self):
-        platform_name = platform.system()
-        if platform_name == 'Linux':
-            return os.path.join(self.root_path,
-                                'prebuilts',
-                                'build-tools',
-                                'linux-x86',
-                                'bin')
-        if platform_name == 'Windows':
-            return os.path.join(self.root_path,
-                                'prebuilts',
-                                'build-tools',
-                                'win-x86',
-                                'bin')
-
-        raise OHOSException(f'unidentified platform: {platform_name}')
+        try:
+            tools_path = BUILD_TOOLS_CFG[self.platform]['build_tools_path']
+            return os.path.join(self.root_path, tools_path)
+        except KeyError:
+            raise OHOSException(f'unidentified platform: {self.platform}')
 
     @property
     def gn_path(self):
         repo_gn_path = os.path.join(self.build_tools_path, 'gn')
+        # gn exist.
         if os.path.isfile(repo_gn_path):
             return repo_gn_path
 
-        env_gn_path = find_executable('gn')
-        if env_gn_path is not None:
-            return env_gn_path
+        # gn not install, download and extract it.
+        makedirs(self.build_tools_path, exist_ok=True)
 
-        raise OHOSException('gn not found, install it please')
+        gn_url = BUILD_TOOLS_CFG[self.platform].get('gn')
+        gn_dst = os.path.join(self.build_tools_path, 'gn_pkg')
+        download_tool(gn_url, gn_dst, tgt_dir=self.build_tools_path)
+
+        return repo_gn_path
 
     @property
     def ninja_path(self):
         repo_ninja_path = os.path.join(self.build_tools_path, 'ninja')
+        # ninja exist.
         if os.path.isfile(repo_ninja_path):
             return repo_ninja_path
 
-        env_ninja_path = find_executable('ninja')
-        if env_ninja_path is not None:
-            return env_ninja_path
+        # ninja not install, download and extract.
+        ninja_url = BUILD_TOOLS_CFG[self.platform].get('ninja')
+        ninja_dst = os.path.join(self.build_tools_path, 'ninja_pkg')
+        download_tool(ninja_url, ninja_dst, tgt_dir=self.build_tools_path)
 
-        raise OHOSException('ninja not found, install it please')
+        return repo_ninja_path
 
     @property
     def clang_path(self):
@@ -204,9 +204,12 @@ class Config(metaclass=Singleton):
                                        'ohos',
                                        'linux-x86_64',
                                        'llvm')
+        # clang exist
         if os.path.isdir(repo_clang_path):
             return f'//{repo_clang_path}'
-
+        # clang installed manually or auto download
+        else:
+            # already installed manually
             env_clang_bin_path = find_executable('clang')
             if env_clang_bin_path is not None:
                 env_clang_path = os.path.abspath(os.path.join(env_clang_bin_path,
@@ -216,7 +219,15 @@ class Config(metaclass=Singleton):
                 if os.path.basename(env_clang_path) == 'llvm':
                     return env_clang_path
 
-        raise OHOSException('clang not found, install it please')
+            # need auto download and extract clang.
+            clang_path = os.path.abspath(os.path.join(repo_clang_path,
+                                                      os.pardir))
+            makedirs(clang_path, exist_ok=True)
+
+            clang_url = BUILD_TOOLS_CFG[self.platform].get('clang')
+            clang_dst = os.path.join(clang_path, 'clang_pkg')
+            download_tool(clang_url, clang_dst, tgt_dir=clang_path)
+            return f'//{repo_clang_path}'
 
     @property
     def patch_cache(self):

hb/common/utils.py

@@ -25,6 +25,9 @@ import json
 from collections import namedtuple
 import yaml
 from datetime import datetime
+import requests
+import tarfile
+import zipfile
 
 
 def encode(data, encoding='utf-8'):
@@ -230,3 +233,50 @@ class Singleton(type):
 
 class OHOSException(Exception):
     pass
+
+
+def download_tool(url, dst, tgt_dir=None):
+    try:
+        res = requests.get(url, stream=True, timeout=(5, 9))
+    except OSError:
+        raise OHOSException(f'download {url} timeout!')
+
+    if res.status_code == 200:
+        hb_info(f'Downloading {url} ...')
+    else:
+        hb_error(f'Downloading {url} failed with code: {res.status_code}!')
+        return res.status_code
+
+    total_size = int(res.headers['content-length'])
+    download_size = 0
+    download_percent = 0
+
+    try:
+        with open(dst, "wb") as f:
+            for chunk in res.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+                    download_size += len(chunk)
+                    download_percent = round(float(download_size / total_size * 100), 2)
+                    print('Progress: %s%%\r' % download_percent, end=' ')
+            hb_info('Download complete!')
+    except OSError:
+        raise OHOSException(f'{url} download failed, please install it manually!')
+
+    if tgt_dir is not None:
+        extract_tool(dst, tgt_dir)
+
+
+def extract_tool(src, tgt_dir):
+    hb_info(f'Extracting to {tgt_dir}, please wait...')
+    try:
+        if tarfile.is_tarfile(src):
+            ef = tarfile.open(src)
+        elif zipfile.is_zipfile(src):
+            ef = zipfile.ZipFile(src)
+        else:
+            raise OHOSException(f'Extract file type not support!')
+        ef.extractall(tgt_dir)
+        ef.close()
+    except OSError:
+        raise OHOSException(f'{src} extract failed, please install it manually!')
\ No newline at end of file

setup.py

@@ -43,7 +43,8 @@ setup(
     install_requires=[
         'prompt_toolkit==1.0.14',
         'kconfiglib>=14.1.0',
-        'PyYAML'
+        'PyYAML',
+        'requests'
     ],
     entry_points={
         'console_scripts': [