from setuptools import setup, Distribution, Extension import subprocess import os import re import shutil class BinaryDistribution(Distribution): def has_ext_modules(foo): return True RC = 0 ext_name = '.dll' if os.name == 'nt' else '.so' def git_commit(): try: cmd = ['git', 'rev-parse', 'HEAD'] git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip() except: git_commit = 'Unknown' git_commit = git_commit.decode() return str(git_commit) def _get_version_detail(idx): assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \ so detail index must less than 3" if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'): version_details = '@PADDLE_VERSION@'.split('.') if len(version_details) >= 3: return version_details[idx] return 0 def get_major(): return int(_get_version_detail(0)) def get_minor(): return int(_get_version_detail(1)) def get_patch(): return str(_get_version_detail(2)) def is_taged(): try: cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null'] git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip() git_tag = git_tag.decode() except: return False if str(git_tag).replace('v', '') == '@PADDLE_VERSION@': return True else: return False def write_version_py(filename='paddle/version.py'): cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY # full_version = '%(major)d.%(minor)d.%(patch)s' major = '%(major)d' minor = '%(minor)d' patch = '%(patch)s' rc = '%(rc)d' istaged = %(istaged)s commit = '%(commit)s' with_mkl = '%(with_mkl)s' def show(): if istaged: print('full_version:', full_version) print('major:', major) print('minor:', minor) print('patch:', patch) print('rc:', rc) else: print('commit:', commit) def mkl(): return with_mkl ''' commit = git_commit() with open(filename, 'w') as f: f.write(cnt % { 'major': get_major(), 'minor': get_minor(), 'patch': get_patch(), 'rc': RC, 'version': '${PADDLE_VERSION}', 'commit': commit, 'istaged': is_taged(), 'with_mkl': '@WITH_MKL@'}) write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py') packages=['paddle', 'paddle.libs', 'paddle.utils', 'paddle.dataset', 'paddle.reader', 'paddle.distributed', 'paddle.fluid', 'paddle.fluid.dygraph', 'paddle.fluid.proto', 'paddle.fluid.proto.profiler', 'paddle.fluid.distributed', 'paddle.fluid.layers', 'paddle.fluid.contrib', 'paddle.fluid.contrib.decoder', 'paddle.fluid.contrib.quantize', 'paddle.fluid.contrib.int8_inference', 'paddle.fluid.contrib.reader', 'paddle.fluid.contrib.slim', 'paddle.fluid.contrib.slim.core', 'paddle.fluid.contrib.slim.graph', 'paddle.fluid.contrib.slim.prune', 'paddle.fluid.contrib.slim.quantization', 'paddle.fluid.contrib.slim.distillation', 'paddle.fluid.contrib.slim.nas', 'paddle.fluid.contrib.slim.searcher', 'paddle.fluid.contrib.utils', 'paddle.fluid.contrib.extend_optimizer', 'paddle.fluid.contrib.mixed_precision', 'paddle.fluid.contrib.layers', 'paddle.fluid.transpiler', 'paddle.fluid.transpiler.details', 'paddle.fluid.incubate', 'paddle.fluid.incubate.data_generator', 'paddle.fluid.incubate.fleet', 'paddle.fluid.incubate.fleet.base', 'paddle.fluid.incubate.fleet.parameter_server', 'paddle.fluid.incubate.fleet.parameter_server.distributed_transpiler', 'paddle.fluid.incubate.fleet.parameter_server.pslib', 'paddle.fluid.incubate.fleet.collective'] with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f: setup_requires = f.read().splitlines() if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']: setup_requires+=['opencv-python'] # the prefix is sys.prefix which should always be usr paddle_bins = '' if not '${WIN32}': paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle'] package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + (ext_name if os.name != 'nt' else '.pyd')]} if '${HAS_NOAVX_CORE}' == 'ON': package_data['paddle.fluid'] += ['core_noavx' + (ext_name if os.name != 'nt' else '.pyd')] package_dir={ '': '${PADDLE_BINARY_DIR}/python', # The paddle.fluid.proto will be generated while compiling. # So that package points to other directory. 'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform', 'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework', 'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid', } # put all thirdparty libraries in paddle.libs libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs' package_data['paddle.libs']= [] package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name] shutil.copy('${WARPCTC_LIBRARIES}', libs_path) if '${WITH_MKL}' == 'ON': shutil.copy('${MKLML_SHARED_LIB}', libs_path) shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path) package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name] # mklml has dependency to vs runtime library if os.name == 'nt': shutil.copy('${MKLML_SHARED_LIB_DEPS}', libs_path) package_data['paddle.libs'] += ['msvcr120.dll'] else: if os.name == 'nt': # copy the openblas.dll shutil.copy(os.path.dirname('${CBLAS_LIBRARIES}') + '/openblas' + ext_name, libs_path) package_data['paddle.libs'] += ['openblas' + ext_name] if '${WITH_MKLDNN}' == 'ON': if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt': # only change rpath in Release mode. # TODO(typhoonzero): use install_name_tool to patch mkl libs once # we can support mkl on mac. # # change rpath of libmkldnn.so.0, add $ORIGIN/ to it. # The reason is that all thirdparty libraries in the same directory, # thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so. command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}" if os.system(command) != 0: raise Exception("patch libmkldnn.so failed, command: %s" % command) package_data['paddle.libs']+=['libmkldnn.so.0' if os.name != 'nt' else ('mkldnn' + ext_name)] shutil.copy('${MKLDNN_SHARED_LIB}', libs_path) if '${WITH_NGRAPH}' == 'ON': # only change rpath in Release mode, # since in Debug mode, nGraph lib may be too large to be changed? if '${CMAKE_BUILD_TYPE}' == 'Release': if os.name != 'nt': if "@APPLE@" == "1": command = "install_name_tool -id \"@loader_path/\" ${NGRAPH_SHARED_LIB}" else: command = "patchelf --set-rpath '$ORIGIN/' ${NGRAPH_SHARED_LIB}" if os.system(command) != 0: raise Exception("patch ${NGRAPH_SHARED_LIB_NAME} failed, command: %s" % command) shutil.copy('${NGRAPH_SHARED_LIB}', libs_path) shutil.copy('${NGRAPH_CPU_LIB}', libs_path) shutil.copy('${NGRAPH_TBB_LIB}', libs_path) package_data['paddle.libs']+=['${NGRAPH_SHARED_LIB_NAME}', '${NGRAPH_CPU_LIB_NAME}', '${NGRAPH_TBB_LIB_NAME}'] # remove unused paddle/libs/__init__.py if os.path.isfile(libs_path+'/__init__.py'): os.remove(libs_path+'/__init__.py') package_dir['paddle.libs']=libs_path # change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it. # The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and # ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries. # This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213 if '${CMAKE_BUILD_TYPE}' == 'Release': if os.name != 'nt': # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed. if "@APPLE@" == "1": command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + ext_name else: command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + ext_name if os.system(command) != 0: raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command)) ext_modules = [Extension('_foo', ['stub.cc'])] if os.name == 'nt': # fix the path separator under windows fix_package_dir = {} for k, v in package_dir.items(): fix_package_dir[k] = v.replace('/', '\\') package_dir = fix_package_dir ext_modules = [] setup(name='${PACKAGE_NAME}', version='${PADDLE_VERSION}', description='Parallel Distributed Deep Learning', install_requires=setup_requires, packages=packages, ext_modules=ext_modules, package_data=package_data, package_dir=package_dir, scripts=paddle_bins, distclass=BinaryDistribution )