setup.py.in 14.5 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import re
4
import shutil
5
import sys
6 7 8 9 10 11 12
import fnmatch

from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


13 14 15
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
16

Y
Yancey 已提交
17 18
RC      = 0

19
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
20 21 22 23

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
24 25
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
26 27
    except:
        git_commit = 'Unknown'
28
    git_commit = git_commit.decode()
29
    return str(git_commit)
Y
Yancey 已提交
30

31
def _get_version_detail(idx):
32 33
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
34

M
minqiyang 已提交
35 36
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
37

M
minqiyang 已提交
38
        if len(version_details) >= 3:
M
minqiyang 已提交
39
            return version_details[idx]
40

41
    return 0
42

M
minqiyang 已提交
43
def get_major():
44
    return int(_get_version_detail(0))
45

M
minqiyang 已提交
46
def get_minor():
47
    return int(_get_version_detail(1))
48

49
def get_patch():
50
    return str(_get_version_detail(2))
51 52

def is_taged():
M
minqiyang 已提交
53
    try:
54
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
55
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
56
        git_tag = git_tag.decode()
M
minqiyang 已提交
57
    except:
58 59
        return False

60
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
61 62
        return True
    else:
M
minqiyang 已提交
63
        return False
64

Y
Yancey 已提交
65
def write_version_py(filename='paddle/version.py'):
66
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
67
#
68
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
69 70
major           = '%(major)d'
minor           = '%(minor)d'
71
patch           = '%(patch)s'
Y
Yancey 已提交
72 73 74
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
75
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
76 77 78

def show():
    if istaged:
79 80 81 82 83
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
84
    else:
85
        print('commit:', commit)
L
Luo Tao 已提交
86 87 88

def mkl():
    return with_mkl
Y
Yancey 已提交
89 90 91 92
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
93 94 95
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
96 97 98
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
99
            'istaged': is_taged(),
L
Luo Tao 已提交
100
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
101

102
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
103 104


Z
zhangjinchao01 已提交
105
packages=['paddle',
106
          'paddle.libs',
Q
qiaolongfei 已提交
107
          'paddle.utils',
108 109
          'paddle.dataset',
          'paddle.reader',
110
          'paddle.distributed',
111
          'paddle.fluid',
L
lujun 已提交
112
          'paddle.fluid.dygraph',
113
          'paddle.fluid.proto',
X
Xin Pan 已提交
114
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
115
          'paddle.fluid.distributed',
Y
Yancey 已提交
116
          'paddle.fluid.layers',
Q
Qingsheng Li 已提交
117 118
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
119
          'paddle.fluid.contrib.quantize',
Q
Qiao Longfei 已提交
120
          'paddle.fluid.contrib.reader',
W
whs 已提交
121 122 123 124
          'paddle.fluid.contrib.slim',
          'paddle.fluid.contrib.slim.core',
          'paddle.fluid.contrib.slim.graph',
          'paddle.fluid.contrib.slim.prune',
W
WangZhen 已提交
125
          'paddle.fluid.contrib.slim.quantization',
126
          'paddle.fluid.contrib.slim.distillation',
W
whs 已提交
127 128
          'paddle.fluid.contrib.slim.nas',
          'paddle.fluid.contrib.slim.searcher',
T
tangwei12 已提交
129
          'paddle.fluid.contrib.utils',
C
chengduo 已提交
130
          'paddle.fluid.contrib.extend_optimizer',
131
          'paddle.fluid.contrib.mixed_precision',
132
          'paddle.fluid.contrib.layers',
Q
qiaolongfei 已提交
133
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
134 135
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
136
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
137 138 139
          'paddle.fluid.incubate.fleet',
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
140
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
141
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
142 143
          'paddle.fluid.incubate.fleet.collective',
          'paddle.fluid.incubate.fleet.utils']
L
Luo Tao 已提交
144

145 146
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
147 148

if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
Y
Yancey 已提交
149
    setup_requires+=['opencv-python']
150

151
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
152
paddle_bins = ''
T
Tao Luo 已提交
153 154
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
155
package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]}
156
if '${HAS_NOAVX_CORE}' == 'ON':
157
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
158

L
Luo Tao 已提交
159
package_dir={
160
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
161 162 163 164
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
165
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
166
}
167

168 169
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
170

P
peizhilin 已提交
171
package_data['paddle.libs']= []
P
peizhilin 已提交
172
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
173
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
174

175 176 177 178 179
if '${TENSORRT_FOUND}' == 'ON' and os.name == 'nt':
    shutil.copy(os.path.join('${TENSORRT_ROOT}', 'lib', '${TR_INFER_RT}'), libs_path)
    shutil.copy(os.path.join('${TENSORRT_ROOT}', 'lib', '${TR_INFER_PLUGIN_RT}'), libs_path)
    package_data['paddle.libs'] += ['${TR_INFER_RT}', '${TR_INFER_PLUGIN_RT}']

180
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
181 182 183
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
184 185 186 187
    # mklml has dependency to vs runtime library
    if os.name == 'nt':
        shutil.copy('${MKLML_SHARED_LIB_DEPS}', libs_path)
        package_data['paddle.libs'] += ['msvcr120.dll']
P
peizhilin 已提交
188 189
else:
    if os.name == 'nt':
P
peizhilin 已提交
190 191
        # copy the openblas.dll
        shutil.copy(os.path.dirname('${CBLAS_LIBRARIES}') + '/openblas' + ext_name, libs_path)
P
peizhilin 已提交
192
        package_data['paddle.libs'] += ['openblas' + ext_name]
P
peizhilin 已提交
193

X
xujiaqi01 已提交
194 195 196 197
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
198
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
199
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
200 201 202 203 204 205 206 207 208 209
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
        # change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
        # The reason is that all thirdparty libraries in the same directory,
        # thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
            raise Exception("patch libmkldnn.so failed, command: %s" % command)
P
peizhilin 已提交
210
    package_data['paddle.libs']+=['libmkldnn.so.0' if os.name != 'nt' else ('mkldnn' + ext_name)]
S
Sang Ik Lee 已提交
211
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
S
Sang Ik Lee 已提交
212
if '${WITH_NGRAPH}' == 'ON':
S
Sang Ik Lee 已提交
213 214
    # only change rpath in Release mode,
    # since in Debug mode, nGraph lib may be too large to be changed?
S
Sang Ik Lee 已提交
215
    if '${CMAKE_BUILD_TYPE}' == 'Release':
S
Sang Ik Lee 已提交
216 217 218 219 220 221 222
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${NGRAPH_SHARED_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${NGRAPH_SHARED_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${NGRAPH_SHARED_LIB_NAME} failed, command: %s" % command)
S
Sang Ik Lee 已提交
223 224 225 226 227 228
    shutil.copy('${NGRAPH_SHARED_LIB}', libs_path)
    shutil.copy('${NGRAPH_CPU_LIB}', libs_path)
    shutil.copy('${NGRAPH_TBB_LIB}', libs_path)
    package_data['paddle.libs']+=['${NGRAPH_SHARED_LIB_NAME}',
                                  '${NGRAPH_CPU_LIB_NAME}',
                                  '${NGRAPH_TBB_LIB_NAME}']
229 230 231 232 233 234 235

# copy libfuild_framework.so to libs
if os.name != 'nt' and sys.platform != 'darwin':
    paddle_framework_lib='${FLUID_FRAMEWORK_SHARED_LIB}'
    shutil.copy(paddle_framework_lib, libs_path)
    package_data['paddle.libs'] += [('libpaddle_framework' if os.name != 'nt' else 'paddle_framework') + ext_name]

236
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
237 238
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
239 240
package_dir['paddle.libs']=libs_path

241

242
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
243
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
244
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
245
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
246
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
247
    if os.name != 'nt':
248
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
249
        if "@APPLE@" == "1":
250
            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
251
        else:
252
            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
253
        if os.system(command) != 0:
254
            raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
255

P
peizhilin 已提交
256
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
257 258 259 260 261 262
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
263
    ext_modules = []
264 265
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
266

267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
def find_files(pattern, root):
    for dirpath, _, files in os.walk(root):
      for filename in fnmatch.filter(files, pattern):
          yield os.path.join(dirpath, filename)

headers = (
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/framework')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/memory')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/string')) +
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/framework')) +
    ['${EIGEN_INCLUDE_DIR}/Eigen/Core'] + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/Eigen/src')) + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/unsupported/Eigen')) + # eigen
    list(find_files('*', '${GFLAGS_INSTALL_DIR}/include')) + # gflags
    list(find_files('*', '${GLOG_INSTALL_DIR}/include')) + # glog
    list(find_files('*', '${BOOST_INCLUDE_DIR}/boost')) + # boost
    list(find_files('*', '${XXHASH_INSTALL_DIR}/include')) + # xxhash
    list(find_files('*', '${PROTOBUF_INCLUDE_DIR}')) + # protobuf
286
    list(find_files('*', '${DLPACK_INCLUDE_DIR}')) + # dlpack 
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
    list(find_files('*.h', '${THREADPOOL_INCLUDE_DIR}'))) # threadpool


class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_headers = os.path.join(self.install_purelib, 'paddle',
                                            'include')
        self.install_lib = self.install_platlib
        return ret
  

class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
  
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
  
    boolean_options = ['force']
  
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
  
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
  
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
            # framework
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
            install_dir = re.sub('src/extern_eigen3/', '', install_dir)
            install_dir = re.sub('src/extern_boost/', '', install_dir)
333
            install_dir = re.sub('src/extern_dlpack/', '', install_dir)
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
  
    def run(self):
        if os.name == 'nt' or sys.platform == 'darwin':
            return
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
  
    def get_inputs(self):
        return self.distribution.headers or []
  
    def get_outputs(self):
        return self.outfiles

356 357 358 359 360 361 362 363
# Saving the installation log generated from setup.py to log_file.
# The log_file is ${PADDLE_BINARY_DIR}/python/setup.py.log.
stdout = sys.stdout
stderr = sys.stderr
log_file = open('setup.py.log', 'w')
sys.stdout = log_file
sys.stderr = log_file

T
typhoonzero 已提交
364
setup(name='${PACKAGE_NAME}',
Z
zhangjinchao01 已提交
365 366
      version='${PADDLE_VERSION}',
      description='Parallel Distributed Deep Learning',
367
      install_requires=setup_requires,
L
Luo Tao 已提交
368
      packages=packages,
P
peizhilin 已提交
369
      ext_modules=ext_modules,
L
Luo Tao 已提交
370 371
      package_data=package_data,
      package_dir=package_dir,
P
peizhilin 已提交
372
      scripts=paddle_bins,
373 374 375 376 377 378
      distclass=BinaryDistribution,
      headers=headers,
      cmdclass={
          'install_headers': InstallHeaders,
          'install': InstallCommand,
      }
Z
zhangjinchao01 已提交
379
)
380 381 382 383 384 385 386 387 388

log_file.close()
# Revert back the stdout/stderr to their default references.
sys.stdout = stdout
sys.stderr = stderr
# As there are a lot of files in purelib which causes many logs,
# we don't print them on the screen, and you can open `setup.py.log` 
# for the full logs.
os.system('grep -v "purelib" setup.py.log')