setup.py.in 23.7 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import os.path
4
import errno
5
import re
6
import shutil
7
import sys
8
import fnmatch
C
Chengmo 已提交
9
import errno
10
import platform
11

12
from contextlib import contextmanager
13 14 15 16 17
from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


18 19 20
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
21

Y
Yancey 已提交
22 23
RC      = 0

24
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
25 26 27 28

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
29 30
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
31 32
    except:
        git_commit = 'Unknown'
33
    git_commit = git_commit.decode()
34
    return str(git_commit)
Y
Yancey 已提交
35

36
def _get_version_detail(idx):
37 38
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
39

M
minqiyang 已提交
40 41
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
42

M
minqiyang 已提交
43
        if len(version_details) >= 3:
M
minqiyang 已提交
44
            return version_details[idx]
45

46
    return 0
47

M
minqiyang 已提交
48
def get_major():
49
    return int(_get_version_detail(0))
50

M
minqiyang 已提交
51
def get_minor():
52
    return int(_get_version_detail(1))
53

54
def get_patch():
55
    return str(_get_version_detail(2))
56

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
def get_cuda_version():
    if '@WITH_GPU@' == 'ON':
        return '@CUDA_VERSION@'
    else:
        return 'False'

def get_cudnn_version():
    if '@WITH_GPU@' == 'ON':
        temp_cudnn_version = ''
        if '@CUDNN_MAJOR_VERSION@':
            temp_cudnn_version += '@CUDNN_MAJOR_VERSION@'
            if '@CUDNN_MINOR_VERSION@':
                temp_cudnn_version += '.@CUDNN_MINOR_VERSION@'
                if '@CUDNN_PATCHLEVEL_VERSION@':
                    temp_cudnn_version += '.@CUDNN_PATCHLEVEL_VERSION@'
        return temp_cudnn_version
    else:
        return 'False'

76
def is_taged():
M
minqiyang 已提交
77
    try:
78
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
79
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
80
        git_tag = git_tag.decode()
M
minqiyang 已提交
81
    except:
82 83
        return False

84
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
85 86
        return True
    else:
M
minqiyang 已提交
87
        return False
88

89
def write_version_py(filename='paddle/version/__init__.py'):
90
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
91
#
92
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
93 94
major           = '%(major)d'
minor           = '%(minor)d'
95
patch           = '%(patch)s'
Y
Yancey 已提交
96
rc              = '%(rc)d'
97 98
cuda_version    = '%(cuda)s'
cudnn_version   = '%(cudnn)s'
Y
Yancey 已提交
99 100
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
101
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
102

103 104
__all__ = ['cuda', 'cudnn']

Y
Yancey 已提交
105 106
def show():
    if istaged:
107 108 109 110 111
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
112
    else:
113
        print('commit:', commit)
L
Luo Tao 已提交
114 115 116

def mkl():
    return with_mkl
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150

def cuda():
    """Get cuda version of paddle package.

    Returns:
        string: Return the version information of cuda. If paddle package is CPU version, it will return False.
    
    Examples:
        .. code-block:: python

            import paddle

            paddle.version.cuda()
            # '10.2'

    """
    return cuda_version

def cudnn():
    """Get cudnn version of paddle package.

    Returns:
        string: Return the version information of cudnn. If paddle package is CPU version, it will return False.
    
    Examples:
        .. code-block:: python

            import paddle

            paddle.version.cudnn()
            # '7.6.5'

    """
    return cudnn_version
Y
Yancey 已提交
151 152
'''
    commit = git_commit()
153 154 155 156 157 158 159 160 161

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

Y
Yancey 已提交
162 163
    with open(filename, 'w') as f:
        f.write(cnt % {
164 165 166
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
167 168
            'rc': RC,
            'version': '${PADDLE_VERSION}',
169 170
            'cuda': get_cuda_version(),
            'cudnn': get_cudnn_version(),
Y
Yancey 已提交
171
            'commit': commit,
172
            'istaged': is_taged(),
L
Luo Tao 已提交
173
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
174

175
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version/__init__.py')
Y
Yancey 已提交
176

177 178 179 180 181 182 183 184 185 186 187 188 189 190
def write_cuda_env_config_py(filename='paddle/cuda_env.py'):
    cnt = ""
    if '${JIT_RELEASE_WHL}' == 'ON':
        cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
#
import os
os.environ['CUDA_CACHE_MAXSIZE'] = '805306368'
'''

    with open(filename, 'w') as f:
        f.write(cnt)

write_cuda_env_config_py(filename='@PADDLE_BINARY_DIR@/python/paddle/cuda_env.py')

C
Chengmo 已提交
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
def write_distributed_training_mode_py(filename='paddle/fluid/incubate/fleet/parameter_server/version.py'):
    cnt = '''from __future__ import print_function

# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY

from paddle.fluid.incubate.fleet.base.mode import Mode

BUILD_MODE=Mode.%(mode)s

def is_transpiler():
    return Mode.TRANSPILER == BUILD_MODE

'''

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    with open(filename, 'w') as f:
        f.write(cnt % {
            'mode': 'PSLIB' if '${WITH_PSLIB}' == 'ON' else 'TRANSPILER'
        })

write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/fluid/incubate/fleet/parameter_server/version.py')
Y
Yancey 已提交
219

220

Z
zhangjinchao01 已提交
221
packages=['paddle',
222
          'paddle.libs',
Q
qiaolongfei 已提交
223
          'paddle.utils',
224
          'paddle.utils.gast',
225
          'paddle.utils.cpp_extension',
226 227
          'paddle.dataset',
          'paddle.reader',
228
          'paddle.distributed',
229
          'paddle.incubate',
230
          'paddle.incubate.optimizer',
231
          'paddle.incubate.checkpoint',
232
          'paddle.incubate.operators',
233
          'paddle.incubate.tensor',
234
          'paddle.incubate.nn',
235 236
          'paddle.distributed.fleet',
          'paddle.distributed.fleet.base',
K
kuizhiqing 已提交
237
          'paddle.distributed.fleet.elastic',
238
          'paddle.distributed.fleet.meta_optimizers',
239
          'paddle.distributed.fleet.meta_optimizers.sharding',
240
          'paddle.distributed.fleet.meta_optimizers.ascend',
241
          'paddle.distributed.fleet.meta_optimizers.dygraph_optimizer',
242 243
          'paddle.distributed.fleet.runtime',
          'paddle.distributed.fleet.dataset',
244
          'paddle.distributed.fleet.data_generator',
245 246 247
          'paddle.distributed.fleet.metrics',
          'paddle.distributed.fleet.proto',
          'paddle.distributed.fleet.utils',
248
          'paddle.distributed.fleet.meta_parallel',
249
          'paddle.distributed.fleet.meta_parallel.pp_utils',
250
          'paddle.distributed.fleet.meta_parallel.parallel_layers',
251
          'paddle.distributed.auto_parallel',
252
          'paddle.distributed.auto_parallel.operators',
253
          'paddle.framework',
254
          'paddle.jit',
255
          'paddle.jit.dy2static',
W
Wilber 已提交
256
          'paddle.inference',
257 258
          'paddle.inference.contrib',
          'paddle.inference.contrib.utils',
259
          'paddle.fluid',
W
Wilber 已提交
260
          'paddle.fluid.inference',
L
lujun 已提交
261
          'paddle.fluid.dygraph',
262
          'paddle.fluid.dygraph.dygraph_to_static',
263
          'paddle.fluid.dygraph.amp',
264
          'paddle.fluid.proto',
X
Xin Pan 已提交
265
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
266
          'paddle.fluid.distributed',
Y
Yancey 已提交
267
          'paddle.fluid.layers',
268
          'paddle.fluid.dataloader',
Q
Qingsheng Li 已提交
269 270
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
271
          'paddle.fluid.contrib.quantize',
W
whs 已提交
272
          'paddle.fluid.contrib.slim',
W
WangZhen 已提交
273
          'paddle.fluid.contrib.slim.quantization',
274
          'paddle.fluid.contrib.slim.quantization.imperative',
C
chengduo 已提交
275
          'paddle.fluid.contrib.extend_optimizer',
276
          'paddle.fluid.contrib.mixed_precision',
277
          'paddle.fluid.contrib.mixed_precision.bf16',
278
          'paddle.fluid.contrib.layers',
279
          'paddle.fluid.contrib.sparsity',
Q
qiaolongfei 已提交
280
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
281 282
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
283
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
284
          'paddle.fluid.incubate.fleet',
285
          'paddle.fluid.incubate.checkpoint',
D
dongdaxiang 已提交
286 287
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
288
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
289
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
290
          'paddle.fluid.incubate.fleet.parameter_server.ir',
291
          'paddle.fluid.incubate.fleet.collective',
292
          'paddle.fluid.incubate.fleet.utils',
293
          'paddle.amp',
294 295 296 297 298 299 300
          'paddle.hapi',
          'paddle.vision',
          'paddle.vision.models',
          'paddle.vision.transforms',
          'paddle.vision.datasets',
          'paddle.text',
          'paddle.text.datasets',
H
hong 已提交
301
          'paddle.incubate',
302 303 304
          'paddle.incubate.nn',
          'paddle.incubate.nn.functional',
          'paddle.incubate.nn.layer',
305
          'paddle.io',
H
hong 已提交
306
          'paddle.optimizer',
307 308
          'paddle.nn',
          'paddle.nn.functional',
309
          'paddle.nn.layer',
310
          'paddle.nn.quant',
H
hong 已提交
311
          'paddle.nn.initializer',
312
          'paddle.nn.utils',
H
hong 已提交
313
          'paddle.metric',
314 315
          'paddle.static',
          'paddle.static.nn',
316
          'paddle.static.amp',
317
          'paddle.tensor',
C
channings 已提交
318
          'paddle.onnx',
319
          'paddle.autograd',
320 321
          'paddle.device',
          'paddle.device.cuda',
322
          'paddle.version',
323
          ]
L
Luo Tao 已提交
324

325 326
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
327

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
# Note(wangzhongpu):
# When compiling paddle under python36, the dependencies belonging to python2.7 will be imported, resulting in errors when installing paddle
if sys.version_info >= (3,6) and sys.version_info < (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,5) and sys.version_info < (3,6):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<=\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i or "<\"3.7\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp

352
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
353
paddle_bins = ''
354

T
Tao Luo 已提交
355 356
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
357 358 359 360 361 362

if os.name != 'nt':
    package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.so']}
else:
    package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.pyd', '${FLUID_CORE_NAME}' + '.lib']}

363
if '${HAS_NOAVX_CORE}' == 'ON':
364
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
365

L
Luo Tao 已提交
366
package_dir={
367
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
368 369 370 371
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
372
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
373
}
374

375 376
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
377

P
peizhilin 已提交
378
package_data['paddle.libs']= []
P
peizhilin 已提交
379
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
380
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
381

382 383 384 385 386 387 388 389 390 391 392 393 394
package_data['paddle.libs']+=[
    os.path.basename('${LAPACK_LIB}'), 
    os.path.basename('${BLAS_LIB}'),
    os.path.basename('${GFORTRAN_LIB}'),
    os.path.basename('${GNU_RT_LIB_1}')]
shutil.copy('${BLAS_LIB}', libs_path)
shutil.copy('${LAPACK_LIB}', libs_path)
shutil.copy('${GFORTRAN_LIB}', libs_path)
shutil.copy('${GNU_RT_LIB_1}', libs_path)
if not sys.platform.startswith("linux"):
    package_data['paddle.libs']+=[os.path.basename('${GNU_RT_LIB_2}')]
    shutil.copy('${GNU_RT_LIB_2}', libs_path)

395
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
396 397 398
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
P
peizhilin 已提交
399 400
else:
    if os.name == 'nt':
P
peizhilin 已提交
401
        # copy the openblas.dll
402
        shutil.copy('${OPENBLAS_SHARED_LIB}', libs_path)
P
peizhilin 已提交
403
        package_data['paddle.libs'] += ['openblas' + ext_name]
W
Wilber 已提交
404
    elif os.name == 'posix' and platform.machine() == 'aarch64' and '${OPENBLAS_LIB}'.endswith('so'):
H
houj04 已提交
405 406
        # copy the libopenblas.so on linux+aarch64
        # special: core_noavx.so depends on 'libopenblas.so.0', not 'libopenblas.so'
W
Wilber 已提交
407 408 409
        if os.path.exists('${OPENBLAS_LIB}' + '.0'):
            shutil.copy('${OPENBLAS_LIB}' + '.0', libs_path)
            package_data['paddle.libs'] += ['libopenblas.so.0']
P
peizhilin 已提交
410

411 412 413
if '${WITH_LITE}' == 'ON':
    shutil.copy('${LITE_SHARED_LIB}', libs_path)
    package_data['paddle.libs']+=['libpaddle_full_api_shared' + ext_name]
414 415 416 417 418 419
    if '${LITE_WITH_NNADAPTER}' == 'ON':
        shutil.copy('${LITE_NNADAPTER_LIB}', libs_path)
        package_data['paddle.libs']+=['libnnadapter' + ext_name]
        if '${NNADAPTER_WITH_HUAWEI_ASCEND_NPU}' == 'ON':
            shutil.copy('${LITE_NNADAPTER_NPU_LIB}', libs_path)
            package_data['paddle.libs']+=['libnnadapter_driver_huawei_ascend_npu' + ext_name]
420

X
xujiaqi01 已提交
421 422
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
X
xujiaqi01 已提交
423 424
    if os.path.exists('${PSLIB_VERSION_PY}'):
        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/')
X
xujiaqi01 已提交
425 426
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
427
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
428
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
429 430 431 432
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
A
Adam 已提交
433
        # change rpath of libdnnl.so.1, add $ORIGIN/ to it.
P
peizhilin 已提交
434
        # The reason is that all thirdparty libraries in the same directory,
A
Adam 已提交
435
        # thus, libdnnl.so.1 will find libmklml_intel.so and libiomp5.so.
P
peizhilin 已提交
436 437
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
A
Adam 已提交
438
            raise Exception("patch libdnnl.so failed, command: %s" % command)
S
Sang Ik Lee 已提交
439
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
A
Adam 已提交
440 441
    if os.name != 'nt':
        shutil.copy('${MKLDNN_SHARED_LIB_1}', libs_path)
442 443
        shutil.copy('${MKLDNN_SHARED_LIB_2}', libs_path)
        package_data['paddle.libs']+=['libmkldnn.so.0', 'libdnnl.so.1', 'libdnnl.so.2']
A
Adam 已提交
444 445 446
    else:
        package_data['paddle.libs']+=['mkldnn.dll']

447 448 449 450 451 452 453 454 455 456 457 458 459
if '${WITH_XPU}' == 'ON':
    # only change rpath in Release mode,
    if '${CMAKE_BUILD_TYPE}' == 'Release':
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${XPU_API_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${XPU_API_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${XPU_API_LIB} failed, command: %s" % command)
    shutil.copy('${XPU_API_LIB}', libs_path)
    shutil.copy('${XPU_RT_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_API_LIB_NAME}',
460
                                  '${XPU_RT_LIB_NAME}']
461

462 463 464 465
if '${WITH_XPU_BKCL}' == 'ON':
    shutil.copy('${XPU_BKCL_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']

466
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
467 468
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
469 470
package_dir['paddle.libs']=libs_path

471

472
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
473
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
474
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
475
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
476
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
477
    if os.name != 'nt':
478
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
479
        if "@APPLE@" == "1":
480 481
            commands = ["install_name_tool -id '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
            commands.append("install_name_tool -add_rpath '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so')
L
luotao1 已提交
482
        else:
483
            commands = ["patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
W
Wilber 已提交
484
        # The sw_64 not suppot patchelf, so we just disable that.
H
houj04 已提交
485
        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
486 487 488
            for command in commands:
                if os.system(command) != 0:
                    raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
489

P
peizhilin 已提交
490
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
491 492 493 494 495 496
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
497
    ext_modules = []
498 499
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
500

501
def find_files(pattern, root, recursive=False):
502
    for dirpath, _, files in os.walk(root):
503 504 505 506
        for filename in fnmatch.filter(files, pattern):
            yield os.path.join(dirpath, filename)
        if not recursive:
            break
507 508

headers = (
509
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle')) +
510 511 512
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/extension/include')) +  # extension
    # For paddle uew custom op, only copy data type headers from `paddle/fluid/platform`
    # to `extension/incude`,
513
    ['@PADDLE_SOURCE_DIR@/paddle/fluid/platform/complex.h'] +
514 515
    ['@PADDLE_SOURCE_DIR@/paddle/fluid/platform/float16.h'] +
    ['@PADDLE_SOURCE_DIR@/paddle/utils/any.h'])
516

517 518 519
if '${WITH_MKLDNN}' == 'ON':
    headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn

520
if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON':
521 522
    # externalErrorMsg.pb for External Error message
    headers += list(find_files('*.pb', '${externalError_INCLUDE_DIR}'))
523 524 525 526 527

class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_lib = self.install_platlib
528 529
        self.install_headers = os.path.join(self.install_platlib, 'paddle', 'include')
        
530
        return ret
531

532 533 534 535
class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
536

537 538 539 540 541
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
542

543
    boolean_options = ['force']
544

545 546 547 548
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
549

550 551 552 553
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
554

555 556 557 558
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
559
            # paddle headers
560
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
561
            if 'fluid' in install_dir or 'utils' in install_dir:
562
                install_dir = "paddle/extension/include/"
563 564 565
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
566
            patterns = ['install/mkldnn/include']
567 568
            for pattern in patterns:
                install_dir = re.sub(pattern, '', install_dir)
569 570 571 572
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
573

574 575 576 577 578 579 580 581
    def run(self):
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
582

583 584
    def get_inputs(self):
        return self.distribution.headers or []
585

586 587 588
    def get_outputs(self):
        return self.outfiles

589 590 591 592
# we redirect setuptools log for non-windows
if sys.platform != 'win32':
    @contextmanager
    def redirect_stdout():
593 594 595 596 597 598 599
        f_log = open('${SETUP_LOG_FILE}', 'w')
        origin_stdout = sys.stdout
        sys.stdout = f_log
        yield
        f_log = sys.stdout
        sys.stdout = origin_stdout
        f_log.close()
600 601 602 603 604
else:
    @contextmanager
    def redirect_stdout():
        yield

605 606
# Log for PYPI
if sys.version_info > (3,0):
607
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.rst", "r", encoding='UTF-8') as f:
608 609
        long_description = f.read()
else:
610
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.rst", "r")as f:
611 612
        long_description = unicode(f.read(), 'UTF-8')

W
wuhuanzhou 已提交
613 614 615 616 617 618
# strip *.so to reduce package size
if '${WITH_STRIP}' == 'ON':
    command = 'find ${PADDLE_BINARY_DIR}/python/paddle -name "*.so" | xargs -i strip {}'
    if os.system(command) != 0:
        raise Exception("strip *.so failed, command: %s" % command)

619 620 621 622
with redirect_stdout():
    setup(name='${PACKAGE_NAME}',
        version='${PADDLE_VERSION}',
        description='Parallel Distributed Deep Learning',
623 624 625 626 627 628 629 630 631 632
        long_description=long_description,
        long_description_content_type="text/markdown",
        author_email="Paddle-better@baidu.com",
        maintainer="PaddlePaddle",
        maintainer_email="Paddle-better@baidu.com",
        project_urls = {
            'Homepage': 'https://www.paddlepaddle.org.cn/',
            'Downloads': 'https://github.com/paddlepaddle/paddle'
        }, 
        license='Apache Software License',
633
        packages=packages,
634
        install_requires=setup_requires,
635 636 637 638 639 640 641 642 643
        ext_modules=ext_modules,
        package_data=package_data,
        package_dir=package_dir,
        scripts=paddle_bins,
        distclass=BinaryDistribution,
        headers=headers,
        cmdclass={
            'install_headers': InstallHeaders,
            'install': InstallCommand,
644 645 646
        },
        entry_points={
            'console_scripts': [
647
                'fleetrun = paddle.distributed.fleet.launch:launch'
648
            ]
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
        },
        classifiers=[
            'Development Status :: 5 - Production/Stable',
            'Operating System :: OS Independent',
            'Intended Audience :: Developers',
            'Intended Audience :: Education',
            'Intended Audience :: Science/Research',
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: C++',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: 3.6',
            'Programming Language :: Python :: 3.7',
            'Programming Language :: Python :: 3.8',
        ],
664 665
    )

666
# As there are a lot of files in purelib which causes many logs,
667
# we don't print them on the screen, and you can open `setup.py.log`
668
# for the full logs.
669 670
if os.path.exists('${SETUP_LOG_FILE}'):
    os.system('grep -v "purelib" ${SETUP_LOG_FILE}')