setup.py.in 23.7 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import os.path
4
import errno
5
import re
6
import shutil
7
import sys
8
import fnmatch
C
Chengmo 已提交
9
import errno
10
import platform
11

12
from contextlib import contextmanager
13 14 15 16 17
from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


18 19 20
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
21

Y
Yancey 已提交
22 23
RC      = 0

24
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
25 26 27 28

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
29 30
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
31 32
    except:
        git_commit = 'Unknown'
33
    git_commit = git_commit.decode()
34
    return str(git_commit)
Y
Yancey 已提交
35

36
def _get_version_detail(idx):
37 38
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
39

M
minqiyang 已提交
40 41
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
42

M
minqiyang 已提交
43
        if len(version_details) >= 3:
M
minqiyang 已提交
44
            return version_details[idx]
45

46
    return 0
47

M
minqiyang 已提交
48
def get_major():
49
    return int(_get_version_detail(0))
50

M
minqiyang 已提交
51
def get_minor():
52
    return int(_get_version_detail(1))
53

54
def get_patch():
55
    return str(_get_version_detail(2))
56

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
def get_cuda_version():
    if '@WITH_GPU@' == 'ON':
        return '@CUDA_VERSION@'
    else:
        return 'False'

def get_cudnn_version():
    if '@WITH_GPU@' == 'ON':
        temp_cudnn_version = ''
        if '@CUDNN_MAJOR_VERSION@':
            temp_cudnn_version += '@CUDNN_MAJOR_VERSION@'
            if '@CUDNN_MINOR_VERSION@':
                temp_cudnn_version += '.@CUDNN_MINOR_VERSION@'
                if '@CUDNN_PATCHLEVEL_VERSION@':
                    temp_cudnn_version += '.@CUDNN_PATCHLEVEL_VERSION@'
        return temp_cudnn_version
    else:
        return 'False'

76
def is_taged():
M
minqiyang 已提交
77
    try:
78
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
79
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
80
        git_tag = git_tag.decode()
M
minqiyang 已提交
81
    except:
82 83
        return False

84
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
85 86
        return True
    else:
M
minqiyang 已提交
87
        return False
88

89
def write_version_py(filename='paddle/version/__init__.py'):
90
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
91
#
92
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
93 94
major           = '%(major)d'
minor           = '%(minor)d'
95
patch           = '%(patch)s'
Y
Yancey 已提交
96
rc              = '%(rc)d'
97 98
cuda_version    = '%(cuda)s'
cudnn_version   = '%(cudnn)s'
Y
Yancey 已提交
99 100
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
101
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
102

103 104
__all__ = ['cuda', 'cudnn']

Y
Yancey 已提交
105 106
def show():
    if istaged:
107 108 109 110 111
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
112
    else:
113
        print('commit:', commit)
L
Luo Tao 已提交
114 115 116

def mkl():
    return with_mkl
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150

def cuda():
    """Get cuda version of paddle package.

    Returns:
        string: Return the version information of cuda. If paddle package is CPU version, it will return False.
    
    Examples:
        .. code-block:: python

            import paddle

            paddle.version.cuda()
            # '10.2'

    """
    return cuda_version

def cudnn():
    """Get cudnn version of paddle package.

    Returns:
        string: Return the version information of cudnn. If paddle package is CPU version, it will return False.
    
    Examples:
        .. code-block:: python

            import paddle

            paddle.version.cudnn()
            # '7.6.5'

    """
    return cudnn_version
Y
Yancey 已提交
151 152
'''
    commit = git_commit()
153 154 155 156 157 158 159 160 161

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

Y
Yancey 已提交
162 163
    with open(filename, 'w') as f:
        f.write(cnt % {
164 165 166
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
167 168
            'rc': RC,
            'version': '${PADDLE_VERSION}',
169 170
            'cuda': get_cuda_version(),
            'cudnn': get_cudnn_version(),
Y
Yancey 已提交
171
            'commit': commit,
172
            'istaged': is_taged(),
L
Luo Tao 已提交
173
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
174

175
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version/__init__.py')
Y
Yancey 已提交
176

177 178 179 180 181 182 183 184 185 186 187 188 189 190
def write_cuda_env_config_py(filename='paddle/cuda_env.py'):
    cnt = ""
    if '${JIT_RELEASE_WHL}' == 'ON':
        cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
#
import os
os.environ['CUDA_CACHE_MAXSIZE'] = '805306368'
'''

    with open(filename, 'w') as f:
        f.write(cnt)

write_cuda_env_config_py(filename='@PADDLE_BINARY_DIR@/python/paddle/cuda_env.py')

C
Chengmo 已提交
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
def write_distributed_training_mode_py(filename='paddle/fluid/incubate/fleet/parameter_server/version.py'):
    cnt = '''from __future__ import print_function

# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY

from paddle.fluid.incubate.fleet.base.mode import Mode

BUILD_MODE=Mode.%(mode)s

def is_transpiler():
    return Mode.TRANSPILER == BUILD_MODE

'''

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    with open(filename, 'w') as f:
        f.write(cnt % {
            'mode': 'PSLIB' if '${WITH_PSLIB}' == 'ON' else 'TRANSPILER'
        })

write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/fluid/incubate/fleet/parameter_server/version.py')
Y
Yancey 已提交
219

220

Z
zhangjinchao01 已提交
221
packages=['paddle',
222
          'paddle.libs',
Q
qiaolongfei 已提交
223
          'paddle.utils',
224
          'paddle.utils.gast',
225
          'paddle.utils.cpp_extension',
226 227
          'paddle.dataset',
          'paddle.reader',
228
          'paddle.distributed',
229
          'paddle.incubate',
230
          'paddle.incubate.optimizer',
231
          'paddle.incubate.checkpoint',
232
          'paddle.incubate.operators',
233
          'paddle.incubate.tensor',
234
          'paddle.incubate.nn',
235 236
          'paddle.distributed.fleet',
          'paddle.distributed.fleet.base',
K
kuizhiqing 已提交
237
          'paddle.distributed.fleet.elastic',
238
          'paddle.distributed.fleet.meta_optimizers',
239
          'paddle.distributed.fleet.meta_optimizers.sharding',
240
          'paddle.distributed.fleet.meta_optimizers.ascend',
241
          'paddle.distributed.fleet.meta_optimizers.dygraph_optimizer',
242 243
          'paddle.distributed.fleet.runtime',
          'paddle.distributed.fleet.dataset',
244
          'paddle.distributed.fleet.data_generator',
245 246 247
          'paddle.distributed.fleet.metrics',
          'paddle.distributed.fleet.proto',
          'paddle.distributed.fleet.utils',
248
          'paddle.distributed.fleet.meta_parallel',
249
          'paddle.distributed.fleet.meta_parallel.pp_utils',
250
          'paddle.distributed.fleet.meta_parallel.parallel_layers',
251
          'paddle.distributed.auto_parallel',
252
          'paddle.distributed.auto_parallel.operators',
253
          'paddle.framework',
254
          'paddle.jit',
255
          'paddle.jit.dy2static',
W
Wilber 已提交
256
          'paddle.inference',
257 258
          'paddle.inference.contrib',
          'paddle.inference.contrib.utils',
259
          'paddle.fluid',
W
Wilber 已提交
260
          'paddle.fluid.inference',
L
lujun 已提交
261
          'paddle.fluid.dygraph',
262
          'paddle.fluid.dygraph.dygraph_to_static',
263
          'paddle.fluid.dygraph.amp',
264
          'paddle.fluid.proto',
X
Xin Pan 已提交
265
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
266
          'paddle.fluid.distributed',
Y
Yancey 已提交
267
          'paddle.fluid.layers',
268
          'paddle.fluid.dataloader',
Q
Qingsheng Li 已提交
269 270
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
271
          'paddle.fluid.contrib.quantize',
W
whs 已提交
272
          'paddle.fluid.contrib.slim',
W
WangZhen 已提交
273
          'paddle.fluid.contrib.slim.quantization',
274
          'paddle.fluid.contrib.slim.quantization.imperative',
C
chengduo 已提交
275
          'paddle.fluid.contrib.extend_optimizer',
276
          'paddle.fluid.contrib.mixed_precision',
277
          'paddle.fluid.contrib.mixed_precision.bf16',
278
          'paddle.fluid.contrib.layers',
279
          'paddle.fluid.contrib.sparsity',
Q
qiaolongfei 已提交
280
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
281 282
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
283
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
284
          'paddle.fluid.incubate.fleet',
285
          'paddle.fluid.incubate.checkpoint',
D
dongdaxiang 已提交
286 287
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
288
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
289
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
290
          'paddle.fluid.incubate.fleet.parameter_server.ir',
291
          'paddle.fluid.incubate.fleet.collective',
292
          'paddle.fluid.incubate.fleet.utils',
293
          'paddle.amp',
294 295 296 297 298 299 300
          'paddle.hapi',
          'paddle.vision',
          'paddle.vision.models',
          'paddle.vision.transforms',
          'paddle.vision.datasets',
          'paddle.text',
          'paddle.text.datasets',
H
hong 已提交
301
          'paddle.incubate',
302 303 304
          'paddle.incubate.nn',
          'paddle.incubate.nn.functional',
          'paddle.incubate.nn.layer',
305
          'paddle.io',
H
hong 已提交
306
          'paddle.optimizer',
307 308
          'paddle.nn',
          'paddle.nn.functional',
309
          'paddle.nn.layer',
310
          'paddle.nn.quant',
H
hong 已提交
311
          'paddle.nn.initializer',
312
          'paddle.nn.utils',
H
hong 已提交
313
          'paddle.metric',
314 315
          'paddle.static',
          'paddle.static.nn',
316
          'paddle.static.amp',
317
          'paddle.static.sparsity',
318
          'paddle.tensor',
C
channings 已提交
319
          'paddle.onnx',
320
          'paddle.autograd',
321 322
          'paddle.device',
          'paddle.device.cuda',
323
          'paddle.version',
324
          ]
L
Luo Tao 已提交
325

326 327
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
328

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
# Note(wangzhongpu):
# When compiling paddle under python36, the dependencies belonging to python2.7 will be imported, resulting in errors when installing paddle
if sys.version_info >= (3,6) and sys.version_info < (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,5) and sys.version_info < (3,6):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<=\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i or "<\"3.7\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp

353
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
354
paddle_bins = ''
355

T
Tao Luo 已提交
356 357
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
358 359 360 361 362 363

if os.name != 'nt':
    package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.so']}
else:
    package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.pyd', '${FLUID_CORE_NAME}' + '.lib']}

364
if '${HAS_NOAVX_CORE}' == 'ON':
365
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
366

L
Luo Tao 已提交
367
package_dir={
368
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
369 370 371 372
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
373
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
374
}
375

376 377
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
378

P
peizhilin 已提交
379
package_data['paddle.libs']= []
P
peizhilin 已提交
380
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
381
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
382

383 384 385 386 387 388 389 390 391 392 393 394 395
package_data['paddle.libs']+=[
    os.path.basename('${LAPACK_LIB}'), 
    os.path.basename('${BLAS_LIB}'),
    os.path.basename('${GFORTRAN_LIB}'),
    os.path.basename('${GNU_RT_LIB_1}')]
shutil.copy('${BLAS_LIB}', libs_path)
shutil.copy('${LAPACK_LIB}', libs_path)
shutil.copy('${GFORTRAN_LIB}', libs_path)
shutil.copy('${GNU_RT_LIB_1}', libs_path)
if not sys.platform.startswith("linux"):
    package_data['paddle.libs']+=[os.path.basename('${GNU_RT_LIB_2}')]
    shutil.copy('${GNU_RT_LIB_2}', libs_path)

396
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
397 398 399
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
P
peizhilin 已提交
400 401
else:
    if os.name == 'nt':
P
peizhilin 已提交
402
        # copy the openblas.dll
403
        shutil.copy('${OPENBLAS_SHARED_LIB}', libs_path)
P
peizhilin 已提交
404
        package_data['paddle.libs'] += ['openblas' + ext_name]
W
Wilber 已提交
405
    elif os.name == 'posix' and platform.machine() == 'aarch64' and '${OPENBLAS_LIB}'.endswith('so'):
H
houj04 已提交
406 407
        # copy the libopenblas.so on linux+aarch64
        # special: core_noavx.so depends on 'libopenblas.so.0', not 'libopenblas.so'
W
Wilber 已提交
408 409 410
        if os.path.exists('${OPENBLAS_LIB}' + '.0'):
            shutil.copy('${OPENBLAS_LIB}' + '.0', libs_path)
            package_data['paddle.libs'] += ['libopenblas.so.0']
P
peizhilin 已提交
411

412 413 414
if '${WITH_LITE}' == 'ON':
    shutil.copy('${LITE_SHARED_LIB}', libs_path)
    package_data['paddle.libs']+=['libpaddle_full_api_shared' + ext_name]
415 416 417 418 419 420
    if '${LITE_WITH_NNADAPTER}' == 'ON':
        shutil.copy('${LITE_NNADAPTER_LIB}', libs_path)
        package_data['paddle.libs']+=['libnnadapter' + ext_name]
        if '${NNADAPTER_WITH_HUAWEI_ASCEND_NPU}' == 'ON':
            shutil.copy('${LITE_NNADAPTER_NPU_LIB}', libs_path)
            package_data['paddle.libs']+=['libnnadapter_driver_huawei_ascend_npu' + ext_name]
421

X
xujiaqi01 已提交
422 423
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
X
xujiaqi01 已提交
424 425
    if os.path.exists('${PSLIB_VERSION_PY}'):
        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/')
X
xujiaqi01 已提交
426 427
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
428
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
429
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
430 431 432 433
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
A
Adam 已提交
434
        # change rpath of libdnnl.so.1, add $ORIGIN/ to it.
P
peizhilin 已提交
435
        # The reason is that all thirdparty libraries in the same directory,
A
Adam 已提交
436
        # thus, libdnnl.so.1 will find libmklml_intel.so and libiomp5.so.
P
peizhilin 已提交
437 438
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
A
Adam 已提交
439
            raise Exception("patch libdnnl.so failed, command: %s" % command)
S
Sang Ik Lee 已提交
440
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
A
Adam 已提交
441 442
    if os.name != 'nt':
        shutil.copy('${MKLDNN_SHARED_LIB_1}', libs_path)
443 444
        shutil.copy('${MKLDNN_SHARED_LIB_2}', libs_path)
        package_data['paddle.libs']+=['libmkldnn.so.0', 'libdnnl.so.1', 'libdnnl.so.2']
A
Adam 已提交
445 446 447
    else:
        package_data['paddle.libs']+=['mkldnn.dll']

448 449 450 451 452 453 454 455 456 457 458 459 460
if '${WITH_XPU}' == 'ON':
    # only change rpath in Release mode,
    if '${CMAKE_BUILD_TYPE}' == 'Release':
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${XPU_API_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${XPU_API_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${XPU_API_LIB} failed, command: %s" % command)
    shutil.copy('${XPU_API_LIB}', libs_path)
    shutil.copy('${XPU_RT_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_API_LIB_NAME}',
461
                                  '${XPU_RT_LIB_NAME}']
462

463 464 465 466
if '${WITH_XPU_BKCL}' == 'ON':
    shutil.copy('${XPU_BKCL_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']

467
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
468 469
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
470 471
package_dir['paddle.libs']=libs_path

472

473
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
474
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
475
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
476
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
477
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
478
    if os.name != 'nt':
479
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
480
        if "@APPLE@" == "1":
481 482
            commands = ["install_name_tool -id '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
            commands.append("install_name_tool -add_rpath '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so')
L
luotao1 已提交
483
        else:
484
            commands = ["patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
W
Wilber 已提交
485
        # The sw_64 not suppot patchelf, so we just disable that.
H
houj04 已提交
486
        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
487 488 489
            for command in commands:
                if os.system(command) != 0:
                    raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
490

P
peizhilin 已提交
491
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
492 493 494 495 496 497
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
498
    ext_modules = []
499 500
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
501

502
def find_files(pattern, root, recursive=False):
503
    for dirpath, _, files in os.walk(root):
504 505 506 507
        for filename in fnmatch.filter(files, pattern):
            yield os.path.join(dirpath, filename)
        if not recursive:
            break
508 509

headers = (
510
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle')) +
511 512 513
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/extension/include')) +  # extension
    # For paddle uew custom op, only copy data type headers from `paddle/fluid/platform`
    # to `extension/incude`,
514
    ['@PADDLE_SOURCE_DIR@/paddle/fluid/platform/complex.h'] +
515 516
    ['@PADDLE_SOURCE_DIR@/paddle/fluid/platform/float16.h'] +
    ['@PADDLE_SOURCE_DIR@/paddle/utils/any.h'])
517

518 519 520
if '${WITH_MKLDNN}' == 'ON':
    headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn

521
if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON':
522 523
    # externalErrorMsg.pb for External Error message
    headers += list(find_files('*.pb', '${externalError_INCLUDE_DIR}'))
524 525 526 527 528

class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_lib = self.install_platlib
529 530
        self.install_headers = os.path.join(self.install_platlib, 'paddle', 'include')
        
531
        return ret
532

533 534 535 536
class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
537

538 539 540 541 542
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
543

544
    boolean_options = ['force']
545

546 547 548 549
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
550

551 552 553 554
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
555

556 557 558 559
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
560
            # paddle headers
561
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
562
            if 'fluid' in install_dir or 'utils' in install_dir:
563
                install_dir = "paddle/extension/include/"
564 565 566
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
567
            patterns = ['install/mkldnn/include']
568 569
            for pattern in patterns:
                install_dir = re.sub(pattern, '', install_dir)
570 571 572 573
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
574

575 576 577 578 579 580 581 582
    def run(self):
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
583

584 585
    def get_inputs(self):
        return self.distribution.headers or []
586

587 588 589
    def get_outputs(self):
        return self.outfiles

590 591 592 593
# we redirect setuptools log for non-windows
if sys.platform != 'win32':
    @contextmanager
    def redirect_stdout():
594 595 596 597 598 599 600
        f_log = open('${SETUP_LOG_FILE}', 'w')
        origin_stdout = sys.stdout
        sys.stdout = f_log
        yield
        f_log = sys.stdout
        sys.stdout = origin_stdout
        f_log.close()
601 602 603 604 605
else:
    @contextmanager
    def redirect_stdout():
        yield

606 607
# Log for PYPI
if sys.version_info > (3,0):
608
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.rst", "r", encoding='UTF-8') as f:
609 610
        long_description = f.read()
else:
611
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.rst", "r")as f:
612 613
        long_description = unicode(f.read(), 'UTF-8')

W
wuhuanzhou 已提交
614 615 616 617 618 619
# strip *.so to reduce package size
if '${WITH_STRIP}' == 'ON':
    command = 'find ${PADDLE_BINARY_DIR}/python/paddle -name "*.so" | xargs -i strip {}'
    if os.system(command) != 0:
        raise Exception("strip *.so failed, command: %s" % command)

620 621 622 623
with redirect_stdout():
    setup(name='${PACKAGE_NAME}',
        version='${PADDLE_VERSION}',
        description='Parallel Distributed Deep Learning',
624 625 626 627 628 629 630 631 632 633
        long_description=long_description,
        long_description_content_type="text/markdown",
        author_email="Paddle-better@baidu.com",
        maintainer="PaddlePaddle",
        maintainer_email="Paddle-better@baidu.com",
        project_urls = {
            'Homepage': 'https://www.paddlepaddle.org.cn/',
            'Downloads': 'https://github.com/paddlepaddle/paddle'
        }, 
        license='Apache Software License',
634
        packages=packages,
635
        install_requires=setup_requires,
636 637 638 639 640 641 642 643 644
        ext_modules=ext_modules,
        package_data=package_data,
        package_dir=package_dir,
        scripts=paddle_bins,
        distclass=BinaryDistribution,
        headers=headers,
        cmdclass={
            'install_headers': InstallHeaders,
            'install': InstallCommand,
645 646 647
        },
        entry_points={
            'console_scripts': [
648
                'fleetrun = paddle.distributed.fleet.launch:launch'
649
            ]
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
        },
        classifiers=[
            'Development Status :: 5 - Production/Stable',
            'Operating System :: OS Independent',
            'Intended Audience :: Developers',
            'Intended Audience :: Education',
            'Intended Audience :: Science/Research',
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: C++',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: 3.6',
            'Programming Language :: Python :: 3.7',
            'Programming Language :: Python :: 3.8',
        ],
665 666
    )

667
# As there are a lot of files in purelib which causes many logs,
668
# we don't print them on the screen, and you can open `setup.py.log`
669
# for the full logs.
670 671
if os.path.exists('${SETUP_LOG_FILE}'):
    os.system('grep -v "purelib" ${SETUP_LOG_FILE}')