setup.py.in 22.7 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import os.path
4
import errno
5
import re
6
import shutil
7
import sys
8
import fnmatch
C
Chengmo 已提交
9
import errno
10
import platform
11

12
from contextlib import contextmanager
13 14 15 16 17
from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


18 19 20
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
21

Y
Yancey 已提交
22 23
RC      = 0

24
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
25 26 27 28

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
29 30
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
31 32
    except:
        git_commit = 'Unknown'
33
    git_commit = git_commit.decode()
34
    return str(git_commit)
Y
Yancey 已提交
35

36
def _get_version_detail(idx):
37 38
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
39

M
minqiyang 已提交
40 41
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
42

M
minqiyang 已提交
43
        if len(version_details) >= 3:
M
minqiyang 已提交
44
            return version_details[idx]
45

46
    return 0
47

M
minqiyang 已提交
48
def get_major():
49
    return int(_get_version_detail(0))
50

M
minqiyang 已提交
51
def get_minor():
52
    return int(_get_version_detail(1))
53

54
def get_patch():
55
    return str(_get_version_detail(2))
56 57

def is_taged():
M
minqiyang 已提交
58
    try:
59
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
60
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
61
        git_tag = git_tag.decode()
M
minqiyang 已提交
62
    except:
63 64
        return False

65
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
66 67
        return True
    else:
M
minqiyang 已提交
68
        return False
69

Y
Yancey 已提交
70
def write_version_py(filename='paddle/version.py'):
71
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
72
#
73
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
74 75
major           = '%(major)d'
minor           = '%(minor)d'
76
patch           = '%(patch)s'
Y
Yancey 已提交
77 78 79
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
80
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
81 82 83

def show():
    if istaged:
84 85 86 87 88
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
89
    else:
90
        print('commit:', commit)
L
Luo Tao 已提交
91 92 93

def mkl():
    return with_mkl
Y
Yancey 已提交
94 95 96 97
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
98 99 100
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
101 102 103
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
104
            'istaged': is_taged(),
L
Luo Tao 已提交
105
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
106

107
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
108

C
Chengmo 已提交
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
def write_distributed_training_mode_py(filename='paddle/fluid/incubate/fleet/parameter_server/version.py'):
    cnt = '''from __future__ import print_function

# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY

from paddle.fluid.incubate.fleet.base.mode import Mode

BUILD_MODE=Mode.%(mode)s

def is_transpiler():
    return Mode.TRANSPILER == BUILD_MODE

'''

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    with open(filename, 'w') as f:
        f.write(cnt % {
            'mode': 'PSLIB' if '${WITH_PSLIB}' == 'ON' else 'TRANSPILER'
        })

write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/fluid/incubate/fleet/parameter_server/version.py')
Y
Yancey 已提交
137

138

Z
zhangjinchao01 已提交
139
packages=['paddle',
140
          'paddle.libs',
Q
qiaolongfei 已提交
141
          'paddle.utils',
142
          'paddle.utils.cpp_extension',
143 144
          'paddle.dataset',
          'paddle.reader',
145
          'paddle.distributed',
146
          'paddle.incubate',
147
          'paddle.incubate.optimizer',
148 149 150
          'paddle.distributed.fleet',
          'paddle.distributed.fleet.base',
          'paddle.distributed.fleet.meta_optimizers',
151
          'paddle.distributed.fleet.meta_optimizers.sharding',
152 153
          'paddle.distributed.fleet.runtime',
          'paddle.distributed.fleet.dataset',
154
          'paddle.distributed.fleet.data_generator',
155 156 157
          'paddle.distributed.fleet.metrics',
          'paddle.distributed.fleet.proto',
          'paddle.distributed.fleet.utils',
158
          'paddle.framework',
159
          'paddle.jit',
160
          'paddle.jit.dy2static',
W
Wilber 已提交
161
          'paddle.inference',
162
          'paddle.fluid',
W
Wilber 已提交
163
          'paddle.fluid.inference',
L
lujun 已提交
164
          'paddle.fluid.dygraph',
165
          'paddle.fluid.dygraph.dygraph_to_static',
166
          'paddle.fluid.dygraph.amp',
167
          'paddle.fluid.proto',
X
Xin Pan 已提交
168
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
169
          'paddle.fluid.distributed',
Y
Yancey 已提交
170
          'paddle.fluid.layers',
171
          'paddle.fluid.dataloader',
Q
Qingsheng Li 已提交
172 173
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
174
          'paddle.fluid.contrib.quantize',
Q
Qiao Longfei 已提交
175
          'paddle.fluid.contrib.reader',
W
whs 已提交
176
          'paddle.fluid.contrib.slim',
W
WangZhen 已提交
177
          'paddle.fluid.contrib.slim.quantization',
178
          'paddle.fluid.contrib.slim.quantization.imperative',
T
tangwei12 已提交
179
          'paddle.fluid.contrib.utils',
C
chengduo 已提交
180
          'paddle.fluid.contrib.extend_optimizer',
181
          'paddle.fluid.contrib.mixed_precision',
182
          'paddle.fluid.contrib.mixed_precision.bf16',
183
          'paddle.fluid.contrib.layers',
Q
qiaolongfei 已提交
184
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
185 186
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
187
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
188
          'paddle.fluid.incubate.fleet',
189
          'paddle.fluid.incubate.checkpoint',
D
dongdaxiang 已提交
190 191
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
192
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
193
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
194
          'paddle.fluid.incubate.fleet.parameter_server.ir',
195
          'paddle.fluid.incubate.fleet.collective',
196
          'paddle.fluid.incubate.fleet.utils',
197
          'paddle.amp',
198 199 200 201 202 203 204
          'paddle.hapi',
          'paddle.vision',
          'paddle.vision.models',
          'paddle.vision.transforms',
          'paddle.vision.datasets',
          'paddle.text',
          'paddle.text.datasets',
H
hong 已提交
205
          'paddle.incubate',
206
          'paddle.io',
H
hong 已提交
207
          'paddle.optimizer',
208 209
          'paddle.nn',
          'paddle.nn.functional',
210
          'paddle.nn.layer',
H
hong 已提交
211
          'paddle.nn.initializer',
212
          'paddle.nn.utils',
H
hong 已提交
213
          'paddle.metric',
214 215
          'paddle.static',
          'paddle.static.nn',
216
          'paddle.static.amp',
217
          'paddle.tensor',
C
channings 已提交
218
          'paddle.onnx',
219
          ]
L
Luo Tao 已提交
220

221 222
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
223

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
# Note(wangzhongpu):
# When compiling paddle under python36, the dependencies belonging to python2.7 will be imported, resulting in errors when installing paddle
if sys.version_info >= (3,6) and sys.version_info < (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,5) and sys.version_info < (3,6):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<=\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i or "<\"3.7\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp

248
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
249
paddle_bins = ''
250

T
Tao Luo 已提交
251 252
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
253
package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]}
254
if '${HAS_NOAVX_CORE}' == 'ON':
255
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
256

257

L
Luo Tao 已提交
258
package_dir={
259
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
260 261 262 263
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
264
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
265
}
266

267 268
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
269

P
peizhilin 已提交
270
package_data['paddle.libs']= []
P
peizhilin 已提交
271
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
272
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
273

274
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
275 276 277
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
P
peizhilin 已提交
278 279
else:
    if os.name == 'nt':
P
peizhilin 已提交
280
        # copy the openblas.dll
281
        shutil.copy('${OPENBLAS_SHARED_LIB}', libs_path)
P
peizhilin 已提交
282
        package_data['paddle.libs'] += ['openblas' + ext_name]
W
Wilber 已提交
283
    elif os.name == 'posix' and platform.machine() == 'aarch64' and '${OPENBLAS_LIB}'.endswith('so'):
H
houj04 已提交
284 285
        # copy the libopenblas.so on linux+aarch64
        # special: core_noavx.so depends on 'libopenblas.so.0', not 'libopenblas.so'
W
Wilber 已提交
286 287 288
        if os.path.exists('${OPENBLAS_LIB}' + '.0'):
            shutil.copy('${OPENBLAS_LIB}' + '.0', libs_path)
            package_data['paddle.libs'] += ['libopenblas.so.0']
P
peizhilin 已提交
289

290 291 292 293
if '${WITH_LITE}' == 'ON':
    shutil.copy('${LITE_SHARED_LIB}', libs_path)
    package_data['paddle.libs']+=['libpaddle_full_api_shared' + ext_name]

X
xujiaqi01 已提交
294 295
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
X
xujiaqi01 已提交
296 297
    if os.path.exists('${PSLIB_VERSION_PY}'):
        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/')
X
xujiaqi01 已提交
298 299
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
300
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
301
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
302 303 304 305
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
A
Adam 已提交
306
        # change rpath of libdnnl.so.1, add $ORIGIN/ to it.
P
peizhilin 已提交
307
        # The reason is that all thirdparty libraries in the same directory,
A
Adam 已提交
308
        # thus, libdnnl.so.1 will find libmklml_intel.so and libiomp5.so.
P
peizhilin 已提交
309 310
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
A
Adam 已提交
311
            raise Exception("patch libdnnl.so failed, command: %s" % command)
S
Sang Ik Lee 已提交
312
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
A
Adam 已提交
313 314
    if os.name != 'nt':
        shutil.copy('${MKLDNN_SHARED_LIB_1}', libs_path)
315 316
        shutil.copy('${MKLDNN_SHARED_LIB_2}', libs_path)
        package_data['paddle.libs']+=['libmkldnn.so.0', 'libdnnl.so.1', 'libdnnl.so.2']
A
Adam 已提交
317 318 319
    else:
        package_data['paddle.libs']+=['mkldnn.dll']

320 321 322 323 324 325 326 327 328 329 330 331 332
if '${WITH_XPU}' == 'ON':
    # only change rpath in Release mode,
    if '${CMAKE_BUILD_TYPE}' == 'Release':
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${XPU_API_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${XPU_API_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${XPU_API_LIB} failed, command: %s" % command)
    shutil.copy('${XPU_API_LIB}', libs_path)
    shutil.copy('${XPU_RT_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_API_LIB_NAME}',
333
                                  '${XPU_RT_LIB_NAME}']
334

335 336 337 338
if '${WITH_XPU_BKCL}' == 'ON':
    shutil.copy('${XPU_BKCL_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']

339 340 341 342 343 344 345 346 347 348 349
# Only for lite xpu inference.
if '${WITH_XPU}' == 'OFF' and '${XPU_SDK_ROOT}' != '':
    xpu_api_lib = os.path.join('${XPU_SDK_ROOT}', 'XTDK/shlib/', 'libxpuapi.so')
    xpu_rt_lib = os.path.join('${XPU_SDK_ROOT}', 'XTDK/runtime/shlib/', 'libxpurt.so')
    if os.path.exists(xpu_api_lib):
        shutil.copy(xpu_api_lib, libs_path)
        package_data['paddle.libs']+=['libxpuapi.so']
    if os.path.exists(xpu_rt_lib):
        shutil.copy(xpu_rt_lib, libs_path)
        package_data['paddle.libs']+=['libxpurt.so']

350
### Old custom op extension mechanism related, will be removed in 2.1.0 ###
351 352 353 354 355
# copy libpaddle_framework.so to libs on linux
if sys.platform.startswith('linux'):
    shutil.copy('${FLUID_FRAMEWORK_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['libpaddle_framework.so']

356
### New custom op extension mechanism related ###
357 358 359 360 361
# copy libpaddle_custom_op.so to libs on linux
if sys.platform.startswith('linux'):
    shutil.copy('${PADDLE_CUSTOM_OP_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['libpaddle_custom_op.so']

362
# copy paddle_custom_op.lib/paddle_custom_op.dll to libs on Windows
363
if os.name == 'nt':
364 365 366
    shutil.copy('${PADDLE_CUSTOM_OP_IMPORT_LIB}', libs_path)
    shutil.copy('${PADDLE_CUSTOM_OP_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['paddle_custom_op.lib', 'paddle_custom_op.dll']
367

368
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
369 370
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
371 372
package_dir['paddle.libs']=libs_path

373

374
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
375
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
376
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
377
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
378
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
379
    if os.name != 'nt':
380
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
381
        if "@APPLE@" == "1":
382
            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
383
        else:
384
            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
W
Wilber 已提交
385
        # The sw_64 not suppot patchelf, so we just disable that.
H
houj04 已提交
386
        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
387 388
          if os.system(command) != 0:
              raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
389

P
peizhilin 已提交
390
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
391 392 393 394 395 396
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
397
    ext_modules = []
398 399
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
400

401 402 403 404 405 406
def find_files(pattern, root):
    for dirpath, _, files in os.walk(root):
      for filename in fnmatch.filter(files, pattern):
          yield os.path.join(dirpath, filename)

headers = (
407 408
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/extension')) +
409
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/framework')) +
410
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/imperative')) +
411 412 413
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/memory')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/string')) +
414
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/platform')) +
415
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/framework')) +
416
    list(find_files('*.pb', '${cudaerror_INCLUDE_DIR}')) + # errorMessage.pb for errormessage
417 418 419 420 421 422 423 424
    ['${EIGEN_INCLUDE_DIR}/Eigen/Core'] + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/Eigen/src')) + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/unsupported/Eigen')) + # eigen
    list(find_files('*', '${GFLAGS_INSTALL_DIR}/include')) + # gflags
    list(find_files('*', '${GLOG_INSTALL_DIR}/include')) + # glog
    list(find_files('*', '${BOOST_INCLUDE_DIR}/boost')) + # boost
    list(find_files('*', '${XXHASH_INSTALL_DIR}/include')) + # xxhash
    list(find_files('*', '${PROTOBUF_INCLUDE_DIR}')) + # protobuf
425
    list(find_files('*', '${DLPACK_INCLUDE_DIR}')) + # dlpack
426 427
    list(find_files('*.h', '${THREADPOOL_INCLUDE_DIR}'))) # threadpool

428 429 430
if '${WITH_MKLDNN}' == 'ON':
    headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn

431
if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON':
432
    headers += list(find_files('*.pb', '${cudaerror_INCLUDE_DIR}'))   # errorMessage.pb for errormessage
433 434 435 436 437

class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_lib = self.install_platlib
438 439
        self.install_headers = os.path.join(self.install_platlib, 'paddle',
                                            'include')
440
        return ret
441

442 443 444 445 446

class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
447

448 449 450 451 452
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
453

454
    boolean_options = ['force']
455

456 457 458 459
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
460

461 462 463 464
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
465

466 467
    def copy_data_type_headers(self, header):
        if os.name == 'nt':
468
            data_type_headers = ['platform\\complex64.h', 'platform\\complex128.h', 'platform\\float16.h']
469
        else:
470
            data_type_headers = ['platform/complex64.h', 'platform/complex128.h', 'platform/float16.h']
471 472
        for dtype_header in data_type_headers:
            if dtype_header in header:
473
                install_dir = os.path.join(self.install_dir, "paddle/fluid/extension/include")
474 475 476 477
                if not os.path.exists(install_dir):
                    self.mkpath(install_dir)
                return self.copy_file(header, install_dir)

478 479 480 481
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
482
            # paddle headers
483
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
484 485 486
            # For paddle data type headers, we also need to copy to `extension/incude`,
            # used for new custom operator
            self.copy_data_type_headers(header)
487 488 489
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
490 491 492 493 494
            patterns = ['eigen3/src/extern_eigen3', 'boost/src/extern_boost',
                       'dlpack/src/extern_dlpack/include',
                       'install/protobuf/include',
                       'install/gflags/include',
                       'install/glog/include', 'install/xxhash/include',
495
                       'install/mkldnn/include',
496 497 498
                       'threadpool/src/extern_threadpool']
            for pattern in patterns:
                install_dir = re.sub(pattern, '', install_dir)
499 500 501 502
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
503

504 505 506 507 508 509 510 511
    def run(self):
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
512

513 514
    def get_inputs(self):
        return self.distribution.headers or []
515

516 517 518
    def get_outputs(self):
        return self.outfiles

519 520 521 522
# we redirect setuptools log for non-windows
if sys.platform != 'win32':
    @contextmanager
    def redirect_stdout():
523 524 525 526 527 528 529
        f_log = open('${SETUP_LOG_FILE}', 'w')
        origin_stdout = sys.stdout
        sys.stdout = f_log
        yield
        f_log = sys.stdout
        sys.stdout = origin_stdout
        f_log.close()
530 531 532 533 534
else:
    @contextmanager
    def redirect_stdout():
        yield

535 536 537 538 539 540 541 542
# Log for PYPI
if sys.version_info > (3,0):
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.md", "r", encoding='UTF-8') as f:
        long_description = f.read()
else:
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.md", "r")as f:
        long_description = unicode(f.read(), 'UTF-8')

543 544 545 546
with redirect_stdout():
    setup(name='${PACKAGE_NAME}',
        version='${PADDLE_VERSION}',
        description='Parallel Distributed Deep Learning',
547 548 549 550 551 552 553 554 555 556
        long_description=long_description,
        long_description_content_type="text/markdown",
        author_email="Paddle-better@baidu.com",
        maintainer="PaddlePaddle",
        maintainer_email="Paddle-better@baidu.com",
        project_urls = {
            'Homepage': 'https://www.paddlepaddle.org.cn/',
            'Downloads': 'https://github.com/paddlepaddle/paddle'
        }, 
        license='Apache Software License',
557
        packages=packages,
558
        install_requires=setup_requires,
559 560 561 562 563 564 565 566 567
        ext_modules=ext_modules,
        package_data=package_data,
        package_dir=package_dir,
        scripts=paddle_bins,
        distclass=BinaryDistribution,
        headers=headers,
        cmdclass={
            'install_headers': InstallHeaders,
            'install': InstallCommand,
568 569 570
        },
        entry_points={
            'console_scripts': [
571
                'fleetrun = paddle.distributed.fleet.launch:launch'
572
            ]
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
        },
        classifiers=[
            'Development Status :: 5 - Production/Stable',
            'Operating System :: OS Independent',
            'Intended Audience :: Developers',
            'Intended Audience :: Education',
            'Intended Audience :: Science/Research',
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: C++',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: 3.6',
            'Programming Language :: Python :: 3.7',
            'Programming Language :: Python :: 3.8',
        ],
588 589
    )

590
# As there are a lot of files in purelib which causes many logs,
591
# we don't print them on the screen, and you can open `setup.py.log`
592
# for the full logs.
593 594
if os.path.exists('${SETUP_LOG_FILE}'):
    os.system('grep -v "purelib" ${SETUP_LOG_FILE}')