setup.py.in 22.2 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import os.path
4
import errno
5
import re
6
import shutil
7
import sys
8
import fnmatch
C
Chengmo 已提交
9
import errno
10
import platform
11

12
from contextlib import contextmanager
13 14 15 16 17
from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


18 19 20
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
21

Y
Yancey 已提交
22 23
RC      = 0

24
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
25 26 27 28

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
29 30
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
31 32
    except:
        git_commit = 'Unknown'
33
    git_commit = git_commit.decode()
34
    return str(git_commit)
Y
Yancey 已提交
35

36
def _get_version_detail(idx):
37 38
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
39

M
minqiyang 已提交
40 41
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
42

M
minqiyang 已提交
43
        if len(version_details) >= 3:
M
minqiyang 已提交
44
            return version_details[idx]
45

46
    return 0
47

M
minqiyang 已提交
48
def get_major():
49
    return int(_get_version_detail(0))
50

M
minqiyang 已提交
51
def get_minor():
52
    return int(_get_version_detail(1))
53

54
def get_patch():
55
    return str(_get_version_detail(2))
56 57

def is_taged():
M
minqiyang 已提交
58
    try:
59
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
60
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
61
        git_tag = git_tag.decode()
M
minqiyang 已提交
62
    except:
63 64
        return False

65
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
66 67
        return True
    else:
M
minqiyang 已提交
68
        return False
69

Y
Yancey 已提交
70
def write_version_py(filename='paddle/version.py'):
71
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
72
#
73
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
74 75
major           = '%(major)d'
minor           = '%(minor)d'
76
patch           = '%(patch)s'
Y
Yancey 已提交
77 78 79
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
80
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
81 82 83

def show():
    if istaged:
84 85 86 87 88
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
89
    else:
90
        print('commit:', commit)
L
Luo Tao 已提交
91 92 93

def mkl():
    return with_mkl
Y
Yancey 已提交
94 95 96 97
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
98 99 100
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
101 102 103
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
104
            'istaged': is_taged(),
L
Luo Tao 已提交
105
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
106

107
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
108

C
Chengmo 已提交
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
def write_distributed_training_mode_py(filename='paddle/fluid/incubate/fleet/parameter_server/version.py'):
    cnt = '''from __future__ import print_function

# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY

from paddle.fluid.incubate.fleet.base.mode import Mode

BUILD_MODE=Mode.%(mode)s

def is_transpiler():
    return Mode.TRANSPILER == BUILD_MODE

'''

    dirname = os.path.dirname(filename)

    try:
        os.makedirs(dirname)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    with open(filename, 'w') as f:
        f.write(cnt % {
            'mode': 'PSLIB' if '${WITH_PSLIB}' == 'ON' else 'TRANSPILER'
        })

write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/fluid/incubate/fleet/parameter_server/version.py')
Y
Yancey 已提交
137

138

Z
zhangjinchao01 已提交
139
packages=['paddle',
140
          'paddle.libs',
Q
qiaolongfei 已提交
141
          'paddle.utils',
142
          'paddle.utils.cpp_extension',
143 144
          'paddle.dataset',
          'paddle.reader',
145
          'paddle.distributed',
146
          'paddle.incubate',
147
          'paddle.incubate.optimizer',
148 149 150
          'paddle.distributed.fleet',
          'paddle.distributed.fleet.base',
          'paddle.distributed.fleet.meta_optimizers',
151
          'paddle.distributed.fleet.meta_optimizers.sharding',
152 153
          'paddle.distributed.fleet.runtime',
          'paddle.distributed.fleet.dataset',
154
          'paddle.distributed.fleet.data_generator',
155 156 157
          'paddle.distributed.fleet.metrics',
          'paddle.distributed.fleet.proto',
          'paddle.distributed.fleet.utils',
158
          'paddle.framework',
159
          'paddle.jit',
160
          'paddle.jit.dy2static',
W
Wilber 已提交
161
          'paddle.inference',
162
          'paddle.fluid',
W
Wilber 已提交
163
          'paddle.fluid.inference',
L
lujun 已提交
164
          'paddle.fluid.dygraph',
165
          'paddle.fluid.dygraph.dygraph_to_static',
166
          'paddle.fluid.dygraph.amp',
167
          'paddle.fluid.proto',
X
Xin Pan 已提交
168
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
169
          'paddle.fluid.distributed',
Y
Yancey 已提交
170
          'paddle.fluid.layers',
171
          'paddle.fluid.dataloader',
Q
Qingsheng Li 已提交
172 173
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
174
          'paddle.fluid.contrib.quantize',
Q
Qiao Longfei 已提交
175
          'paddle.fluid.contrib.reader',
W
whs 已提交
176
          'paddle.fluid.contrib.slim',
W
WangZhen 已提交
177
          'paddle.fluid.contrib.slim.quantization',
178
          'paddle.fluid.contrib.slim.quantization.imperative',
T
tangwei12 已提交
179
          'paddle.fluid.contrib.utils',
C
chengduo 已提交
180
          'paddle.fluid.contrib.extend_optimizer',
181
          'paddle.fluid.contrib.mixed_precision',
182
          'paddle.fluid.contrib.layers',
Q
qiaolongfei 已提交
183
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
184 185
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
186
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
187
          'paddle.fluid.incubate.fleet',
188
          'paddle.fluid.incubate.checkpoint',
D
dongdaxiang 已提交
189 190
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
191
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
192
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
193
          'paddle.fluid.incubate.fleet.parameter_server.ir',
194
          'paddle.fluid.incubate.fleet.collective',
195
          'paddle.fluid.incubate.fleet.utils',
196
          'paddle.amp',
197 198 199 200 201 202 203
          'paddle.hapi',
          'paddle.vision',
          'paddle.vision.models',
          'paddle.vision.transforms',
          'paddle.vision.datasets',
          'paddle.text',
          'paddle.text.datasets',
H
hong 已提交
204
          'paddle.incubate',
205
          'paddle.io',
H
hong 已提交
206
          'paddle.optimizer',
207 208
          'paddle.nn',
          'paddle.nn.functional',
209
          'paddle.nn.layer',
H
hong 已提交
210
          'paddle.nn.initializer',
211
          'paddle.nn.utils',
H
hong 已提交
212
          'paddle.metric',
213 214
          'paddle.static',
          'paddle.static.nn',
215
          'paddle.static.amp',
216
          'paddle.tensor',
C
channings 已提交
217
          'paddle.onnx',
218
          ]
L
Luo Tao 已提交
219

220 221
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
222

223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
# Note(wangzhongpu):
# When compiling paddle under python36, the dependencies belonging to python2.7 will be imported, resulting in errors when installing paddle
if sys.version_info >= (3,6) and sys.version_info < (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,5) and sys.version_info < (3,6):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.5\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp
if sys.version_info >= (3,7):
    setup_requires_tmp = []
    for setup_requires_i in setup_requires:
        if "<\"3.6\"" in setup_requires_i or "<=\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i or "<\"3.7\"" in setup_requires_i:
            continue
        setup_requires_tmp+=[setup_requires_i]
    setup_requires = setup_requires_tmp

247
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
248
paddle_bins = ''
249

T
Tao Luo 已提交
250 251
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
252
package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]}
253
if '${HAS_NOAVX_CORE}' == 'ON':
254
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
255

256

L
Luo Tao 已提交
257
package_dir={
258
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
259 260 261 262
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
263
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
264
}
265

266 267
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
268

P
peizhilin 已提交
269
package_data['paddle.libs']= []
P
peizhilin 已提交
270
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
271
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
272

273
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
274 275 276
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
P
peizhilin 已提交
277 278
else:
    if os.name == 'nt':
P
peizhilin 已提交
279
        # copy the openblas.dll
280
        shutil.copy('${OPENBLAS_SHARED_LIB}', libs_path)
P
peizhilin 已提交
281
        package_data['paddle.libs'] += ['openblas' + ext_name]
W
Wilber 已提交
282
    elif os.name == 'posix' and platform.machine() == 'aarch64' and '${OPENBLAS_LIB}'.endswith('so'):
H
houj04 已提交
283 284
        # copy the libopenblas.so on linux+aarch64
        # special: core_noavx.so depends on 'libopenblas.so.0', not 'libopenblas.so'
W
Wilber 已提交
285 286 287
        if os.path.exists('${OPENBLAS_LIB}' + '.0'):
            shutil.copy('${OPENBLAS_LIB}' + '.0', libs_path)
            package_data['paddle.libs'] += ['libopenblas.so.0']
P
peizhilin 已提交
288

289 290 291 292
if '${WITH_LITE}' == 'ON':
    shutil.copy('${LITE_SHARED_LIB}', libs_path)
    package_data['paddle.libs']+=['libpaddle_full_api_shared' + ext_name]

X
xujiaqi01 已提交
293 294
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
X
xujiaqi01 已提交
295 296
    if os.path.exists('${PSLIB_VERSION_PY}'):
        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/')
X
xujiaqi01 已提交
297 298
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
299
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
300
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
301 302 303 304
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
A
Adam 已提交
305
        # change rpath of libdnnl.so.1, add $ORIGIN/ to it.
P
peizhilin 已提交
306
        # The reason is that all thirdparty libraries in the same directory,
A
Adam 已提交
307
        # thus, libdnnl.so.1 will find libmklml_intel.so and libiomp5.so.
P
peizhilin 已提交
308 309
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
A
Adam 已提交
310
            raise Exception("patch libdnnl.so failed, command: %s" % command)
S
Sang Ik Lee 已提交
311
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
A
Adam 已提交
312 313
    if os.name != 'nt':
        shutil.copy('${MKLDNN_SHARED_LIB_1}', libs_path)
314 315
        shutil.copy('${MKLDNN_SHARED_LIB_2}', libs_path)
        package_data['paddle.libs']+=['libmkldnn.so.0', 'libdnnl.so.1', 'libdnnl.so.2']
A
Adam 已提交
316 317 318
    else:
        package_data['paddle.libs']+=['mkldnn.dll']

319 320 321 322 323 324 325 326 327 328 329 330 331
if '${WITH_XPU}' == 'ON':
    # only change rpath in Release mode,
    if '${CMAKE_BUILD_TYPE}' == 'Release':
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${XPU_API_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${XPU_API_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${XPU_API_LIB} failed, command: %s" % command)
    shutil.copy('${XPU_API_LIB}', libs_path)
    shutil.copy('${XPU_RT_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_API_LIB_NAME}',
332
                                  '${XPU_RT_LIB_NAME}']
333

334 335 336 337
if '${WITH_XPU_BKCL}' == 'ON':
    shutil.copy('${XPU_BKCL_LIB}', libs_path)
    package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']

338
### Old custom op extension mechanism related, will be removed in 2.1.0 ###
339 340 341 342 343
# copy libpaddle_framework.so to libs on linux
if sys.platform.startswith('linux'):
    shutil.copy('${FLUID_FRAMEWORK_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['libpaddle_framework.so']

344
### New custom op extension mechanism related ###
345 346 347 348 349
# copy libpaddle_custom_op.so to libs on linux
if sys.platform.startswith('linux'):
    shutil.copy('${PADDLE_CUSTOM_OP_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['libpaddle_custom_op.so']

350
# copy paddle_custom_op.lib/paddle_custom_op.dll to libs on Windows
351
if os.name == 'nt':
352 353 354
    shutil.copy('${PADDLE_CUSTOM_OP_IMPORT_LIB}', libs_path)
    shutil.copy('${PADDLE_CUSTOM_OP_SHARED_LIB}', libs_path)
    package_data['paddle.libs'] += ['paddle_custom_op.lib', 'paddle_custom_op.dll']
355

356
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
357 358
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
359 360
package_dir['paddle.libs']=libs_path

361

362
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
363
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
364
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
365
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
366
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
367
    if os.name != 'nt':
368
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
369
        if "@APPLE@" == "1":
370
            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
371
        else:
372
            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
W
Wilber 已提交
373
        # The sw_64 not suppot patchelf, so we just disable that.
H
houj04 已提交
374
        if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
375 376
          if os.system(command) != 0:
              raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
377

P
peizhilin 已提交
378
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
379 380 381 382 383 384
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
385
    ext_modules = []
386 387
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
388

389 390 391 392 393 394
def find_files(pattern, root):
    for dirpath, _, files in os.walk(root):
      for filename in fnmatch.filter(files, pattern):
          yield os.path.join(dirpath, filename)

headers = (
395 396
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/extension')) +
397
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/framework')) +
398
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/imperative')) +
399 400 401
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/memory')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/string')) +
402
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/platform')) +
403
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/framework')) +
404
    list(find_files('*.pb', '${cudaerror_INCLUDE_DIR}')) + # errorMessage.pb for errormessage
405 406 407 408 409 410 411 412
    ['${EIGEN_INCLUDE_DIR}/Eigen/Core'] + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/Eigen/src')) + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/unsupported/Eigen')) + # eigen
    list(find_files('*', '${GFLAGS_INSTALL_DIR}/include')) + # gflags
    list(find_files('*', '${GLOG_INSTALL_DIR}/include')) + # glog
    list(find_files('*', '${BOOST_INCLUDE_DIR}/boost')) + # boost
    list(find_files('*', '${XXHASH_INSTALL_DIR}/include')) + # xxhash
    list(find_files('*', '${PROTOBUF_INCLUDE_DIR}')) + # protobuf
413
    list(find_files('*', '${DLPACK_INCLUDE_DIR}')) + # dlpack
414 415
    list(find_files('*.h', '${THREADPOOL_INCLUDE_DIR}'))) # threadpool

416 417 418
if '${WITH_MKLDNN}' == 'ON':
    headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn

419
if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON':
420
    headers += list(find_files('*.pb', '${cudaerror_INCLUDE_DIR}'))   # errorMessage.pb for errormessage
421 422 423 424 425

class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_lib = self.install_platlib
426 427
        self.install_headers = os.path.join(self.install_platlib, 'paddle',
                                            'include')
428
        return ret
429

430 431 432 433 434

class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
435

436 437 438 439 440
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
441

442
    boolean_options = ['force']
443

444 445 446 447
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
448

449 450 451 452
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
453

454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
    def copy_data_type_headers(self, header):
        if os.name == 'nt':
            data_type_headers = ['platform\\complex64.h', 'platform\\complex128.h']
        else:
            data_type_headers = ['platform/complex64.h', 'platform/complex128.h']
        for dtype_header in data_type_headers:
            if dtype_header in header:
                if os.name == 'nt':
                    install_dir = os.path.join(self.install_dir, "paddle\\fluid\\extension\\include")
                else:
                    install_dir = os.path.join(self.install_dir, "paddle/fluid/extension/include")
                if not os.path.exists(install_dir):
                    self.mkpath(install_dir)
                return self.copy_file(header, install_dir)

469 470 471 472
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
473
            # paddle headers
474
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
475 476 477
            # For paddle data type headers, we also need to copy to `extension/incude`,
            # used for new custom operator
            self.copy_data_type_headers(header)
478 479 480
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
481 482 483 484 485
            patterns = ['eigen3/src/extern_eigen3', 'boost/src/extern_boost',
                       'dlpack/src/extern_dlpack/include',
                       'install/protobuf/include',
                       'install/gflags/include',
                       'install/glog/include', 'install/xxhash/include',
486
                       'install/mkldnn/include',
487 488 489
                       'threadpool/src/extern_threadpool']
            for pattern in patterns:
                install_dir = re.sub(pattern, '', install_dir)
490 491 492 493
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
494

495 496 497 498 499 500 501 502
    def run(self):
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
503

504 505
    def get_inputs(self):
        return self.distribution.headers or []
506

507 508 509
    def get_outputs(self):
        return self.outfiles

510 511 512 513
# we redirect setuptools log for non-windows
if sys.platform != 'win32':
    @contextmanager
    def redirect_stdout():
514 515 516 517 518 519 520
        f_log = open('${SETUP_LOG_FILE}', 'w')
        origin_stdout = sys.stdout
        sys.stdout = f_log
        yield
        f_log = sys.stdout
        sys.stdout = origin_stdout
        f_log.close()
521 522 523 524 525
else:
    @contextmanager
    def redirect_stdout():
        yield

526 527 528 529 530 531 532 533
# Log for PYPI
if sys.version_info > (3,0):
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.md", "r", encoding='UTF-8') as f:
        long_description = f.read()
else:
    with open("@PADDLE_BINARY_DIR@/python/paddle/README.md", "r")as f:
        long_description = unicode(f.read(), 'UTF-8')

534 535 536 537
with redirect_stdout():
    setup(name='${PACKAGE_NAME}',
        version='${PADDLE_VERSION}',
        description='Parallel Distributed Deep Learning',
538 539 540 541 542 543 544 545 546 547
        long_description=long_description,
        long_description_content_type="text/markdown",
        author_email="Paddle-better@baidu.com",
        maintainer="PaddlePaddle",
        maintainer_email="Paddle-better@baidu.com",
        project_urls = {
            'Homepage': 'https://www.paddlepaddle.org.cn/',
            'Downloads': 'https://github.com/paddlepaddle/paddle'
        }, 
        license='Apache Software License',
548
        packages=packages,
549
        install_requires=setup_requires,
550 551 552 553 554 555 556 557 558
        ext_modules=ext_modules,
        package_data=package_data,
        package_dir=package_dir,
        scripts=paddle_bins,
        distclass=BinaryDistribution,
        headers=headers,
        cmdclass={
            'install_headers': InstallHeaders,
            'install': InstallCommand,
559 560 561
        },
        entry_points={
            'console_scripts': [
562
                'fleetrun = paddle.distributed.fleet.launch:launch'
563
            ]
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
        },
        classifiers=[
            'Development Status :: 5 - Production/Stable',
            'Operating System :: OS Independent',
            'Intended Audience :: Developers',
            'Intended Audience :: Education',
            'Intended Audience :: Science/Research',
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: C++',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: 3.6',
            'Programming Language :: Python :: 3.7',
            'Programming Language :: Python :: 3.8',
        ],
579 580
    )

581
# As there are a lot of files in purelib which causes many logs,
582
# we don't print them on the screen, and you can open `setup.py.log`
583
# for the full logs.
584 585
if os.path.exists('${SETUP_LOG_FILE}'):
    os.system('grep -v "purelib" ${SETUP_LOG_FILE}')