setup.py.in 15.2 KB
Newer Older
Y
Yancey 已提交
1
import subprocess
2
import os
3
import os.path
4
import re
5
import shutil
6
import sys
7 8
import fnmatch

9
from contextlib import contextmanager
10 11 12 13 14
from setuptools import Command
from setuptools import setup, Distribution, Extension
from setuptools.command.install import install as InstallCommandBase


15 16 17
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
18

Y
Yancey 已提交
19 20
RC      = 0

21
ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
Y
Yancey 已提交
22 23 24 25

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
26 27
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
28 29
    except:
        git_commit = 'Unknown'
30
    git_commit = git_commit.decode()
31
    return str(git_commit)
Y
Yancey 已提交
32

33
def _get_version_detail(idx):
34 35
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
36

M
minqiyang 已提交
37 38
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
39

M
minqiyang 已提交
40
        if len(version_details) >= 3:
M
minqiyang 已提交
41
            return version_details[idx]
42

43
    return 0
44

M
minqiyang 已提交
45
def get_major():
46
    return int(_get_version_detail(0))
47

M
minqiyang 已提交
48
def get_minor():
49
    return int(_get_version_detail(1))
50

51
def get_patch():
52
    return str(_get_version_detail(2))
53 54

def is_taged():
M
minqiyang 已提交
55
    try:
56
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
57
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
58
        git_tag = git_tag.decode()
M
minqiyang 已提交
59
    except:
60 61
        return False

62
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
63 64
        return True
    else:
M
minqiyang 已提交
65
        return False
66

Y
Yancey 已提交
67
def write_version_py(filename='paddle/version.py'):
68
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
69
#
70
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
71 72
major           = '%(major)d'
minor           = '%(minor)d'
73
patch           = '%(patch)s'
Y
Yancey 已提交
74 75 76
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
77
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
78 79 80

def show():
    if istaged:
81 82 83 84 85
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
86
    else:
87
        print('commit:', commit)
L
Luo Tao 已提交
88 89 90

def mkl():
    return with_mkl
Y
Yancey 已提交
91 92 93 94
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
95 96 97
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
98 99 100
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
101
            'istaged': is_taged(),
L
Luo Tao 已提交
102
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
103

104
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
105 106


Z
zhangjinchao01 已提交
107
packages=['paddle',
108
          'paddle.libs',
Q
qiaolongfei 已提交
109
          'paddle.utils',
110 111
          'paddle.dataset',
          'paddle.reader',
112
          'paddle.distributed',
113
          'paddle.fluid',
L
lujun 已提交
114
          'paddle.fluid.dygraph',
115
          'paddle.fluid.proto',
X
Xin Pan 已提交
116
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
117
          'paddle.fluid.distributed',
Y
Yancey 已提交
118
          'paddle.fluid.layers',
Q
Qingsheng Li 已提交
119 120
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
121
          'paddle.fluid.contrib.quantize',
Q
Qiao Longfei 已提交
122
          'paddle.fluid.contrib.reader',
W
whs 已提交
123 124 125 126
          'paddle.fluid.contrib.slim',
          'paddle.fluid.contrib.slim.core',
          'paddle.fluid.contrib.slim.graph',
          'paddle.fluid.contrib.slim.prune',
W
WangZhen 已提交
127
          'paddle.fluid.contrib.slim.quantization',
128
          'paddle.fluid.contrib.slim.distillation',
W
whs 已提交
129 130
          'paddle.fluid.contrib.slim.nas',
          'paddle.fluid.contrib.slim.searcher',
T
tangwei12 已提交
131
          'paddle.fluid.contrib.utils',
C
chengduo 已提交
132
          'paddle.fluid.contrib.extend_optimizer',
133
          'paddle.fluid.contrib.mixed_precision',
134
          'paddle.fluid.contrib.layers',
Q
qiaolongfei 已提交
135
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
136 137
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
138
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
139 140 141
          'paddle.fluid.incubate.fleet',
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
T
tangwei12 已提交
142
          'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
143
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
144 145
          'paddle.fluid.incubate.fleet.collective',
          'paddle.fluid.incubate.fleet.utils']
L
Luo Tao 已提交
146

147 148
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
149 150

if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
Y
Yancey 已提交
151
    setup_requires+=['opencv-python']
152

153
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
154
paddle_bins = ''
T
Tao Luo 已提交
155 156
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
157
package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + ('.so' if os.name != 'nt' else '.pyd')]}
158
if '${HAS_NOAVX_CORE}' == 'ON':
159
    package_data['paddle.fluid'] += ['core_noavx' + ('.so' if os.name != 'nt' else '.pyd')]
P
peizhilin 已提交
160

L
Luo Tao 已提交
161
package_dir={
162
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
163 164 165 166
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
167
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
168
}
169

170 171
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
172

P
peizhilin 已提交
173
package_data['paddle.libs']= []
P
peizhilin 已提交
174
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
175
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
176

177
if '${TENSORRT_FOUND}' == 'ON' and os.name == 'nt':
J
Jeng Bai-Cheng 已提交
178 179
    shutil.copy(os.path.join('${TENSORRT_LIBRARY_DIR}', '${TR_INFER_RT}'), libs_path)
    shutil.copy(os.path.join('${TENSORRT_LIBRARY_DIR}', '${TR_INFER_PLUGIN_RT}'), libs_path)
180 181
    package_data['paddle.libs'] += ['${TR_INFER_RT}', '${TR_INFER_PLUGIN_RT}']

182
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
183 184 185
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
186 187 188 189
    # mklml has dependency to vs runtime library
    if os.name == 'nt':
        shutil.copy('${MKLML_SHARED_LIB_DEPS}', libs_path)
        package_data['paddle.libs'] += ['msvcr120.dll']
P
peizhilin 已提交
190 191
else:
    if os.name == 'nt':
P
peizhilin 已提交
192 193
        # copy the openblas.dll
        shutil.copy(os.path.dirname('${CBLAS_LIBRARIES}') + '/openblas' + ext_name, libs_path)
P
peizhilin 已提交
194
        package_data['paddle.libs'] += ['openblas' + ext_name]
P
peizhilin 已提交
195

X
xujiaqi01 已提交
196 197 198 199
if '${WITH_PSLIB}' == 'ON':
    shutil.copy('${PSLIB_LIB}', libs_path)
    package_data['paddle.libs'] += ['libps' + ext_name]

S
Sang Ik Lee 已提交
200
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
201
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
202 203 204 205 206 207 208 209 210 211
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
        # change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
        # The reason is that all thirdparty libraries in the same directory,
        # thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
            raise Exception("patch libmkldnn.so failed, command: %s" % command)
P
peizhilin 已提交
212
    package_data['paddle.libs']+=['libmkldnn.so.0' if os.name != 'nt' else ('mkldnn' + ext_name)]
S
Sang Ik Lee 已提交
213
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
S
Sang Ik Lee 已提交
214
if '${WITH_NGRAPH}' == 'ON':
S
Sang Ik Lee 已提交
215 216
    # only change rpath in Release mode,
    # since in Debug mode, nGraph lib may be too large to be changed?
S
Sang Ik Lee 已提交
217
    if '${CMAKE_BUILD_TYPE}' == 'Release':
S
Sang Ik Lee 已提交
218 219 220 221 222 223 224
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${NGRAPH_SHARED_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${NGRAPH_SHARED_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${NGRAPH_SHARED_LIB_NAME} failed, command: %s" % command)
S
Sang Ik Lee 已提交
225 226 227 228 229 230
    shutil.copy('${NGRAPH_SHARED_LIB}', libs_path)
    shutil.copy('${NGRAPH_CPU_LIB}', libs_path)
    shutil.copy('${NGRAPH_TBB_LIB}', libs_path)
    package_data['paddle.libs']+=['${NGRAPH_SHARED_LIB_NAME}',
                                  '${NGRAPH_CPU_LIB_NAME}',
                                  '${NGRAPH_TBB_LIB_NAME}']
231 232 233 234 235 236 237

# copy libfuild_framework.so to libs
if os.name != 'nt' and sys.platform != 'darwin':
    paddle_framework_lib='${FLUID_FRAMEWORK_SHARED_LIB}'
    shutil.copy(paddle_framework_lib, libs_path)
    package_data['paddle.libs'] += [('libpaddle_framework' if os.name != 'nt' else 'paddle_framework') + ext_name]

238
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
239 240
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
241 242
package_dir['paddle.libs']=libs_path

243

244
# change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
245
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
246
# ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
247
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
248
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
249
    if os.name != 'nt':
250
        # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
L
luotao1 已提交
251
        if "@APPLE@" == "1":
252
            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
253
        else:
254
            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so'
L
luotao1 已提交
255
        if os.system(command) != 0:
256
            raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
257

P
peizhilin 已提交
258
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
259 260 261 262 263 264
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
265
    ext_modules = []
266 267
elif sys.platform == 'darwin':
    ext_modules = []
T
tensor-tang 已提交
268

269 270 271 272 273 274 275
def find_files(pattern, root):
    for dirpath, _, files in os.walk(root):
      for filename in fnmatch.filter(files, pattern):
          yield os.path.join(dirpath, filename)

headers = (
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/framework')) +
276
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/imperative')) +
277 278 279
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/memory')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform')) +
    list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/string')) +
280
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/platform')) +
281 282 283 284 285 286 287 288
    list(find_files('*.pb.h', '${PADDLE_BINARY_DIR}/paddle/fluid/framework')) +
    ['${EIGEN_INCLUDE_DIR}/Eigen/Core'] + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/Eigen/src')) + # eigen
    list(find_files('*', '${EIGEN_INCLUDE_DIR}/unsupported/Eigen')) + # eigen
    list(find_files('*', '${GFLAGS_INSTALL_DIR}/include')) + # gflags
    list(find_files('*', '${GLOG_INSTALL_DIR}/include')) + # glog
    list(find_files('*', '${BOOST_INCLUDE_DIR}/boost')) + # boost
    list(find_files('*', '${XXHASH_INSTALL_DIR}/include')) + # xxhash
289
    list(find_files('mkldnn.h', '${MKLDNN_INSTALL_DIR}/include')) + # mkldnn
290
    list(find_files('*', '${PROTOBUF_INCLUDE_DIR}')) + # protobuf
291
    list(find_files('*', '${DLPACK_INCLUDE_DIR}')) + # dlpack 
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
    list(find_files('*.h', '${THREADPOOL_INCLUDE_DIR}'))) # threadpool


class InstallCommand(InstallCommandBase):
    def finalize_options(self):
        ret = InstallCommandBase.finalize_options(self)
        self.install_headers = os.path.join(self.install_purelib, 'paddle',
                                            'include')
        self.install_lib = self.install_platlib
        return ret
  

class InstallHeaders(Command):
    """Override how headers are copied.
    """
    description = 'install C/C++ header files'
  
    user_options = [('install-dir=', 'd',
                     'directory to install header files to'),
                    ('force', 'f',
                     'force installation (overwrite existing files)'),
                   ]
  
    boolean_options = ['force']
  
    def initialize_options(self):
        self.install_dir = None
        self.force = 0
        self.outfiles = []
  
    def finalize_options(self):
        self.set_undefined_options('install',
                                   ('install_headers', 'install_dir'),
                                   ('force', 'force'))
  
    def mkdir_and_copy_file(self, header):
        if 'pb.h' in header:
            install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
        elif 'third_party' not in header:
            # framework
            install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
        else:
            # third_party
            install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
336 337 338 339 340
            patterns = ['eigen3/src/extern_eigen3', 'boost/src/extern_boost',
                       'dlpack/src/extern_dlpack/include',
                       'install/protobuf/include',
                       'install/gflags/include',
                       'install/glog/include', 'install/xxhash/include',
341
                       'install/mkldnn/include',
342 343 344
                       'threadpool/src/extern_threadpool']
            for pattern in patterns:
                install_dir = re.sub(pattern, '', install_dir)
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
        install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
        if not os.path.exists(install_dir):
            self.mkpath(install_dir)
        return self.copy_file(header, install_dir)
  
    def run(self):
        if os.name == 'nt' or sys.platform == 'darwin':
            return
        hdrs = self.distribution.headers
        if not hdrs:
            return
        self.mkpath(self.install_dir)
        for header in hdrs:
            (out, _) = self.mkdir_and_copy_file(header)
            self.outfiles.append(out)
  
    def get_inputs(self):
        return self.distribution.headers or []
  
    def get_outputs(self):
        return self.outfiles

367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
# we redirect setuptools log for non-windows
if sys.platform != 'win32':
    @contextmanager
    def redirect_stdout():
        with open('${SETUP_LOG_FILE}', 'w') as f:
            origin_stdout = sys.stdout
            sys.stdout = f
            yield
            sys.stdout = origin_stdout
else:
    @contextmanager
    def redirect_stdout():
        yield

with redirect_stdout():
    setup(name='${PACKAGE_NAME}',
        version='${PADDLE_VERSION}',
        description='Parallel Distributed Deep Learning',
        install_requires=setup_requires,
        packages=packages,
        ext_modules=ext_modules,
        package_data=package_data,
        package_dir=package_dir,
        scripts=paddle_bins,
        distclass=BinaryDistribution,
        headers=headers,
        cmdclass={
            'install_headers': InstallHeaders,
            'install': InstallCommand,
        }
    )

399 400 401
# As there are a lot of files in purelib which causes many logs,
# we don't print them on the screen, and you can open `setup.py.log` 
# for the full logs.
402 403
if os.path.exists('${SETUP_LOG_FILE}'):
    os.system('grep -v "purelib" ${SETUP_LOG_FILE}')