setup.py.in 9.2 KB
Newer Older
T
typhoonzero 已提交
1
from setuptools import setup, Distribution, Extension
Y
Yancey 已提交
2
import subprocess
3
import os
4
import re
5
import shutil
6 7 8
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
9

Y
Yancey 已提交
10 11
RC      = 0

P
peizhilin 已提交
12
ext_name = '.dll' if os.name == 'nt' else '.so'
Y
Yancey 已提交
13 14 15 16

def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
17 18
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
Y
Yancey 已提交
19 20
    except:
        git_commit = 'Unknown'
21
    git_commit = git_commit.decode()
22
    return str(git_commit)
Y
Yancey 已提交
23

24
def _get_version_detail(idx):
25 26
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
27

M
minqiyang 已提交
28 29
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
30

M
minqiyang 已提交
31
        if len(version_details) >= 3:
M
minqiyang 已提交
32
            return version_details[idx]
33

34
    return 0
35

M
minqiyang 已提交
36
def get_major():
37
    return int(_get_version_detail(0))
38

M
minqiyang 已提交
39
def get_minor():
40
    return int(_get_version_detail(1))
41

42
def get_patch():
43
    return str(_get_version_detail(2))
44 45

def is_taged():
M
minqiyang 已提交
46
    try:
47
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
48
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
49
        git_tag = git_tag.decode()
M
minqiyang 已提交
50
    except:
51 52
        return False

53
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
54 55
        return True
    else:
M
minqiyang 已提交
56
        return False
57

Y
Yancey 已提交
58
def write_version_py(filename='paddle/version.py'):
59
    cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
Y
Yancey 已提交
60
#
61
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
62 63
major           = '%(major)d'
minor           = '%(minor)d'
64
patch           = '%(patch)s'
Y
Yancey 已提交
65 66 67
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
68
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
69 70 71

def show():
    if istaged:
72 73 74 75 76
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
77
    else:
78
        print('commit:', commit)
L
Luo Tao 已提交
79 80 81

def mkl():
    return with_mkl
Y
Yancey 已提交
82 83 84 85
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
86 87 88
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
89 90 91
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
92
            'istaged': is_taged(),
L
Luo Tao 已提交
93
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
94

95
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
96 97


Z
zhangjinchao01 已提交
98
packages=['paddle',
99
          'paddle.libs',
Q
qiaolongfei 已提交
100
          'paddle.utils',
101 102
          'paddle.dataset',
          'paddle.reader',
103
          'paddle.distributed',
104
          'paddle.fluid',
L
lujun 已提交
105
          'paddle.fluid.dygraph',
106
          'paddle.fluid.proto',
X
Xin Pan 已提交
107
          'paddle.fluid.proto.profiler',
H
heqiaozhi 已提交
108
          'paddle.fluid.distributed',
Y
Yancey 已提交
109
          'paddle.fluid.layers',
Q
Qingsheng Li 已提交
110 111
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
112
          'paddle.fluid.contrib.quantize',
113
          'paddle.fluid.contrib.int8_inference',
Q
Qiao Longfei 已提交
114
          'paddle.fluid.contrib.reader',
W
whs 已提交
115 116 117 118
          'paddle.fluid.contrib.slim',
          'paddle.fluid.contrib.slim.core',
          'paddle.fluid.contrib.slim.graph',
          'paddle.fluid.contrib.slim.prune',
W
WangZhen 已提交
119
          'paddle.fluid.contrib.slim.quantization',
120
          'paddle.fluid.contrib.slim.distillation',
W
whs 已提交
121 122
          'paddle.fluid.contrib.slim.nas',
          'paddle.fluid.contrib.slim.searcher',
T
tangwei12 已提交
123
          'paddle.fluid.contrib.utils',
C
chengduo 已提交
124
          'paddle.fluid.contrib.extend_optimizer',
125
          'paddle.fluid.contrib.mixed_precision',
126
          'paddle.fluid.contrib.layers',
Q
qiaolongfei 已提交
127
          'paddle.fluid.transpiler',
D
dongdaxiang 已提交
128 129
          'paddle.fluid.transpiler.details',
          'paddle.fluid.incubate',
130
          'paddle.fluid.incubate.data_generator',
D
dongdaxiang 已提交
131 132 133
          'paddle.fluid.incubate.fleet',
          'paddle.fluid.incubate.fleet.base',
          'paddle.fluid.incubate.fleet.parameter_server',
134 135 136
          'paddle.fluid.incubate.fleet.parameter_server.distributed_transpiler',
          'paddle.fluid.incubate.fleet.parameter_server.pslib',
          'paddle.fluid.incubate.fleet.collective']
L
Luo Tao 已提交
137

138 139
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
140 141

if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
Y
Yancey 已提交
142
    setup_requires+=['opencv-python']
143

144
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
145
paddle_bins = ''
T
Tao Luo 已提交
146 147
if not '${WIN32}':
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
P
peizhilin 已提交
148 149
package_data={'paddle.fluid': ['core' + (ext_name if os.name != 'nt' else '.pyd')]}

L
Luo Tao 已提交
150
package_dir={
151
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
152 153 154 155
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
156
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
157
}
158

159 160
# put all thirdparty libraries in paddle.libs
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
P
peizhilin 已提交
161

P
peizhilin 已提交
162
package_data['paddle.libs']= []
P
peizhilin 已提交
163
package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name]
P
peizhilin 已提交
164
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
P
peizhilin 已提交
165

166
if '${WITH_MKL}' == 'ON':
P
peizhilin 已提交
167 168 169
    shutil.copy('${MKLML_SHARED_LIB}', libs_path)
    shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
170 171 172 173
    # mklml has dependency to vs runtime library
    if os.name == 'nt':
        shutil.copy('${MKLML_SHARED_LIB_DEPS}', libs_path)
        package_data['paddle.libs'] += ['msvcr120.dll']
P
peizhilin 已提交
174 175
else:
    if os.name == 'nt':
P
peizhilin 已提交
176 177
        # copy the openblas.dll
        shutil.copy(os.path.dirname('${CBLAS_LIBRARIES}') + '/openblas' + ext_name, libs_path)
P
peizhilin 已提交
178
        package_data['paddle.libs'] += ['openblas' + ext_name]
P
peizhilin 已提交
179

S
Sang Ik Lee 已提交
180
if '${WITH_MKLDNN}' == 'ON':
P
peizhilin 已提交
181
    if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
P
peizhilin 已提交
182 183 184 185 186 187 188 189 190 191
        # only change rpath in Release mode.
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
        # change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
        # The reason is that all thirdparty libraries in the same directory,
        # thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
            raise Exception("patch libmkldnn.so failed, command: %s" % command)
P
peizhilin 已提交
192
    package_data['paddle.libs']+=['libmkldnn.so.0' if os.name != 'nt' else ('mkldnn' + ext_name)]
S
Sang Ik Lee 已提交
193
    shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
S
Sang Ik Lee 已提交
194
if '${WITH_NGRAPH}' == 'ON':
S
Sang Ik Lee 已提交
195 196
    # only change rpath in Release mode,
    # since in Debug mode, nGraph lib may be too large to be changed?
S
Sang Ik Lee 已提交
197
    if '${CMAKE_BUILD_TYPE}' == 'Release':
S
Sang Ik Lee 已提交
198 199 200 201 202 203 204
        if os.name != 'nt':
            if "@APPLE@" == "1":
                command = "install_name_tool -id \"@loader_path/\" ${NGRAPH_SHARED_LIB}"
            else:
                command = "patchelf --set-rpath '$ORIGIN/' ${NGRAPH_SHARED_LIB}"
            if os.system(command) != 0:
                raise Exception("patch ${NGRAPH_SHARED_LIB_NAME} failed, command: %s" % command)
S
Sang Ik Lee 已提交
205 206 207 208 209 210
    shutil.copy('${NGRAPH_SHARED_LIB}', libs_path)
    shutil.copy('${NGRAPH_CPU_LIB}', libs_path)
    shutil.copy('${NGRAPH_TBB_LIB}', libs_path)
    package_data['paddle.libs']+=['${NGRAPH_SHARED_LIB_NAME}',
                                  '${NGRAPH_CPU_LIB_NAME}',
                                  '${NGRAPH_TBB_LIB_NAME}']
211
# remove unused paddle/libs/__init__.py
P
peizhilin 已提交
212 213
if os.path.isfile(libs_path+'/__init__.py'):
    os.remove(libs_path+'/__init__.py')
214 215
package_dir['paddle.libs']=libs_path

216 217 218
# change rpath of core.ext, add $ORIGIN/../libs/ to it.
# The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
# core.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
219
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
220
if '${CMAKE_BUILD_TYPE}' == 'Release':
P
peizhilin 已提交
221
    if os.name != 'nt':
222
        # only change rpath in Release mode, since in Debug mode, core.xx is too large to be changed.
L
luotao1 已提交
223
        if "@APPLE@" == "1":
P
peizhilin 已提交
224
            command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/core" + ext_name
L
luotao1 已提交
225
        else:
P
peizhilin 已提交
226
            command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/core" + ext_name
L
luotao1 已提交
227
        if os.system(command) != 0:
228
            raise Exception("patch core.%s failed, command: %s" % (ext_name, command))
P
peizhilin 已提交
229

P
peizhilin 已提交
230
ext_modules = [Extension('_foo', ['stub.cc'])]
P
peizhilin 已提交
231 232 233 234 235 236
if os.name == 'nt':
    # fix the path separator under windows
    fix_package_dir = {}
    for k, v in package_dir.items():
        fix_package_dir[k] = v.replace('/', '\\')
    package_dir = fix_package_dir
P
peizhilin 已提交
237
    ext_modules = []
T
tensor-tang 已提交
238

T
typhoonzero 已提交
239
setup(name='${PACKAGE_NAME}',
Z
zhangjinchao01 已提交
240 241
      version='${PADDLE_VERSION}',
      description='Parallel Distributed Deep Learning',
242
      install_requires=setup_requires,
L
Luo Tao 已提交
243
      packages=packages,
P
peizhilin 已提交
244
      ext_modules=ext_modules,
L
Luo Tao 已提交
245 246
      package_data=package_data,
      package_dir=package_dir,
P
peizhilin 已提交
247 248
      scripts=paddle_bins,
      distclass=BinaryDistribution
Z
zhangjinchao01 已提交
249
)