setup.py.in 7.3 KB
Newer Older
T
typhoonzero 已提交
1
from setuptools import setup, Distribution, Extension
Y
Yancey 已提交
2
import subprocess
3
import os
4
import re
5
import shutil
6 7 8
class BinaryDistribution(Distribution):
    def has_ext_modules(foo):
        return True
Z
zhangjinchao01 已提交
9

Y
Yancey 已提交
10 11 12 13 14 15 16 17 18 19
RC      = 0



def git_commit():
    try:
        cmd = ['git', 'rev-parse', 'HEAD']
        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0].strip()
    except:
        git_commit = 'Unknown'
20
    git_commit = git_commit.decode()
21
    return str(git_commit)
Y
Yancey 已提交
22

23
def _get_version_detail(idx):
24 25
    assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
        so detail index must less than 3"
26

M
minqiyang 已提交
27 28
    if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
        version_details = '@PADDLE_VERSION@'.split('.')
29

M
minqiyang 已提交
30 31
        if len(version_details) == 3:
            return version_details[idx]
32

33
    return 0
34

M
minqiyang 已提交
35
def get_major():
36
    return int(_get_version_detail(0))
37

M
minqiyang 已提交
38
def get_minor():
39
    return int(_get_version_detail(1))
40

41
def get_patch():
42
    return str(_get_version_detail(2))
43 44

def is_taged():
M
minqiyang 已提交
45
    try:
46
        cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
47
        git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0].strip()
48
        git_tag = git_tag.decode()
M
minqiyang 已提交
49
    except:
50 51
        return False

52
    if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
53 54
        return True
    else:
M
minqiyang 已提交
55
        return False
56

Y
Yancey 已提交
57 58 59 60
def write_version_py(filename='paddle/version.py'):
    cnt = '''
# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
#
61
full_version    = '%(major)d.%(minor)d.%(patch)s'
Y
Yancey 已提交
62 63
major           = '%(major)d'
minor           = '%(minor)d'
64
patch           = '%(patch)s'
Y
Yancey 已提交
65 66 67
rc              = '%(rc)d'
istaged         = %(istaged)s
commit          = '%(commit)s'
L
Luo Tao 已提交
68
with_mkl        = '%(with_mkl)s'
Y
Yancey 已提交
69 70 71

def show():
    if istaged:
72 73 74 75 76
        print('full_version:', full_version)
        print('major:', major)
        print('minor:', minor)
        print('patch:', patch)
        print('rc:', rc)
Y
Yancey 已提交
77
    else:
78
        print('commit:', commit)
L
Luo Tao 已提交
79 80 81

def mkl():
    return with_mkl
Y
Yancey 已提交
82 83 84 85
'''
    commit = git_commit()
    with open(filename, 'w') as f:
        f.write(cnt % {
86 87 88
            'major': get_major(),
            'minor': get_minor(),
            'patch': get_patch(),
Y
Yancey 已提交
89 90 91
            'rc': RC,
            'version': '${PADDLE_VERSION}',
            'commit': commit,
92
            'istaged': is_taged(),
L
Luo Tao 已提交
93
            'with_mkl': '@WITH_MKL@'})
Y
Yancey 已提交
94

95
write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
Y
Yancey 已提交
96 97


Z
zhangjinchao01 已提交
98
packages=['paddle',
99
          'paddle.libs',
Q
qiaolongfei 已提交
100
          'paddle.utils',
101 102
          'paddle.dataset',
          'paddle.reader',
103 104
          'paddle.fluid',
          'paddle.fluid.proto',
X
Xin Pan 已提交
105
          'paddle.fluid.proto.profiler',
Y
Yancey 已提交
106
          'paddle.fluid.layers',
Q
Qingsheng Li 已提交
107 108
          'paddle.fluid.contrib',
          'paddle.fluid.contrib.decoder',
D
Dang Qingqing 已提交
109
          'paddle.fluid.contrib.quantize',
Q
qiaolongfei 已提交
110 111
          'paddle.fluid.transpiler',
          'paddle.fluid.transpiler.details']
L
Luo Tao 已提交
112

113
if '${WITH_FLUID_ONLY}'== 'OFF':
L
Luo Tao 已提交
114 115 116 117 118 119
    packages+=['paddle.proto',
               'paddle.trainer',
               'paddle.trainer_config_helpers',
               'paddle.v2',
               'paddle.v2.master',
               'paddle.v2.plot',
120 121
               'paddle.v2.reader',
               'paddle.v2.dataset',
L
Luo Tao 已提交
122
               'py_paddle']
Z
zhangjinchao01 已提交
123

124 125
with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
    setup_requires = f.read().splitlines()
126 127

if '${CMAKE_SYSTEM_PROCESSOR}' not in ['arm', 'armv7-a', 'aarch64']:
Y
Yancey 已提交
128
    setup_requires+=['opencv-python']
129

130
# the prefix is sys.prefix which should always be usr
L
Luo Tao 已提交
131
paddle_bins = ''
132
if '${WITH_FLUID_ONLY}'== 'OFF':
L
Luo Tao 已提交
133
    paddle_bin_dir = 'opt/paddle/bin'
X
Xin Pan 已提交
134 135
    paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/legacy/trainer/paddle_trainer',
                   '${PADDLE_BINARY_DIR}/paddle/legacy/trainer/paddle_merge_model',
X
Xin Pan 已提交
136
                   '${PADDLE_BINARY_DIR}/paddle/legacy/pserver/paddle_pserver_main',
L
Luo Tao 已提交
137 138 139
                   '${PADDLE_BINARY_DIR}/paddle/scripts/paddle']

package_data={'paddle.fluid': ['core.so']}
140
if '${WITH_FLUID_ONLY}'== 'OFF':
L
Luo Tao 已提交
141 142 143 144
    package_data['paddle.v2.master']=['libpaddle_master.so']
    package_data['py_paddle']=['*.py','_swig_paddle.so']

package_dir={
145
    '': '${PADDLE_BINARY_DIR}/python',
L
Luo Tao 已提交
146 147 148 149
    # The paddle.fluid.proto will be generated while compiling.
    # So that package points to other directory.
    'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
    'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
Q
qiaolongfei 已提交
150
    'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
L
Luo Tao 已提交
151
}
152
if '${WITH_FLUID_ONLY}'== 'OFF':
153
    package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle'
154

155 156 157 158 159 160 161 162
# put all thirdparty libraries in paddle.libs
package_data['paddle.libs']=['libwarpctc.so']
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
if '${WITH_MKL}' == 'ON':
    shutil.copy('${MKLML_LIB}', libs_path)
    shutil.copy('${MKLML_IOMP_LIB}', libs_path)
    package_data['paddle.libs']+=['libmklml_intel.so','libiomp5.so']
L
luotao1 已提交
163 164 165 166 167 168 169 170 171 172 173 174 175 176
if '${CMAKE_BUILD_TYPE}' == 'Release':
    # only change rpath in Release mode.
    if '${WITH_MKLDNN}' == 'ON':
        # TODO(typhoonzero): use install_name_tool to patch mkl libs once
        # we can support mkl on mac.
        #
        # change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
        # The reason is that all thirdparty libraries in the same directory,
        # thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
        command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
        if os.system(command) != 0:
            raise Exception("patch libmkldnn.so failed, command: %s" % command)
        package_data['paddle.libs']+=['libmkldnn.so.0']
        shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
177 178 179 180 181 182 183 184
# remove unused paddle/libs/__init__.py
os.remove(libs_path+'/__init__.py')
package_dir['paddle.libs']=libs_path

# change rpath of core.so, add $ORIGIN/../libs/ to it.
# The reason is that libwarpctc.so, libiomp5.so etc are in paddle.libs, and
# core.so is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
L
luotao1 已提交
185 186
if '${CMAKE_BUILD_TYPE}' == 'Release':
    # only change rpath in Release mode, since in Debug mode, core.so is too large to be changed.
L
Luo Tao 已提交
187
    if "@APPLE@" == "1":
L
luotao1 已提交
188
        command = "install_name_tool -id \"@loader_path/../libs/\" ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so"
L
Luo Tao 已提交
189
    else:
L
luotao1 已提交
190
        command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so"
L
Luo Tao 已提交
191
    if os.system(command) != 0:
L
luotao1 已提交
192 193 194 195 196 197 198 199 200
        raise Exception("patch core.so failed, command: %s" % command)
    if '${WITH_FLUID_ONLY}'== 'OFF':
        # change rpath of _swig_paddle.so.
        if "@APPLE@" == "1":
            command = "install_name_tool -id \"@loader_path/../paddle/libs/\" ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle.so"
        else:
            command = "patchelf --set-rpath '$ORIGIN/../paddle/libs/' ${PADDLE_BINARY_DIR}/python/py_paddle/_swig_paddle.so"
        if os.system(command) != 0:
            raise Exception("patch _swig_paddle.so failed, command: %s" % command)
T
tensor-tang 已提交
201

T
typhoonzero 已提交
202
setup(name='${PACKAGE_NAME}',
Z
zhangjinchao01 已提交
203 204
      version='${PADDLE_VERSION}',
      description='Parallel Distributed Deep Learning',
205
      install_requires=setup_requires,
L
Luo Tao 已提交
206
      packages=packages,
T
typhoonzero 已提交
207
      ext_modules=[Extension('_foo', ['stub.cc'])],
L
Luo Tao 已提交
208 209
      package_data=package_data,
      package_dir=package_dir,
210
      scripts=paddle_bins
Z
zhangjinchao01 已提交
211
)