remove legacy WITH_SWIG_PY option

CMakeLists.txt

@@ -49,7 +49,6 @@ option(WITH_MKL         "Compile PaddlePaddle with MKL support."        ${AVX_FO
 option(WITH_NGRAPH      "Compile PaddlePaddle with nGraph support."     OFF)
 option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" ON)
 option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        OFF)
-option(WITH_SWIG_PY     "Compile PaddlePaddle with inference api"       ON)
 option(WITH_PYTHON      "Compile PaddlePaddle with python interpreter"  ON)
 option(WITH_DOUBLE      "Compile PaddlePaddle with double precision"    OFF)
 option(WITH_RDMA        "Compile PaddlePaddle with RDMA support"        OFF)
@@ -176,7 +175,6 @@ include(external/python)    # download, build, install python
 include(external/openblas)  # download, build, install openblas
 include(external/mkldnn)    # download, build, install mkldnn
 include(external/ngraph)    # download, build, install nGraph
-include(external/swig)      # download, build, install swig
 include(external/boost)     # download boost
 include(external/any)       # download libn::any
 include(external/eigen)     # download eigen3

cmake/external/swig.cmake

-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-IF(NOT WITH_SWIG_PY)
-    return()
-ENDIF()
-
-FIND_PACKAGE(SWIG)
-
-IF(NOT SWIG_FOUND)
-    # build swig as an external project
-    INCLUDE(ExternalProject)
-
-    SET(SWIG_SOURCES_DIR ${THIRD_PARTY_PATH}/swig)
-    SET(SWIG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/swig)
-    SET(SWIG_TARGET_VERSION "3.0.2")
-    SET(SWIG_DOWNLOAD_SRC_MD5 "62f9b0d010cef36a13a010dc530d0d41")
-    SET(SWIG_DOWNLOAD_WIN_MD5 "3f18de4fc09ab9abb0d3be37c11fbc8f")
-
-    IF(WIN32)
-        # swig.exe available as pre-built binary on Windows:
-        ExternalProject_Add(swig
-            URL                 http://prdownloads.sourceforge.net/swig/swigwin-${SWIG_TARGET_VERSION}.zip
-            URL_MD5             ${SWIG_DOWNLOAD_WIN_MD5}
-            SOURCE_DIR          ${SWIG_SOURCES_DIR}
-            CONFIGURE_COMMAND   ""
-            BUILD_COMMAND       ""
-            INSTALL_COMMAND     ""
-            UPDATE_COMMAND      ""
-        )
-        SET(SWIG_DIR ${SWIG_SOURCES_DIR} CACHE FILEPATH "SWIG Directory" FORCE)
-        SET(SWIG_EXECUTABLE ${SWIG_SOURCES_DIR}/swig.exe  CACHE FILEPATH "SWIG Executable" FORCE)
-    ELSE(WIN32)
-        # swig uses bison find it by cmake and pass it down
-        FIND_PACKAGE(BISON)
-
-        # From SWIG configure
-        ExternalProject_Add(swig
-            GIT_REPOSITORY      https://github.com/swig/swig.git
-            GIT_TAG             rel-3.0.10
-            PREFIX              ${SWIG_SOURCES_DIR}
-            CONFIGURE_COMMAND   cd <SOURCE_DIR> && ./autogen.sh && ./configure
-                                --prefix=${SWIG_INSTALL_DIR} --without-pcre
-            BUILD_COMMAND       cd <SOURCE_DIR> && make
-            INSTALL_COMMAND     cd <SOURCE_DIR> && make install
-            UPDATE_COMMAND      ""
-        )
-
-        SET(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION})
-        SET(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig)
-    ENDIF(WIN32)
-
-    LIST(APPEND external_project_dependencies swig)
-ENDIF(NOT SWIG_FOUND)

paddle/py_paddle/.gitignore

-swig_paddle.py
-_swig_paddle.so

paddle/py_paddle/__init__.py

-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from util import DataProviderWrapperConverter
-from dataprovider_converter import DataProviderConverter
-
-__all__ = [
-    'paddle',
-    'DataProviderConverter',
-    'DataProviderWrapperConverter',  # for deprecated usage.
-    'loadParameterFile'
-]
-util.monkeypatches()

paddle/py_paddle/dataprovider_converter.py

-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import paddle.trainer.PyDataProvider2 as dp2
-import collections
-import swig_paddle
-import numpy
-import itertools
-from functools import reduce
-
-__all__ = ['DataProviderConverter']
-
-
-class IScanner(object):
-    """
-    The scanner will scan Python object two passes, then convert it to Paddle's
-    argument.
-
-    In the first pass, `pre_scan` will be invoked by every data instance, and
-    then invoke `finish_pre_scan` to arguments. And the second pass do the same
-    thing except the functions changed to `scan`, `finish_scan`.
-
-    During the first pass, a scanner may count the shape of input matrix and
-    allocate memory for this argument. Then fill the data into this  argument
-    in second pass.
-    """
-
-    def __init__(self, input_type, pos):
-        self.input_type = input_type
-        if not isinstance(self.input_type, dp2.InputType):
-            raise ValueError("input type should be dataprovider2.InputType")
-        self.pos = pos
-        # data_in_gpu is used to indicate whether to create argument on GPU
-        # or not in GPU mode. Now if using one thread (trainer_count=1),
-        # trainer uses NeuralNetwork which needs to create argument on GPU
-        # before calling forward function. So, set data_in_gpu to True.
-        # Otherwise, trainer uses MultiGradientMachine which will transfer
-        # data from CPU to GPU in the forward function, set data_in_gpu to
-        # False in this case.
-        self.data_in_gpu = swig_paddle.isUsingGpu(
-        ) and swig_paddle.getTrainerCount() == 1
-
-    def pre_scan(self, dat):
-        """
-        First pass scan method. During this method, the scanner could count the
-        data number, and get the total memory size this batch would use.
-
-        :param dat: The python object.
-        """
-        pass
-
-    def finish_pre_scan(self, argument):
-        """
-        Finish first scan pass. Allocate the memory.
-
-        :param argument: Output arguments object.
-        :type argument: swig_paddle.Arguments
-        :param dat: Output arguments object.
-        :type dat: The Python object, numpy.array or List.
-        :return:
-        """
-        pass
-
-    def scan(self, dat):
-        """
-        Second pass scan method. Copy the data to arguments.
-
-        :param dat: The python object.
-        """
-        pass
-
-    def finish_scan(self, argument):
-        """
-        Finish second pass. Finalize the resources, etc.
-
-        :param argument: Output arguments object.
-        :type argument: swig_paddle.Arguments
-        """
-        pass
-
-
-class DenseScanner(IScanner):
-    """
-    :type __mat__: numpy.ndarray
-    """
-
-    def __init__(self, input_type, pos):
-        IScanner.__init__(self, input_type, pos)
-        self.__mat__ = None
-        self.__shape__ = None
-        self.__height__ = 0
-        self.__dim__ = 0
-
-    def pre_scan(self, dat):
-        self.__height__ += 1
-        if self.__shape__ is None:
-            self.__shape__ = numpy.array(dat).shape
-            if len(self.__shape__) > 3:
-                raise ValueError(
-                    "The dimension of input cannot be greater than 3.")
-            if len(self.__shape__) == 0:
-                raise ValueError(
-                    "The input should be a vector, please check your input data."
-                )
-            self.__dim__ = reduce(lambda x, y: x * y, self.__shape__)
-            if len(self.__shape__) == 1 and self.__dim__ != self.input_type.dim:
-                raise ValueError(
-                    "The data size must be equal to it in data layer.")
-        else:
-            if self.__shape__ != numpy.array(dat).shape:
-                raise ValueError(
-                    "The data shape must be same in one mini-batch.")
-
-    def finish_pre_scan(self, argument):
-        self.__mat__ = numpy.ndarray(
-            shape=(self.__height__, self.__dim__), dtype=numpy.float32)
-        self.__height__ = 0
-
-    def scan(self, dat):
-        # It's better to use NumPy array for speed.
-        dat = numpy.array(dat)
-        dat = dat.flatten()
-        self.__mat__[self.__height__] = dat
-        self.__height__ += 1
-
-    def finish_scan(self, argument):
-        assert isinstance(argument, swig_paddle.Arguments)
-        if self.__mat__.dtype != numpy.float32:
-            self.__mat__ = self.__mat__.astype(numpy.float32)
-        m = swig_paddle.Matrix.createDenseFromNumpy(self.__mat__, True,
-                                                    self.data_in_gpu)
-        argument.setSlotValue(self.pos, m)
-        if len(self.__shape__) > 1:
-            # The last-two dimenstions are the frame height and width.
-            # For example, the layout is CHW for 3-D feature of image.
-            # The H and W are the frame height and width.
-            h, w = self.__shape__[-2:]
-            argument.setSlotFrameHeight(self.pos, h)
-            argument.setSlotFrameWidth(self.pos, w)
-        self.__shape__ = None
-
-
-class SparseBinaryScanner(IScanner):
-    def __init__(self, input_type, pos):
-        IScanner.__init__(self, input_type, pos)
-        self.__rows__ = [0]
-        self.__cols__ = []
-        self.__height__ = 0
-        self.__value__ = []
-
-    def scan(self, dat):
-        self.extend_cols(dat)
-        self.__rows__.append(len(self.__cols__))
-        self.__height__ += 1
-
-    def extend_cols(self, dat):
-        self.__cols__.extend(dat)
-
-    def finish_scan(self, argument):
-        assert isinstance(argument, swig_paddle.Arguments)
-        m = swig_paddle.Matrix.createSparse(
-            self.__height__,
-            self.input_type.dim,
-            len(self.__cols__),
-            len(self.__value__) == 0,
-            False,  # trans
-            False)  # TODO supoort GPU
-        assert isinstance(m, swig_paddle.Matrix)
-        m.sparseCopyFrom(self.__rows__, self.__cols__, self.__value__)
-        argument.setSlotValue(self.pos, m)
-
-
-class SparseFloatScanner(SparseBinaryScanner):
-    def __init__(self, input_type, pos):
-        SparseBinaryScanner.__init__(self, input_type, pos)
-
-    def extend_cols(self, dat):
-        self.__cols__.extend((x[0] for x in dat))
-        self.__value__.extend((x[1] for x in dat))
-
-
-class IndexScanner(IScanner):
-    def __init__(self, input_type, pos):
-        IScanner.__init__(self, input_type, pos)
-        self.__ids__ = None
-        self.__idx__ = 0
-
-    def pre_scan(self, dat):
-        self.__idx__ += 1
-
-    def finish_pre_scan(self, argument):
-        self.__ids__ = [0] * self.__idx__
-        self.__idx__ = 0
-
-    def scan(self, dat):
-        self.__ids__[self.__idx__] = dat
-        self.__idx__ += 1
-
-    def finish_scan(self, argument):
-        ids = swig_paddle.IVector.create(self.__ids__, self.data_in_gpu)
-        assert isinstance(argument, swig_paddle.Arguments)
-        argument.setSlotIds(self.pos, ids)
-
-
-class SequenceScanner(IScanner):
-    def __init__(self, input_type, pos, inner_scanner, setter):
-        IScanner.__init__(self, input_type, pos)
-        self.__seq__ = [0]
-        self.__inner_scanner__ = inner_scanner
-        self.__setter__ = setter
-
-    def pre_scan(self, dat):
-        for each in dat:
-            self.__inner_scanner__.pre_scan(each)
-
-    def finish_pre_scan(self, argument):
-        self.__inner_scanner__.finish_pre_scan(argument)
-
-    def scan(self, dat):
-        self.__seq__.append(self.__seq__[-1] + self.get_size(dat))
-        for each in dat:
-            self.__inner_scanner__.scan(each)
-
-    def finish_scan(self, argument):
-        seq = swig_paddle.IVector.create(self.__seq__, False)
-        self.__setter__(argument, self.pos, seq)
-        self.__inner_scanner__.finish_scan(argument)
-
-    def get_size(self, dat):
-        if isinstance(self.__inner_scanner__, SequenceScanner):
-            return sum(self.__inner_scanner__.get_size(item) for item in dat)
-        else:
-            return len(dat)
-
-
-class DataProviderConverter(object):
-    def __init__(self, input_types):
-        self.input_types = input_types
-        assert isinstance(self.input_types, collections.Sequence)
-        for each in self.input_types:
-            assert isinstance(each, dp2.InputType)
-
-    def convert(self, dat, argument=None):
-        if argument is None:
-            argument = swig_paddle.Arguments.createArguments(0)
-        assert isinstance(argument, swig_paddle.Arguments)
-        argument.resize(len(self.input_types))
-
-        scanners = [
-            DataProviderConverter.create_scanner(i, each_type)
-            for i, each_type in enumerate(self.input_types)
-        ]
-
-        for each_sample in dat:
-            for each_step, scanner in itertools.izip(each_sample, scanners):
-                scanner.pre_scan(each_step)
-
-        for scanner in scanners:
-            scanner.finish_pre_scan(argument)
-
-        for each_sample in dat:
-            for each_step, scanner in itertools.izip(each_sample, scanners):
-                scanner.scan(each_step)
-
-        for scanner in scanners:
-            scanner.finish_scan(argument)
-
-        return argument
-
-    def __call__(self, dat, argument=None):
-        return self.convert(dat, argument)
-
-    @staticmethod
-    def create_scanner(i, each):
-        assert isinstance(each, dp2.InputType)
-        retv = None
-        if each.type == dp2.DataType.Dense:
-            retv = DenseScanner(each, i)
-        elif each.type == dp2.DataType.Index:
-            retv = IndexScanner(each, i)
-        elif each.type == dp2.DataType.SparseNonValue:
-            retv = SparseBinaryScanner(each, i)
-        elif each.type == dp2.DataType.SparseValue:
-            retv = SparseFloatScanner(each, i)
-        assert retv is not None
-
-        if each.seq_type == dp2.SequenceType.SUB_SEQUENCE:
-            retv = SequenceScanner(
-                each, i, retv,
-                lambda a, p, seq: a.setSlotSubSequenceStartPositions(p, seq))
-
-        if each.seq_type in [
-                dp2.SequenceType.SUB_SEQUENCE, dp2.SequenceType.SEQUENCE
-        ]:
-            retv = SequenceScanner(
-                each, i, retv,
-                lambda a, p, seq: a.setSlotSequenceStartPositions(p, seq))
-        return retv

paddle/scripts/README.md

@@ -68,7 +68,6 @@ Users can specify the following Docker build arguments with either "ON" or "OFF"
 | `WITH_TESTING` | OFF | Build unit tests binaries. |
 | `WITH_MKL` | ON | Build with [Intel® MKL](https://software.intel.com/en-us/mkl) and [Intel® MKL-DNN](https://github.com/01org/mkl-dnn) support. |
 | `WITH_GOLANG` | OFF | Build fault-tolerant parameter server written in go. |
-| `WITH_SWIG_PY` | ON | Build with SWIG python API support. |
 | `WITH_PYTHON` | ON | Build with python support. Turn this off if build is only for capi. |
 | `WITH_STYLE_CHECK` | ON | Check the code style when building. |
 | `PYTHON_ABI` | "" | Build for different python ABI support, can be cp27-cp27m or cp27-cp27mu |

paddle/scripts/paddle_build.sh

@@ -180,7 +180,6 @@ function cmake_gen() {
         -DWITH_GOLANG=${WITH_GOLANG:-OFF}
         -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All}
         -DWITH_PYTHON=${WITH_PYTHON:-ON}
-        -DWITH_SWIG_PY=${WITH_SWIG_PY:-ON}
         -DCUDNN_ROOT=/usr/
         -DWITH_TESTING=${WITH_TESTING:-ON}
         -DCMAKE_MODULE_PATH=/opt/rocm/hip/cmake
@@ -214,7 +213,6 @@ EOF
         -DWITH_AVX=${WITH_AVX:-OFF} \
         -DWITH_GOLANG=${WITH_GOLANG:-OFF} \
         -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-All} \
-        -DWITH_SWIG_PY=${WITH_SWIG_PY:-ON} \
         -DWITH_PYTHON=${WITH_PYTHON:-ON} \
         -DCUDNN_ROOT=/usr/ \
         -DWITH_TESTING=${WITH_TESTING:-ON} \

python/CMakeLists.txt

@@ -39,7 +39,6 @@ add_custom_target(copy_paddle_pybind ALL DEPENDS ${FLUID_CORE})
 IF(WIN32)
     add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
             COMMAND ${CMAKE_COMMAND} -E copy_directory ${PADDLE_SOURCE_DIR}/python/paddle ${PADDLE_BINARY_DIR}/python/paddle/
-            COMMAND ${CMAKE_COMMAND} -E copy_directory ${PADDLE_SOURCE_DIR}/paddle/py_paddle ${PADDLE_BINARY_DIR}/python/
             COMMAND ${CMAKE_COMMAND} -E env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
             COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
             COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python
@@ -48,7 +47,6 @@ ELSE(WIN32)
 	add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
 		COMMAND touch stub.cc
 		COMMAND cp -r ${PADDLE_SOURCE_DIR}/python/paddle ${PADDLE_BINARY_DIR}/python
-		COMMAND cp -r ${PADDLE_SOURCE_DIR}/paddle/py_paddle ${PADDLE_BINARY_DIR}/python/
 		COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
 		COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp
 		COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python