Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into rnn

paddle/api/Evaluator.cpp

@@ -27,3 +27,18 @@ std::string Evaluator::toString() {
   m->rawPtr->printStats(sout);
   return sout.str();
 }
+
+std::vector<std::string> Evaluator::getNames() const {
+  std::vector<std::string> retv;
+  m->rawPtr->getNames(&retv);
+  return retv;
+}
+
+double Evaluator::getValue(const std::string name) const {
+  paddle::Error err;
+  double v = m->rawPtr->getValue(name, &err);
+  if (err) {
+    throw std::runtime_error(err.msg());
+  }
+  return v;
+}

paddle/api/PaddleAPI.h

@@ -900,6 +900,10 @@ public:
    */
   std::string toString();
 
+  std::vector<std::string> getNames() const;
+
+  double getValue(const std::string name) const;
+
 private:
   EvaluatorPrivate* m;
 

paddle/api/test/testTrain.py

@@ -89,9 +89,14 @@ def main():
             except Exception as e:
                 print e
 
+        ev = m.makeEvaluator()
+        ev.start()
         m.forwardBackward(inArgs, outArgs, swig_paddle.PASS_TRAIN,
                           update_callback)
-
+        m.eval(ev)
+        ev.finish()
+        for name in ev.getNames():
+            print name, ev.getValue(name)
         for optimizer in optimizers:
             optimizer.finishBatch()
 

paddle/scripts/docker/Dockerfile

@@ -10,28 +10,30 @@ RUN apt-get update && \
     apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
     apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
     apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
-    apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \
+    apt-get install -y automake && \
     apt-get clean -y
 
 RUN pip install --upgrade pip && \
-    pip install -U protobuf && \
+    pip install -U "protobuf==3.1.0" && \
     pip install -U wheel pillow BeautifulSoup && \
     pip install -U docopt PyYAML sphinx && \
     pip install -U sphinx_rtd_theme recommonmark jupyter
 
 RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
-    cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \
+    cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
     cd .. && rm -rf cmake-3.4.1
 
+ARG BUILD_WOBOQ
 ARG BUILD_AND_INSTALL
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_STYLE_CHECK
 
+ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
 ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
 ENV WITH_GPU=OFF
 ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
 ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
 
 RUN mkdir /paddle

paddle/scripts/docker/Dockerfile.gpu

@@ -10,28 +10,30 @@ RUN apt-get update && \
     apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
     apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
     apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
-    apt-get install -y automake clang-3.8 llvm-3.8 libclang-3.8-dev && \
+    apt-get install -y automake && \
     apt-get clean -y
 
 RUN pip install --upgrade pip && \
-    pip install -U protobuf && \
+    pip install -U "protobuf==3.1.0" && \
     pip install -U wheel pillow BeautifulSoup && \
     pip install -U docopt PyYAML sphinx && \
     pip install -U sphinx_rtd_theme recommonmark jupyter
 
 RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
-    cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \
+    cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
     cd .. && rm -rf cmake-3.4.1
 
+ARG BUILD_WOBOQ
 ARG BUILD_AND_INSTALL
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_STYLE_CHECK
 
+ENV BUILD_WOBOQ=${BUILD_WOBOQ:-OFF}
 ENV BUILD_AND_INSTALL=${BUILD_AND_INSTALL:-OFF}
 ENV WITH_GPU=ON
 ENV WITH_AVX=${WITH_AVX:-ON}
-ENV WITH_DOC=${WITH_DOC:-ON}
+ENV WITH_DOC=${WITH_DOC:-OFF}
 ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
 
 RUN mkdir /paddle

paddle/scripts/docker/build.sh

@@ -11,7 +11,7 @@ set -e
 # If Dockerfile.* sets BUILD_AND_INSTALL to 'ON', it would have copied
 # source tree to /paddle, and this scripts should build it into
 # /paddle/build.
-if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then
+if [[ ${BUILD_AND_INSTALL:-OFF} == 'ON' ]]; then
     if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
 	ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
     fi
@@ -19,7 +19,7 @@ if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then
     mkdir -p /paddle/build # -p means no error if exists
     cd /paddle/build
     cmake .. \
-	  -DWITH_DOC=ON \
+	  -DWITH_DOC=${WITH_DOC:-OFF} \
 	  -DWITH_GPU=${WITH_GPU:-OFF} \
 	  -DWITH_AVX=${WITH_AVX:-OFF} \
 	  -DWITH_SWIG_PY=ON \
@@ -29,28 +29,32 @@ if [[ ${BUILD_AND_INSTALL:-ON} == 'ON' ]]; then
     make -j `nproc`
     make install
 
-    # Install woboq_codebrowser.
-    git clone https://github.com/woboq/woboq_codebrowser /woboq
-    cd /woboq
-    cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \
-	  -DCMAKE_BUILD_TYPE=Release \
-	  .
-    make
-
-    export WOBOQ_OUT=/usr/share/nginx/html/paddle
-    export BUILD_DIR=/paddle/build
-    mkdir -p $WOBOQ_OUT
-    cp -rv /woboq/data $WOBOQ_OUT/../data
-    /woboq/generator/codebrowser_generator \
-	-b /paddle/build \
-	-a \
-	-o $WOBOQ_OUT \
-	-p paddle:/paddle
-    /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
-    cd /woboq
-    make clean
-
-    pip install /usr/local/opt/paddle/share/wheels/*.whl
+    if [[ ${BUILD_WOBOQ:-OFF} == 'ON' ]]; then
+        apt-get install -y clang-3.8 llvm-3.8 libclang-3.8-dev 
+        # Install woboq_codebrowser.
+        git clone https://github.com/woboq/woboq_codebrowser /woboq
+        cd /woboq
+        cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \
+        -DCMAKE_BUILD_TYPE=Release \
+        .
+        make
+
+        export WOBOQ_OUT=/usr/share/nginx/html/paddle
+        export BUILD_DIR=/paddle/build
+        mkdir -p $WOBOQ_OUT
+        cp -rv /woboq/data $WOBOQ_OUT/../data
+        /woboq/generator/codebrowser_generator \
+        -b /paddle/build \
+        -a \
+        -o $WOBOQ_OUT \
+        -p paddle:/paddle
+        /woboq/indexgenerator/codebrowser_indexgenerator $WOBOQ_OUT
+        cd /woboq
+        make clean
+    fi
+
+    pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl
+    pip install /usr/local/opt/paddle/share/wheels/paddle*.whl
     paddle version
 fi
 

python/paddle/reader/__init__.py

@@ -21,3 +21,5 @@
 #
 #     r = paddle.reader.buffered(paddle.reader.creator.text("hello.txt"))
 from decorator import *
+
+import creator

python/paddle/reader/creator.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ['np_array', 'text_file']
+
+
+def np_array(x):
+    """
+    Creates a reader that yields elements of x, if it is a
+    numpy vector. Or rows of x, if it is a numpy matrix.
+    Or any sub-hyperplane indexed by the highest dimension.
+
+    :param x: the numpy array to create reader from.
+    :returns: data reader created from x.
+    """
+
+    def reader():
+        if x.ndim < 1:
+            yield x
+
+        for e in x:
+            yield e
+
+    return reader
+
+
+def text_file(path):
+    """
+    Creates a data reader that outputs text line by line from given text file.
+    Trailing new line ('\n') of each line will be removed.
+
+    :path: path of the text file.
+    :returns: data reader of text file
+    """
+
+    def reader():
+        f = open(path, "r")
+        for l in f:
+            yield l.rstrip('\n')
+        f.close()
+
+    return reader

python/paddle/reader/tests/CMakeLists.txt

@@ -2,3 +2,8 @@ add_test(NAME reader_decorator_test
   COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
         ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/tests/decorator_test.py
     WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
+
+add_test(NAME reader_creator_test
+  COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
+        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/reader/tests/creator_test.py
+    WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)

python/paddle/reader/tests/creator_test.py

+# Copyright PaddlePaddle contributors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+import paddle.reader.creator
+import numpy as np
+import os
+
+
+class TestNumpyArray(unittest.TestCase):
+    def test_numpy_array(self):
+        l = [[1, 2, 3], [4, 5, 6]]
+        x = np.array(l, np.int32)
+        reader = paddle.reader.creator.np_array(x)
+        for idx, e in enumerate(reader()):
+            self.assertItemsEqual(e, l[idx])
+
+
+class TestTextFile(unittest.TestCase):
+    def test_text_file(self):
+        path = os.path.join(os.path.dirname(__file__), "test_data_creator.txt")
+        reader = paddle.reader.creator.text_file(path)
+        for idx, e in enumerate(reader()):
+            self.assertEqual(e, str(idx * 2) + " " + str(idx * 2 + 1))
+
+
+if __name__ == '__main__':
+    unittest.main()

python/paddle/reader/tests/test_data_creator.txt

+0 1
+2 3
+4 5