Merge pull request #245 from gongweibao/builddockerimage3

Pack paddle.dataset cache dir into Docker image

.dockerignore

+Dockerfile

.gitignore

@@ -4,3 +4,5 @@ pandoc.template
 .DS_Store
 .idea
 py_env*
+*.ipynb
+build

.tools/build_docker.sh

@@ -5,36 +5,39 @@ cd $cur_path/../
 #convert md to ipynb
 .tools/convert-markdown-into-ipynb-and-test.sh
 
-paddle_version=0.10.0rc2
+paddle_tag=0.10.0rc2
+book_tag=latest
 
 #generate docker file
 if [ ${USE_UBUNTU_REPO_MIRROR} ]; then
-  UPDATE_MIRROR_CMD="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\"
+  update_mirror_cmd="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\"
 else
-  UPDATE_MIRROR_CMD="\\"
+  update_mirror_cmd="\\"
 fi
 
-mkdir -p build
-cat > build/Dockerfile <<EOF
-FROM paddlepaddle/paddle:${paddle_version}
-MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+#build docker image
+echo "paddle_tag:"$paddle_tag
+echo "book_tag:"$book_tag
 
-RUN ${UPDATE_MIRROR_CMD}
-        apt-get install locales
-RUN localedef -f UTF-8 -i en_US en_US.UTF-8
+cat > Dockerfile <<EOF
+FROM paddlepaddle/paddle:${paddle_tag}
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
 
-RUN  apt-get -y install gcc && \
-        apt-get -y clean
+COPY . /book
 
-RUN pip install -U matplotlib jupyter numpy requests scipy
+RUN pip install -U nltk \
+    && python /book/.tools/cache_dataset.py
 
-COPY . /book
-RUN rm -rf /book/build
+RUN ${update_mirror_cmd}
+    apt-get update && \
+    apt-get install -y locales && \
+    apt-get -y install gcc && \
+    apt-get -y clean && \
+    localedef -f UTF-8 -i en_US en_US.UTF-8 && \
+    pip install -U matplotlib jupyter numpy requests scipy
 
 EXPOSE 8888
 CMD ["sh", "-c", "jupyter notebook --ip=0.0.0.0 --no-browser --NotebookApp.token='' --NotebookApp.disable_check_xsrf=True /book/"]
 EOF
 
-#build docker image
-echo "paddle_version:"$paddle_version
-docker build --no-cache -t paddlepaddle/book:${paddle_version}  -t paddlepaddle/book:latest  -f ./build/Dockerfile .
+docker build --no-cache  -t paddlepaddle/book:${paddle_tag}  -t paddlepaddle/book:${book_tag} .

.tools/cache_dataset.py

+#!/bin/env python
+import paddle.v2.dataset as dataset
+import nltk
+
+#cifar
+dataset.common.download(dataset.cifar.CIFAR100_URL, 'cifar',
+                        dataset.cifar.CIFAR100_MD5)
+dataset.common.download(dataset.cifar.CIFAR100_URL, 'cifar',
+                        dataset.cifar.CIFAR100_MD5)
+dataset.common.download(dataset.cifar.CIFAR10_URL, 'cifar',
+                        dataset.cifar.CIFAR10_MD5)
+dataset.common.download(dataset.cifar.CIFAR10_URL, 'cifar',
+                        dataset.cifar.CIFAR10_MD5)
+
+# Cache conll05
+dataset.common.download(dataset.conll05.WORDDICT_URL, 'conll05st', \
+                        dataset.conll05.WORDDICT_MD5)
+dataset.common.download(dataset.conll05.VERBDICT_URL, 'conll05st', \
+                        dataset.conll05.VERBDICT_MD5)
+dataset.common.download(dataset.conll05.TRGDICT_URL, 'conll05st', \
+                        dataset.conll05.TRGDICT_MD5)
+
+# Cache imdb
+dataset.common.download(dataset.imdb.URL, "imdb", dataset.imdb.MD5)
+
+# Cache imikolov
+dataset.common.download(dataset.imikolov.URL, "imikolov", dataset.imikolov.MD5)
+
+# Cache movielens
+dataset.common.download('http://files.grouplens.org/datasets/movielens/ml-1m.zip',\
+                        'movielens','c4d9eecfca2ab87c1945afe126590906')
+
+# Cache nltk
+nltk.download('movie_reviews', download_dir=dataset.common.DATA_HOME)
+
+# Cache uci housing
+dataset.common.download(dataset.uci_housing.URL, "uci_housing", \
+                        dataset.uci_housing.MD5)
+
+# Cache vmt14
+dataset.common.download(dataset.wmt14.URL_TRAIN, "wmt14",\
+                        dataset.wmt14.MD5_TRAIN)

.tools/convert-markdown-into-ipynb-and-test.sh

@@ -5,14 +5,14 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-GOPATH=~/.go go get -u github.com/wangkuiyi/ipynb/markdown-to-ipynb
+export GOPATH=~/go; go get -u github.com/wangkuiyi/ipynb/markdown-to-ipynb
 
 cur_path="$(cd "$(dirname "$0")" && pwd -P)"
 cd $cur_path/../
 
 #convert md to ipynb
 for file in */{README,README\.en}.md ; do
-    /tmp/go/bin/markdown-to-ipynb < $file > ${file%.*}".ipynb"
+    ~/go/bin/markdown-to-ipynb < $file > ${file%.*}".ipynb"
     if [ $? -ne 0 ]; then
         echo >&2 "markdown-to-ipynb $file error"
         exit 1