+# Distributed under the GNU General Public License, version 2.0.
+#
+# This script allows you to see repository status in your prompt.
+#
+# To enable:
+#
+# 1) Copy this file to somewhere (e.g. ~/.git-prompt.sh).
+# 2) Add the following line to your .bashrc/.zshrc:
+# source ~/.git-prompt.sh
+# 3a) Change your PS1 to call __git_ps1 as
+# command-substitution:
+# Bash: PS1='[\u@\h \W$(__git_ps1 " (%s)")]\$ '
+# ZSH: setopt PROMPT_SUBST ; PS1='[%n@%m %c$(__git_ps1 " (%s)")]\$ '
+# the optional argument will be used as format string.
+# 3b) Alternatively, for a slightly faster prompt, __git_ps1 can
+# be used for PROMPT_COMMAND in Bash or for precmd() in Zsh
+# with two parameters, and , which are strings
+# you would put in $PS1 before and after the status string
+# generated by the git-prompt machinery. e.g.
+# Bash: PROMPT_COMMAND='__git_ps1 "\u@\h:\w" "\\\$ "'
+# will show username, at-sign, host, colon, cwd, then
+# various status string, followed by dollar and SP, as
+# your prompt.
+# ZSH: precmd () { __git_ps1 "%n" ":%~$ " "|%s" }
+# will show username, pipe, then various status string,
+# followed by colon, cwd, dollar and SP, as your prompt.
+# Optionally, you can supply a third argument with a printf
+# format string to finetune the output of the branch status
+#
+# The repository status will be displayed only if you are currently in a
+# git repository. The %s token is the placeholder for the shown status.
+#
+# The prompt status always includes the current branch name.
+#
+# In addition, if you set GIT_PS1_SHOWDIRTYSTATE to a nonempty value,
+# unstaged (*) and staged (+) changes will be shown next to the branch
+# name. You can configure this per-repository with the
+# bash.showDirtyState variable, which defaults to true once
+# GIT_PS1_SHOWDIRTYSTATE is enabled.
+#
+# You can also see if currently something is stashed, by setting
+# GIT_PS1_SHOWSTASHSTATE to a nonempty value. If something is stashed,
+# then a '$' will be shown next to the branch name.
+#
+# If you would like to see if there're untracked files, then you can set
+# GIT_PS1_SHOWUNTRACKEDFILES to a nonempty value. If there're untracked
+# files, then a '%' will be shown next to the branch name. You can
+# configure this per-repository with the bash.showUntrackedFiles
+# variable, which defaults to true once GIT_PS1_SHOWUNTRACKEDFILES is
+# enabled.
+#
+# If you would like to see the difference between HEAD and its upstream,
+# set GIT_PS1_SHOWUPSTREAM="auto". A "<" indicates you are behind, ">"
+# indicates you are ahead, "<>" indicates you have diverged and "="
+# indicates that there is no difference. You can further control
+# behaviour by setting GIT_PS1_SHOWUPSTREAM to a space-separated list
+# of values:
+#
+# verbose show number of commits ahead/behind (+/-) upstream
+# legacy don't use the '--count' option available in recent
+# versions of git-rev-list
+# git always compare HEAD to @{upstream}
+# svn always compare HEAD to your SVN upstream
+#
+# By default, __git_ps1 will compare HEAD to your SVN upstream if it can
+# find one, or @{upstream} otherwise. Once you have set
+# GIT_PS1_SHOWUPSTREAM, you can override it on a per-repository basis by
+# setting the bash.showUpstream config variable.
+#
+# If you would like to see more information about the identity of
+# commits checked out as a detached HEAD, set GIT_PS1_DESCRIBE_STYLE
+# to one of these values:
+#
+# contains relative to newer annotated tag (v1.6.3.2~35)
+# branch relative to newer tag or branch (master~4)
+# describe relative to older annotated tag (v1.6.3.1-13-gdd42c2f)
+# default exactly matching tag
+#
+# If you would like a colored hint about the current dirty state, set
+# GIT_PS1_SHOWCOLORHINTS to a nonempty value. The colors are based on
+# the colored output of "git status -sb" and are available only when
+# using __git_ps1 for PROMPT_COMMAND or precmd.
+
+# stores the divergence from upstream in $p
+# used by GIT_PS1_SHOWUPSTREAM
+__git_ps1_show_upstream ()
+{
+ local key value
+ local svn_remote svn_url_pattern count n
+ local upstream=git legacy="" verbose=""
+
+ svn_remote=()
+ # get some config options from git-config
+ local output="$(git config -z --get-regexp '^(svn-remote\..*\.url|bash\.showupstream)$' 2>/dev/null | tr '\0\n' '\n ')"
+ while read -r key value; do
+ case "$key" in
+ bash.showupstream)
+ GIT_PS1_SHOWUPSTREAM="$value"
+ if [[ -z "${GIT_PS1_SHOWUPSTREAM}" ]]; then
+ p=""
+ return
+ fi
+ ;;
+ svn-remote.*.url)
+ svn_remote[$((${#svn_remote[@]} + 1))]="$value"
+ svn_url_pattern+="\\|$value"
+ upstream=svn+git # default upstream is SVN if available, else git
+ ;;
+ esac
+ done <<< "$output"
+
+ # parse configuration values
+ for option in ${GIT_PS1_SHOWUPSTREAM}; do
+ case "$option" in
+ git|svn) upstream="$option" ;;
+ verbose) verbose=1 ;;
+ legacy) legacy=1 ;;
+ esac
+ done
+
+ # Find our upstream
+ case "$upstream" in
+ git) upstream="@{upstream}" ;;
+ svn*)
+ # get the upstream from the "git-svn-id: ..." in a commit message
+ # (git-svn uses essentially the same procedure internally)
+ local -a svn_upstream
+ svn_upstream=($(git log --first-parent -1 \
+ --grep="^git-svn-id: \(${svn_url_pattern#??}\)" 2>/dev/null))
+ if [[ 0 -ne ${#svn_upstream[@]} ]]; then
+ svn_upstream=${svn_upstream[${#svn_upstream[@]} - 2]}
+ svn_upstream=${svn_upstream%@*}
+ local n_stop="${#svn_remote[@]}"
+ for ((n=1; n <= n_stop; n++)); do
+ svn_upstream=${svn_upstream#${svn_remote[$n]}}
+ done
+
+ if [[ -z "$svn_upstream" ]]; then
+ # default branch name for checkouts with no layout:
+ upstream=${GIT_SVN_ID:-git-svn}
+ else
+ upstream=${svn_upstream#/}
+ fi
+ elif [[ "svn+git" = "$upstream" ]]; then
+ upstream="@{upstream}"
+ fi
+ ;;
+ esac
+
+ # Find how many commits we are ahead/behind our upstream
+ if [[ -z "$legacy" ]]; then
+ count="$(git rev-list --count --left-right \
+ "$upstream"...HEAD 2>/dev/null)"
+ else
+ # produce equivalent output to --count for older versions of git
+ local commits
+ if commits="$(git rev-list --left-right "$upstream"...HEAD 2>/dev/null)"
+ then
+ local commit behind=0 ahead=0
+ for commit in $commits
+ do
+ case "$commit" in
+ "<"*) ((behind++)) ;;
+ *) ((ahead++)) ;;
+ esac
+ done
+ count="$behind $ahead"
+ else
+ count=""
+ fi
+ fi
+
+ # calculate the result
+ if [[ -z "$verbose" ]]; then
+ case "$count" in
+ "") # no upstream
+ p="" ;;
+ "0 0") # equal to upstream
+ p="=" ;;
+ "0 "*) # ahead of upstream
+ p=">" ;;
+ *" 0") # behind upstream
+ p="<" ;;
+ *) # diverged from upstream
+ p="<>" ;;
+ esac
+ else
+ case "$count" in
+ "") # no upstream
+ p="" ;;
+ "0 0") # equal to upstream
+ p=" u=" ;;
+ "0 "*) # ahead of upstream
+ p=" u+${count#0 }" ;;
+ *" 0") # behind upstream
+ p=" u-${count% 0}" ;;
+ *) # diverged from upstream
+ p=" u+${count#* }-${count% *}" ;;
+ esac
+ fi
+
+}
+
+# Helper function that is meant to be called from __git_ps1. It
+# injects color codes into the appropriate gitstring variables used
+# to build a gitstring.
+__git_ps1_colorize_gitstring ()
+{
+ if [[ -n ${ZSH_VERSION-} ]]; then
+ local c_red='%F{red}'
+ local c_green='%F{green}'
+ local c_lblue='%F{blue}'
+ local c_clear='%f'
+ else
+ # Using \[ and \] around colors is necessary to prevent
+ # issues with command line editing/browsing/completion!
+ local c_red='\[\e[31m\]'
+ local c_green='\[\e[32m\]'
+ local c_lblue='\[\e[1;34m\]'
+ local c_clear='\[\e[0m\]'
+ fi
+ local bad_color=$c_red
+ local ok_color=$c_green
+ local flags_color="$c_lblue"
+
+ local branch_color=""
+ if [ $detached = no ]; then
+ branch_color="$ok_color"
+ else
+ branch_color="$bad_color"
+ fi
+ c="$branch_color$c"
+
+ z="$c_clear$z"
+ if [ "$w" = "*" ]; then
+ w="$bad_color$w"
+ fi
+ if [ -n "$i" ]; then
+ i="$ok_color$i"
+ fi
+ if [ -n "$s" ]; then
+ s="$flags_color$s"
+ fi
+ if [ -n "$u" ]; then
+ u="$bad_color$u"
+ fi
+ r="$c_clear$r"
+}
+
+# __git_ps1 accepts 0 or 1 arguments (i.e., format string)
+# when called from PS1 using command substitution
+# in this mode it prints text to add to bash PS1 prompt (includes branch name)
+#
+# __git_ps1 requires 2 or 3 arguments when called from PROMPT_COMMAND (pc)
+# in that case it _sets_ PS1. The arguments are parts of a PS1 string.
+# when two arguments are given, the first is prepended and the second appended
+# to the state string when assigned to PS1.
+# The optional third parameter will be used as printf format string to further
+# customize the output of the git-status string.
+# In this mode you can request colored hints using GIT_PS1_SHOWCOLORHINTS=true
+__git_ps1 ()
+{
+ local pcmode=no
+ local detached=no
+ local ps1pc_start='\u@\h:\w '
+ local ps1pc_end='\$ '
+ local printf_format=' (%s)'
+
+ case "$#" in
+ 2|3) pcmode=yes
+ ps1pc_start="$1"
+ ps1pc_end="$2"
+ printf_format="${3:-$printf_format}"
+ ;;
+ 0|1) printf_format="${1:-$printf_format}"
+ ;;
+ *) return
+ ;;
+ esac
+
+ local repo_info rev_parse_exit_code
+ repo_info="$(git rev-parse --git-dir --is-inside-git-dir \
+ --is-bare-repository --is-inside-work-tree \
+ --short HEAD 2>/dev/null)"
+ rev_parse_exit_code="$?"
+
+ if [ -z "$repo_info" ]; then
+ if [ $pcmode = yes ]; then
+ #In PC mode PS1 always needs to be set
+ PS1="$ps1pc_start$ps1pc_end"
+ fi
+ return
+ fi
+
+ local short_sha
+ if [ "$rev_parse_exit_code" = "0" ]; then
+ short_sha="${repo_info##*$'\n'}"
+ repo_info="${repo_info%$'\n'*}"
+ fi
+ local inside_worktree="${repo_info##*$'\n'}"
+ repo_info="${repo_info%$'\n'*}"
+ local bare_repo="${repo_info##*$'\n'}"
+ repo_info="${repo_info%$'\n'*}"
+ local inside_gitdir="${repo_info##*$'\n'}"
+ local g="${repo_info%$'\n'*}"
+
+ local r=""
+ local b=""
+ local step=""
+ local total=""
+ if [ -d "$g/rebase-merge" ]; then
+ read b 2>/dev/null <"$g/rebase-merge/head-name"
+ read step 2>/dev/null <"$g/rebase-merge/msgnum"
+ read total 2>/dev/null <"$g/rebase-merge/end"
+ if [ -f "$g/rebase-merge/interactive" ]; then
+ r="|REBASE-i"
+ else
+ r="|REBASE-m"
+ fi
+ else
+ if [ -d "$g/rebase-apply" ]; then
+ read step 2>/dev/null <"$g/rebase-apply/next"
+ read total 2>/dev/null <"$g/rebase-apply/last"
+ if [ -f "$g/rebase-apply/rebasing" ]; then
+ read b 2>/dev/null <"$g/rebase-apply/head-name"
+ r="|REBASE"
+ elif [ -f "$g/rebase-apply/applying" ]; then
+ r="|AM"
+ else
+ r="|AM/REBASE"
+ fi
+ elif [ -f "$g/MERGE_HEAD" ]; then
+ r="|MERGING"
+ elif [ -f "$g/CHERRY_PICK_HEAD" ]; then
+ r="|CHERRY-PICKING"
+ elif [ -f "$g/REVERT_HEAD" ]; then
+ r="|REVERTING"
+ elif [ -f "$g/BISECT_LOG" ]; then
+ r="|BISECTING"
+ fi
+
+ if [ -n "$b" ]; then
+ :
+ elif [ -h "$g/HEAD" ]; then
+ # symlink symbolic ref
+ b="$(git symbolic-ref HEAD 2>/dev/null)"
+ else
+ local head=""
+ if ! read head 2>/dev/null <"$g/HEAD"; then
+ if [ $pcmode = yes ]; then
+ PS1="$ps1pc_start$ps1pc_end"
+ fi
+ return
+ fi
+ # is it a symbolic ref?
+ b="${head#ref: }"
+ if [ "$head" = "$b" ]; then
+ detached=yes
+ b="$(
+ case "${GIT_PS1_DESCRIBE_STYLE-}" in
+ (contains)
+ git describe --contains HEAD ;;
+ (branch)
+ git describe --contains --all HEAD ;;
+ (describe)
+ git describe HEAD ;;
+ (* | default)
+ git describe --tags --exact-match HEAD ;;
+ esac 2>/dev/null)" ||
+
+ b="$short_sha..."
+ b="($b)"
+ fi
+ fi
+ fi
+
+ if [ -n "$step" ] && [ -n "$total" ]; then
+ r="$r $step/$total"
+ fi
+
+ local w=""
+ local i=""
+ local s=""
+ local u=""
+ local c=""
+ local p=""
+
+ if [ "true" = "$inside_gitdir" ]; then
+ if [ "true" = "$bare_repo" ]; then
+ c="BARE:"
+ else
+ b="GIT_DIR!"
+ fi
+ elif [ "true" = "$inside_worktree" ]; then
+ if [ -n "${GIT_PS1_SHOWDIRTYSTATE-}" ] &&
+ [ "$(git config --bool bash.showDirtyState)" != "false" ]
+ then
+ git diff --no-ext-diff --quiet --exit-code || w="*"
+ if [ -n "$short_sha" ]; then
+ git diff-index --cached --quiet HEAD -- || i="+"
+ else
+ i="#"
+ fi
+ fi
+ if [ -n "${GIT_PS1_SHOWSTASHSTATE-}" ] &&
+ [ -r "$g/refs/stash" ]; then
+ s="$"
+ fi
+
+ if [ -n "${GIT_PS1_SHOWUNTRACKEDFILES-}" ] &&
+ [ "$(git config --bool bash.showUntrackedFiles)" != "false" ] &&
+ git ls-files --others --exclude-standard --error-unmatch -- '*' >/dev/null 2>/dev/null
+ then
+ u="%${ZSH_VERSION+%}"
+ fi
+
+ if [ -n "${GIT_PS1_SHOWUPSTREAM-}" ]; then
+ __git_ps1_show_upstream
+ fi
+ fi
+
+ local z="${GIT_PS1_STATESEPARATOR-" "}"
+
+ # NO color option unless in PROMPT_COMMAND mode
+ if [ $pcmode = yes ] && [ -n "${GIT_PS1_SHOWCOLORHINTS-}" ]; then
+ __git_ps1_colorize_gitstring
+ fi
+
+ local f="$w$i$s$u"
+ local gitstring="$c${b##refs/heads/}${f:+$z$f}$r$p"
+
+ if [ $pcmode = yes ]; then
+ if [[ -n ${ZSH_VERSION-} ]]; then
+ gitstring=$(printf -- "$printf_format" "$gitstring")
+ else
+ printf -v gitstring -- "$printf_format" "$gitstring"
+ fi
+ PS1="$ps1pc_start$gitstring$ps1pc_end"
+ else
+ printf -- "$printf_format" "$gitstring"
+ fi
+}
diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh
deleted file mode 100755
index 80f031a74e7052d183b5ef21d432476ff1cce722..0000000000000000000000000000000000000000
--- a/paddle/scripts/travis/before_install.osx.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-brew update
-brew tap homebrew/science
-brew install openblas swig md5sha1sum
diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh
index 5e6350b57458594163f23cca41a546d7bd9b1eda..7deb3e62e88de7e1306fcbfc5a28aa4372d678e6 100755
--- a/paddle/scripts/travis/build_and_test.sh
+++ b/paddle/scripts/travis/build_and_test.sh
@@ -2,18 +2,11 @@
source ./common.sh
NPROC=1
-if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
- export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
- export PYTHONHOME=/opt/python/2.7.12
- export PATH=/opt/python/2.7.12/bin:${PATH}
- cmake .. -DCMAKE_Fortran_COMPILER=/usr/bin/gfortran-4.8 -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON ${EXTRA_CMAKE_OPTS}
- NRPOC=`nproc`
- make -j $NPROC
- make coveralls
- sudo make install
-elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
- export PYTHONPATH=/usr/local/lib/python2.7/site-packages
- cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON ${EXTRA_CMAKE_OPTS}
- NPROC=`sysctl -n hw.ncpu`
- make -j $NPROC
-fi
+export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
+export PYTHONHOME=/opt/python/2.7.12
+export PATH=/opt/python/2.7.12/bin:${PATH}
+cmake .. -DCMAKE_Fortran_COMPILER=/usr/bin/gfortran-4.8 -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON ${EXTRA_CMAKE_OPTS}
+NRPOC=`nproc`
+make -j $NPROC
+make coveralls
+sudo make install
diff --git a/paddle/scripts/travis/docs.sh b/paddle/scripts/travis/docs.sh
index 6b43cad20b76e9abeb3cb10a726d3d8e3da5f8e2..53e998ef6c1b96d9e7d82b7effd12a27e6dc69f2 100755
--- a/paddle/scripts/travis/docs.sh
+++ b/paddle/scripts/travis/docs.sh
@@ -2,8 +2,12 @@
# Add set -e, cd to directory.
source ./common.sh
-
# Compile Documentation only.
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_Fortran_COMPILER=/usr/bin/gfortran-4.8 -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_STYLE_CHECK=OFF ${EXTRA_CMAKE_OPTS}
+mkdir output
+make DESTDIR=./output install -j `nproc`
+pip install ./output/usr/local/opt/paddle/share/wheels/*
+rm -rf *
cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_Fortran_COMPILER=/usr/bin/gfortran-4.8 -DWITH_GPU=OFF -DWITH_DOC=ON ${EXTRA_CMAKE_OPTS}
make paddle_docs paddle_docs_cn
@@ -25,26 +29,41 @@ TARGET_BRANCH="gh-pages"
# Only deploy master branch to build latest documentation.
SOURCE_BRANCH="master"
-# If is not a Github pull request, and in master branch.
-if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
- exit 0
-fi
-
# Clone the repo to output directory
git clone $REPO output
cd output
-# checkout github page branch
-git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH
+function deploy_docs() {
+ SOURCE_BRANCH=$1
+ DIR=$2
+ # If is not a Github pull request
+ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
+ exit 0
+ fi
+ # If it is not watched branch.
+ if [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
+ return
+ fi
-# remove old docs. mv new docs.
-rm -rf doc doc_cn
-mv ../doc/cn/html doc_cn
-mv ../doc/en/html doc
+ # checkout github page branch
+ git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH
+
+ mkdir -p ${DIR}
+ # remove old docs. mv new docs.
+ set +e
+ rm -rf ${DIR}/doc ${DIR}/doc_cn
+ set -e
+ mv ../doc/cn/html ${DIR}/doc_cn
+ mv ../doc/en/html ${DIR}/doc
+ git add .
+}
+
+deploy_docs "master" "."
+deploy_docs "develop" "./develop/"
# Check is there anything changed.
set +e
-git diff --exit-code >/dev/null
+git diff --cached --exit-code >/dev/null
if [ $? -eq 0 ]; then
echo "No changes to the output on this push; exiting."
exit 0
@@ -57,7 +76,6 @@ if [ -n $SSL_KEY ]; then # Only push updated docs for github.com/PaddlePaddle/P
git config user.name "Travis CI"
git config user.email "paddle-dev@baidu.com"
git commit -m "Deploy to GitHub Pages: ${SHA}"
-
# Set ssh private key
openssl aes-256-cbc -K $SSL_KEY -iv $SSL_IV -in ../../paddle/scripts/travis/deploy_key.enc -out deploy_key -d
chmod 600 deploy_key
diff --git a/paddle/setup.py.in b/paddle/setup.py.in
index 38621af065913c9edd44958e9fb767c983c00dbb..382d5be6ecfc26b4a524bb6a775bd1a805a34d96 100644
--- a/paddle/setup.py.in
+++ b/paddle/setup.py.in
@@ -72,6 +72,7 @@ setup(name="py_paddle",
packages=['py_paddle'],
include_dirs = include_dirs,
install_requires = [
+ 'nltk>=3.2.2',
'numpy>=1.8.0', # The numpy is required.
'protobuf>=3.0.0' # The paddle protobuf version
],
diff --git a/paddle/trainer/Tester.cpp b/paddle/trainer/Tester.cpp
index 13aa28ae5d9699d267858d48e46797c756487ddd..80664fa877b324af73e3e3effa11e46eac6294e2 100644
--- a/paddle/trainer/Tester.cpp
+++ b/paddle/trainer/Tester.cpp
@@ -208,7 +208,7 @@ real Tester::forwardOneBatch(const DataBatch& dataBatch,
return 0.0; // In this case, there is no meaning to calculate cost
}
- return Argument::sumCosts(outArgs);
+ return Argument::sum(outArgs);
}
void Tester::testOnePassBatch(int passId) {
diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp
index bd84545375117b178d4324f0ad03f5bc35ae925d..b68e29cd5ea223272151e7a8b52d998832f47103 100644
--- a/paddle/trainer/Trainer.cpp
+++ b/paddle/trainer/Trainer.cpp
@@ -310,7 +310,7 @@ real Trainer::checkGradient() {
std::vector outArgs;
trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
- real cost = Argument::sumCosts(outArgs);
+ real cost = Argument::sum(outArgs);
LOG(INFO) << "original cost=" << cost;
trainerInternal_.getGradientMachine()->backward();
@@ -340,7 +340,7 @@ real Trainer::checkGradient() {
parameter->getBuf(PARAMETER_VALUE)->copyFrom(newPara);
parameter->setValueUpdated();
trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
- real newCost1 = Argument::sumCosts(outArgs);
+ real newCost1 = Argument::sum(outArgs);
for (size_t i = 0; i < dim; ++i) {
newp[i] = oldp[i] - step * d[i];
@@ -349,7 +349,7 @@ real Trainer::checkGradient() {
parameter->getBuf(PARAMETER_VALUE)->copyFrom(newPara);
parameter->setValueUpdated();
trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
- real newCost2 = Argument::sumCosts(outArgs);
+ real newCost2 = Argument::sum(outArgs);
real trueDelta = 0.5 * (newCost1 - newCost2);
real diff = (1e-20 + trueDelta) / (1e-20 + delta) - 1;
@@ -575,7 +575,7 @@ real Trainer::calcGradient(const DataBatch& dataBatch,
trainerInternal_.getGradientMachine()->forwardBackward(
inArgs, &outArgs, PASS_TRAIN);
- real cost = Argument::sumCosts(outArgs);
+ real cost = Argument::sum(outArgs);
offset = 0;
for (auto& para : parameters) {
diff --git a/paddle/trainer/TrainerInternal.cpp b/paddle/trainer/TrainerInternal.cpp
index f3b465b444167d4624a5e99c30e1257eda53ca2c..4c5d4a0913aaf3a9932b3d67806378ece4245304 100644
--- a/paddle/trainer/TrainerInternal.cpp
+++ b/paddle/trainer/TrainerInternal.cpp
@@ -134,7 +134,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId,
real cost = 0;
{
REGISTER_TIMER("sumCost");
- cost = Argument::sumCosts(*outArgs);
+ cost = Argument::sum(*outArgs);
}
if (batchId % intconfig_->log_period == 0) {
diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py
index bd24c68b6fe88eab03c814f8cac70db3880316f4..0e752c117c1ecfab72e2da2f830380e9524236e7 100644
--- a/python/paddle/trainer/PyDataProvider2.py
+++ b/python/paddle/trainer/PyDataProvider2.py
@@ -45,6 +45,23 @@ class CacheType(object):
class InputType(object):
+ """
+ InputType is the base class for paddle input types.
+
+ .. note::
+
+ this is a base class, and should never be used by user.
+
+ :param dim: dimension of input. If the input is an integer, it means the
+ value range. Otherwise, it means the size of layer.
+ :type dim: int
+ :param seq_type: sequence type of input. 0 means it is not a sequence. 1
+ means it is a variable length sequence. 2 means it is a
+ nested sequence.
+ :type seq_type: int
+ :param type: data type of input.
+ :type type: int
+ """
__slots__ = ['dim', 'seq_type', 'type']
def __init__(self, dim, seq_type, tp):
@@ -54,19 +71,63 @@ class InputType(object):
def dense_slot(dim, seq_type=SequenceType.NO_SEQUENCE):
+ """
+ Dense Vector. It means the input feature is dense float vector. For example,
+ if the input is an image with 28*28 pixels, the input of Paddle neural
+ network should be a dense vector with dimension 784.
+
+ :param dim: dimension of this vector.
+ :type dim: int
+ :param seq_type: sequence type of input.
+ :type seq_type: int
+ :return: An input type object.
+ :rtype: InputType
+ """
return InputType(dim, seq_type, DataType.Dense)
def sparse_non_value_slot(dim, seq_type=SequenceType.NO_SEQUENCE):
+ """
+ Sparse binary vector. It means the input feature is a sparse vector and the
+ every element in this vector is either zero or one.
+
+ :param dim: dimension of this vector.
+ :type dim: int
+ :param seq_type: sequence type of this input.
+ :type seq_type: int
+ :return: An input type object.
+ :rtype: InputType
+ """
return InputType(dim, seq_type, DataType.SparseNonValue)
def sparse_value_slot(dim, seq_type=SequenceType.NO_SEQUENCE):
+ """
+ Sparse vector. It means the input feature is a sparse vector. Most of the
+ elements in this vector are zero, others could be any float value.
+
+ :param dim: dimension of this vector.
+ :type dim: int
+ :param seq_type: sequence type of this input.
+ :type seq_type: int
+ :return: An input type object.
+ :rtype: InputType
+ """
return InputType(dim, seq_type, DataType.SparseValue)
-def index_slot(dim, seq_type=SequenceType.NO_SEQUENCE):
- return InputType(dim, seq_type, DataType.Index)
+def index_slot(value_range, seq_type=SequenceType.NO_SEQUENCE):
+ """
+ Data type of integer.
+
+ :param seq_type: sequence type of this input.
+ :type seq_type: int
+ :param value_range: range of this integer.
+ :type value_range: int
+ :return: An input type object
+ :rtype: InputType
+ """
+ return InputType(value_range, seq_type, DataType.Index)
dense_vector = dense_slot
@@ -76,6 +137,14 @@ integer_value = index_slot
def dense_vector_sequence(dim):
+ """
+ Data type of a sequence of dense vector.
+
+ :param dim: dimension of dense vector.
+ :type dim: int
+ :return: An input type object
+ :rtype: InputType
+ """
return dense_vector(dim, seq_type=SequenceType.SEQUENCE)
@@ -84,6 +153,15 @@ def dense_vector_sub_sequence(dim):
def sparse_binary_vector_sequence(dim):
+ """
+ Data type of a sequence of sparse vector, which every element is either zero
+ or one.
+
+ :param dim: dimension of sparse vector.
+ :type dim: int
+ :return: An input type object
+ :rtype: InputType
+ """
return sparse_binary_vector(dim, seq_type=SequenceType.SEQUENCE)
@@ -92,6 +170,15 @@ def sparse_binary_vector_sub_sequence(dim):
def sparse_vector_sequence(dim):
+ """
+ Data type of a sequence of sparse vector, which most elements are zero,
+ others could be any float value.
+
+ :param dim: dimension of sparse vector.
+ :type dim: int
+ :return: An input type object
+ :rtype: InputType
+ """
return sparse_vector(dim, seq_type=SequenceType.SEQUENCE)
@@ -99,8 +186,14 @@ def sparse_vector_sub_sequence(dim):
return sparse_vector(dim, seq_type=SequenceType.SUB_SEQUENCE)
-def integer_value_sequence(dim):
- return integer_value(dim, seq_type=SequenceType.SEQUENCE)
+def integer_value_sequence(value_range):
+ """
+ Data type of a sequence of integer.
+
+ :param value_range: range of each element.
+ :type value_range: int
+ """
+ return integer_value(value_range, seq_type=SequenceType.SEQUENCE)
def integer_value_sub_sequence(dim):
diff --git a/python/paddle/trainer_config_helpers/layer_math.py b/python/paddle/trainer_config_helpers/layer_math.py
index 2d9e36f2b0d379d907634208a45c69efa9dbba3d..544b443825393c9a31c0375724d4ca63dac5c5eb 100644
--- a/python/paddle/trainer_config_helpers/layer_math.py
+++ b/python/paddle/trainer_config_helpers/layer_math.py
@@ -39,6 +39,7 @@ register_unary_math_op('abs', act.AbsActivation())
register_unary_math_op('sigmoid', act.SigmoidActivation())
register_unary_math_op('tanh', act.TanhActivation())
register_unary_math_op('square', act.SquareActivation())
+register_unary_math_op('relu', act.ReluActivation())
def add(layeroutput, other):
diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py
index b68460b6a3ab621904f4dc4e48352044ab265a38..b94f8f9a783552519ca73e7cfc0937b302d3445b 100755
--- a/python/paddle/trainer_config_helpers/layers.py
+++ b/python/paddle/trainer_config_helpers/layers.py
@@ -795,17 +795,16 @@ def data_layer(name, size, height=None, width=None, layer_attr=None):
.. code-block:: python
- data = data_layer(name="input",
- size=1000)
+ data = data_layer(name="input", size=1000)
:param name: Name of this data layer.
:type name: basestring
:param size: Size of this data layer.
:type size: int
:param height: Height of this data layer, used for image
- :type size: int|None
+ :type height: int|None
:param width: Width of this data layer, used for image
- :type size: int|None
+ :type width: int|None
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute.
:return: LayerOutput object.
diff --git a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py
index 3331c10d6497f58eb135208bd7abe48aacfb10ae..24c901c8ee3ab1c90fc14fbff761db06345a6313 100644
--- a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py
+++ b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py
@@ -7,8 +7,9 @@ x = layer_math.exp(x)
x = layer_math.log(x)
x = layer_math.abs(x)
x = layer_math.sigmoid(x)
+x = layer_math.tanh(x)
x = layer_math.square(x)
-x = layer_math.square(x)
+x = layer_math.relu(x)
y = 1 + x
y = y + 1
y = x + y
diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr
index da8da1b541f37a09654202f68232b99e4dac9f61..9b8a2ad9687d313e6c5017c2d7331eddf539af92 100644
--- a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr
+++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr
@@ -65,13 +65,28 @@ layers {
}
}
}
+layers {
+ name: "__tanh_0__"
+ type: "mixed"
+ size: 100
+ active_type: "tanh"
+ inputs {
+ input_layer_name: "__sigmoid_0__"
+ proj_conf {
+ type: "identity"
+ name: "___tanh_0__.w0"
+ input_size: 100
+ output_size: 100
+ }
+ }
+}
layers {
name: "__square_0__"
type: "mixed"
size: 100
active_type: "square"
inputs {
- input_layer_name: "__sigmoid_0__"
+ input_layer_name: "__tanh_0__"
proj_conf {
type: "identity"
name: "___square_0__.w0"
@@ -81,15 +96,15 @@ layers {
}
}
layers {
- name: "__square_1__"
+ name: "__relu_0__"
type: "mixed"
size: 100
- active_type: "square"
+ active_type: "relu"
inputs {
input_layer_name: "__square_0__"
proj_conf {
type: "identity"
- name: "___square_1__.w0"
+ name: "___relu_0__.w0"
input_size: 100
output_size: 100
}
@@ -101,7 +116,7 @@ layers {
size: 100
active_type: ""
inputs {
- input_layer_name: "__square_1__"
+ input_layer_name: "__relu_0__"
}
slope: 1.0
intercept: 1
@@ -123,7 +138,7 @@ layers {
size: 100
active_type: ""
inputs {
- input_layer_name: "__square_1__"
+ input_layer_name: "__relu_0__"
proj_conf {
type: "identity"
name: "___mixed_0__.w0"
@@ -147,7 +162,7 @@ layers {
size: 100
active_type: ""
inputs {
- input_layer_name: "__square_1__"
+ input_layer_name: "__relu_0__"
}
slope: -1.0
intercept: 0.0
@@ -339,8 +354,9 @@ sub_models {
layer_names: "__log_0__"
layer_names: "__abs_0__"
layer_names: "__sigmoid_0__"
+ layer_names: "__tanh_0__"
layer_names: "__square_0__"
- layer_names: "__square_1__"
+ layer_names: "__relu_0__"
layer_names: "__slope_intercept_layer_0__"
layer_names: "__slope_intercept_layer_1__"
layer_names: "__mixed_0__"
diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py
index d548d1adaafacdb097dbe476fdc76651c9f46b6b..25526bf409cf82f26979a84700ce948ac969df0c 100644
--- a/python/paddle/v2/__init__.py
+++ b/python/paddle/v2/__init__.py
@@ -20,18 +20,20 @@ import event
import data_type
import topology
import data_feeder
+import networks
from . import dataset
from . import reader
import attr
import pooling
-import inferencer
+import inference
import networks
import py_paddle.swig_paddle as api
+import minibatch
__all__ = [
'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer',
'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader',
- 'topology', 'networks', 'inferencer', 'infer'
+ 'topology', 'networks', 'infer'
]
@@ -43,4 +45,5 @@ def init(**kwargs):
api.initPaddle(*args)
-infer = inferencer.infer
+infer = inference.infer
+batch = minibatch.batch
diff --git a/python/paddle/v2/activation.py b/python/paddle/v2/activation.py
index 1f3aab9ef3c5f69e22d7e83250d0ff46c1ff718a..21261a178203b633ca6cf59a5fc89edc24a868b9 100644
--- a/python/paddle/v2/activation.py
+++ b/python/paddle/v2/activation.py
@@ -12,26 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from paddle.trainer_config_helpers.activations import *
+import paddle.trainer_config_helpers.activations
+import copy
-__all__ = [
- "Base", "Tanh", "Sigmoid", "Softmax", "Identity", "Linear",
- 'SequenceSoftmax', "Exp", "Relu", "BRelu", "SoftRelu", "STanh", "Abs",
- "Square", "Log"
-]
+__all__ = []
-Base = BaseActivation
-Tanh = TanhActivation
-Sigmoid = SigmoidActivation
-Softmax = SoftmaxActivation
-SequenceSoftmax = SequenceSoftmaxActivation
-Identity = IdentityActivation
-Linear = Identity
-Relu = ReluActivation
-BRelu = BReluActivation
-SoftRelu = SoftReluActivation
-STanh = STanhActivation
-Abs = AbsActivation
-Square = SquareActivation
-Exp = ExpActivation
-Log = LogActivation
+suffix = 'Activation'
+for act in paddle.trainer_config_helpers.activations.__all__:
+ new_name = act[:-len(suffix)]
+ globals()[new_name] = copy.copy(
+ getattr(paddle.trainer_config_helpers.activations, act))
+ globals()[new_name].__name__ = new_name
+ __all__.append(new_name)
diff --git a/python/paddle/v2/attr.py b/python/paddle/v2/attr.py
index 40c64f621b443d5613468b27d030ab08776641b2..32f78614e7f8abe7cffdc7a50a9fa77f1fc1a780 100644
--- a/python/paddle/v2/attr.py
+++ b/python/paddle/v2/attr.py
@@ -12,12 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from paddle.trainer_config_helpers.attrs import *
+import paddle.trainer_config_helpers.attrs
__all__ = [
"Param",
"Extra",
]
-Param = ParameterAttribute
-Extra = ExtraLayerAttribute
+Param = paddle.trainer_config_helpers.attrs.ParameterAttribute
+Extra = paddle.trainer_config_helpers.attrs.ExtraLayerAttribute
+
+for each in paddle.trainer_config_helpers.attrs.__all__:
+ globals()[each] = getattr(paddle.trainer_config_helpers.attrs, each)
+ __all__.append(each)
diff --git a/python/paddle/v2/config_base.py b/python/paddle/v2/config_base.py
index 035f96b0f2e978a413a1ebe0ec115f75ff07befc..1ec1d7bbdf912b940ca4b8e7b20eb11310f0e74f 100644
--- a/python/paddle/v2/config_base.py
+++ b/python/paddle/v2/config_base.py
@@ -13,15 +13,59 @@
# limitations under the License.
import collections
-
+import re
from paddle.trainer_config_helpers.default_decorators import wrap_name_default
import paddle.trainer_config_helpers as conf_helps
+class LayerType(type):
+ def __new__(cls, name, bases, attrs):
+ method_name = attrs.get('METHOD_NAME', None)
+ if method_name is not None:
+ method = getattr(conf_helps, method_name)
+ if method.__doc__ is not None:
+ mapper = attrs.get("__map_docstr__", None)
+ if mapper is not None:
+ attrs['__doc__'] = LayerType.__map_docstr__(
+ mapper(method.__doc__),
+ method_name=method_name,
+ name=name)
+ else:
+ attrs['__doc__'] = LayerType.__map_docstr__(
+ method.__doc__, method_name=method_name, name=name)
+ return super(LayerType, cls).__new__(cls, name, bases, attrs)
+
+ @staticmethod
+ def __map_docstr__(doc, name, method_name):
+ assert isinstance(doc, basestring)
+
+ # replace LayerOutput to paddle.v2.config_base.Layer
+ doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer")
+
+ doc = doc.replace('ParameterAttribute',
+ 'paddle.v2.attr.ParameterAttribute')
+
+ doc = re.sub(r'ExtraLayerAttribute[^\s]?',
+ 'paddle.v2.attr.ExtraAttribute', doc)
+
+ # xxx_layer to xxx
+ doc = re.sub(r"(?P[a-z]+)_layer", r"\g", doc)
+
+ # XxxxActivation to paddle.v2.Activation.Xxxx
+ doc = re.sub(r"(?P[A-Z][a-zA-Z]+)Activation",
+ r"paddle.v2.Activation.\g", doc)
+
+ # TODO(yuyang18): Add more rules if needed.
+ return doc
+
+
class Layer(object):
+ __metaclass__ = LayerType
+
def __init__(self, name=None, parent_layers=None):
assert isinstance(parent_layers, dict)
self.name = name
+ self.__contex__ = {}
self.__parent_layers__ = parent_layers
def to_proto(self, context):
@@ -39,16 +83,38 @@ class Layer(object):
self.__parent_layers__[layer_name])
kwargs[layer_name] = v1_layer
- if self.name is None:
+ if self.context_name() is None:
return self.to_proto_impl(**kwargs)
- elif self.name not in context:
- context[self.name] = self.to_proto_impl(**kwargs)
-
- return context[self.name]
+ elif self.context_name() not in context:
+ context[self.context_name()] = self.to_proto_impl(**kwargs)
+ self.__contex__ = context
+ if self.use_context_name():
+ return context[self.context_name()]
+ else:
+ return context[self.name]
def to_proto_impl(self, **kwargs):
raise NotImplementedError()
+ def context_name(self):
+ """
+ Context name means the context which stores `to_proto_impl` result.
+ If multiple layer share same context_name, the `to_proto_impl` of them
+ will be invoked only once.
+ """
+ return self.name
+
+ def use_context_name(self):
+ return False
+
+ def calculate_size(self):
+ """
+ lazy calculate size of the layer, should be called when to_proto_impl of
+ this layer is called.
+ :return:
+ """
+ return self.__contex__[self.context_name()].size
+
def __convert_to_v2__(method_name, parent_names, is_default_name=True):
if is_default_name:
@@ -57,6 +123,8 @@ def __convert_to_v2__(method_name, parent_names, is_default_name=True):
wrapper = None
class V2LayerImpl(Layer):
+ METHOD_NAME = method_name
+
def __init__(self, **kwargs):
parent_layers = dict()
other_kwargs = dict()
diff --git a/python/paddle/v2/data_feeder.py b/python/paddle/v2/data_feeder.py
index 3b106e100cff7539611d95bb4123b4e0dfbfa6cb..bda8e22fd282f8ff4a820e4ecb6b3bb421d57890 100644
--- a/python/paddle/v2/data_feeder.py
+++ b/python/paddle/v2/data_feeder.py
@@ -12,13 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from py_paddle import swig_paddle
from py_paddle import DataProviderConverter
-import data_type
+
+import paddle.trainer.PyDataProvider2 as pydp2
__all__ = ['DataFeeder']
+def default_feeding_map(data_types):
+ reader_dict = dict()
+ for i, tp in enumerate(data_types):
+ reader_dict[tp[0]] = i
+ return reader_dict
+
+
class DataFeeder(DataProviderConverter):
"""
DataFeeder converts the data returned by paddle.reader into a data structure
@@ -29,7 +36,10 @@ class DataFeeder(DataProviderConverter):
to feed it to C++ interface.
The example usage:
-
+
+
+ .. code-block:: python
+
data_types = [('image', paddle.data_type.dense_vector(784)),
('label', paddle.data_type.integer_value(10))]
reader_dict = {'image':0, 'label':1}
@@ -43,49 +53,51 @@ class DataFeeder(DataProviderConverter):
# [ [1.0,2.0,3.0,4.0], 5, [6,7,8] ] # second sample
# ]
arg = feeder(minibatch_data)
+
+ .. note::
+
+ This module is for internal use only. Users should use the `reader`
+ interface.
+
+
+
+ :param data_types: A list to specify data name and type. Each item is
+ a tuple of (data_name, data_type).
+
+ :type data_types: list
+ :param reader_dict: A dictionary to specify the position of each data
+ in the input data.
+ :type feeding: dict
"""
- def __init__(self, data_types, reader_dict):
- """
- :param data_types: A list to specify data name and type. Each item is
- a tuple of (data_name, data_type). For example:
- [('image', paddle.data_type.dense_vector(784)),
- ('label', paddle.data_type.integer_value(10))]
-
- :type data_types: A list of tuple
- :param reader_dict: A dictionary to specify the position of each data
- in the input data.
- :type reader_dict: dict()
- """
+ def __init__(self, data_types, feeding=None):
self.input_names = []
input_types = []
- self.reader_dict = reader_dict
+ if feeding is None:
+ feeding = default_feeding_map(data_types)
+
+ self.feeding = feeding
for each in data_types:
self.input_names.append(each[0])
- assert isinstance(each[1], data_type.InputType)
+ if not isinstance(each[1], pydp2.InputType):
+ raise TypeError("second item in each data_type should be an "
+ "InputType")
input_types.append(each[1])
DataProviderConverter.__init__(self, input_types)
+ def __len__(self):
+ return len(self.input_names)
+
def convert(self, dat, argument=None):
"""
:param dat: A list of mini-batch data. Each sample is a list or tuple
one feature or multiple features.
- for example:
- [
- ([0.2, 0.2], ), # first sample
- ([0.8, 0.3], ), # second sample
- ]
- or,
- [
- [[0.2, 0.2], ], # first sample
- [[0.8, 0.3], ], # second sample
- ]
-
- :type dat: List
+
+ :type dat: list
:param argument: An Arguments object contains this mini-batch data with
one or multiple features. The Arguments definition is
in the API.
- :type argument: swig_paddle.Arguments
+ :type argument: py_paddle.swig_paddle.Arguments
"""
def reorder_data(data):
@@ -93,7 +105,7 @@ class DataFeeder(DataProviderConverter):
for each in data:
reorder = []
for name in self.input_names:
- reorder.append(each[self.reader_dict[name]])
+ reorder.append(each[self.feeding[name]])
retv.append(reorder)
return retv
diff --git a/python/paddle/v2/data_type.py b/python/paddle/v2/data_type.py
index 522ddfdaacce44be7cf27bdbfc1009d4a0c0bbe6..d582f76ddf01ed3430a1d075624bbb8e0bf3f2a9 100644
--- a/python/paddle/v2/data_type.py
+++ b/python/paddle/v2/data_type.py
@@ -12,11 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from paddle.trainer.PyDataProvider2 import \
- InputType, DataType, dense_vector, sparse_binary_vector,\
- sparse_vector, integer_value, integer_value_sequence
+import paddle.trainer.PyDataProvider2 as pydp2
-__all__ = [
- 'InputType', 'DataType', 'dense_vector', 'sparse_binary_vector',
- 'sparse_vector', 'integer_value', 'integer_value_sequence'
+import_list = [
+ nm for nm in dir(pydp2)
+ if '_' in nm and nm[0] != '_' and ('value' in nm or 'vector' in nm)
]
+import_list.extend(['InputType'])
+
+for nm in import_list:
+ globals()[nm] = getattr(pydp2, nm)
+
+__all__ = import_list
diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py
index 9647e9850332b795f6480d26e4f9c736129782ae..80ff6295c34e853d8f69b9e78719af23a56d1fbb 100644
--- a/python/paddle/v2/dataset/__init__.py
+++ b/python/paddle/v2/dataset/__init__.py
@@ -1,3 +1,31 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Dataset package.
+"""
+
import mnist
+import imikolov
+import imdb
+import cifar
+import movielens
+import conll05
+import uci_housing
+import sentiment
+import wmt14
-__all__ = ['mnist']
+__all__ = [
+ 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment'
+ 'uci_housing', 'wmt14'
+]
diff --git a/python/paddle/v2/dataset/cifar.py b/python/paddle/v2/dataset/cifar.py
index 77c54bd268b5d988b0802a3edca91605e56f730e..d9f7a830ee60a331b55a1e218923e690103e1c5b 100644
--- a/python/paddle/v2/dataset/cifar.py
+++ b/python/paddle/v2/dataset/cifar.py
@@ -1,6 +1,22 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
"""
CIFAR dataset: https://www.cs.toronto.edu/~kriz/cifar.html
+
+TODO(yuyang18): Complete the comments.
"""
+
import cPickle
import itertools
import numpy
diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py
index fcf4437ffaf329f52cc5bc997eff45dee200873c..3021b68ddb02ecaa874e21681796c0912ad4cc06 100644
--- a/python/paddle/v2/dataset/common.py
+++ b/python/paddle/v2/dataset/common.py
@@ -1,7 +1,22 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import requests
import hashlib
import os
import shutil
+import sys
__all__ = ['DATA_HOME', 'download', 'md5file']
@@ -27,9 +42,24 @@ def download(url, module_name, md5sum):
filename = os.path.join(dirname, url.split('/')[-1])
if not (os.path.exists(filename) and md5file(filename) == md5sum):
+ print "Cache file %s not found, downloading %s" % (filename, url)
r = requests.get(url, stream=True)
- with open(filename, 'w') as f:
- shutil.copyfileobj(r.raw, f)
+ total_length = r.headers.get('content-length')
+
+ if total_length is None:
+ with open(filename, 'w') as f:
+ shutil.copyfileobj(r.raw, f)
+ else:
+ with open(filename, 'w') as f:
+ dl = 0
+ total_length = int(total_length)
+ for data in r.iter_content(chunk_size=4096):
+ dl += len(data)
+ f.write(data)
+ done = int(50 * dl / total_length)
+ sys.stdout.write("\r[%s%s]" % ('=' * done,
+ ' ' * (50 - done)))
+ sys.stdout.flush()
return filename
diff --git a/python/paddle/v2/dataset/conll05.py b/python/paddle/v2/dataset/conll05.py
new file mode 100644
index 0000000000000000000000000000000000000000..9eab49ee39325c1c60fc511e0bd834e83aa987f0
--- /dev/null
+++ b/python/paddle/v2/dataset/conll05.py
@@ -0,0 +1,198 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tarfile
+import gzip
+import itertools
+from common import download
+"""
+Conll 2005 dataset. Paddle semantic role labeling Book and demo use this
+dataset as an example. Because Conll 2005 is not free in public, the default
+downloaded URL is test set of Conll 2005 (which is public). Users can change
+URL and MD5 to their Conll dataset.
+
+TODO(yuyang18): Complete comments.
+"""
+
+__all__ = ['test, get_dict', 'get_embedding']
+
+DATA_URL = 'http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz'
+DATA_MD5 = '387719152ae52d60422c016e92a742fc'
+WORDDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/wordDict.txt'
+WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa'
+VERBDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/verbDict.txt'
+VERBDICT_MD5 = '0d2977293bbb6cbefab5b0f97db1e77c'
+TRGDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/targetDict.txt'
+TRGDICT_MD5 = 'd8c7f03ceb5fc2e5a0fa7503a4353751'
+EMB_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/emb'
+EMB_MD5 = 'bf436eb0faa1f6f9103017f8be57cdb7'
+
+UNK_IDX = 0
+
+
+def load_dict(filename):
+ d = dict()
+ with open(filename, 'r') as f:
+ for i, line in enumerate(f):
+ d[line.strip()] = i
+ return d
+
+
+def corpus_reader(data_path, words_name, props_name):
+ """
+ Read one corpus. It returns an iterator. Each element of
+ this iterator is a tuple including sentence and labels. The sentence is
+ consist of a list of word IDs. The labels include a list of label IDs.
+ :return: a iterator of data.
+ :rtype: iterator
+ """
+
+ def reader():
+ tf = tarfile.open(data_path)
+ wf = tf.extractfile(words_name)
+ pf = tf.extractfile(props_name)
+ with gzip.GzipFile(fileobj=wf) as words_file, gzip.GzipFile(
+ fileobj=pf) as props_file:
+ sentences = []
+ labels = []
+ one_seg = []
+ for word, label in itertools.izip(words_file, props_file):
+ word = word.strip()
+ label = label.strip().split()
+
+ if len(label) == 0: # end of sentence
+ for i in xrange(len(one_seg[0])):
+ a_kind_lable = [x[i] for x in one_seg]
+ labels.append(a_kind_lable)
+
+ if len(labels) >= 1:
+ verb_list = []
+ for x in labels[0]:
+ if x != '-':
+ verb_list.append(x)
+
+ for i, lbl in enumerate(labels[1:]):
+ cur_tag = 'O'
+ is_in_bracket = False
+ lbl_seq = []
+ verb_word = ''
+ for l in lbl:
+ if l == '*' and is_in_bracket == False:
+ lbl_seq.append('O')
+ elif l == '*' and is_in_bracket == True:
+ lbl_seq.append('I-' + cur_tag)
+ elif l == '*)':
+ lbl_seq.append('I-' + cur_tag)
+ is_in_bracket = False
+ elif l.find('(') != -1 and l.find(')') != -1:
+ cur_tag = l[1:l.find('*')]
+ lbl_seq.append('B-' + cur_tag)
+ is_in_bracket = False
+ elif l.find('(') != -1 and l.find(')') == -1:
+ cur_tag = l[1:l.find('*')]
+ lbl_seq.append('B-' + cur_tag)
+ is_in_bracket = True
+ else:
+ raise RuntimeError('Unexpected label: %s' %
+ l)
+
+ yield sentences, verb_list[i], lbl_seq
+
+ sentences = []
+ labels = []
+ one_seg = []
+ else:
+ sentences.append(word)
+ one_seg.append(label)
+
+ pf.close()
+ wf.close()
+ tf.close()
+
+ return reader
+
+
+def reader_creator(corpus_reader,
+ word_dict=None,
+ predicate_dict=None,
+ label_dict=None):
+ def reader():
+ for sentence, predicate, labels in corpus_reader():
+
+ sen_len = len(sentence)
+
+ verb_index = labels.index('B-V')
+ mark = [0] * len(labels)
+ if verb_index > 0:
+ mark[verb_index - 1] = 1
+ ctx_n1 = sentence[verb_index - 1]
+ else:
+ ctx_n1 = 'bos'
+
+ if verb_index > 1:
+ mark[verb_index - 2] = 1
+ ctx_n2 = sentence[verb_index - 2]
+ else:
+ ctx_n2 = 'bos'
+
+ mark[verb_index] = 1
+ ctx_0 = sentence[verb_index]
+
+ if verb_index < len(labels) - 1:
+ mark[verb_index + 1] = 1
+ ctx_p1 = sentence[verb_index + 1]
+ else:
+ ctx_p1 = 'eos'
+
+ if verb_index < len(labels) - 2:
+ mark[verb_index + 2] = 1
+ ctx_p2 = sentence[verb_index + 2]
+ else:
+ ctx_p2 = 'eos'
+
+ word_idx = [word_dict.get(w, UNK_IDX) for w in sentence]
+
+ ctx_n2_idx = [word_dict.get(ctx_n2, UNK_IDX)] * sen_len
+ ctx_n1_idx = [word_dict.get(ctx_n1, UNK_IDX)] * sen_len
+ ctx_0_idx = [word_dict.get(ctx_0, UNK_IDX)] * sen_len
+ ctx_p1_idx = [word_dict.get(ctx_p1, UNK_IDX)] * sen_len
+ ctx_p2_idx = [word_dict.get(ctx_p2, UNK_IDX)] * sen_len
+
+ pred_idx = [predicate_dict.get(predicate)] * sen_len
+ label_idx = [label_dict.get(w) for w in labels]
+
+ yield word_idx, ctx_n2_idx, ctx_n1_idx, \
+ ctx_0_idx, ctx_p1_idx, ctx_p2_idx, pred_idx, mark, label_idx
+
+ return reader
+
+
+def get_dict():
+ word_dict = load_dict(download(WORDDICT_URL, 'conll05st', WORDDICT_MD5))
+ verb_dict = load_dict(download(VERBDICT_URL, 'conll05st', VERBDICT_MD5))
+ label_dict = load_dict(download(TRGDICT_URL, 'conll05st', TRGDICT_MD5))
+ return word_dict, verb_dict, label_dict
+
+
+def get_embedding():
+ return download(EMB_URL, 'conll05st', EMB_MD5)
+
+
+def test():
+ word_dict, verb_dict, label_dict = get_dict()
+ reader = corpus_reader(
+ download(DATA_URL, 'conll05st', DATA_MD5),
+ words_name='conll05st-release/test.wsj/words/test.wsj.words.gz',
+ props_name='conll05st-release/test.wsj/props/test.wsj.props.gz')
+ return reader_creator(reader, word_dict, verb_dict, label_dict)
diff --git a/python/paddle/v2/dataset/imdb.py b/python/paddle/v2/dataset/imdb.py
index 433e37380f840f5b7ff619a5f64b99d2ad724b17..76019d9f54020ff6f02c17eb6047cbd014a8ccf2 100644
--- a/python/paddle/v2/dataset/imdb.py
+++ b/python/paddle/v2/dataset/imdb.py
@@ -1,6 +1,3 @@
-# /usr/bin/env python
-# -*- coding:utf-8 -*-
-
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +13,10 @@
# limitations under the License.
"""
IMDB dataset: http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz
+
+TODO(yuyang18): Complete comments.
"""
+
import paddle.v2.dataset.common
import tarfile
import Queue
@@ -118,3 +118,8 @@ def test(word_idx):
return reader_creator(
re.compile("aclImdb/test/pos/.*\.txt$"),
re.compile("aclImdb/test/neg/.*\.txt$"), word_idx, 1000)
+
+
+def word_dict():
+ return build_dict(
+ re.compile("aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150)
diff --git a/python/paddle/v2/dataset/imikolov.py b/python/paddle/v2/dataset/imikolov.py
index b3791ddad66e588356338150fccadbcc8fa113ca..97c160f111d09d61eb860c7f02552e635f2400a7 100644
--- a/python/paddle/v2/dataset/imikolov.py
+++ b/python/paddle/v2/dataset/imikolov.py
@@ -1,10 +1,25 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
"""
imikolov's simple dataset: http://www.fit.vutbr.cz/~imikolov/rnnlm/
+
+Complete comments.
"""
import paddle.v2.dataset.common
import tarfile
-__all__ = ['train', 'test']
+__all__ = ['train', 'test', 'build_dict']
URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz'
MD5 = '30177ea32e27c525793142b6bf2c8e2d'
@@ -24,7 +39,9 @@ def word_count(f, word_freq=None):
return word_freq
-def build_dict(train_filename, test_filename):
+def build_dict():
+ train_filename = './simple-examples/data/ptb.train.txt'
+ test_filename = './simple-examples/data/ptb.valid.txt'
with tarfile.open(
paddle.v2.dataset.common.download(
paddle.v2.dataset.imikolov.URL, 'imikolov',
@@ -32,27 +49,22 @@ def build_dict(train_filename, test_filename):
trainf = tf.extractfile(train_filename)
testf = tf.extractfile(test_filename)
word_freq = word_count(testf, word_count(trainf))
+ if '' in word_freq:
+ # remove for now, since we will set it as last index
+ del word_freq['']
TYPO_FREQ = 50
word_freq = filter(lambda x: x[1] > TYPO_FREQ, word_freq.items())
- dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0]))
- words, _ = list(zip(*dictionary))
+ word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0]))
+ words, _ = list(zip(*word_freq_sorted))
word_idx = dict(zip(words, xrange(len(words))))
word_idx[''] = len(words)
return word_idx
-word_idx = {}
-
-
-def reader_creator(filename, n):
- global word_idx
- if len(word_idx) == 0:
- word_idx = build_dict('./simple-examples/data/ptb.train.txt',
- './simple-examples/data/ptb.valid.txt')
-
+def reader_creator(filename, word_idx, n):
def reader():
with tarfile.open(
paddle.v2.dataset.common.download(
@@ -71,9 +83,9 @@ def reader_creator(filename, n):
return reader
-def train(n):
- return reader_creator('./simple-examples/data/ptb.train.txt', n)
+def train(word_idx, n):
+ return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n)
-def test(n):
- return reader_creator('./simple-examples/data/ptb.valid.txt', n)
+def test(word_idx, n):
+ return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n)
diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py
index ebcdff78b317ceb4811048ac78982e072962fa9c..16f2fcb99de4cb1971a7375a97b5daa209ee95ef 100644
--- a/python/paddle/v2/dataset/mnist.py
+++ b/python/paddle/v2/dataset/mnist.py
@@ -1,5 +1,21 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
"""
MNIST dataset.
+
+This module will download dataset from http://yann.lecun.com/exdb/mnist/ and
+parse train set and test set into paddle reader creators.
"""
import paddle.v2.dataset.common
import subprocess
@@ -59,6 +75,15 @@ def reader_creator(image_filename, label_filename, buffer_size):
def train():
+ """
+ MNIST train set creator.
+
+ It returns a reader creator, each sample in the reader is image pixels in
+ [0, 1] and label in [0, 9].
+
+ :return: Train reader creator
+ :rtype: callable
+ """
return reader_creator(
paddle.v2.dataset.common.download(TRAIN_IMAGE_URL, 'mnist',
TRAIN_IMAGE_MD5),
@@ -67,6 +92,15 @@ def train():
def test():
+ """
+ MNIST test set cretor.
+
+ It returns a reader creator, each sample in the reader is image pixels in
+ [0, 1] and label in [0, 9].
+
+ :return: Test reader creator.
+ :rtype: callable
+ """
return reader_creator(
paddle.v2.dataset.common.download(TEST_IMAGE_URL, 'mnist',
TEST_IMAGE_MD5),
diff --git a/python/paddle/v2/dataset/movielens.py b/python/paddle/v2/dataset/movielens.py
index dcffcff2f58c63d451761d37f14127d730faf621..25fd8227da2f219d75c6b830e65627ecf35be453 100644
--- a/python/paddle/v2/dataset/movielens.py
+++ b/python/paddle/v2/dataset/movielens.py
@@ -1,10 +1,34 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Movielens 1-M dataset.
+
+TODO(yuyang18): Complete comments.
+"""
+
import zipfile
from common import download
import re
import random
import functools
-__all__ = ['train_creator', 'test_creator']
+__all__ = [
+ 'train', 'test', 'get_movie_title_dict', 'max_movie_id', 'max_user_id',
+ 'age_table', 'movie_categories', 'max_job_id', 'user_info', 'movie_info'
+]
+
+age_table = [1, 18, 25, 35, 45, 50, 56]
class MovieInfo(object):
@@ -19,17 +43,32 @@ class MovieInfo(object):
[MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()]
]
+ def __str__(self):
+ return "" % (
+ self.index, self.title, self.categories)
+
+ def __repr__(self):
+ return self.__str__()
+
class UserInfo(object):
def __init__(self, index, gender, age, job_id):
self.index = int(index)
self.is_male = gender == 'M'
- self.age = [1, 18, 25, 35, 45, 50, 56].index(int(age))
+ self.age = age_table.index(int(age))
self.job_id = int(job_id)
def value(self):
return [self.index, 0 if self.is_male else 1, self.age, self.job_id]
+ def __str__(self):
+ return "" % (
+ self.index, "M"
+ if self.is_male else "F", age_table[self.age], self.job_id)
+
+ def __repr__(self):
+ return str(self)
+
MOVIE_INFO = None
MOVIE_TITLE_DICT = None
@@ -40,7 +79,8 @@ USER_INFO = None
def __initialize_meta_info__():
fn = download(
url='http://files.grouplens.org/datasets/movielens/ml-1m.zip',
- md5='c4d9eecfca2ab87c1945afe126590906')
+ module_name='movielens',
+ md5sum='c4d9eecfca2ab87c1945afe126590906')
global MOVIE_INFO
if MOVIE_INFO is None:
pattern = re.compile(r'^(.*)\((\d+)\)$')
@@ -103,14 +143,63 @@ def __reader_creator__(**kwargs):
return lambda: __reader__(**kwargs)
-train_creator = functools.partial(__reader_creator__, is_test=False)
-test_creator = functools.partial(__reader_creator__, is_test=True)
+train = functools.partial(__reader_creator__, is_test=False)
+test = functools.partial(__reader_creator__, is_test=True)
+
+
+def get_movie_title_dict():
+ __initialize_meta_info__()
+ return MOVIE_TITLE_DICT
+
+
+def __max_index_info__(a, b):
+ if a.index > b.index:
+ return a
+ else:
+ return b
+
+
+def max_movie_id():
+ __initialize_meta_info__()
+ return reduce(__max_index_info__, MOVIE_INFO.viewvalues()).index
+
+
+def max_user_id():
+ __initialize_meta_info__()
+ return reduce(__max_index_info__, USER_INFO.viewvalues()).index
+
+
+def __max_job_id_impl__(a, b):
+ if a.job_id > b.job_id:
+ return a
+ else:
+ return b
+
+
+def max_job_id():
+ __initialize_meta_info__()
+ return reduce(__max_job_id_impl__, USER_INFO.viewvalues()).job_id
+
+
+def movie_categories():
+ __initialize_meta_info__()
+ return CATEGORIES_DICT
+
+
+def user_info():
+ __initialize_meta_info__()
+ return USER_INFO
+
+
+def movie_info():
+ __initialize_meta_info__()
+ return MOVIE_INFO
def unittest():
- for train_count, _ in enumerate(train_creator()()):
+ for train_count, _ in enumerate(train()()):
pass
- for test_count, _ in enumerate(test_creator()()):
+ for test_count, _ in enumerate(test()()):
pass
print train_count, test_count
diff --git a/python/paddle/v2/dataset/sentiment.py b/python/paddle/v2/dataset/sentiment.py
new file mode 100644
index 0000000000000000000000000000000000000000..71689fd61b6b14a7b5072caff4e2fd48a7f74072
--- /dev/null
+++ b/python/paddle/v2/dataset/sentiment.py
@@ -0,0 +1,127 @@
+# /usr/bin/env python
+# -*- coding:utf-8 -*-
+
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+The script fetch and preprocess movie_reviews data set that provided by NLTK
+
+TODO(yuyang18): Complete dataset.
+"""
+
+import collections
+from itertools import chain
+
+import nltk
+from nltk.corpus import movie_reviews
+
+import common
+
+__all__ = ['train', 'test', 'get_word_dict']
+NUM_TRAINING_INSTANCES = 1600
+NUM_TOTAL_INSTANCES = 2000
+
+
+def download_data_if_not_yet():
+ """
+ Download the data set, if the data set is not download.
+ """
+ try:
+ # make sure that nltk can find the data
+ if common.DATA_HOME not in nltk.data.path:
+ nltk.data.path.append(common.DATA_HOME)
+ movie_reviews.categories()
+ except LookupError:
+ print "Downloading movie_reviews data set, please wait....."
+ nltk.download('movie_reviews', download_dir=common.DATA_HOME)
+ print "Download data set success....."
+ print "Path is " + nltk.data.find('corpora/movie_reviews').path
+
+
+def get_word_dict():
+ """
+ Sorted the words by the frequency of words which occur in sample
+ :return:
+ words_freq_sorted
+ """
+ words_freq_sorted = list()
+ word_freq_dict = collections.defaultdict(int)
+ download_data_if_not_yet()
+
+ for category in movie_reviews.categories():
+ for field in movie_reviews.fileids(category):
+ for words in movie_reviews.words(field):
+ word_freq_dict[words] += 1
+ words_sort_list = word_freq_dict.items()
+ words_sort_list.sort(cmp=lambda a, b: b[1] - a[1])
+ for index, word in enumerate(words_sort_list):
+ words_freq_sorted.append((word[0], index))
+ return words_freq_sorted
+
+
+def sort_files():
+ """
+ Sorted the sample for cross reading the sample
+ :return:
+ files_list
+ """
+ files_list = list()
+ neg_file_list = movie_reviews.fileids('neg')
+ pos_file_list = movie_reviews.fileids('pos')
+ files_list = list(chain.from_iterable(zip(neg_file_list, pos_file_list)))
+ return files_list
+
+
+def load_sentiment_data():
+ """
+ Load the data set
+ :return:
+ data_set
+ """
+ data_set = list()
+ download_data_if_not_yet()
+ words_ids = dict(get_word_dict())
+ for sample_file in sort_files():
+ words_list = list()
+ category = 0 if 'neg' in sample_file else 1
+ for word in movie_reviews.words(sample_file):
+ words_list.append(words_ids[word.lower()])
+ data_set.append((words_list, category))
+ return data_set
+
+
+def reader_creator(data):
+ """
+ Reader creator, generate an iterator for data set
+ :param data:
+ train data set or test data set
+ """
+ for each in data:
+ yield each[0], each[1]
+
+
+def train():
+ """
+ Default train set reader creator
+ """
+ data_set = load_sentiment_data()
+ return reader_creator(data_set[0:NUM_TRAINING_INSTANCES])
+
+
+def test():
+ """
+ Default test set reader creator
+ """
+ data_set = load_sentiment_data()
+ return reader_creator(data_set[NUM_TRAINING_INSTANCES:])
diff --git a/python/paddle/v2/dataset/tests/cifar_test.py b/python/paddle/v2/dataset/tests/cifar_test.py
index a2af45ecf508462fe4b596b5d8d6401c5b974eff..e0e18229da7818be5752ee592e094a00da286ad9 100644
--- a/python/paddle/v2/dataset/tests/cifar_test.py
+++ b/python/paddle/v2/dataset/tests/cifar_test.py
@@ -1,3 +1,17 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import paddle.v2.dataset.cifar
import unittest
diff --git a/python/paddle/v2/dataset/tests/common_test.py b/python/paddle/v2/dataset/tests/common_test.py
index 7d8406171b8478e4a8331637c5e867c18d5eb3d8..5babcef0eb4345d243904877d323c37d4889a643 100644
--- a/python/paddle/v2/dataset/tests/common_test.py
+++ b/python/paddle/v2/dataset/tests/common_test.py
@@ -1,3 +1,17 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import paddle.v2.dataset.common
import unittest
import tempfile
diff --git a/python/paddle/v2/dataset/tests/imdb_test.py b/python/paddle/v2/dataset/tests/imdb_test.py
index e887af16634d2db04b8cf5fa0269a69991d8baac..c4d82f26895d77d05c6e936bd636b1239e1a0cd8 100644
--- a/python/paddle/v2/dataset/tests/imdb_test.py
+++ b/python/paddle/v2/dataset/tests/imdb_test.py
@@ -1,3 +1,17 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import paddle.v2.dataset.imdb
import unittest
import re
diff --git a/python/paddle/v2/dataset/tests/imikolov_test.py b/python/paddle/v2/dataset/tests/imikolov_test.py
index 9b1748eaaa7f913a6b94f2087a8089fb998570aa..009e55243a594e5e235c36fb0223ec70754d17f3 100644
--- a/python/paddle/v2/dataset/tests/imikolov_test.py
+++ b/python/paddle/v2/dataset/tests/imikolov_test.py
@@ -1,6 +1,8 @@
import paddle.v2.dataset.imikolov
import unittest
+WORD_DICT = paddle.v2.dataset.imikolov.build_dict()
+
class TestMikolov(unittest.TestCase):
def check_reader(self, reader, n):
@@ -9,11 +11,15 @@ class TestMikolov(unittest.TestCase):
def test_train(self):
n = 5
- self.check_reader(paddle.v2.dataset.imikolov.train(n), n)
+ self.check_reader(paddle.v2.dataset.imikolov.train(WORD_DICT, n), n)
def test_test(self):
n = 5
- self.check_reader(paddle.v2.dataset.imikolov.test(n), n)
+ self.check_reader(paddle.v2.dataset.imikolov.test(WORD_DICT, n), n)
+
+ def test_total(self):
+ _, idx = zip(*WORD_DICT.items())
+ self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1)
if __name__ == '__main__':
diff --git a/python/paddle/v2/dataset/tests/mnist_test.py b/python/paddle/v2/dataset/tests/mnist_test.py
index b4408cc2f590d4d8da4ce5e98213cf7b208cfc15..1d344cac3e7483a351033570fbec75a4d19f4a55 100644
--- a/python/paddle/v2/dataset/tests/mnist_test.py
+++ b/python/paddle/v2/dataset/tests/mnist_test.py
@@ -1,3 +1,17 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import paddle.v2.dataset.mnist
import unittest
diff --git a/python/paddle/v2/dataset/tests/test_sentiment.py b/python/paddle/v2/dataset/tests/test_sentiment.py
new file mode 100644
index 0000000000000000000000000000000000000000..407405290734609059c1767600748d530e8a13a6
--- /dev/null
+++ b/python/paddle/v2/dataset/tests/test_sentiment.py
@@ -0,0 +1,55 @@
+# /usr/bin/env python
+# -*- coding:utf-8 -*-
+
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import nltk
+import paddle.v2.dataset.sentiment as st
+from nltk.corpus import movie_reviews
+
+
+class TestSentimentMethods(unittest.TestCase):
+ def test_get_word_dict(self):
+ word_dict = st.get_word_dict()[0:10]
+ test_word_list = [(u',', 0), (u'the', 1), (u'.', 2), (u'a', 3),
+ (u'and', 4), (u'of', 5), (u'to', 6), (u"'", 7),
+ (u'is', 8), (u'in', 9)]
+ for idx, each in enumerate(word_dict):
+ self.assertEqual(each, test_word_list[idx])
+ self.assertTrue("/root/.cache/paddle/dataset" in nltk.data.path)
+
+ def test_sort_files(self):
+ last_label = ''
+ for sample_file in st.sort_files():
+ current_label = sample_file.split("/")[0]
+ self.assertNotEqual(current_label, last_label)
+ last_label = current_label
+
+ def test_data_set(self):
+ data_set = st.load_sentiment_data()
+ last_label = -1
+ for each in st.test():
+ self.assertNotEqual(each[1], last_label)
+ last_label = each[1]
+ self.assertEqual(len(data_set), st.NUM_TOTAL_INSTANCES)
+ self.assertEqual(len(list(st.train())), st.NUM_TRAINING_INSTANCES)
+ self.assertEqual(
+ len(list(st.test())),
+ (st.NUM_TOTAL_INSTANCES - st.NUM_TRAINING_INSTANCES))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/dataset/uci_housing.py b/python/paddle/v2/dataset/uci_housing.py
new file mode 100644
index 0000000000000000000000000000000000000000..27f454b137e3a40febd19cf085e2f4034cc16b24
--- /dev/null
+++ b/python/paddle/v2/dataset/uci_housing.py
@@ -0,0 +1,91 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+UCI Housing dataset.
+
+TODO(yuyang18): Complete comments.
+"""
+
+import numpy as np
+import os
+from common import download
+
+__all__ = ['train', 'test']
+
+URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
+MD5 = 'd4accdce7a25600298819f8e28e8d593'
+feature_names = [
+ 'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',
+ 'PTRATIO', 'B', 'LSTAT'
+]
+
+UCI_TRAIN_DATA = None
+UCI_TEST_DATA = None
+
+
+def feature_range(maximums, minimums):
+ import matplotlib
+ matplotlib.use('Agg')
+ import matplotlib.pyplot as plt
+ fig, ax = plt.subplots()
+ feature_num = len(maximums)
+ ax.bar(range(feature_num), maximums - minimums, color='r', align='center')
+ ax.set_title('feature scale')
+ plt.xticks(range(feature_num), feature_names)
+ plt.xlim([-1, feature_num])
+ fig.set_figheight(6)
+ fig.set_figwidth(10)
+ if not os.path.exists('./image'):
+ os.makedirs('./image')
+ fig.savefig('image/ranges.png', dpi=48)
+ plt.close(fig)
+
+
+def load_data(filename, feature_num=14, ratio=0.8):
+ global UCI_TRAIN_DATA, UCI_TEST_DATA
+ if UCI_TRAIN_DATA is not None and UCI_TEST_DATA is not None:
+ return
+
+ data = np.fromfile(filename, sep=' ')
+ data = data.reshape(data.shape[0] / feature_num, feature_num)
+ maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum(
+ axis=0) / data.shape[0]
+ feature_range(maximums[:-1], minimums[:-1])
+ for i in xrange(feature_num - 1):
+ data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
+ offset = int(data.shape[0] * ratio)
+ UCI_TRAIN_DATA = data[:offset]
+ UCI_TEST_DATA = data[offset:]
+
+
+def train():
+ global UCI_TRAIN_DATA
+ load_data(download(URL, 'uci_housing', MD5))
+
+ def reader():
+ for d in UCI_TRAIN_DATA:
+ yield d[:-1], d[-1:]
+
+ return reader
+
+
+def test():
+ global UCI_TEST_DATA
+ load_data(download(URL, 'uci_housing', MD5))
+
+ def reader():
+ for d in UCI_TEST_DATA:
+ yield d[:-1], d[-1:]
+
+ return reader
diff --git a/python/paddle/v2/dataset/wmt14.py b/python/paddle/v2/dataset/wmt14.py
new file mode 100644
index 0000000000000000000000000000000000000000..f5a16d51477f9cfbf0cd32af54098406fbbd2b41
--- /dev/null
+++ b/python/paddle/v2/dataset/wmt14.py
@@ -0,0 +1,105 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+wmt14 dataset
+"""
+import tarfile
+
+import paddle.v2.dataset.common
+
+__all__ = ['train', 'test', 'build_dict']
+
+URL_DEV_TEST = 'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz'
+MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5'
+# this is a small set of data for test. The original data is too large and will be add later.
+URL_TRAIN = 'http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz'
+MD5_TRAIN = 'a755315dd01c2c35bde29a744ede23a6'
+
+START = ""
+END = ""
+UNK = ""
+UNK_IDX = 2
+
+
+def __read_to_dict__(tar_file, dict_size):
+ def __to_dict__(fd, size):
+ out_dict = dict()
+ for line_count, line in enumerate(fd):
+ if line_count < size:
+ out_dict[line.strip()] = line_count
+ else:
+ break
+ return out_dict
+
+ with tarfile.open(tar_file, mode='r') as f:
+ names = [
+ each_item.name for each_item in f
+ if each_item.name.endswith("src.dict")
+ ]
+ assert len(names) == 1
+ src_dict = __to_dict__(f.extractfile(names[0]), dict_size)
+ names = [
+ each_item.name for each_item in f
+ if each_item.name.endswith("trg.dict")
+ ]
+ assert len(names) == 1
+ trg_dict = __to_dict__(f.extractfile(names[0]), dict_size)
+ return src_dict, trg_dict
+
+
+def reader_creator(tar_file, file_name, dict_size):
+ def reader():
+ src_dict, trg_dict = __read_to_dict__(tar_file, dict_size)
+ with tarfile.open(tar_file, mode='r') as f:
+ names = [
+ each_item.name for each_item in f
+ if each_item.name.endswith(file_name)
+ ]
+ for name in names:
+ for line in f.extractfile(name):
+ line_split = line.strip().split('\t')
+ if len(line_split) != 2:
+ continue
+ src_seq = line_split[0] # one source sequence
+ src_words = src_seq.split()
+ src_ids = [
+ src_dict.get(w, UNK_IDX)
+ for w in [START] + src_words + [END]
+ ]
+
+ trg_seq = line_split[1] # one target sequence
+ trg_words = trg_seq.split()
+ trg_ids = [trg_dict.get(w, UNK_IDX) for w in trg_words]
+
+ # remove sequence whose length > 80 in training mode
+ if len(src_ids) > 80 or len(trg_ids) > 80:
+ continue
+ trg_ids_next = trg_ids + [trg_dict[END]]
+ trg_ids = [trg_dict[START]] + trg_ids
+
+ yield src_ids, trg_ids, trg_ids_next
+
+ return reader
+
+
+def train(dict_size):
+ return reader_creator(
+ paddle.v2.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN),
+ 'train/train', dict_size)
+
+
+def test(dict_size):
+ return reader_creator(
+ paddle.v2.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN),
+ 'test/test', dict_size)
diff --git a/python/paddle/v2/event.py b/python/paddle/v2/event.py
index a78bcf076cc65e0dfdfc5760e099900418162f35..1ad52b8baa411269d29732685871a875df5185cc 100644
--- a/python/paddle/v2/event.py
+++ b/python/paddle/v2/event.py
@@ -34,8 +34,13 @@ class WithMetric(object):
class TestResult(WithMetric):
- def __init__(self, evaluator):
+ """
+ Result that trainer.test return.
+ """
+
+ def __init__(self, evaluator, cost):
super(TestResult, self).__init__(evaluator)
+ self.cost = cost
class BeginPass(object):
diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py
new file mode 100644
index 0000000000000000000000000000000000000000..ec3c67d89548f68d705a9b5de80e28597e9829da
--- /dev/null
+++ b/python/paddle/v2/inference.py
@@ -0,0 +1,88 @@
+import numpy
+import py_paddle.swig_paddle as api
+import collections
+import topology
+import minibatch
+from data_feeder import DataFeeder
+
+__all__ = ['infer']
+
+
+class Inference(object):
+ def __init__(self, output_layer, parameters):
+ topo = topology.Topology(output_layer)
+ gm = api.GradientMachine.createFromConfigProto(
+ topo.proto(), api.CREATE_MODE_TESTING, [api.PARAMETER_VALUE])
+ for param in gm.getParameters():
+ val = param.getBuf(api.PARAMETER_VALUE)
+ name = param.getName()
+ assert isinstance(val, api.Vector)
+ val.copyFromNumpyArray(parameters.get(name).flatten())
+ self.__gradient_machine__ = gm
+ self.__data_types__ = topo.data_type()
+
+ def iter_infer(self, input, feeding=None):
+ feeder = DataFeeder(self.__data_types__, feeding)
+ batch_size = len(input)
+
+ def __reader_impl__():
+ for each_sample in input:
+ yield each_sample
+
+ reader = minibatch.batch(__reader_impl__, batch_size=batch_size)
+
+ self.__gradient_machine__.start()
+ for data_batch in reader():
+ yield self.__gradient_machine__.forwardTest(feeder(data_batch))
+ self.__gradient_machine__.finish()
+
+ def iter_infer_field(self, field, **kwargs):
+ for result in self.iter_infer(**kwargs):
+ yield [each_result[field] for each_result in result]
+
+ def infer(self, field='value', **kwargs):
+ retv = None
+ for result in self.iter_infer_field(field=field, **kwargs):
+ if retv is None:
+ retv = [[]] * len(result)
+ for i, item in enumerate(result):
+ retv[i].append(item)
+ retv = [numpy.concatenate(out) for out in retv]
+ if len(retv) == 1:
+ return retv[0]
+ else:
+ return retv
+
+
+def infer(output_layer, parameters, input, feeding=None, field='value'):
+ """
+ Infer a neural network by given neural network output and parameters. The
+ user should pass either a batch of input data or reader method.
+
+ Example usages:
+
+ .. code-block:: python
+
+ result = paddle.infer(prediction, parameters, input=SomeData,
+ batch_size=32)
+ print result
+
+ :param output_layer: output of the neural network that would be inferred
+ :type output_layer: paddle.v2.config_base.Layer
+ :param parameters: parameters of the neural network.
+ :type parameters: paddle.v2.parameters.Parameters
+ :param input: input data batch. Should be a python iterable object, and each
+ element is the data batch.
+ :type input: collections.Iterable
+ :param feeding: Reader dictionary. Default could generate from input
+ value.
+ :param field: The prediction field. It should in [`value`, `ids`]. `value`
+ means return the prediction probabilities, `ids` means return
+ the prediction labels. Default is `value`
+ :type field: str
+ :return: a numpy array
+ :rtype: numpy.ndarray
+ """
+
+ inferer = Inference(output_layer=output_layer, parameters=parameters)
+ return inferer.infer(field=field, input=input, feeding=feeding)
diff --git a/python/paddle/v2/inferencer.py b/python/paddle/v2/inferencer.py
deleted file mode 100644
index ac03b016c9b8bfbc586072855402ed3a373e9b54..0000000000000000000000000000000000000000
--- a/python/paddle/v2/inferencer.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import py_paddle.swig_paddle as api
-
-import topology
-from data_feeder import DataFeeder
-import itertools
-import numpy
-
-__all__ = ['Inference', 'infer']
-
-
-class Inference(object):
- def __init__(self, output, parameters):
- topo = topology.Topology(output)
- gm = api.GradientMachine.createFromConfigProto(
- topo.proto(), api.CREATE_MODE_TESTING, [api.PARAMETER_VALUE])
- for param in gm.getParameters():
- val = param.getBuf(api.PARAMETER_VALUE)
- name = param.getName()
- assert isinstance(val, api.Vector)
- val.copyFromNumpyArray(parameters.get(name).flatten())
- self.__gradient_machine__ = gm
- self.__data_types__ = topo.data_type()
-
- def iter_infer(self, reader, reader_dict=None):
- if reader_dict is None:
- reader_dict = self.default_reader_dict()
- feeder = DataFeeder(self.__data_types__, reader_dict)
- self.__gradient_machine__.start()
- for data_batch in reader():
- yield self.__gradient_machine__.forwardTest(feeder(data_batch))
- self.__gradient_machine__.finish()
-
- def iter_infer_field(self, field, **kwargs):
- for result in self.iter_infer(**kwargs):
- yield [each_result[field] for each_result in result]
-
- def infer(self, field='value', **kwargs):
- retv = None
- for result in self.iter_infer_field(field=field, **kwargs):
- if retv is None:
- retv = [[]] * len(result)
- for i, item in enumerate(result):
- retv[i].append(item)
- retv = [numpy.concatenate(out) for out in retv]
- if len(retv) == 1:
- return retv[0]
- else:
- return retv
-
- def default_reader_dict(self):
- reader_dict = dict()
- for i, tp in enumerate(self.__data_types__):
- reader_dict[tp[0]] = i
- return reader_dict
-
-
-def infer(output, parameters, reader, reader_dict=None, field='value'):
- inferer = Inference(output=output, parameters=parameters)
- return inferer.infer(field=field, reader=reader, reader_dict=reader_dict)
diff --git a/python/paddle/v2/layer.py b/python/paddle/v2/layer.py
index 67111f1315fbb0f55c1db0f6fe89fc988c8d83f6..1e4efedde363f20fde168941adcb6e8a594b533a 100644
--- a/python/paddle/v2/layer.py
+++ b/python/paddle/v2/layer.py
@@ -12,91 +12,67 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
-Before this new package paddle.v2.layer, users would need to use functions
-in paddle.trainer_config_helpers.layers to configure networks.
-
-The Old Way:
-=========
-This old way requires that the creation of a network be defined in a Python
-function, say network_config, and that this Python function being passed to
-paddle.trainer_config_helpers.parse_network_config for the creation of
-protobuf message description of this network.
-
-```python
-def network_config():
- img = paddle.trainer_config_helpers.data_layer(name="pixel", size=784)
- inference = paddle.trainer_config_helpers.fc_layer(
- input=img,
- size=10,
- act=paddle.trainer_config_helpers.SoftmaxActivation())
- cost = paddle.trainer_config_helpers.classification_cost(
- input=inference,
- label=paddle.trainer_config_helpers.data_layer(name="label", size=10))
-
-proto_desc = parse_network_config(network_config)
-```
-
-When parse_network_config executes network_config, those layer definition
-functions like data_layer and fc_layer would change some Python global variables,
-so that after the execution, parse_network_config could collect information from
-these global variables and generates the protobuf message.
-
-
-
-The New Way:
-=========
-In this PR, we define a function in paddle.v2.layer which creates a Python
-class for each layer creation function in paddle.trainer_config_helpers.layers.
-Users can use create a network as follows:
-
-```python
-img = paddle.v2.layer.data(name="pixel", size=784)
-inference = paddle.v2.layer.fc(input=img, size=10, act=paddle.v2.layer.Softmax())
-cost = paddle.v2.layer.classification(
- input=inference,
- label=paddle.v2.layer.data(name="label", size=10))
-
-parameters = paddle.v2.parameters.create(cost)
-```
-
-This new way doesn't require those invocations to layer definition functions
-to be in a Python function but could be anywhere.
-
-Also, the creation of a protobuf message is hidden in the invocation of
-paddle.v2.parameters.create, no longer exposed to users.
+`paddle.v2.layer` is a part of model config packages in paddle.v2. In API v2,
+we want to make Paddle a plain Python package. The model config package defined
+the way how to configure a neural network topology in Paddle Python code.
+
+The primary usage shows below.
+
+.. code-block:: python
+
+ import paddle.v2 as paddle
+
+ img = paddle.layer.data(name='img', type=paddle.data_type.dense_vector(784))
+ hidden = paddle.layer.fc(input=img, size=200)
+ prediction = paddle.layer.fc(input=hidden, size=10,
+ act=paddle.activation.Softmax())
+
+ # use prediction instance where needed.
+ parameters = paddle.parameters.create(cost)
"""
+
+import collections
+import inspect
from config_base import Layer, __convert_to_v2__
import paddle.trainer_config_helpers as conf_helps
from paddle.trainer_config_helpers.config_parser_utils import \
parse_network_config as __parse__
-
-from paddle.trainer_config_helpers.default_decorators import wrap_name_default
from paddle.trainer_config_helpers.default_decorators import wrap_act_default
from paddle.trainer_config_helpers.default_decorators import \
wrap_bias_attr_default
+from paddle.trainer_config_helpers.default_decorators import wrap_name_default
from paddle.trainer_config_helpers.layers import layer_support
+from paddle.trainer.config_parser import \
+ RecurrentLayerGroupWithoutOutLinksBegin, RecurrentLayerGroupSetOutLink, \
+ RecurrentLayerGroupEnd, model_type
-import data_type
import activation
+import re
+import data_type
__all__ = ['parse_network', 'data']
-__projection_names__ = filter(lambda x: x.endswith('_projection'),
- dir(conf_helps))
-__all__ += __projection_names__
-
-__operator_names__ = filter(lambda x: x.endswith('_operator'), dir(conf_helps))
-__all__ += __operator_names__
-
def parse_network(*outputs):
"""
- parse all output layers and then generate a model config proto.
- :param outputs:
- :return:
+ Parse all output layers and then generate a ModelConfig object.
+
+ .. note::
+
+ This function is used internally in paddle.v2 module. User should never
+ invoke this method.
+
+ :param outputs: Output layers.
+ :type outputs: Layer
+ :return: A ModelConfig object instance.
+ :rtype: ModelConfig
"""
def __real_func__():
+ """
+ __real_func__ is the function that config_parser.parse invoked. It is
+ the plain old paddle configuration function.
+ """
context = dict()
real_output = [each.to_proto(context=context) for each in outputs]
conf_helps.outputs(real_output)
@@ -111,6 +87,8 @@ So we also need to implement some special LayerV2.
class DataLayerV2(Layer):
+ METHOD_NAME = 'data_layer'
+
def __init__(self, name, type, **kwargs):
assert isinstance(type, data_type.InputType)
@@ -129,6 +107,148 @@ class DataLayerV2(Layer):
args[each] = self.__kwargs__[each]
return getattr(conf_helps, self.__method_name__)(name=self.name, **args)
+ def __map_docstr__(doc):
+ doc = re.sub(r'(data = [^\)]+)\).*',
+ "data = paddle.layer.data(name=\"input\", "
+ "type=paddle.data_type.dense_vector(1000))", doc)
+
+ doc = re.sub(r':param size:.*',
+ ':param type: Data type of this data layer', doc)
+ doc = re.sub(r':type size:.*',
+ ":type size: paddle.v2.data_type.InputType", doc)
+ return doc
+
+
+class WithExtraParent(Layer):
+ def extra_parent(self):
+ return self.__extra_parent__
+
+ def __init__(self, name=None, parent_layers=None):
+ self.__extra_parent__ = []
+ super(WithExtraParent, self).__init__(
+ name=name, parent_layers=parent_layers)
+
+ def append_extra_parent(self, parent):
+ self.__extra_parent__.append(parent)
+
+ def to_proto(self, context):
+ """
+ function to set proto attribute
+ """
+ kwargs = dict()
+ for p in self.__extra_parent__:
+ p.to_proto(context=context)
+
+ for layer_name in self.__parent_layers__:
+ if not isinstance(self.__parent_layers__[layer_name],
+ collections.Sequence):
+ v1_layer = self.__parent_layers__[layer_name].to_proto(
+ context=context)
+ else:
+ v1_layer = map(lambda x: x.to_proto(context=context),
+ self.__parent_layers__[layer_name])
+ kwargs[layer_name] = v1_layer
+
+ if self.context_name() is None:
+ return self.to_proto_impl(context=context, **kwargs)
+ elif self.context_name() not in context:
+ context[self.context_name()] = self.to_proto_impl(
+ context=context, **kwargs)
+
+ if self.use_context_name():
+ return context[self.context_name()]
+ else:
+ return context[self.name]
+
+
+class MemoryV2(WithExtraParent):
+ def __init__(self, name, **kwargs):
+ self.name = name
+ super(MemoryV2, self).__init__(name=name, parent_layers=dict())
+ self.__kwargs__ = kwargs
+ self.__boot_layer_name__ = None
+ if 'boot_layer' in kwargs:
+ begin_of_current_rnn = []
+ # TODO(yuyang18): Fix inspect, it could be wrong when user invoke a
+ # function inside step.
+ st = inspect.stack()
+ for i in xrange(len(st)):
+ locs = inspect.stack()[i][0].f_locals
+ keys = locs.keys()
+ for key in keys:
+ val = locs[key]
+ if isinstance(val, RecurrentLayerInput):
+ begin_of_current_rnn.append(val)
+ elif isinstance(val, collections.Sequence):
+ for v in val:
+ if isinstance(v, RecurrentLayerInput):
+ begin_of_current_rnn.append(v)
+
+ if begin_of_current_rnn:
+ break
+ assert begin_of_current_rnn is not None
+ for extra in begin_of_current_rnn:
+ self.append_extra_parent(extra)
+ assert isinstance(extra, WithExtraParent)
+ extra.append_extra_parent(kwargs['boot_layer'])
+ self.__boot_layer_name__ = kwargs['boot_layer'].name
+
+ def to_proto_impl(self, context, **kwargs):
+ args = dict()
+ for each in kwargs:
+ args[each] = kwargs[each]
+ for each in self.__kwargs__:
+ args[each] = self.__kwargs__[each]
+
+ if self.__boot_layer_name__ is not None:
+ args['boot_layer'] = context[self.__boot_layer_name__]
+
+ size = args.get('size', None)
+ if size is not None:
+ if callable(size):
+ real_size = size()
+ else:
+ real_size = size
+ args['size'] = real_size
+ return conf_helps.memory(name=self.name, **args)
+
+ def context_name(self):
+ return self.name + "#memory"
+
+ def use_context_name(self):
+ """
+ memory layer will have the same name with some layer
+ :return:
+ """
+ return True
+
+
+class LayerOutputV2(Layer):
+ """
+ LayerOutputV2 is used to store the result of LayerOutput in v1 api.
+ It will not store it's parents because layer_output has been parsed already.
+ """
+
+ def __init__(self, layer_output):
+ assert isinstance(layer_output, conf_helps.LayerOutput)
+ self.layer_output = layer_output
+ super(LayerOutputV2, self).__init__(
+ name=layer_output.name, parent_layers=dict())
+
+ def to_proto_impl(self):
+ return self.layer_output
+
+
+class StaticInputV2(object):
+ def __init__(self, input, is_seq=False, size=None):
+ assert isinstance(input, LayerV2)
+ self.name = input.name
+ self.input = input
+ self.is_seq = is_seq
+ self.size = size
+ # TODO(add size check)
+ # assert input.size is not None or size is not None
+
class MixedLayerV2(Layer):
"""
@@ -161,7 +281,6 @@ class MixedLayerV2(Layer):
other_kwargs['act'] = act
other_kwargs['bias_attr'] = bias_attr
other_kwargs['layer_attr'] = layer_attr
-
parent_layers = {"input": self.__inputs__}
super(MixedLayerV2, self).__init__(name, parent_layers)
self.__other_kwargs__ = other_kwargs
@@ -171,7 +290,7 @@ class MixedLayerV2(Layer):
self.__inputs__.append(other)
return self
else:
- raise MixedLayerTypeV2.AddToSealedMixedLayerExceptionV2()
+ raise MixedLayerV2.AddToSealedMixedLayerExceptionV2()
def __enter__(self):
assert len(self.__inputs__) == 0
@@ -186,6 +305,13 @@ class MixedLayerV2(Layer):
args[each] = kwargs[each]
for each in self.__other_kwargs__:
args[each] = self.__other_kwargs__[each]
+ size = args.get('size', None)
+ if size is not None:
+ if callable(size):
+ real_size = size()
+ else:
+ real_size = size
+ args['size'] = real_size
return getattr(conf_helps, self.__method_name__)(**args)
@@ -202,14 +328,52 @@ def mixed(size=0,
return MixedLayerV2(size, input, name, act, bias_attr, layer_attr)
+class RecurrentLayerInput(WithExtraParent):
+ def __init__(self, recurrent_name, index, parent_layers):
+ assert len(parent_layers) == 1
+ self.__parents__ = parent_layers.values()[0]
+ super(RecurrentLayerInput, self).__init__(
+ name=self.__parents__[index].name, parent_layers=parent_layers)
+ self.__recurrent_name__ = recurrent_name
+
+ def context_name(self):
+ return self.__recurrent_name__ + ".begin"
+
+ def to_proto_impl(self, context, **kwargs):
+ model_type('recurrent_nn')
+ RecurrentLayerGroupWithoutOutLinksBegin(
+ name=self.__recurrent_name__,
+ in_links=map(lambda x: x.name, self.__parents__))
+ return self
+
+
+class RecurrentLayerOutput(Layer):
+ def __init__(self, recurrent_name, index, parent_layers):
+ assert len(parent_layers) == 1
+ self.__parents__ = parent_layers.values()[0]
+ super(RecurrentLayerOutput, self).__init__(
+ name=self.__parents__[index].name, parent_layers=parent_layers)
+ self.__recurrent_name__ = recurrent_name
+
+ def context_name(self):
+ return self.__recurrent_name__ + ".end"
+
+ def to_proto_impl(self, **kwargs):
+ for l in self.__parents__:
+ RecurrentLayerGroupSetOutLink(l.name)
+ RecurrentLayerGroupEnd(name=self.__recurrent_name__)
+
+
LayerV2 = Layer
data = DataLayerV2
+data.__name__ = 'data'
AggregateLevel = conf_helps.layers.AggregateLevel
ExpandLevel = conf_helps.layers.ExpandLevel
+memory = MemoryV2
def __layer_name_mapping__(inname):
- if inname in ['data_layer', 'memory', 'mixed_layer']:
+ if inname in ['data_layer', 'memory', 'mixed_layer', 'recurrent_group']:
# Do Not handle these layers
return
elif inname == 'maxid_layer':
@@ -231,8 +395,10 @@ def __layer_name_mapping__(inname):
def __layer_name_mapping_parent_names__(inname):
all_args = getattr(conf_helps, inname).argspec.args
return filter(
- lambda x: x in ['input1', 'input2','label', 'input', 'a', 'b', 'expand_as',
- 'weights', 'vectors', 'weight', 'score', 'left', 'right'],
+ lambda x: x in ['input1', 'input2', 'label', 'input', 'a', 'b',
+ 'expand_as',
+ 'weights', 'vectors', 'weight', 'score', 'left',
+ 'right', 'output_mem'],
all_args)
@@ -240,6 +406,7 @@ def __convert_layer__(_new_name_, _old_name_, _parent_names_):
global __all__
__all__.append(_new_name_)
globals()[new_name] = __convert_to_v2__(_old_name_, _parent_names_)
+ globals()[new_name].__name__ = new_name
for each_layer_name in dir(conf_helps):
@@ -253,10 +420,71 @@ del parent_names
del new_name
del each_layer_name
+
+@wrap_name_default()
+def recurrent_group(step, input, name=None):
+ if not isinstance(input, collections.Sequence):
+ input = [input]
+
+ non_static_inputs = filter(lambda x: not isinstance(x, StaticInputV2),
+ input)
+ actual_input = [
+ RecurrentLayerInput(
+ recurrent_name=name,
+ index=i,
+ parent_layers={'recurrent_inputs': non_static_inputs})
+ for i in xrange(len(non_static_inputs))
+ ]
+
+ def __real_step__(*args):
+ rnn_input = list(args)
+ static_inputs = filter(lambda x: isinstance(x, StaticInputV2), input)
+ for static_input in static_inputs:
+ mem_name = "__%s_memory__" % static_input.input.name
+ mem = memory(
+ name=mem_name,
+ is_seq=static_input.is_seq,
+ size=static_input.input.calculate_size,
+ boot_layer=static_input.input)
+ with mixed(
+ name=mem_name,
+ size=static_input.input.calculate_size,
+ act=activation.Identity()) as mix:
+ mix += identity_projection(input=mem)
+ rnn_input.insert(input.index(static_input), mix)
+ return step(*rnn_input)
+
+ actual_output = __real_step__(*actual_input)
+
+ if not isinstance(actual_output, collections.Sequence):
+ actual_output = [actual_output]
+
+ retv = [
+ RecurrentLayerOutput(
+ recurrent_name=name,
+ index=i,
+ parent_layers={'recurrent_outputs': actual_output})
+ for i in xrange(len(actual_output))
+ ]
+ if len(retv) == 1:
+ return retv[0]
+ else:
+ return retv
+
+
+__projection_names__ = filter(lambda x: x.endswith('_projection'),
+ dir(conf_helps))
+
+__all__ += __projection_names__
+
+__operator_names__ = filter(lambda x: x.endswith('_operator'), dir(conf_helps))
+__all__ += __operator_names__
+
# convert projection
for prj in __projection_names__:
globals()[prj] = __convert_to_v2__(
prj, parent_names=['input'], is_default_name=False)
+ globals()[prj].__name__ = prj
# convert operator
operator_list = [
@@ -267,3 +495,4 @@ operator_list = [
for op in operator_list:
globals()[op[0]] = __convert_to_v2__(
op[0], parent_names=op[1], is_default_name=False)
+ globals()[op[0]].__name__ = op[0]
diff --git a/python/paddle/v2/minibatch.py b/python/paddle/v2/minibatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..317cf037c69f8639e3760fbfce20565127794fcb
--- /dev/null
+++ b/python/paddle/v2/minibatch.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ['batch']
+
+
+def batch(reader, batch_size):
+ """
+ Create a batched reader.
+
+ :param reader: the data reader to read from.
+ :type reader: callable
+ :param batch_size: size of each mini-batch
+ :type batch_size: int
+ :return: the batched reader.
+ :rtype: callable
+ """
+
+ def batch_reader():
+ r = reader()
+ b = []
+ for instance in r:
+ b.append(instance)
+ if len(b) == batch_size:
+ yield b
+ b = []
+ if b:
+ yield b
+
+ return batch_reader
diff --git a/python/paddle/v2/networks.py b/python/paddle/v2/networks.py
index 74d91593d8551bc1592a78efb704ab3b89f0e0d9..9e6644196c8242cc3fed7a4fb1503697e5b59ffb 100644
--- a/python/paddle/v2/networks.py
+++ b/python/paddle/v2/networks.py
@@ -38,6 +38,7 @@ def __initialize__():
parent_names=parents,
is_default_name='name' in argspec.args)
globals()[each_subnetwork] = v2_subnet
+ globals()[each_subnetwork].__name__ = each_subnetwork
global __all__
__all__.append(each_subnetwork)
diff --git a/python/paddle/v2/optimizer.py b/python/paddle/v2/optimizer.py
index 10e255dc945efb8b20f09dc1806d2ba7ef856c55..1a01d95c205c0626374e1814a170ce2d58f23a60 100644
--- a/python/paddle/v2/optimizer.py
+++ b/python/paddle/v2/optimizer.py
@@ -1,7 +1,12 @@
import py_paddle.swig_paddle as swig_api
-import paddle.trainer_config_helpers.optimizers as v1_optimizers
+
import paddle.trainer_config_helpers.config_parser_utils as config_parser_utils
-import paddle.v2
+import paddle.trainer_config_helpers.optimizers as v1_optimizers
+"""
+Optimizers(update equation) for SGD method.
+
+TODO(yuyang18): Complete comments.
+"""
__all__ = [
'Momentum', 'Adam', 'Adamax', 'AdaGrad', 'DecayedAdaGrad', 'AdaDelta',
@@ -44,7 +49,7 @@ class Optimizer(object):
class Momentum(Optimizer):
def __init__(self, momentum=None, sparse=False, **kwargs):
learning_method = v1_optimizers.MomentumOptimizer(
- momentum=None, sparse=False)
+ momentum=momentum, sparse=sparse)
super(Momentum, self).__init__(
learning_method=learning_method, **kwargs)
diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py
index 2a6026bcab1c8a373d8dd5eac480dec62a8eb3b9..05dc5c68dd97b00fb15b74564a32313430c45345 100644
--- a/python/paddle/v2/parameters.py
+++ b/python/paddle/v2/parameters.py
@@ -1,7 +1,9 @@
import numpy as np
import py_paddle.swig_paddle as api
from paddle.proto.ParameterConfig_pb2 import ParameterConfig
-
+import struct
+import tarfile
+import cStringIO
from topology import Topology
__all__ = ['Parameters', 'create']
@@ -10,6 +12,7 @@ __all__ = ['Parameters', 'create']
def create(layers):
"""
Create parameter pool by topology.
+
:param layers:
:return:
"""
@@ -67,6 +70,7 @@ class Parameters(object):
def keys(self):
"""
keys are the names of each parameter.
+
:return: list of parameter name
:rtype: list
"""
@@ -75,6 +79,7 @@ class Parameters(object):
def names(self):
"""
names of each parameter.
+
:return: list of parameter name
:rtype: list
"""
@@ -83,6 +88,7 @@ class Parameters(object):
def has_key(self, key):
"""
has_key return true if there are such parameter name == key
+
:param key: Parameter name
:type key: basestring
:return: True if contains such key
@@ -118,6 +124,12 @@ class Parameters(object):
if len(self.__gradient_machines__) == 0:
# create new parameter in python numpy.
+ if len(self.__tmp_params__) != 0:
+ ret_list = [
+ mat for name, mat in self.__tmp_params__ if name == key
+ ]
+ if len(ret_list) == 1:
+ return ret_list[0]
return np.ndarray(shape=shape, dtype=np.float32)
else:
for each_gradient_machine in self.__gradient_machines__:
@@ -136,6 +148,7 @@ class Parameters(object):
def get_shape(self, key):
"""
get shape of the parameter.
+
:param key: parameter name
:type key: basestring
:return: parameter's shape
@@ -190,6 +203,7 @@ class Parameters(object):
def set(self, parameter_name, value):
"""
Set parameter by parameter name & matrix.
+
:param parameter_name: parameter name
:type parameter_name: basestring
:param value: parameter matrix
@@ -222,6 +236,67 @@ class Parameters(object):
self.__gradient_machines__.append(gradient_machine)
+ def serialize(self, name, f):
+ """
+
+ :param name:
+ :param f:
+ :type f: file
+ :return:
+ """
+ param = self.get(name)
+ size = reduce(lambda a, b: a * b, param.shape)
+ f.write(struct.pack("IIQ", 0, 4, size))
+ param = param.astype(np.float32)
+ f.write(param.tobytes())
+
+ def deserialize(self, name, f):
+ """
+
+ :param name:
+ :param f:
+ :type f: file
+ :return:
+ """
+ f.read(16) # header
+ arr = np.frombuffer(f.read(), dtype=np.float32)
+ self.set(name, arr.reshape(self.get_shape(name)))
+
+ def to_tar(self, f):
+ tar = tarfile.TarFile(fileobj=f, mode='w')
+ for nm in self.names():
+ buf = cStringIO.StringIO()
+ self.serialize(nm, buf)
+ tarinfo = tarfile.TarInfo(name=nm)
+ buf.seek(0)
+ tarinfo.size = len(buf.getvalue())
+ tar.addfile(tarinfo, buf)
+
+ conf = self.__param_conf__[nm]
+ confStr = conf.SerializeToString()
+ tarinfo = tarfile.TarInfo(name="%s.protobuf" % nm)
+ tarinfo.size = len(confStr)
+ buf = cStringIO.StringIO(confStr)
+ buf.seek(0)
+ tar.addfile(tarinfo, fileobj=buf)
+
+ @staticmethod
+ def from_tar(f):
+ params = Parameters()
+ tar = tarfile.TarFile(fileobj=f, mode='r')
+ for finfo in tar:
+ assert isinstance(finfo, tarfile.TarInfo)
+ if finfo.name.endswith('.protobuf'):
+ f = tar.extractfile(finfo)
+ conf = ParameterConfig()
+ conf.ParseFromString(f.read())
+ params.__append_config__(conf)
+
+ for param_name in params.names():
+ f = tar.extractfile(param_name)
+ params.deserialize(param_name, f)
+ return params
+
def __get_parameter_in_gradient_machine__(gradient_machine, name):
"""
diff --git a/python/paddle/v2/pooling.py b/python/paddle/v2/pooling.py
index 9076a159bb4f2c58bf52cba1657c58510637f4f8..4881c27d1d6d3d926f12aab096f377164debf1ef 100644
--- a/python/paddle/v2/pooling.py
+++ b/python/paddle/v2/pooling.py
@@ -12,13 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from paddle.trainer_config_helpers.poolings import *
+import paddle.trainer_config_helpers.poolings
+import copy
-__all__ = ["Max", "CudnnMax", "Avg", "CudnnAvg", "Sum", "SquareRootN"]
+__all__ = []
+suffix = 'Pooling'
-Max = MaxPooling
-CudnnMax = CudnnMaxPooling
-Avg = AvgPooling
-CudnnAvg = CudnnAvgPooling
-Sum = SumPooling
-SquareRootN = SquareRootNPooling
+for name in paddle.trainer_config_helpers.poolings.__all__:
+ new_name = name[:-len(suffix)]
+ globals()[new_name] = copy.copy(
+ getattr(paddle.trainer_config_helpers.poolings, name))
+ globals()[new_name].__name__ = new_name
+ __all__.append(new_name)
diff --git a/python/paddle/v2/reader/__init__.py b/python/paddle/v2/reader/__init__.py
index 7373dc461b1d3115c03b37c5102a469a52aa7441..3b059735a924d58714cd88a761eb83143f1192d6 100644
--- a/python/paddle/v2/reader/__init__.py
+++ b/python/paddle/v2/reader/__init__.py
@@ -11,15 +11,64 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+At training and testing time, PaddlePaddle programs need to read data. To ease
+the users' work to write data reading code, we define that
-# It would be too lengthy to require our users to prefix decorators with `decorator`.
-# For example, we want the following line
-#
-# r = paddle.reader.decorator.bufferd(paddle.reader.creator.text("hello.txt"))
-#
-# to be a shorter version:
-#
-# r = paddle.reader.buffered(paddle.reader.creator.text("hello.txt"))
+- A *reader* is a function that reads data (from file, network, random number
+ generator, etc) and yields data items.
+- A *reader creator* is a function that returns a reader function.
+- A *reader decorator* is a function, which accepts one or more readers, and
+ returns a reader.
+- A *batch reader* is a function that reads data (from *reader*, file, network,
+ random number generator, etc) and yields a batch of data items.
+
+#####################
+Data Reader Interface
+#####################
+
+Indeed, *data reader* doesn't have to be a function that reads and yields data
+items. It can be any function with no parameter that creates a iterable
+(anything can be used in :code:`for x in iterable`)\:
+
+.. code-block:: python
+
+ iterable = data_reader()
+
+Element produced from the iterable should be a **single** entry of data,
+**not** a mini batch. That entry of data could be a single item, or a tuple of
+items.
+Item should be of `supported type `_ (e.g., numpy 1d
+array of float32, int, list of int)
+
+An example implementation for single item data reader creator:
+
+.. code-block:: python
+
+ def reader_creator_random_image(width, height):
+ def reader():
+ while True:
+ yield numpy.random.uniform(-1, 1, size=width*height)
+ return reader
+
+An example implementation for multiple item data reader creator:
+
+.. code-block:: python
+
+ def reader_creator_random_image_and_label(width, height, label):
+ def reader():
+ while True:
+ yield numpy.random.uniform(-1, 1, size=width*height), label
+ return reader
+
+
+TODO(yuyang18): Should we add whole design doc here?
+"""
+
+import decorator
from decorator import *
import creator
+
+__all__ = decorator.__all__ + ['creator']
diff --git a/python/paddle/v2/reader/creator.py b/python/paddle/v2/reader/creator.py
index 5a91bb0b8ef6d1874737386897f6c555eaec18d4..07142056f872db5113acdd296b17c52b343c1be6 100644
--- a/python/paddle/v2/reader/creator.py
+++ b/python/paddle/v2/reader/creator.py
@@ -11,6 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Creator package contains some simple reader creator, which could be used in user
+program.
+"""
__all__ = ['np_array', 'text_file']
@@ -38,7 +42,7 @@ def np_array(x):
def text_file(path):
"""
Creates a data reader that outputs text line by line from given text file.
- Trailing new line ('\n') of each line will be removed.
+ Trailing new line ('\\\\n') of each line will be removed.
:path: path of the text file.
:returns: data reader of text file
diff --git a/python/paddle/v2/reader/decorator.py b/python/paddle/v2/reader/decorator.py
index b7657e27764f099334ba3030c493a7607f323abe..104ce9a0411413bb8fc65eedf5821f98d6acdba3 100644
--- a/python/paddle/v2/reader/decorator.py
+++ b/python/paddle/v2/reader/decorator.py
@@ -14,7 +14,7 @@
__all__ = [
'map_readers', 'buffered', 'compose', 'chain', 'shuffle',
- 'ComposeNotAligned', 'batched', 'firstn'
+ 'ComposeNotAligned', 'firstn'
]
import itertools
@@ -28,9 +28,11 @@ def map_readers(func, *readers):
Creates a data reader that outputs return value of function using
output of each data readers as arguments.
- :param func: function to use.
- :param *readers: readers whose outputs will be used as arguments of func.
- :returns: the created data reader.
+ :param func: function to use. The type of func should be (Sample) => Sample
+ :type: callable
+ :param readers: readers whose outputs will be used as arguments of func.
+ :return: the created data reader.
+ :rtype: callable
"""
def reader():
@@ -45,16 +47,19 @@ def map_readers(func, *readers):
def shuffle(reader, buf_size):
"""
- Creates a data reader whose data output is suffled.
+ Creates a data reader whose data output is shuffled.
Output from the iterator that created by original reader will be
buffered into shuffle buffer, and then shuffled. The size of shuffle buffer
is determined by argument buf_size.
:param reader: the original reader whose output will be shuffled.
+ :type reader: callable
:param buf_size: shuffle buffer size.
+ :type buf_size: int
- :returns:the new reader whose output is shuffled.
+ :return: the new reader whose output is shuffled.
+ :rtype: callable
"""
def data_reader():
@@ -88,7 +93,8 @@ def chain(*readers):
[0, 0, 0, 1, 1, 1, 2, 2, 2]
:param readers: input readers.
- :returns: the new data reader.
+ :return: the new data reader.
+ :rtype: callable
"""
def reader():
@@ -115,12 +121,13 @@ def compose(*readers, **kwargs):
The composed reader will output:
(1, 2, 3, 4, 5)
- :*readers: readers that will be composed together.
- :check_alignment: if True, will check if input readers are aligned
+ :param readers: readers that will be composed together.
+ :param check_alignment: if True, will check if input readers are aligned
correctly. If False, will not check alignment and trailing outputs
will be discarded. Defaults to True.
+ :type check_alignment: bool
- :returns: the new data reader.
+ :return: the new data reader.
:raises ComposeNotAligned: outputs of readers are not aligned.
Will not raise when check_alignment is set to False.
@@ -161,7 +168,9 @@ def buffered(reader, size):
as the buffer is not empty.
:param reader: the data reader to read from.
+ :type reader: callable
:param size: max buffer size.
+ :type size: int
:returns: the buffered data reader.
"""
@@ -193,31 +202,16 @@ def buffered(reader, size):
return data_reader
-def batched(reader, batch_size):
- """
- Create a batched reader.
- :param reader: the data reader to read from.
- :param batch_size: batch_size
- :return: the batched reader.
- """
-
- def batched_reader():
- r = reader()
- batch = []
- for instance in r:
- batch.append(instance)
- if len(batch) == batch_size:
- yield batch
- batch = []
- if batch:
- yield batch
-
- return batched_reader
-
-
def firstn(reader, n):
"""
Limit the max number of samples that reader could return.
+
+ :param reader: the data reader to read from.
+ :type reader: callable
+ :param n: the max number of samples that return.
+ :type n: int
+ :return: the decorated reader.
+ :rtype: callable
"""
# TODO(yuyang18): Check if just drop the reader, could clean the opened
diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt
index 46b5d08b8761ea58530f5aefe5d1947408727f85..572deaff356712cac23cd7911cdf289db100564c 100644
--- a/python/paddle/v2/tests/CMakeLists.txt
+++ b/python/paddle/v2/tests/CMakeLists.txt
@@ -1,12 +1,16 @@
+add_test(NAME test_v2_api
+ COMMAND bash ${PROJ_ROOT}/python/paddle/v2/tests/run_tests.sh ${PYTHON_EXECUTABLE})
+
add_test(NAME test_v2_layer
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_layer.py
WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
-add_test(NAME test_v2_api
- COMMAND bash ${PROJ_ROOT}/python/paddle/v2/tests/run_tests.sh ${PYTHON_EXECUTABLE})
+add_test(NAME test_v2_rnn_layer
+ COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
+ ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_rnn_layer.py)
-add_test(NAME topology_test
+add_test(NAME test_topology
COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py
WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)
diff --git a/python/paddle/v2/tests/run_tests.sh b/python/paddle/v2/tests/run_tests.sh
index b96f54fe9cc78a436bc67e6c542b6e842aba997b..dda1b1bd222a9f226db1a4bd730e9637ab882196 100755
--- a/python/paddle/v2/tests/run_tests.sh
+++ b/python/paddle/v2/tests/run_tests.sh
@@ -22,7 +22,7 @@ cd $SCRIPTPATH
$1 -m pip install ../../../../paddle/dist/*.whl
-test_list="test_data_feeder.py"
+test_list="test_data_feeder.py test_parameters.py"
export PYTHONPATH=$PWD/../../../../python/
diff --git a/python/paddle/v2/tests/test_data_feeder.py b/python/paddle/v2/tests/test_data_feeder.py
index ab2bc5df76cd839b5b0184e9559f0c2e03baf38b..71eb3bf31425c22b47accc11c9550042e077ef12 100644
--- a/python/paddle/v2/tests/test_data_feeder.py
+++ b/python/paddle/v2/tests/test_data_feeder.py
@@ -110,14 +110,14 @@ class DataFeederTest(unittest.TestCase):
self.assertAlmostEqual(value.all(), w[i].all())
def test_integer(self):
- dim = 100
+ value_range = 100
batch_size = 32
index = []
for i in xrange(batch_size):
each_sample = []
- each_sample.append(np.random.randint(dim))
+ each_sample.append(np.random.randint(value_range))
index.append(each_sample)
- feeder = DataFeeder([('input', data_type.integer_value(dim))],
+ feeder = DataFeeder([('input', data_type.integer_value(value_range))],
{'input': 0})
arg = feeder(index)
output = arg.getSlotIds(0).copyToNumpyArray()
@@ -125,7 +125,7 @@ class DataFeederTest(unittest.TestCase):
self.assertEqual(output.all(), index.flatten().all())
def test_integer_sequence(self):
- dim = 10000
+ value_range = 10000
batch_size = 32
start = [0]
data = []
@@ -133,11 +133,12 @@ class DataFeederTest(unittest.TestCase):
each_sample = []
each_sample.append(
self.sparse_binary_reader(
- dim, 30, non_empty=True))
+ value_range, 30, non_empty=True))
data.append(each_sample)
start.append(len(each_sample[0]) + start[-1])
- feeder = DataFeeder([('input', data_type.integer_value_sequence(dim))],
- {'input': 0})
+ feeder = DataFeeder(
+ [('input', data_type.integer_value_sequence(value_range))],
+ {'input': 0})
arg = feeder(data)
output_data = arg.getSlotIds(0).copyToNumpyArray()
output_start = arg.getSlotSequenceStartPositions(0).copyToNumpyArray()
diff --git a/python/paddle/v2/tests/test_parameters.py b/python/paddle/v2/tests/test_parameters.py
new file mode 100644
index 0000000000000000000000000000000000000000..ebb182caab6430862a8e4da2ae4ea6b1e72f726c
--- /dev/null
+++ b/python/paddle/v2/tests/test_parameters.py
@@ -0,0 +1,60 @@
+import unittest
+import sys
+
+try:
+ import py_paddle
+
+ del py_paddle
+except ImportError:
+ print >> sys.stderr, "It seems swig of Paddle is not installed, this " \
+ "unittest will not be run."
+ sys.exit(0)
+
+import paddle.v2.parameters as parameters
+from paddle.proto.ParameterConfig_pb2 import ParameterConfig
+import random
+import cStringIO
+import numpy
+
+
+def __rand_param_config__(name):
+ conf = ParameterConfig()
+ conf.name = name
+ size = 1
+ for i in xrange(2):
+ dim = random.randint(1, 1000)
+ conf.dims.append(dim)
+ size *= dim
+ conf.size = size
+ assert conf.IsInitialized()
+ return conf
+
+
+class TestParameters(unittest.TestCase):
+ def test_serialization(self):
+ params = parameters.Parameters()
+ params.__append_config__(__rand_param_config__("param_0"))
+ params.__append_config__(__rand_param_config__("param_1"))
+
+ for name in params.names():
+ param = params.get(name)
+ param[:] = numpy.random.uniform(
+ -1.0, 1.0, size=params.get_shape(name))
+ params.set(name, param)
+
+ tmp_file = cStringIO.StringIO()
+ params.to_tar(tmp_file)
+ tmp_file.seek(0)
+ params_dup = parameters.Parameters.from_tar(tmp_file)
+
+ self.assertEqual(params_dup.names(), params.names())
+
+ for name in params.names():
+ self.assertEqual(params.get_shape(name), params_dup.get_shape(name))
+ p0 = params.get(name)
+ p1 = params_dup.get(name)
+ self.assertTrue(numpy.isclose(p0, p1).all())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/tests/test_rnn_layer.py b/python/paddle/v2/tests/test_rnn_layer.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fbbd20eb76bb9daab2bcf98c4adad989106a377
--- /dev/null
+++ b/python/paddle/v2/tests/test_rnn_layer.py
@@ -0,0 +1,155 @@
+# Copyright PaddlePaddle contributors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import difflib
+import unittest
+
+import paddle.trainer_config_helpers as conf_helps
+import paddle.v2.activation as activation
+import paddle.v2.data_type as data_type
+import paddle.v2.layer as layer
+from paddle.trainer_config_helpers.config_parser_utils import \
+ parse_network_config as parse_network
+
+
+class RNNTest(unittest.TestCase):
+ def test_simple_rnn(self):
+ dict_dim = 10
+ word_dim = 8
+ hidden_dim = 8
+
+ def parse_old_rnn():
+ def step(y):
+ mem = conf_helps.memory(name="rnn_state", size=hidden_dim)
+ out = conf_helps.fc_layer(
+ input=[y, mem],
+ size=hidden_dim,
+ act=activation.Tanh(),
+ bias_attr=True,
+ name="rnn_state")
+ return out
+
+ def test():
+ data = conf_helps.data_layer(name="word", size=dict_dim)
+ embd = conf_helps.embedding_layer(input=data, size=word_dim)
+ conf_helps.recurrent_group(name="rnn", step=step, input=embd)
+
+ return str(parse_network(test))
+
+ def parse_new_rnn():
+ def new_step(y):
+ mem = layer.memory(name="rnn_state", size=hidden_dim)
+ out = layer.fc(input=[y, mem],
+ size=hidden_dim,
+ act=activation.Tanh(),
+ bias_attr=True,
+ name="rnn_state")
+ return out
+
+ data = layer.data(
+ name="word", type=data_type.integer_value(dict_dim))
+ embd = layer.embedding(input=data, size=word_dim)
+ rnn_layer = layer.recurrent_group(
+ name="rnn", step=new_step, input=embd)
+ return str(layer.parse_network(rnn_layer))
+
+ diff = difflib.unified_diff(parse_old_rnn().splitlines(1),
+ parse_new_rnn().splitlines(1))
+ print ''.join(diff)
+
+ def test_sequence_rnn_multi_input(self):
+ dict_dim = 10
+ word_dim = 8
+ hidden_dim = 8
+ label_dim = 3
+
+ def parse_old_rnn():
+ def test():
+ data = conf_helps.data_layer(name="word", size=dict_dim)
+ label = conf_helps.data_layer(name="label", size=label_dim)
+ emb = conf_helps.embedding_layer(input=data, size=word_dim)
+ boot_layer = conf_helps.data_layer(name="boot", size=10)
+ boot_layer = conf_helps.fc_layer(
+ name='boot_fc', input=boot_layer, size=10)
+
+ def step(y, wid):
+ z = conf_helps.embedding_layer(input=wid, size=word_dim)
+ mem = conf_helps.memory(
+ name="rnn_state",
+ size=hidden_dim,
+ boot_layer=boot_layer)
+ out = conf_helps.fc_layer(
+ input=[y, z, mem],
+ size=hidden_dim,
+ act=conf_helps.TanhActivation(),
+ bias_attr=True,
+ name="rnn_state")
+ return out
+
+ out = conf_helps.recurrent_group(
+ name="rnn", step=step, input=[emb, data])
+
+ rep = conf_helps.last_seq(input=out)
+ prob = conf_helps.fc_layer(
+ size=label_dim,
+ input=rep,
+ act=conf_helps.SoftmaxActivation(),
+ bias_attr=True)
+
+ conf_helps.outputs(
+ conf_helps.classification_cost(
+ input=prob, label=label))
+
+ return str(parse_network(test))
+
+ def parse_new_rnn():
+ data = layer.data(
+ name="word", type=data_type.dense_vector(dict_dim))
+ label = layer.data(
+ name="label", type=data_type.dense_vector(label_dim))
+ emb = layer.embedding(input=data, size=word_dim)
+ boot_layer = layer.data(
+ name="boot", type=data_type.dense_vector(10))
+ boot_layer = layer.fc(name='boot_fc', input=boot_layer, size=10)
+
+ def step(y, wid):
+ z = layer.embedding(input=wid, size=word_dim)
+ mem = layer.memory(
+ name="rnn_state", size=hidden_dim, boot_layer=boot_layer)
+ out = layer.fc(input=[y, z, mem],
+ size=hidden_dim,
+ act=activation.Tanh(),
+ bias_attr=True,
+ name="rnn_state")
+ return out
+
+ out = layer.recurrent_group(
+ name="rnn", step=step, input=[emb, data])
+
+ rep = layer.last_seq(input=out)
+ prob = layer.fc(size=label_dim,
+ input=rep,
+ act=activation.Softmax(),
+ bias_attr=True)
+
+ cost = layer.classification_cost(input=prob, label=label)
+
+ return str(layer.parse_network(cost))
+
+ diff = difflib.unified_diff(parse_old_rnn().splitlines(1),
+ parse_new_rnn().splitlines(1))
+ print ''.join(diff)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/tests/test_topology.py b/python/paddle/v2/tests/test_topology.py
index 1bf55a5bc68dfdb837773b3120e5b55d304f644d..5c6dbcdb4f49b960fb8b71aecbad4f013d2cd283 100644
--- a/python/paddle/v2/tests/test_topology.py
+++ b/python/paddle/v2/tests/test_topology.py
@@ -16,6 +16,7 @@ import paddle.v2.layer as layer
import paddle.v2.topology as topology
import paddle.v2.data_type as data_type
import paddle.trainer_config_helpers as conf_helps
+import paddle.trainer.PyDataProvider2 as pydp2
class TestTopology(unittest.TestCase):
@@ -35,13 +36,13 @@ class TestTopology(unittest.TestCase):
pixel_data_type = filter(lambda type: type[0] == "pixel", data_types)
self.assertEqual(len(pixel_data_type), 1)
pixel_data_type = pixel_data_type[0]
- self.assertEqual(pixel_data_type[1].type, data_type.DataType.Dense)
+ self.assertEqual(pixel_data_type[1].type, pydp2.DataType.Dense)
self.assertEqual(pixel_data_type[1].dim, 784)
label_data_type = filter(lambda type: type[0] == "label", data_types)
self.assertEqual(len(label_data_type), 1)
label_data_type = label_data_type[0]
- self.assertEqual(label_data_type[1].type, data_type.DataType.Index)
+ self.assertEqual(label_data_type[1].type, pydp2.DataType.Index)
self.assertEqual(label_data_type[1].dim, 10)
def test_get_layer(self):
diff --git a/python/paddle/v2/topology.py b/python/paddle/v2/topology.py
index 4c211254319bbdf46b02a2cee56b6a98b01819a2..f0679c5675b0c0f24f28f3df22efd4eb51ccbb3a 100644
--- a/python/paddle/v2/topology.py
+++ b/python/paddle/v2/topology.py
@@ -17,6 +17,7 @@ import collections
from paddle.proto.ModelConfig_pb2 import ModelConfig
import layer as v2_layer
+from layer import WithExtraParent
__all__ = ['Topology']
@@ -40,7 +41,10 @@ def __bfs_travel__(callback, *layers):
__break__ = callback(each_layer)
if __break__:
return
- __bfs_travel__(callback, *each_layer.__parent_layers__.values())
+ __layers__ = each_layer.__parent_layers__.values()
+ if isinstance(each_layer, WithExtraParent):
+ __layers__ = __layers__ + each_layer.extra_parent()
+ __bfs_travel__(callback, *__layers__)
class Topology(object):
diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py
index 5003f55f3e0d15149d28d1478e0487d6873d6e0a..7bd3e2c565ee00c91402e7dea36c7393fb1a9bdf 100644
--- a/python/paddle/v2/trainer.py
+++ b/python/paddle/v2/trainer.py
@@ -8,7 +8,11 @@ from . import event as v2_event
from . import optimizer as v2_optimizer
from . import parameters as v2_parameters
-__all__ = ['ITrainer', 'SGD']
+__all__ = ['SGD']
+"""
+Trainer package
+TODO(yuyang18): Complete comments.
+"""
def default_event_handler(event):
@@ -22,33 +26,20 @@ def default_event_handler(event):
pass
-class ITrainer(object):
+class SGD(object):
"""
- The interface of Trainer. The only exposed method is `train`.
+ Simple SGD Trainer.
+ TODO(yuyang18): Complete comments
+
+ :param update_equation: The optimizer object.
+ :type update_equation: paddle.v2.optimizer.Optimizer
+ :param cost: Target cost that neural network should be optimized.
+ :type cost: paddle.v2.config_base.Layer
+ :param parameters: The parameters dictionary.
+ :type parameters: paddle.v2.parameters.Parameters
"""
- def train(self, reader, topology, parameters, event_handler=None):
- """
- train method.
-
- :param reader:
- :param topology:
- :param parameters:
- :param event_handler:
- :return:
- """
-
- raise NotImplementedError()
-
-
-class SGD(ITrainer):
def __init__(self, cost, parameters, update_equation):
- """
- Simple SGD Trainer.
-
- :param update_equation: The optimizer object.
- :type update_equation: v2_optimizer.Optimizer
- """
if not isinstance(parameters, v2_parameters.Parameters):
raise TypeError('parameters should be parameters')
@@ -66,29 +57,26 @@ class SGD(ITrainer):
self.__topology_in_proto__, api.CREATE_MODE_NORMAL,
self.__optimizer__.enable_types())
assert isinstance(gm, api.GradientMachine)
- parameters.append_gradient_machine(gm)
self.__gradient_machine__ = gm
self.__gradient_machine__.randParameters()
+ parameters.append_gradient_machine(gm)
- def train(self, reader, num_passes=1, event_handler=None, reader_dict=None):
+ def train(self, reader, num_passes=1, event_handler=None, feeding=None):
"""
Training method. Will train num_passes of input data.
:param reader:
- :param topology: Network Topology, use one or more Layers to represent it.
- :param parameters: The parameter pools.
:param num_passes: The total train passes.
:param event_handler: Event handler. A method will be invoked when event
occurred.
:type event_handler: (BaseEvent) => None
+ :param feeding: Feeding is a map of neural network input name and array
+ index that reader returns.
+ :type feeding: dict
:return:
"""
if event_handler is None:
event_handler = default_event_handler
-
- if reader_dict is None:
- reader_dict = self.default_reader_dict()
-
__check_train_args__(**locals())
updater = self.__optimizer__.create_local_updater()
@@ -100,17 +88,12 @@ class SGD(ITrainer):
pass_evaluator = self.__gradient_machine__.makeEvaluator()
assert isinstance(pass_evaluator, api.Evaluator)
out_args = api.Arguments.createArguments(0)
-
- feeder = DataFeeder(self.__data_types__, reader_dict)
-
+ feeder = DataFeeder(self.__data_types__, feeding)
for pass_id in xrange(num_passes):
event_handler(v2_event.BeginPass(pass_id))
pass_evaluator.start()
updater.startPass()
for batch_id, data_batch in enumerate(reader()):
- pass_type = updater.startBatch(len(data_batch))
- self.__gradient_machine__.forwardBackward(
- feeder(data_batch), out_args, pass_type)
batch_evaluator.start()
event_handler(
v2_event.BeginIteration(
@@ -120,12 +103,11 @@ class SGD(ITrainer):
feeder(data_batch), out_args, pass_type)
self.__gradient_machine__.eval(pass_evaluator)
self.__gradient_machine__.eval(batch_evaluator)
- for each_param in self.__gradient_machine__.getParameters():
+ for each_param in self.__gradient_machine__.getNonStaticParameters(
+ ):
updater.update(each_param)
- # Get cost. We use numpy to calculate total cost for this batch.
- cost_vec = out_args.getSlotValue(0)
- cost_vec = cost_vec.copyToNumpyMat()
- cost = cost_vec.sum() / len(data_batch)
+ cost_sum = out_args.sum()
+ cost = cost_sum / len(data_batch)
updater.finishBatch(cost)
batch_evaluator.finish()
event_handler(
@@ -140,27 +122,23 @@ class SGD(ITrainer):
event_handler(v2_event.EndPass(pass_id, evaluator=pass_evaluator))
self.__gradient_machine__.finish()
- def default_reader_dict(self):
- reader_dict = dict()
- for i, tp in enumerate(self.__data_types__):
- reader_dict[tp[0]] = i
- return reader_dict
-
- def test(self, reader, reader_dict=None):
- if reader_dict is None:
- reader_dict = self.default_reader_dict()
-
- feeder = DataFeeder(self.__data_types__, reader_dict)
+ def test(self, reader, feeding=None):
+ feeder = DataFeeder(self.__data_types__, feeding)
evaluator = self.__gradient_machine__.makeEvaluator()
out_args = api.Arguments.createArguments(0)
evaluator.start()
+ total_cost = 0
+ num_samples = 0.0
for data_batch in reader():
+ num_samples += len(data_batch)
self.__gradient_machine__.forward(
feeder(data_batch), out_args, api.PASS_TEST)
+ total_cost += out_args.sum()
self.__gradient_machine__.eval(evaluator)
evaluator.finish()
- return v2_event.TestResult(evaluator=evaluator)
+ return v2_event.TestResult(
+ evaluator=evaluator, cost=total_cost / num_samples)
def __check_train_args__(reader, event_handler, **kwargs):