diff --git a/Dockerfile b/Dockerfile index c3ad0c9c2f6d619f2b2ef2bcf21429937d06dd6e..6d5797da9e06a514e03dee52bd248478fa41813f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN apt-get update && \ apt-get install -y git python-pip python-dev openssh-server bison && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ + apt-get install -y python-numpy python-matplotlib gcc g++ liblapack-dev liblapacke-dev && \ apt-get install -y automake locales clang-format-3.8 swig doxygen && \ apt-get clean -y diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 9fd3afd0998b38c18b4490e6fb1c6fe0222ed142..fc66d6b2154b73d8f6a259ecfa55c7ef5ce999fa 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,7 @@ FIND_PACKAGE(PythonInterp 2.7) FIND_PACKAGE(PythonLibs 2.7) SET(py_env "") - +SET(USE_VIRTUALENV_FOR_TEST 1) IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND) find_python_module(pip REQUIRED) find_python_module(numpy REQUIRED) diff --git a/cmake/make_resource.py b/cmake/make_resource.py new file mode 100644 index 0000000000000000000000000000000000000000..a9241b0e3e36c2e79c79e46b4f9114b7f6947341 --- /dev/null +++ b/cmake/make_resource.py @@ -0,0 +1,11 @@ +import os +import re +import sys + +res = sys.argv[1] +out = sys.argv[2] +var = re.sub(r'[ .-]', '_', os.path.basename(res)) + +open(out, "w").write("const unsigned char " + var + "[] = {" + ",".join([ + "0x%02x" % ord(c) for c in open(res).read() +]) + ",0};\n" + "const unsigned " + var + "_size = sizeof(" + var + ");\n") diff --git a/cmake/util.cmake b/cmake/util.cmake index 099a85809d93e01772bf8c5c329fd9055ee4f054..966e0a7bf60fdeaac575b9d9c19c7095e104b13c 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -138,17 +138,20 @@ macro(add_simple_unittest TARGET_NAME) endmacro() # Creates C resources file from files in given resource file -function(create_resources res_file output) - # Create empty output file - file(WRITE ${output} "") - # Get short filename - string(REGEX MATCH "([^/]+)$" filename ${res_file}) - # Replace filename spaces & extension separator for C compatibility - string(REGEX REPLACE "\\.| |-" "_" filename ${filename}) - # Read hex data from file - file(READ ${res_file} filedata HEX) - # Convert hex data for C compatibility - string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) - # Append data to output file - file(APPEND ${output} "const unsigned char ${filename}[] = {${filedata}0};\nconst unsigned ${filename}_size = sizeof(${filename});\n") +function(create_resources res_file output_file) + add_custom_command( + OUTPUT ${output_file} + COMMAND python ARGS ${PROJ_ROOT}/cmake/make_resource.py ${res_file} ${output_file} + DEPENDS ${res_file} ${PROJ_ROOT}/cmake/make_resource.py) +endfunction() + + +# Create a python unittest using run_python_tests.sh, +# which takes care of making correct running environment +function(add_python_test TEST_NAME) + message("PYTHON: ${PYTHON_EXECUTABLE}") + add_test(NAME ${TEST_NAME} + COMMAND bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh + ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endfunction() diff --git a/doc/api/v1/trainer_config_helpers/activations.rst b/doc/api/v1/trainer_config_helpers/activations.rst deleted file mode 100644 index 269e6491e7ebe3899c3fb24fca756a393043473b..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/activations.rst +++ /dev/null @@ -1,108 +0,0 @@ -=========== -Activations -=========== - -BaseActivation -============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: BaseActivation - :noindex: - -AbsActivation -=============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: AbsActivation - :noindex: - -ExpActivation -=============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: ExpActivation - :noindex: - -IdentityActivation -================== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: IdentityActivation - :noindex: - -LinearActivation -================== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: LinearActivation - :noindex: - -LogActivation -================== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: LogActivation - :noindex: - -SquareActivation -================ - -.. automodule:: paddle.trainer_config_helpers.activations - :members: SquareActivation - :noindex: - -SigmoidActivation -================= - -.. automodule:: paddle.trainer_config_helpers.activations - :members: SigmoidActivation - :noindex: - -SoftmaxActivation -================= - -.. automodule:: paddle.trainer_config_helpers.activations - :members: SoftmaxActivation - :noindex: - -SequenceSoftmaxActivation -========================= - -.. automodule:: paddle.trainer_config_helpers.activations - :members: SequenceSoftmaxActivation - :noindex: - -ReluActivation -============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: ReluActivation - :noindex: - -BReluActivation -=============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: BReluActivation - :noindex: - -SoftReluActivation -================== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: SoftReluActivation - :noindex: - -TanhActivation -============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: TanhActivation - :noindex: - -STanhActivation -=============== - -.. automodule:: paddle.trainer_config_helpers.activations - :members: STanhActivation - :noindex: diff --git a/doc/api/v1/trainer_config_helpers/attrs.rst b/doc/api/v1/trainer_config_helpers/attrs.rst deleted file mode 100644 index ac63127bf7d9db6351365ab7b58f43db12347a8e..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/attrs.rst +++ /dev/null @@ -1,5 +0,0 @@ -Parameter Attributes -======================= - -.. automodule:: paddle.trainer_config_helpers.attrs - :members: diff --git a/doc/api/v1/trainer_config_helpers/data_sources.rst b/doc/api/v1/trainer_config_helpers/data_sources.rst deleted file mode 100644 index b9dd4dda01ae59d1260356aff50ddf298d02c87f..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/data_sources.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _api_trainer_config_helpers_data_sources: - -DataSources -=========== - -.. automodule:: paddle.trainer_config_helpers.data_sources - :members: diff --git a/doc/api/v1/trainer_config_helpers/evaluators.rst b/doc/api/v1/trainer_config_helpers/evaluators.rst deleted file mode 100644 index 11dc735164284d6ed1d661fab1e7690d263b3a7c..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/evaluators.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. _api_trainer_config_helpers_evaluators: - -========== -Evaluators -========== - -Base -==== -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: evaluator_base - :noindex: - -Classification -============== - -classification_error_evaluator ------------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: classification_error_evaluator - :noindex: - -auc_evaluator -------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: auc_evaluator - :noindex: - -ctc_error_evaluator -------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: ctc_error_evaluator - :noindex: - -chunk_evaluator ---------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: chunk_evaluator - :noindex: - -precision_recall_evaluator --------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: precision_recall_evaluator - :noindex: - -Rank -==== - -pnpair_evaluator ----------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: pnpair_evaluator - :noindex: - -Utils -===== - -sum_evaluator -------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: sum_evaluator - :noindex: - -column_sum_evaluator --------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: column_sum_evaluator - :noindex: - -Print -===== - -classification_error_printer_evaluator --------------------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: classification_error_printer_evaluator - :noindex: - -gradient_printer_evaluator --------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: gradient_printer_evaluator - :noindex: - -maxid_printer_evaluator ------------------------ -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: maxid_printer_evaluator - :noindex: - -maxframe_printer_evaluator ---------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: maxframe_printer_evaluator - :noindex: - -seqtext_printer_evaluator -------------------------- -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: seqtext_printer_evaluator - :noindex: - -value_printer_evaluator ------------------------ -.. automodule:: paddle.trainer_config_helpers.evaluators - :members: value_printer_evaluator - :noindex: - diff --git a/doc/api/v1/trainer_config_helpers/layers.rst b/doc/api/v1/trainer_config_helpers/layers.rst deleted file mode 100644 index 75c1b35246486528524fd5baa04704249d5a9275..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/layers.rst +++ /dev/null @@ -1,514 +0,0 @@ -.. _api_trainer_config_helpers_layers: - -====== -Layers -====== - -Base -====== - -LayerType ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: LayerType - :noindex: - -LayerOutput ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: LayerOutput - :noindex: - -Data layer -=========== - -.. _api_trainer_config_helpers_layers_data_layer: - -data_layer ----------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: data_layer - :noindex: - -Fully Connected Layers -====================== - -.. _api_trainer_config_helpers_layers_fc_layer: - -fc_layer --------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: fc_layer - :noindex: - -selective_fc_layer ------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: selective_fc_layer - :noindex: - -Conv Layers -=========== - -conv_operator -------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: conv_operator - :noindex: - -conv_projection ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: conv_projection - :noindex: - -conv_shift_layer ------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: conv_shift_layer - :noindex: - -img_conv_layer --------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: img_conv_layer - :noindex: - -.. _api_trainer_config_helpers_layers_context_projection: - -context_projection ------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: context_projection - :noindex: - -Image Pooling Layer -=================== - -img_pool_layer --------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: img_pool_layer - :noindex: - -spp_layer --------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: spp_layer - :noindex: - -maxout_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: maxout_layer - :noindex: - -Norm Layer -========== - -img_cmrnorm_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: img_cmrnorm_layer - :noindex: - -batch_norm_layer ---------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: batch_norm_layer - :noindex: - -sum_to_one_norm_layer ---------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: sum_to_one_norm_layer - :noindex: - -Recurrent Layers -================ - -recurrent_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: recurrent_layer - :noindex: - -lstmemory ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: lstmemory - :noindex: - -grumemory ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: grumemory - :noindex: - -Recurrent Layer Group -===================== - -memory ------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: memory - :noindex: - -recurrent_group ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: recurrent_group - :noindex: - -lstm_step_layer ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: lstm_step_layer - :noindex: - -gru_step_layer ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: gru_step_layer - :noindex: - -beam_search ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: beam_search - :noindex: - -get_output_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: get_output_layer - :noindex: - -Mixed Layer -=========== - -.. _api_trainer_config_helpers_layers_mixed_layer: - -mixed_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: mixed_layer - :noindex: - -.. _api_trainer_config_helpers_layers_embedding_layer: - -embedding_layer ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: embedding_layer - :noindex: - -scaling_projection ------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: scaling_projection - :noindex: - -dotmul_projection ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: dotmul_projection - :noindex: - -dotmul_operator ---------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: dotmul_operator - :noindex: - -full_matrix_projection ----------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: full_matrix_projection - :noindex: - -identity_projection -------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: identity_projection - :noindex: - - -table_projection ----------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: table_projection - :noindex: - -trans_full_matrix_projection ----------------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: trans_full_matrix_projection - :noindex: - -Aggregate Layers -================ - -.. _api_trainer_config_helpers_layers_pooling_layer: - -pooling_layer -------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: pooling_layer - :noindex: - -.. _api_trainer_config_helpers_layers_last_seq: - -last_seq --------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: last_seq - :noindex: - -.. _api_trainer_config_helpers_layers_first_seq: - -first_seq ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: first_seq - :noindex: - -concat_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: concat_layer - :noindex: - -seq_concat_layer ----------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: seq_concat_layer - :noindex: - -Reshaping Layers -================ - -block_expand_layer ------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: block_expand_layer - :noindex: - -.. _api_trainer_config_helpers_layers_expand_layer: - -expand_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: expand_layer - :noindex: - -repeat_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: repeat_layer - :noindex: - -rotate_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: rotate_layer - :noindex: - -seq_reshape_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: seq_reshape_layer - :noindex: - -Math Layers -=========== - -addto_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: addto_layer - :noindex: - -linear_comb_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: linear_comb_layer - :noindex: - -interpolation_layer -------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: interpolation_layer - :noindex: - -bilinear_interp_layer ----------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: bilinear_interp_layer - :noindex: - -power_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: power_layer - :noindex: - -scaling_layer -------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: scaling_layer - :noindex: - -slope_intercept_layer ----------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: slope_intercept_layer - :noindex: - -tensor_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: tensor_layer - :noindex: - -.. _api_trainer_config_helpers_layers_cos_sim: - -cos_sim -------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: cos_sim - :noindex: - -trans_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: trans_layer - :noindex: - -Sampling Layers -=============== - -maxid_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: maxid_layer - :noindex: - -sampling_id_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: sampling_id_layer - :noindex: - -Slicing and Joining Layers -========================== - -pad_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: pad_layer - :noindex: - -.. _api_trainer_config_helpers_layers_cost_layers: - -Cost Layers -=========== - -cross_entropy -------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: cross_entropy - :noindex: - -cross_entropy_with_selfnorm ---------------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: cross_entropy_with_selfnorm - :noindex: - -multi_binary_label_cross_entropy --------------------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: multi_binary_label_cross_entropy - :noindex: - -mse_cost ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: mse_cost - :noindex: - -huber_cost ----------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: huber_cost - :noindex: - -lambda_cost ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: lambda_cost - :noindex: - -rank_cost ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: rank_cost - :noindex: - -sum_cost ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: sum_cost - :noindex: - -crf_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: crf_layer - :noindex: - -crf_decoding_layer -------------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: crf_decoding_layer - :noindex: - -ctc_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: ctc_layer - :noindex: - -warp_ctc_layer --------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: warp_ctc_layer - :noindex: - -nce_layer ------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: nce_layer - :noindex: - -hsigmoid ---------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: hsigmoid - :noindex: - -smooth_l1_cost --------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: smooth_l1_cost - :noindex: - -Check Layer -============ - -eos_layer ------------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: eos_layer - :noindex: diff --git a/doc/api/v1/trainer_config_helpers/networks.rst b/doc/api/v1/trainer_config_helpers/networks.rst deleted file mode 100644 index edb53acbf0c31532aa34bda044066fed72eaa426..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/networks.rst +++ /dev/null @@ -1,123 +0,0 @@ -======== -Networks -======== - -The networks module contains pieces of neural network that combine multiple layers. - -NLP -=== - -sequence_conv_pool ------------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: sequence_conv_pool - :noindex: - -.. _api_trainer_config_helpers_network_text_conv_pool: - -text_conv_pool --------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: text_conv_pool - :noindex: - -Images -====== - -img_conv_bn_pool ----------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: img_conv_bn_pool - :noindex: - -img_conv_group --------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: img_conv_group - :noindex: - -.. _api_trainer_config_helpers_network_simple_img_conv_pool: - -simple_img_conv_pool --------------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: simple_img_conv_pool - :noindex: - -vgg_16_network ---------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: vgg_16_network - :noindex: - -Recurrent -========= - -LSTM ----- - -lstmemory_unit -`````````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: lstmemory_unit - :noindex: - -lstmemory_group -``````````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: lstmemory_group - :noindex: - -simple_lstm -``````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: simple_lstm - :noindex: - -bidirectional_lstm -`````````````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: bidirectional_lstm - :noindex: - -GRU ---- - -gru_unit -```````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: gru_unit - :noindex: - -gru_group -````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: gru_group - :noindex: - -simple_gru -`````````` -.. automodule:: paddle.trainer_config_helpers.networks - :members: simple_gru - :noindex: - -simple_attention ----------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: simple_attention - :noindex: - -Miscs -===== - -dropout_layer --------------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: dropout_layer - :noindex: - -outputs -------- -.. automodule:: paddle.trainer_config_helpers.networks - :members: outputs - :noindex: diff --git a/doc/api/v1/trainer_config_helpers/optimizers.rst b/doc/api/v1/trainer_config_helpers/optimizers.rst deleted file mode 100644 index d2f4958c92b8e3b7426945f1af07112ab4071136..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/optimizers.rst +++ /dev/null @@ -1,61 +0,0 @@ -.. _api_trainer_config_helpers_optimizers: - -========== -Optimizers -========== - -BaseSGDOptimizer -================ -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: BaseSGDOptimizer - :noindex: - -MomentumOptimizer -================= -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: MomentumOptimizer - :noindex: - -AdamOptimizer -============= -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: AdamOptimizer - :noindex: - -AdamaxOptimizer -================ -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: AdamaxOptimizer - :noindex: - -AdaGradOptimizer -================ -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: AdaGradOptimizer - :noindex: - -DecayedAdaGradOptimizer -======================= -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: DecayedAdaGradOptimizer - :noindex: - -AdaDeltaOptimizer -================= -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: AdaDeltaOptimizer - :noindex: - -RMSPropOptimizer -================ -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: RMSPropOptimizer - :noindex: - -.. _api_trainer_config_helpers_optimizers_settings: - -settings -======== -.. automodule:: paddle.trainer_config_helpers.optimizers - :members: settings - :noindex: diff --git a/doc/api/v1/trainer_config_helpers/poolings.rst b/doc/api/v1/trainer_config_helpers/poolings.rst deleted file mode 100644 index 66566809d26f59263597b5286c5b27e0bbc9415a..0000000000000000000000000000000000000000 --- a/doc/api/v1/trainer_config_helpers/poolings.rst +++ /dev/null @@ -1,33 +0,0 @@ -======== -Poolings -======== - -BasePoolingType -=============== -.. automodule:: paddle.trainer_config_helpers.poolings - :members: BasePoolingType - :noindex: - -AvgPooling -========== -.. automodule:: paddle.trainer_config_helpers.poolings - :members: AvgPooling - :noindex: - -MaxPooling -========== -.. automodule:: paddle.trainer_config_helpers.poolings - :members: MaxPooling - :noindex: - -SumPooling -========== -.. automodule:: paddle.trainer_config_helpers.poolings - :members: SumPooling - :noindex: - -SquareRootNPooling -================== -.. automodule:: paddle.trainer_config_helpers.poolings - :members: SquareRootNPooling - :noindex: diff --git a/doc/api/v2/config/evaluators.rst b/doc/api/v2/config/evaluators.rst new file mode 100644 index 0000000000000000000000000000000000000000..39db51fa4abc370855ca3f2778b47464f33b6fce --- /dev/null +++ b/doc/api/v2/config/evaluators.rst @@ -0,0 +1,101 @@ +.. _api_v2: + +========== +Evaluators +========== + +Classification +============== + +classification_error +-------------------- +.. automodule:: paddle.v2.evaluator + :members: classification_error + :noindex: + +auc +--- +.. automodule:: paddle.v2.evaluator + :members: auc + :noindex: + +ctc_error +--------- +.. automodule:: paddle.v2.evaluator + :members: ctc_error + :noindex: + +chunk +----- +.. automodule:: paddle.v2.evaluator + :members: chunk + :noindex: + +precision_recall +---------------- +.. automodule:: paddle.v2.evaluator + :members: precision_recall + :noindex: + +Rank +==== + +pnpair +------ +.. automodule:: paddle.v2.evaluator + :members: pnpair + :noindex: + +Utils +===== + +sum +--- +.. automodule:: paddle.v2.evaluator + :members: sum + :noindex: + +column_sum +---------- +.. automodule:: paddle.v2.evaluator + :members: column_sum + :noindex: + +Print +===== + +classification_error_printer +---------------------------- +.. automodule:: paddle.v2.evaluator + :members: classification_error_printer + :noindex: + +gradient_printer +---------------- +.. automodule:: paddle.v2.evaluator + :members: gradient_printer + :noindex: + +maxid_printer +------------- +.. automodule:: paddle.v2.evaluator + :members: maxid_printer + :noindex: + +maxframe_printer +---------------- +.. automodule:: paddle.v2.evaluator + :members: maxframe_printer + :noindex: + +seqtext_printer +--------------- +.. automodule:: paddle.v2.evaluator + :members: seqtext_printer + :noindex: + +value_printer +------------- +.. automodule:: paddle.v2.evaluator + :members: value_printer + :noindex: diff --git a/doc/api/v2/config/networks.rst b/doc/api/v2/config/networks.rst index 6f209bc95bec7279051118bb857a96515e0371a9..b2a617fff134035c04eeabbbaf6d9cbe2a525f1c 100644 --- a/doc/api/v2/config/networks.rst +++ b/doc/api/v2/config/networks.rst @@ -44,6 +44,12 @@ simple_img_conv_pool :members: simple_img_conv_pool :noindex: +small_vgg +--------- +.. automodule:: paddle.v2.networks + :members: small_vgg + :noindex: + vgg_16_network --------------- .. automodule:: paddle.v2.networks @@ -101,6 +107,18 @@ simple_gru :members: simple_gru :noindex: +simple_gru2 +``````````` +.. automodule:: paddle.v2.networks + :members: simple_gru2 + :noindex: + +bidirectional_gru +`````````````````` +.. automodule:: paddle.v2.networks + :members: bidirectional_gru + :noindex: + simple_attention ---------------- .. automodule:: paddle.v2.networks diff --git a/doc/api/v2/model_configs.rst b/doc/api/v2/model_configs.rst index a5fae7e29e56d9e236d489353a5c8967d1954641..992b559cbd87244612521d4c96f84f997d6c4196 100644 --- a/doc/api/v2/model_configs.rst +++ b/doc/api/v2/model_configs.rst @@ -6,6 +6,7 @@ Model Configuration config/activation.rst config/layer.rst + config/evaluators.rst config/optimizer.rst config/pooling.rst config/networks.rst diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index a2fa623c80087d42e6a2a5c05f62eba4997f8ec4..f3b1c2c4d438b5d3e776ef27ce8f8b78f710f2ab 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,2 +1,2 @@ -add_test(NAME test_swig_api - COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE}) +add_python_test(test_swig_api + testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py) diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index 4db30f37a5bc92d4348caed0aebdd8a589b55712..ff1c195cee60b6b2773f1059c0c3fb41a36c609d 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -261,9 +261,12 @@ TEST(Compare, img_conv2) { std::string config_file_a = "./gserver/tests/img_conv_a.conf"; std::string config_file_b = "./gserver/tests/img_conv_c.conf"; bool useGpu = FLAGS_use_gpu; + double eps = FLAGS_checkgrad_eps; FLAGS_use_gpu = true; + FLAGS_checkgrad_eps = 1e-3; compareNetwork(config_file_a, config_file_b); FLAGS_use_gpu = useGpu; + FLAGS_checkgrad_eps = eps; } #endif diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 4172063d923f939dac7229573bc087ec8c62b844..9e72c68d4bb1e3708b01885460b10a0f9dc5afe8 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe # Set BASE_IMAGE according to env variables if [ ${WITH_GPU} == "ON" ]; then @@ -34,7 +34,8 @@ cmake .. \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then - make test + pip uninstall -y py-paddle paddle || true + ctest -V fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl @@ -71,7 +72,7 @@ if [[ ${WOBOQ:-OFF} == 'ON' ]]; then cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ -DCMAKE_BUILD_TYPE=Release \ . - make + make -j `nproc` export WOBOQ_OUT=/woboq_out/paddle export BUILD_DIR=/paddle/build diff --git a/paddle/api/test/run_tests.sh b/paddle/scripts/run_python_tests.sh similarity index 63% rename from paddle/api/test/run_tests.sh rename to paddle/scripts/run_python_tests.sh index bcf06afa86aaa1a3151aeb966b54f69657c541e3..05c187522cabc5d9b7688f2a1999c25672e4c550 100755 --- a/paddle/api/test/run_tests.sh +++ b/paddle/scripts/run_python_tests.sh @@ -18,19 +18,29 @@ pushd `dirname $0` > /dev/null SCRIPTPATH=$PWD popd > /dev/null -cd $SCRIPTPATH +USE_VIRTUALENV_FOR_TEST=$1; shift +PYTHON=$1; shift -$1 -m pip install ../../dist/*.whl +if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then + rm -rf .test_env + virtualenv .test_env + source .test_env/bin/activate + PYTHON=python +fi -test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py" +export PYTHONPATH=$SCRIPTPATH/../../python/ +$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib ipython==5.3 -export PYTHONPATH=$PWD/../../../python/ - -for fn in $test_list +for fn in "$@" do echo "test $fn" - $1 $fn + $PYTHON $fn if [ $? -ne 0 ]; then exit 1 fi done + +if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then + deactivate + rm -rf .test_env +fi diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 8fba4a19ba2cecd551aa4bbc764acc94615dd115..12bf629ea920832f96bc5f7cc0b38abfddd34d97 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -50,7 +50,7 @@ if [ -z "${PADDLE_NO_STAT+x}" ]; then -c ${PADDLE_CONF_HOME}/paddle.cookie \ http://api.paddlepaddle.org/version 2>/dev/null` if [ $? -eq 0 ] && [ "$(ver2num @PADDLE_VERSION@)" -lt $(ver2num $SERVER_VER) ]; then - echo "Paddle release a new version ${SERVER_VER}, you can get the install package in http://www.paddlepaddle.org" + echo "Paddle release a new version ${SERVER_VER}, you can get the install package in http://www.paddlepaddle.org" fi fi @@ -95,7 +95,7 @@ if [ $? -eq 1 ]; then # Older version installed, or not installed at all echo "First time run paddle, need to install some python dependencies." # setuptools normalizes package version, so we need to use normalized # package version for paddle python package - PYTHON_PADDLE_VERSION=$(python -c 'import packaging + PYTHON_PADDLE_VERSION=$(python -c 'import packaging.version import setuptools print str(packaging.version.Version("@PADDLE_VERSION@")) ' 2>/dev/null) diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 171eae381af70e9e76210a77a409e56a527cab06..af59951752d1799c95e293d3eae233e6aa26e5f3 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -1,8 +1,9 @@ # The utilities for paddle file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) -create_resources(enable_virtualenv.py enable_virtualenv.c) -set(UTIL_RES enable_virtualenv.c) +create_resources(${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.py + ${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.c) +set(UTIL_RES ${CMAKE_CURRENT_SOURCE_DIR}/enable_virtualenv.c) if(APPLE) file(GLOB UTIL_ARCH_SOURCES . arch/osx/*.cpp) diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py index c178336303f53769863063922868cd2a22e4b957..1fea7917e1553f63a6e6df50e1a8c6473018085f 100644 --- a/python/paddle/v2/inference.py +++ b/python/paddle/v2/inference.py @@ -79,8 +79,9 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): .. code-block:: python - result = paddle.infer(prediction, parameters, input=SomeData, - batch_size=32) + result = paddle.infer(outptut_layer=prediction, + parameters=parameters, + input=SomeData) print result :param output_layer: output of the neural network that would be inferred diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index d686d09f220671fce50be0784e354f97cb109f32..64805d0c504b876f4d1f6657fe94457534a0b278 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -249,7 +249,7 @@ class Parameters(object): size = reduce(lambda a, b: a * b, param.shape) f.write(struct.pack("IIQ", 0, 4, size)) param = param.astype(np.float32) - f.write(param.tobytes()) + f.write(param.tostring()) def deserialize(self, name, f): """ diff --git a/python/paddle/v2/plot/tests/CMakeLists.txt b/python/paddle/v2/plot/tests/CMakeLists.txt index da550a178ce0fe4832b640e0c23505279dedd27a..b1132f131737e26bfeeb31f6b3f062710bdf6f75 100644 --- a/python/paddle/v2/plot/tests/CMakeLists.txt +++ b/python/paddle/v2/plot/tests/CMakeLists.txt @@ -1,3 +1 @@ -add_test(NAME test_ploter - COMMAND bash ${PROJ_ROOT}/python/paddle/v2/plot/tests/run_tests.sh - ${PYTHON_EXECUTABLE}) +add_python_test(test_ploter test_ploter.py) diff --git a/python/paddle/v2/plot/tests/run_tests.sh b/python/paddle/v2/plot/tests/run_tests.sh deleted file mode 100755 index 9c1a4a71ce43f285c4f970eddf6af46a2821a40a..0000000000000000000000000000000000000000 --- a/python/paddle/v2/plot/tests/run_tests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -pushd `dirname $0` > /dev/null -SCRIPTPATH=$PWD -popd > /dev/null - -cd $SCRIPTPATH -$1 -m pip install ../../../../../paddle/dist/*.whl - -export DISABLE_PLOT="True" -test_list="test_ploter.py" - -export PYTHONPATH=$PWD/../../../../../python/ - -for fn in $test_list -do - echo "test $fn" - $1 $fn - if [ $? -ne 0 ]; then - exit 1 - fi -done diff --git a/python/paddle/v2/reader/tests/CMakeLists.txt b/python/paddle/v2/reader/tests/CMakeLists.txt index a51f700406b48f8186e45f1ced94765e343a8b5e..6a1d337b232c7a849a8793894bf16d26d609d3dd 100644 --- a/python/paddle/v2/reader/tests/CMakeLists.txt +++ b/python/paddle/v2/reader/tests/CMakeLists.txt @@ -1,3 +1 @@ -add_test(NAME reader_tests - COMMAND bash ${PROJ_ROOT}/python/paddle/v2/reader/tests/run_tests.sh - ${PYTHON_EXECUTABLE}) +add_python_test(reader_tests creator_test.py decorator_test.py) diff --git a/python/paddle/v2/reader/tests/run_tests.sh b/python/paddle/v2/reader/tests/run_tests.sh deleted file mode 100755 index a544a5636021bcf8bd9a35966c91ae343c149d14..0000000000000000000000000000000000000000 --- a/python/paddle/v2/reader/tests/run_tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -pushd `dirname $0` > /dev/null -SCRIPTPATH=$PWD -popd > /dev/null - -cd $SCRIPTPATH -$1 -m pip install ../../../../../paddle/dist/*.whl - -test_list="creator_test.py decorator_test.py" - -export PYTHONPATH=$PWD/../../../../../python/ - -for fn in $test_list -do - echo "test $fn" - $1 $fn - if [ $? -ne 0 ]; then - exit 1 - fi -done diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index 572deaff356712cac23cd7911cdf289db100564c..5554a37df03cb31b37e9bbf68faa606d7c2cadde 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1,16 +1 @@ -add_test(NAME test_v2_api - COMMAND bash ${PROJ_ROOT}/python/paddle/v2/tests/run_tests.sh ${PYTHON_EXECUTABLE}) - -add_test(NAME test_v2_layer - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_layer.py - WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) - -add_test(NAME test_v2_rnn_layer - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_rnn_layer.py) - -add_test(NAME test_topology - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py - WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) +add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py) diff --git a/python/paddle/v2/tests/run_tests.sh b/python/paddle/v2/tests/run_tests.sh deleted file mode 100755 index dda1b1bd222a9f226db1a4bd730e9637ab882196..0000000000000000000000000000000000000000 --- a/python/paddle/v2/tests/run_tests.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -pushd `dirname $0` > /dev/null -SCRIPTPATH=$PWD -popd > /dev/null - -cd $SCRIPTPATH - -$1 -m pip install ../../../../paddle/dist/*.whl - -test_list="test_data_feeder.py test_parameters.py" - -export PYTHONPATH=$PWD/../../../../python/ - -for fn in $test_list -do - echo "test $fn" - $1 $fn - if [ $? -ne 0 ]; then - exit 1 - fi -done