Add check api Examples (#1744)

doc/fluid/api_cn/chinese_samplecode_processor.py

+import math
+import os
+import pickle
+import shutil
+import subprocess
+import multiprocessing
+import sys
+
+
+def remove_desc_code(srcls, filename):
+    if filename == 'fluid_cn/one_hot_cn.rst':
+        srcls.pop(13)
+        srcls.pop(28)
+        srcls.pop(44)
+    if filename == 'layers_cn/one_hot_cn.rst':
+        srcls.pop(15)
+        srcls.pop(30)
+        srcls.pop(46)
+    if filename == 'profiler_cn/profiler_cn.rst':
+        srcls.pop(41)
+    if filename == 'layers_cn/natural_exp_decay_cn.rst':
+        srcls.pop(13)
+    if filename == 'layers_cn/transpose_cn.rst':
+        srcls.pop(20)
+    if filename == 'layers_cn/array_length_cn.rst':
+        srcls.pop(36)
+    if filename == 'layers_cn/inverse_time_decay_cn.rst':
+        srcls.pop(13)
+    if filename == 'layers_cn/stack_cn.rst':
+        srcls.pop(12)
+        srcls.pop(33)
+    if filename == 'layers_cn/sums_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/sum_cn.rst':
+        for i in range(len(srcls)-1, 61, -1):
+            srcls.pop(i)
+    if filename == 'layers_cn/softmax_cn.rst':
+        srcls.pop(30)
+        srcls.pop(57)
+    if filename == 'layers_cn/array_write_cn.rst':
+        srcls.pop(37)
+    if filename == 'layers_cn/lod_append_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/reorder_lod_tensor_by_rank_cn.rst':
+        srcls.pop(25)
+    if filename == 'layers_cn/round_cn.rst':
+        srcls.pop(10)
+    if filename == 'layers_cn/squeeze_cn.rst':
+        srcls.pop(11)
+        srcls.pop(19)
+        srcls.pop(27)
+    if filename == 'layers_cn/unsqueeze_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/array_read_cn.rst':
+        srcls.pop(51)
+    if filename == 'layers_cn/scatter_cn.rst':
+        srcls.pop(9)
+    if filename == 'layers_cn/topk_cn.rst':
+        srcls.pop(11)
+    if filename == 'optimizer_cn/ModelAverage_cn.rst':
+        srcls.pop(15)
+    return srcls
+
+
+def check_indent(code_line):
+    indent = ""
+    for c in code_line:
+        if c == '\t':
+            indent += '    '
+        elif c == ' ':
+            indent += ' '
+        if c != ' ' and c != '\t':
+            break
+    return indent
+
+
+def find_all(src_str, substr):
+    indices = []
+    get_one = src_str.find(substr)
+    while get_one != -1:
+        indices.append(get_one)
+        get_one = src_str.find(substr, get_one + 1)
+    return indices
+
+
+def extract_sample_code(srcfile, status_all):
+    filename = srcfile.name
+    srcc = srcfile.read()
+    srcfile.seek(0, 0)
+    srcls = srcfile.readlines()
+    srcls = remove_desc_code(srcls, filename) # remove description info for samplecode
+    status = []
+    sample_code_begins = find_all(srcc, " code-block:: python")
+    if len(sample_code_begins) == 0:
+        status.append(-1)
+
+    else:
+        for i in range(0, len(srcls)):
+            if srcls[i].find(".. code-block:: python") != -1:
+                content = ""
+                start = i
+
+                blank_line = 1
+                while srcls[start + blank_line].strip() == '':
+                    blank_line += 1
+
+                startindent = ""
+                # remove indent error
+                if srcls[start + blank_line].find("from") != -1:
+                    startindent += srcls[start + blank_line][:srcls[start + blank_line].find("from")]
+                elif srcls[start + blank_line].find("import") != -1:
+                    startindent += srcls[start + blank_line][:srcls[start + blank_line].find("import")]
+                else:
+                    startindent += check_indent(srcls[start + blank_line])
+                content += srcls[start + blank_line][len(startindent):]
+                for j in range(start + blank_line + 1, len(srcls)):
+                    # planish a blank line
+                    if not srcls[j].startswith(startindent) and srcls[j] != '\n':
+                        break
+                    if srcls[j].find(" code-block:: python") != -1:
+                        break
+                    content += srcls[j].replace(startindent, "", 1)
+                status.append(run_sample_code(content, filename))
+
+    status_all[filename] = status
+    return status_all
+
+
+def run_sample_code(content, filename):
+    # three status ,-1:no sample code; 1: running error; 0:normal
+    fname = filename.split("/")[-1].replace("_cn", "").replace(".rst", "") + ".py"
+    tempf = open("temp/" + fname, 'w')
+    content = "# -*- coding: utf-8 -*-\n" + content
+    tempf.write(content)
+    tempf.close()
+    cmd = ["python", "temp/" + fname]
+
+    subprc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, error = subprc.communicate()
+    err = "".join(error.decode(encoding='utf-8'))
+
+    if subprc.returncode != 0:
+        print("\nSample code error found in ", filename, ":\n")
+        print(err)
+        status = 1
+    else:
+        status = 0
+    os.remove("temp/" + fname)
+    return status
+
+def test(file):
+    temp = []
+    src = open(file, 'r')
+    status_all = {}
+    extract_sample_code(src, status_all)
+    temp.append(status_all)
+    src.close()
+    return temp
+
+
+if os.path.isdir("temp"):
+    shutil.rmtree("temp")
+if os.path.isdir("infer_model"):
+    shutil.rmtree("infer_model")
+if os.path.isdir("image"):
+    shutil.rmtree("image")
+if os.path.isdir("my_paddle_model"):
+    shutil.rmtree("my_paddle_model")
+if os.path.isdir("my_paddle_vars"):
+    shutil.rmtree("my_paddle_vars")
+
+
+if not os.path.isdir("temp"):
+    os.mkdir("temp")
+
+output = []
+
+if len(sys.argv) < 2:
+    print("Error: inadequate number of arguments")
+    print("Please one file")
+    sys.exit(1)
+else:
+    if not os.path.exists(sys.argv[1]):
+        print("File not found")
+        sys.exit(1)
+    res = test(sys.argv[1])    
+    output.append(res)
+
+
+status_groups = {-1: [], 0: [], 1: []}
+# polishes show format
+ci_pass = True
+for one_file in output:
+    for dicts in one_file:
+        for key in dicts:
+            status = dicts[key]
+            for ele in status:
+                if ele != 0:
+                    ci_pass = False
+                    break
+            if len(status) == 1:
+                status_groups[status[0]].append(key)
+            else:
+                for u in range(0, len(status)):
+                    status_groups[status[u]].append(key + '_' + str(u + 1))
+
+error_api = status_groups[-1] + status_groups[1]
+total_error_number = len(error_api)
+
+
+print("****************************************************")
+print("----------------End of the Check--------------------")
+print("****************************************************")
+if total_error_number > 0:
+    print("Error sample code number is:{}".format(total_error_number))
+    type_one_number = len(status_groups[-1])
+    type_two_number = len(status_groups[1])
+    if type_one_number > 0:
+        print("Error type one sample number is:{}".format(type_one_number))
+        print("Error raised from type one:no sample code.", str(status_groups[-1]))
+    if type_two_number > 0:
+        print("Error type two sample number is:{}".format(type_two_number))
+        print("Error raised from type two:running error sample code.", str(status_groups[1]))
+if not ci_pass:
+    print("Mistakes found in sample codes.")
+    exit(1)
+else:
+    print("Sample code check is successful!")

doc/fluid/api_cn/profiler_cn/reset_profiler_cn.rst

@@ -14,7 +14,7 @@ reset_profiler
     import paddle.fluid as fluid
     import paddle.fluid.profiler as profiler
     with profiler.profiler('CPU', 'total', '/tmp/profile'):
-    for iter in range(10):
-        if iter == 2:
-            profiler.reset_profiler()
-        # ...
+        for iter in range(10):
+            if iter == 2:
+                profiler.reset_profiler()
+            # ...

scripts/api_white_list.txt

+fluid_cn/DistributeTranspiler_cn.rst
+transpiler_cn/DistributeTranspiler_cn.rst
+transpiler_cn/DistributeTranspilerConfig_cn.rst
+transpiler_cn/HashName_cn.rst
+transpiler_cn/memory_optimize_cn.rst
+transpiler_cn/release_memory_cn.rst
+transpiler_cn/RoundRobin_cn.rst

scripts/check_api_cn.sh

+#!/bin/bash
+
+git_files=`git diff --numstat upstream/$BRANCH`
+
+
+if [ "$night" == "develop" ];then
+   wget -q https://paddle-wheel.bj.bcebos.com/0.0.0-gpu-cuda9-cudnn7-mkl/paddlepaddle_gpu-0.0.0-cp27-cp27mu-linux_x86_64.whl
+   pip install -U paddlepaddle_gpu-0.0.0-cp27-cp27mu-linux_x86_64.whl
+else
+   git clone https://github.com/PaddlePaddle/Paddle.git
+   mkdir Paddle/build && cd Paddle/build
+   cmake .. -DWITH_GPU=ON -DWITH_COVERAGE=OFF -DWITH_TESTING=OFF -DCMAKE_BUILD_TYPE=Release
+   make -j`nproc`
+   pip install -U python/dist/paddlepaddle_gpu-0.0.0-cp27-cp27mu-linux_x86_64.whl 
+fi
+
+for files in `echo $git_files`;do
+    echo $files|grep 'doc/fluid/api_cn/.*/.*.rst'
+    if [ $? -eq 0 ];then
+        api_file=`echo $files|sed 's#doc/fluid/api_cn/##g'`
+        cd /FluidDoc/doc/fluid/api_cn/
+        grep -w "$api_file" /FluidDoc/scripts/api_white_list.txt
+        if [ $? -ne 0 ];then
+            python chinese_samplecode_processor.py $api_file
+            if [ $? -ne 0 ];then
+                exit 5
+            fi
+        fi 
+    fi
+done
+

scripts/start.sh

+#!/bin/bash
+
+DIR_PATH="/FluidDoc"
+
+/bin/bash ${DIR_PATH}/scripts/checkapproval.sh
+/bin/bash ${DIR_PATH}/scripts/check_api_cn.sh