add new ci_scripts for new paddle doc dirs (#2556)

ci_scripts/api_white_list.txt

+paddle/fluid/DistributeTranspiler_cn.rst
+paddle/fluid/DistributeTranspilerConfig_cn.rst
+paddle/fluid/transpiler/HashName_cn.rst
+paddle/fluid/memory_optimize_cn.rst
+paddle/fluid/release_memory_cn.rst
+paddle/optimizer/Dpsgd_cn.rst
+paddle/reader/ComposeNotAligned_cn.rst
+paddle/fluid/layers/scatter_cn.rst
+paddle/tensor/manipulation/scatter_cn.rst

ci_scripts/check_api_cn.sh

+#!/bin/bash
+
+git_files=`git diff --numstat upstream/$BRANCH | awk '{print $NF}'`
+
+for file in `echo $git_files`;do
+  grep "code-block" $files
+  if [ $? -eq 0 ] ;then 
+    echo $file | grep "doc/paddle/api/paddle/.*_cn.rst"
+    if [ $? -eq 0 ];then
+        api_file=`echo $file | sed 's#doc/paddle/api/##g'`
+        grep -w "${api_file}" ${DIR_PATH}/api_white_list.txt
+        if [ $? -ne 0 ];then
+            python chinese_samplecode_processor.py $file
+            if [ $? -ne 0 ];then
+                echo "chinese sample code failed"
+                exit 5
+            fi
+        fi 
+    fi
+  fi
+done
+

ci_scripts/check_code.sh

+#!/usr/bin/env bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#=================================================
+#                   Utils
+#=================================================
+
+set -ex
+
+if [ -z ${BRANCH} ]; then
+    BRANCH="develop"
+fi
+
+BENCHMARK_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/.." && pwd )"
+echo ${BENCHMARK_ROOT}
+
+function prepare_env(){
+    # Install tensorflow and other packages
+    pip install pre-commit==1.21 pylint==1.9.5 pytest==4.6.9
+}
+
+function abort(){
+    echo "Your change doesn't follow benchmark's code style." 1>&2
+    echo "Please use pre-commit to check what is wrong." 1>&2
+    exit 1
+}
+
+
+function check_style(){
+	trap 'abort' 0
+	pre-commit install
+	commit_files=on
+    	for file_name in `git diff --numstat upstream/$BRANCH| awk '{print $NF}'`;do
+        	if  ! pre-commit run --files $file_name ; then
+            		git diff
+            		commit_files=off
+        	fi
+    	done
+    	if [ $commit_files == 'off' ];then
+        	echo "code format error"
+        	exit 1
+    	fi
+    	trap 0
+}
+
+prepare_env
+check_style

ci_scripts/check_pr_approval.py

+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import sys
+import json
+
+
+def check_approval(count, required_reviewers):
+    json_buff = ""
+    for line in sys.stdin:
+        json_buff = "".join([json_buff, line])
+    json_resp = json.loads(json_buff)
+    approves = 0
+    approved_user_ids = []
+    for review in json_resp:
+        if review["state"] == "APPROVED":
+            approves += 1
+            approved_user_ids.append(review["user"]["id"])
+
+    # convert to int
+    required_reviewers_int = set()
+    for rr in required_reviewers:
+        required_reviewers_int.add(int(rr))
+
+    if len(set(approved_user_ids) & required_reviewers_int) >= count:
+        print("TRUE")
+    else:
+        print("FALSE")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1].isdigit():
+        check_approval(int(sys.argv[1]), sys.argv[2:])
+    else:
+        print(
+            "Usage: python check_pr_approval.py [count] [required reviewer id] ..."
+        )

ci_scripts/checkapproval.sh

+#!/bin/bash
+
+API_FILES=("doc/paddle/api/paddle")
+
+for API_FILE in ${API_FILES[*]}; do
+  API_CHANGE=`git diff --name-only upstream/$BRANCH | grep "${API_FILE}"
+  if [ "${API_CHANGE}" ];then
+    approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/FluidDoc/pulls/${GIT_PR_ID}/reviews?per_page=10000`
+    if [ "${API_FILE}" == "doc/paddle/api/paddle" ];then
+      APPROVALS=`echo ${approval_line} | python ./check_pr_approval.py 1 2870059 27208573 29231 28379894 23093488 11935832`
+    fi
+  fi
+  if [ "${APPROVALS}" == "FALSE" ]; then
+    if [ "${API_FILE}" == "doc/paddle/api/paddle" ];then
+      echo "You must have one TPM (saxon-zh or swtkiwi or jzhang533 or Heeenrrry or dingjiaweiww or TCChenlong) approval for the api change! ${API_FILE} for the management reason of API interface and API document."
+    fi
+    exit 1
+  fi
+done

ci_scripts/chinese_samplecode_processor.py

+import math
+import os
+import pickle
+import shutil
+import subprocess
+import multiprocessing
+import sys
+
+
+def remove_desc_code(srcls, filename):
+    if filename == 'fluid_cn/one_hot_cn.rst':
+        srcls.pop(13)
+        srcls.pop(28)
+        srcls.pop(44)
+    if filename == 'layers_cn/one_hot_cn.rst':
+        srcls.pop(15)
+        srcls.pop(30)
+        srcls.pop(46)
+    if filename == 'profiler_cn/profiler_cn.rst':
+        srcls.pop(41)
+    if filename == 'layers_cn/natural_exp_decay_cn.rst':
+        srcls.pop(13)
+    if filename == 'layers_cn/transpose_cn.rst':
+        srcls.pop(20)
+    if filename == 'layers_cn/array_length_cn.rst':
+        srcls.pop(36)
+    if filename == 'layers_cn/inverse_time_decay_cn.rst':
+        srcls.pop(13)
+    if filename == 'layers_cn/stack_cn.rst':
+        srcls.pop(12)
+        srcls.pop(33)
+    if filename == 'layers_cn/sums_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/sum_cn.rst':
+        for i in range(len(srcls) - 1, 61, -1):
+            srcls.pop(i)
+    if filename == 'layers_cn/softmax_cn.rst':
+        srcls.pop(30)
+        srcls.pop(57)
+    if filename == 'layers_cn/array_write_cn.rst':
+        srcls.pop(37)
+    if filename == 'layers_cn/lod_append_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/reorder_lod_tensor_by_rank_cn.rst':
+        srcls.pop(25)
+    if filename == 'layers_cn/round_cn.rst':
+        srcls.pop(10)
+    if filename == 'layers_cn/squeeze_cn.rst':
+        srcls.pop(11)
+        srcls.pop(19)
+        srcls.pop(27)
+    if filename == 'layers_cn/unsqueeze_cn.rst':
+        srcls.pop(11)
+    if filename == 'layers_cn/array_read_cn.rst':
+        srcls.pop(51)
+    if filename == 'layers_cn/scatter_cn.rst':
+        srcls.pop(9)
+    if filename == 'layers_cn/topk_cn.rst':
+        srcls.pop(11)
+    if filename == 'optimizer_cn/ModelAverage_cn.rst':
+        srcls.pop(15)
+    return srcls
+
+
+def check_indent(code_line):
+    indent = ""
+    for c in code_line:
+        if c == '\t':
+            indent += '    '
+        elif c == ' ':
+            indent += ' '
+        if c != ' ' and c != '\t':
+            break
+    return indent
+
+
+def find_all(src_str, substr):
+    indices = []
+    get_one = src_str.find(substr)
+    while get_one != -1:
+        indices.append(get_one)
+        get_one = src_str.find(substr, get_one + 1)
+    return indices
+
+
+def extract_sample_code(srcfile, status_all):
+    filename = srcfile.name
+    srcc = srcfile.read()
+    srcfile.seek(0, 0)
+    srcls = srcfile.readlines()
+    srcls = remove_desc_code(
+        srcls, filename)  # remove description info for samplecode
+    status = []
+    sample_code_begins = find_all(srcc, " code-block:: python")
+    if len(sample_code_begins) == 0:
+        status.append(-1)
+
+    else:
+        for i in range(0, len(srcls)):
+            if srcls[i].find(".. code-block:: python") != -1:
+                content = ""
+                start = i
+
+                blank_line = 1
+                while srcls[start + blank_line].strip() == '':
+                    blank_line += 1
+
+                startindent = ""
+                # remove indent error
+                if srcls[start + blank_line].find("from") != -1:
+                    startindent += srcls[start + blank_line][:srcls[
+                        start + blank_line].find("from")]
+                elif srcls[start + blank_line].find("import") != -1:
+                    startindent += srcls[start + blank_line][:srcls[
+                        start + blank_line].find("import")]
+                else:
+                    startindent += check_indent(srcls[start + blank_line])
+                content += srcls[start + blank_line][len(startindent):]
+                for j in range(start + blank_line + 1, len(srcls)):
+                    # planish a blank line
+                    if not srcls[j].startswith(startindent) and srcls[
+                            j] != '\n':
+                        break
+                    if srcls[j].find(" code-block:: python") != -1:
+                        break
+                    content += srcls[j].replace(startindent, "", 1)
+                status.append(run_sample_code(content, filename))
+
+    status_all[filename] = status
+    return status_all
+
+
+def run_sample_code(content, filename):
+    # three status ,-1:no sample code; 1: running error; 0:normal
+    fname = filename.split("/")[-1].replace("_cn", "").replace(".rst",
+                                                               "") + ".py"
+    tempf = open("temp/" + fname, 'w')
+    content = "# -*- coding: utf-8 -*-\n" + content
+    tempf.write(content)
+    tempf.close()
+    cmd = ["python", "temp/" + fname]
+
+    subprc = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, error = subprc.communicate()
+    err = "".join(error.decode(encoding='utf-8'))
+
+    if subprc.returncode != 0:
+        print("\nSample code error found in ", filename, ":\n")
+        print(err)
+        status = 1
+    else:
+        status = 0
+    os.remove("temp/" + fname)
+    return status
+
+
+def test(file):
+    temp = []
+    src = open(file, 'r')
+    status_all = {}
+    extract_sample_code(src, status_all)
+    temp.append(status_all)
+    src.close()
+    return temp
+
+
+if os.path.isdir("temp"):
+    shutil.rmtree("temp")
+if os.path.isdir("infer_model"):
+    shutil.rmtree("infer_model")
+if os.path.isdir("image"):
+    shutil.rmtree("image")
+if os.path.isdir("my_paddle_model"):
+    shutil.rmtree("my_paddle_model")
+if os.path.isdir("my_paddle_vars"):
+    shutil.rmtree("my_paddle_vars")
+
+if not os.path.isdir("temp"):
+    os.mkdir("temp")
+
+output = []
+
+if len(sys.argv) < 2:
+    print("Error: inadequate number of arguments")
+    print("Please one file")
+    sys.exit(1)
+else:
+    if not os.path.exists(sys.argv[1]):
+        print("File not found")
+        sys.exit(1)
+    res = test(sys.argv[1])
+    output.append(res)
+
+status_groups = {-1: [], 0: [], 1: []}
+# polishes show format
+ci_pass = True
+for one_file in output:
+    for dicts in one_file:
+        for key in dicts:
+            status = dicts[key]
+            for ele in status:
+                if ele != 0:
+                    ci_pass = False
+                    break
+            if len(status) == 1:
+                status_groups[status[0]].append(key)
+            else:
+                for u in range(0, len(status)):
+                    status_groups[status[u]].append(key + '_' + str(u + 1))
+
+error_api = status_groups[-1] + status_groups[1]
+total_error_number = len(error_api)
+
+print("****************************************************")
+print("----------------End of the Check--------------------")
+print("****************************************************")
+if total_error_number > 0:
+    print("Error sample code number is:{}".format(total_error_number))
+    type_one_number = len(status_groups[-1])
+    type_two_number = len(status_groups[1])
+    if type_one_number > 0:
+        print("Error type one sample number is:{}".format(type_one_number))
+        print("Error raised from type one:no sample code.",
+              str(status_groups[-1]))
+    if type_two_number > 0:
+        print("Error type two sample number is:{}".format(type_two_number))
+        print("Error raised from type two:running error sample code.",
+              str(status_groups[1]))
+if not ci_pass:
+    print("Mistakes found in sample codes.")
+    exit(1)
+else:
+    print("Sample code check is successful!")

ci_scripts/ci_start.sh

+#!/bin/bash
+
+export DIR_PATH=${PWD}
+
+/bin/bash  ${DIR_PATH}/check_code.sh
+if [ $? -ne 0 ];then
+    echo "code format error"
+    exit 1
+fi
+
+/bin/bash -x ${DIR_PATH}/check_api_cn.sh
+if [ $? -ne 0 ];then
+  exit 1
+fi
+
+/bin/bash  ${DIR_PATH}/checkapproval.sh
\ No newline at end of file