import math import os import pickle import shutil import subprocess import multiprocessing import sys def remove_desc_code(srcls, filename): if filename == 'fluid_cn/one_hot_cn.rst': srcls.pop(13) srcls.pop(28) srcls.pop(44) if filename == 'layers_cn/one_hot_cn.rst': srcls.pop(15) srcls.pop(30) srcls.pop(46) if filename == 'profiler_cn/profiler_cn.rst': srcls.pop(41) if filename == 'layers_cn/natural_exp_decay_cn.rst': srcls.pop(13) if filename == 'layers_cn/transpose_cn.rst': srcls.pop(20) if filename == 'layers_cn/array_length_cn.rst': srcls.pop(36) if filename == 'layers_cn/inverse_time_decay_cn.rst': srcls.pop(13) if filename == 'layers_cn/stack_cn.rst': srcls.pop(12) srcls.pop(33) if filename == 'layers_cn/sums_cn.rst': srcls.pop(11) if filename == 'layers_cn/sum_cn.rst': for i in range(len(srcls) - 1, 61, -1): srcls.pop(i) if filename == 'layers_cn/softmax_cn.rst': srcls.pop(30) srcls.pop(57) if filename == 'layers_cn/array_write_cn.rst': srcls.pop(37) if filename == 'layers_cn/lod_append_cn.rst': srcls.pop(11) if filename == 'layers_cn/reorder_lod_tensor_by_rank_cn.rst': srcls.pop(25) if filename == 'layers_cn/round_cn.rst': srcls.pop(10) if filename == 'layers_cn/squeeze_cn.rst': srcls.pop(11) srcls.pop(19) srcls.pop(27) if filename == 'layers_cn/unsqueeze_cn.rst': srcls.pop(11) if filename == 'layers_cn/array_read_cn.rst': srcls.pop(51) if filename == 'layers_cn/scatter_cn.rst': srcls.pop(9) if filename == 'layers_cn/topk_cn.rst': srcls.pop(11) if filename == 'optimizer_cn/ModelAverage_cn.rst': srcls.pop(15) return srcls def check_indent(code_line): indent = "" for c in code_line: if c == '\t': indent += ' ' elif c == ' ': indent += ' ' if c != ' ' and c != '\t': break return indent def find_all(src_str, substr): indices = [] get_one = src_str.find(substr) while get_one != -1: indices.append(get_one) get_one = src_str.find(substr, get_one + 1) return indices def extract_sample_code(srcfile, status_all): filename = srcfile.name srcc = srcfile.read() srcfile.seek(0, 0) srcls = srcfile.readlines() srcls = remove_desc_code( srcls, filename) # remove description info for samplecode status = [] sample_code_begins = find_all(srcc, " code-block:: python") if len(sample_code_begins) == 0: status.append(-1) else: for i in range(0, len(srcls)): if srcls[i].find(".. code-block:: python") != -1: content = "" start = i blank_line = 1 while srcls[start + blank_line].strip() == '': blank_line += 1 startindent = "" # remove indent error if srcls[start + blank_line].find("from") != -1: startindent += srcls[start + blank_line][:srcls[ start + blank_line].find("from")] elif srcls[start + blank_line].find("import") != -1: startindent += srcls[start + blank_line][:srcls[ start + blank_line].find("import")] else: startindent += check_indent(srcls[start + blank_line]) content += srcls[start + blank_line][len(startindent):] for j in range(start + blank_line + 1, len(srcls)): # planish a blank line if not srcls[j].startswith(startindent) and srcls[ j] != '\n': break if srcls[j].find(" code-block:: python") != -1: break content += srcls[j].replace(startindent, "", 1) status.append(run_sample_code(content, filename)) status_all[filename] = status return status_all def run_sample_code(content, filename): # three status ,-1:no sample code; 1: running error; 0:normal fname = filename.split("/")[-1].replace("_cn", "").replace(".rst", "") + ".py" tempf = open("temp/" + fname, 'w') content = "# -*- coding: utf-8 -*-\n" + content tempf.write(content) tempf.close() cmd = ["python", "temp/" + fname] subprc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, error = subprc.communicate() err = "".join(error.decode(encoding='utf-8')) if subprc.returncode != 0: print("\nSample code error found in ", filename, ":\n") print(err) status = 1 else: status = 0 os.remove("temp/" + fname) return status def test(file): temp = [] src = open(file, 'r') status_all = {} extract_sample_code(src, status_all) temp.append(status_all) src.close() return temp if os.path.isdir("temp"): shutil.rmtree("temp") if os.path.isdir("infer_model"): shutil.rmtree("infer_model") if os.path.isdir("image"): shutil.rmtree("image") if os.path.isdir("my_paddle_model"): shutil.rmtree("my_paddle_model") if os.path.isdir("my_paddle_vars"): shutil.rmtree("my_paddle_vars") if not os.path.isdir("temp"): os.mkdir("temp") output = [] if len(sys.argv) < 2: print("Error: inadequate number of arguments") print("Please one file") sys.exit(1) else: if not os.path.exists(sys.argv[1]): print("File not found") sys.exit(1) res = test(sys.argv[1]) output.append(res) status_groups = {-1: [], 0: [], 1: []} # polishes show format ci_pass = True for one_file in output: for dicts in one_file: for key in dicts: status = dicts[key] for ele in status: if ele != 0: ci_pass = False break if len(status) == 1: status_groups[status[0]].append(key) else: for u in range(0, len(status)): status_groups[status[u]].append(key + '_' + str(u + 1)) error_api = status_groups[-1] + status_groups[1] total_error_number = len(error_api) print("****************************************************") print("----------------End of the Check--------------------") print("****************************************************") if total_error_number > 0: print("Error sample code number is:{}".format(total_error_number)) type_one_number = len(status_groups[-1]) type_two_number = len(status_groups[1]) if type_one_number > 0: print("Error type one sample number is:{}".format(type_one_number)) print("Error raised from type one:no sample code.", str(status_groups[-1])) if type_two_number > 0: print("Error type two sample number is:{}".format(type_two_number)) print("Error raised from type two:running error sample code.", str(status_groups[1])) if not ci_pass: print("Mistakes found in sample codes.") exit(1) else: print("Sample code check is successful!")