sampcd_processor.py 25.1 KB
Newer Older
1
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6 7 8 9 10 11 12 13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14 15 16 17 18 19 20 21
"""
please make sure to run in the tools path
usage: python sample_test.py {cpu or gpu} 
    {cpu or gpu}: running in cpu version or gpu version

for example, you can run cpu version python2 testing like this:

    python sampcd_processor.py cpu 
T
tianshuo78520a 已提交
22

23
"""
T
tianshuo78520a 已提交
24
import os
25
import sys
T
tianshuo78520a 已提交
26
import subprocess
27 28
import multiprocessing
import platform
29
import inspect
30 31 32 33
import argparse
import shutil
import re
import logging
34
import time
T
tianshuo78520a 已提交
35

36 37 38 39 40
logger = logging.getLogger()
if logger.handlers:
    console = logger.handlers[
        0]  # we assume the first handler is the one we want to configure
else:
41
    console = logging.StreamHandler(stream=sys.stderr)
42
    logger.addHandler(console)
43
console.setFormatter(logging.Formatter("%(message)s"))
44 45

RUN_ON_DEVICE = 'cpu'
46
SAMPLE_CODE_TEST_CAPACITY = set()
47 48 49 50 51 52
GPU_ID = 0
whl_error = []
API_DEV_SPEC_FN = 'paddle/fluid/API_DEV.spec'
API_PR_SPEC_FN = 'paddle/fluid/API_PR.spec'
API_DIFF_SPEC_FN = 'dev_pr_diff_api.spec'
SAMPLECODE_TEMPDIR = 'samplecode_temp'
53 54 55 56 57 58 59 60 61
ENV_KEY_CODES_FRONTEND = 'CODES_INSERTED_INTO_FRONTEND'
ENV_KEY_TEST_CAPACITY = 'SAMPLE_CODE_TEST_CAPACITY'
SUMMARY_INFO = {
    'success': [],
    'failed': [],
    'skiptest': [],
    'nocodes': [],
    # ... required not-match
}
62

T
tianshuo78520a 已提交
63 64

def find_all(srcstr, substr):
65
    """
66 67 68 69 70 71
    to find all desired substring in the source string
     and return their starting indices as a list

    Args:
        srcstr(str): the parent string
        substr(str): substr
72

73
    Returns:
74
        list: a list of the indices of the substrings
75
              found
76
    """
T
tianshuo78520a 已提交
77 78 79 80 81 82 83 84
    indices = []
    gotone = srcstr.find(substr)
    while (gotone != -1):
        indices.append(gotone)
        gotone = srcstr.find(substr, gotone + 1)
    return indices


85 86 87 88 89 90 91 92
def find_last_future_line_end(cbstr):
    """
    find the last `__future__` line.

    Args:
        docstr(str): docstring
    Return:
        index of the line end or None.
93
    """
94 95 96 97 98 99 100 101 102 103 104 105
    pat = re.compile('__future__.*\n')
    lastmo = None
    it = re.finditer(pat, cbstr)
    while True:
        try:
            lastmo = next(it)
        except StopIteration:
            break
    if lastmo:
        return lastmo.end()
    else:
        return None
106 107


108 109 110 111 112 113
def extract_code_blocks_from_docstr(docstr):
    """
    extract code-blocks from the given docstring.

    DON'T include the multiline-string definition in code-blocks.
    The *Examples* section must be the last.
114

115
    Args:
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
        docstr(str): docstring
    Return:
        code_blocks: A list of code-blocks, indent removed. 
                     element {'name': the code-block's name, 'id': sequence id.
                              'codes': codes, 'required': 'gpu'}
    """
    code_blocks = []

    mo = re.search(r"Examples:", docstr)
    if mo is None:
        return code_blocks
    ds_list = docstr[mo.start():].replace("\t", '    ').split("\n")
    lastlineindex = len(ds_list) - 1

    cb_start_pat = re.compile(r"code-block::\s*python")
    cb_param_pat = re.compile(r"^\s*:(\w+):\s*(\S*)\s*$")
    cb_required_pat = re.compile(r"^\s*#\s*require[s|d]\s*:\s*(\S+)\s*$")

    cb_info = {}
    cb_info['cb_started'] = False
    cb_info['cb_cur'] = []
    cb_info['cb_cur_indent'] = -1
    cb_info['cb_cur_name'] = None
    cb_info['cb_cur_seq_id'] = 0
    cb_info['cb_required'] = None

    def _cb_started():
        # nonlocal cb_started, cb_cur_name, cb_required, cb_cur_seq_id
        cb_info['cb_started'] = True
        cb_info['cb_cur_seq_id'] += 1
        cb_info['cb_cur_name'] = None
        cb_info['cb_required'] = None

    def _append_code_block():
        # nonlocal code_blocks, cb_cur, cb_cur_name, cb_cur_seq_id, cb_required
        code_blocks.append({
            'codes': inspect.cleandoc("\n".join(cb_info['cb_cur'])),
            'name': cb_info['cb_cur_name'],
            'id': cb_info['cb_cur_seq_id'],
            'required': cb_info['cb_required'],
        })

    for lineno, linecont in enumerate(ds_list):
        if re.search(cb_start_pat, linecont):
            if not cb_info['cb_started']:
                _cb_started()
                continue
            else:
                # cur block end
                if len(cb_info['cb_cur']):
                    _append_code_block()
                _cb_started()  # another block started
                cb_info['cb_cur_indent'] = -1
                cb_info['cb_cur'] = []
        else:
            if cb_info['cb_started']:
                # handle the code-block directive's options
                mo_p = cb_param_pat.match(linecont)
                if mo_p:
                    if mo_p.group(1) == 'name':
                        cb_info['cb_cur_name'] = mo_p.group(2)
                    continue
                # read the required directive
                mo_r = cb_required_pat.match(linecont)
                if mo_r:
                    cb_info['cb_required'] = mo_r.group(1)
                # docstring end
                if lineno == lastlineindex:
                    mo = re.search(r"\S", linecont)
                    if mo is not None and cb_info['cb_cur_indent'] <= mo.start(
                    ):
                        cb_info['cb_cur'].append(linecont)
                    if len(cb_info['cb_cur']):
                        _append_code_block()
                    break
                # check indent for cur block start and end.
                mo = re.search(r"\S", linecont)
                if mo is None:
                    continue
                if cb_info['cb_cur_indent'] < 0:
                    # find the first non empty line
                    cb_info['cb_cur_indent'] = mo.start()
                    cb_info['cb_cur'].append(linecont)
                else:
                    if cb_info['cb_cur_indent'] <= mo.start():
                        cb_info['cb_cur'].append(linecont)
                    else:
                        if linecont[mo.start()] == '#':
                            continue
                        else:
                            # block end
                            if len(cb_info['cb_cur']):
                                _append_code_block()
                            cb_info['cb_started'] = False
                            cb_info['cb_cur_indent'] = -1
                            cb_info['cb_cur'] = []
    return code_blocks


def get_test_capacity():
    """
    collect capacities and set to SAMPLE_CODE_TEST_CAPACITY
    """
    global SAMPLE_CODE_TEST_CAPACITY  # write
    global ENV_KEY_TEST_CAPACITY, RUN_ON_DEVICE  # readonly
    if ENV_KEY_TEST_CAPACITY in os.environ:
        for r in os.environ[ENV_KEY_TEST_CAPACITY].split(','):
            rr = r.strip().lower()
            if r:
                SAMPLE_CODE_TEST_CAPACITY.add(rr)
    if 'cpu' not in SAMPLE_CODE_TEST_CAPACITY:
        SAMPLE_CODE_TEST_CAPACITY.add('cpu')
228

229 230 231 232 233
    if RUN_ON_DEVICE:
        SAMPLE_CODE_TEST_CAPACITY.add(RUN_ON_DEVICE)


def is_required_match(requirestr, cbtitle='not-specified'):
234
    """
235 236 237 238 239 240 241 242 243 244 245 246 247
    search the required instruction in the code-block, and check it match the current running environment.
    
    environment values of equipped: cpu, gpu, xpu, distributed, skip
    the 'skip' is the special flag to skip the test, so is_required_match will return False directly.

    Args:
        requirestr(str): the required string.
        cbtitle(str): the title of the code-block.
    returns:
        True - yes, matched
        False - not match
        None - skipped  # trick
    """
248
    global SAMPLE_CODE_TEST_CAPACITY, RUN_ON_DEVICE  # readonly
249 250 251 252 253 254
    requires = set(['cpu'])
    if requirestr:
        for r in requirestr.split(','):
            rr = r.strip().lower()
            if rr:
                requires.add(rr)
255 256
    else:
        requires.add(RUN_ON_DEVICE)
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
    if 'skip' in requires or 'skiptest' in requires:
        logger.info('%s: skipped', cbtitle)
        return None

    if all([
            k in SAMPLE_CODE_TEST_CAPACITY for k in requires
            if k not in ['skip', 'skiptest']
    ]):
        return True

    logger.info('%s: the equipments [%s] not match the required [%s].', cbtitle,
                ','.join(SAMPLE_CODE_TEST_CAPACITY), ','.join(requires))
    return False


def insert_codes_into_codeblock(codeblock, apiname='not-specified'):
    """
    insert some codes in the frontend and backend into the code-block.
    """
    global ENV_KEY_CODES_FRONTEND, GPU_ID, RUN_ON_DEVICE  # readonly
    inserted_codes_f = ''
    inserted_codes_b = ''
    if ENV_KEY_CODES_FRONTEND in os.environ and os.environ[
            ENV_KEY_CODES_FRONTEND]:
        inserted_codes_f = os.environ[ENV_KEY_CODES_FRONTEND]
    else:
        cpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = ""\n'
        gpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format(
            GPU_ID)
286 287
        if 'required' in codeblock and codeblock['required']:
            if codeblock['required'] == 'cpu':
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
                inserted_codes_f = cpu_str
            elif codeblock['required'] == 'gpu':
                inserted_codes_f = gpu_str
        else:
            if RUN_ON_DEVICE == "cpu":
                inserted_codes_f = cpu_str
            elif RUN_ON_DEVICE == "gpu":
                inserted_codes_f = gpu_str
    inserted_codes_b = '\nprint("{}\'s sample code (name:{}, id:{}) is executed successfully!")'.format(
        apiname, codeblock['name'], codeblock['id'])

    cb = codeblock['codes']
    last_future_line_end = find_last_future_line_end(cb)
    if last_future_line_end:
        return cb[:last_future_line_end] + inserted_codes_f + cb[
            last_future_line_end:] + inserted_codes_b
    else:
        return inserted_codes_f + cb + inserted_codes_b
T
tianshuo78520a 已提交
306 307


308 309 310
def sampcd_extract_to_file(srccom, name, htype="def", hname=""):
    """
    Extract sample codes from __doc__, and write them to files.
311

312 313 314 315 316 317
    Args:
        srccom(str): the source comment of some API whose
                     example codes will be extracted and run.
        name(str): the name of the API.
        htype(str): the type of hint banners, def/class/method.
        hname(str): the name of the hint  banners , e.t. def hname.
318

319 320 321
    Returns:
        sample_code_filenames(list of str)
    """
322 323
    global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR  # readonly
    global SUMMARY_INFO  # update
324

325 326 327
    codeblocks = extract_code_blocks_from_docstr(srccom)
    if len(codeblocks) == 0:
        SUMMARY_INFO['nocodes'].append(name)
328
        # detect sample codes using >>> to format and consider this situation as wrong
329 330
        logger.info(htype + " name:" + name)
        logger.info("-----------------------")
331
        if srccom.find("Examples:") != -1:
332
            logger.info("----example code check----")
333
            if srccom.find(">>>") != -1:
334 335 336 337 338 339
                logger.warning(r"""Deprecated sample code style:
    Examples:
        >>>codeline
        >>>codeline

Please use '.. code-block:: python' to format the sample code.""")
340
                return []
T
tianshuo78520a 已提交
341
        else:
342 343 344
            logger.error(
                "Error: No sample code found! Please check if the API comment contais string 'Examples:' correctly"
            )
345
            return []
346

347
    sample_code_filenames = []
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
    for y, cb in enumerate(codeblocks):
        matched = is_required_match(cb['required'], name)
        # matched has three states:
        # True - please execute it;
        # None - no sample code found;
        # False - it need other special equipment or environment.
        # so, the following conditional statements are intentionally arranged.
        if matched == True:
            tfname = os.path.join(SAMPLECODE_TEMPDIR, '{}_example{}'.format(
                name, '.py'
                if len(codeblocks) == 1 else '_{}.py'.format(y + 1)))
            with open(tfname, 'w') as tempf:
                sampcd = insert_codes_into_codeblock(cb, name)
                tempf.write(sampcd)
            sample_code_filenames.append(tfname)
        elif matched is None:
            logger.info('{}\' code block (name:{}, id:{}) is skipped.'.format(
                name, cb['name'], cb['id']))
            SUMMARY_INFO['skiptest'].append("{}-{}".format(name, cb['id']))
        elif matched == False:
            logger.info(
                '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).'.
                format(name, cb['name'], cb['id'], cb['required'],
                       SAMPLE_CODE_TEST_CAPACITY))
            if cb['required'] not in SUMMARY_INFO:
                SUMMARY_INFO[cb['required']] = []
            SUMMARY_INFO[cb['required']].append("{}-{}".format(name, cb['id']))

376 377 378
    return sample_code_filenames


379 380
def execute_samplecode(tfname):
    """
381
    Execute a sample-code test
382 383

    Args:
384
        tfname: the filename of the sample code
385 386 387 388
    
    Returns:
        result: success or not
        tfname: same as the input argument
389 390
        msg: the stdout output of the sample code executing
        time: time consumed by sample code
391
    """
392 393
    result = True
    msg = None
T
tianshuo78520a 已提交
394
    if platform.python_version()[0] in ["3"]:
395 396
        cmd = [sys.executable, tfname]
    else:
397
        logger.error("Error: fail to parse python version!")
398 399 400
        result = False
        exit(1)

401 402 403
    logger.info("----example code check----")
    logger.info("executing sample code: %s", tfname)
    start_time = time.time()
404 405 406 407 408
    subprc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, error = subprc.communicate()
    msg = "".join(output.decode(encoding='utf-8'))
    err = "".join(error.decode(encoding='utf-8'))
409
    end_time = time.time()
410 411

    if subprc.returncode != 0:
412 413 414 415 416 417 418 419 420 421 422
        with open(tfname, 'r') as f:
            logger.warning("""Sample code error found in %s:
-----------------------
%s
-----------------------
subprocess return code: %d
Error Raised from Sample Code:
stderr: %s
stdout: %s
""", tfname, f.read(), subprc.returncode, err, msg)
        logger.info("----example code check failed----")
423 424
        result = False
    else:
425
        logger.info("----example code check success----")
426 427

    # msg is the returned code execution report
428
    return result, tfname, msg, end_time - start_time
T
tianshuo78520a 已提交
429 430


431
def get_filenames(full_test=False):
432
    '''
433
    this function will get the sample code files that pending for check.
434

435 436 437
    Args:
        full_test: the full apis or the increment

438 439
    Returns:

440
        dict: the sample code files pending for check .
441 442

    '''
443
    global whl_error
444
    import paddle
445
    import paddle.fluid.contrib.slim.quantization
446
    whl_error = []
447
    if full_test:
448
        get_full_api_from_pr_spec()
449 450
    else:
        get_incrementapi()
451 452
    all_sample_code_filenames = {}
    with open(API_DIFF_SPEC_FN) as f:
453
        for line in f.readlines():
454
            api = line.replace('\n', '')
455
            try:
456
                api_obj = eval(api)
457
            except AttributeError:
458
                whl_error.append(api)
459
                continue
460 461 462 463
            except SyntaxError:
                logger.warning('line:%s, api:%s', line, api)
                # paddle.Tensor.<lambda>
                continue
464 465 466 467 468 469
            if hasattr(api_obj, '__doc__') and api_obj.__doc__:
                sample_code_filenames = sampcd_extract_to_file(api_obj.__doc__,
                                                               api)
                for tfname in sample_code_filenames:
                    all_sample_code_filenames[tfname] = api
    return all_sample_code_filenames
470 471


472
def get_api_md5(path):
473 474 475 476 477 478 479 480 481
    """
    read the api spec file, and scratch the md5sum value of every api's docstring.

    Args:
        path: the api spec file. ATTENTION the path relative
    
    Returns:
        api_md5(dict): key is the api's real fullname, value is the md5sum.
    """
482
    api_md5 = {}
483 484 485
    API_spec = os.path.abspath(os.path.join(os.getcwd(), "..", path))
    if not os.path.isfile(API_spec):
        return api_md5
486 487 488
    pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})')
    patArgSpec = re.compile(
        r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})')
489 490
    with open(API_spec) as f:
        for line in f.readlines():
491 492 493 494 495
            mo = pat.search(line)
            if not mo:
                mo = patArgSpec.search(line)
            if mo:
                api_md5[mo.group(1)] = mo.group(2)
496 497 498
    return api_md5


499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
def get_full_api():
    """
    get all the apis
    """
    global API_DIFF_SPEC_FN  ## readonly
    from print_signatures import get_all_api_from_modulelist
    member_dict = get_all_api_from_modulelist()
    with open(API_DIFF_SPEC_FN, 'w') as f:
        f.write("\n".join(member_dict.keys()))


def get_full_api_by_walk():
    """
    get all the apis
    """
    global API_DIFF_SPEC_FN  ## readonly
    from print_signatures import get_all_api
    apilist = get_all_api()
    with open(API_DIFF_SPEC_FN, 'w') as f:
518 519 520 521 522 523 524 525 526 527 528 529 530 531
        f.write("\n".join([ai[0] for ai in apilist]))


def get_full_api_from_pr_spec():
    """
    get all the apis
    """
    global API_PR_SPEC_FN, API_DIFF_SPEC_FN  ## readonly
    pr_api = get_api_md5(API_PR_SPEC_FN)
    if len(pr_api):
        with open(API_DIFF_SPEC_FN, 'w') as f:
            f.write("\n".join(pr_api.keys()))
    else:
        get_full_api_by_walk()
532 533


534 535 536 537
def get_incrementapi():
    '''
    this function will get the apis that difference between API_DEV.spec and API_PR.spec.
    '''
538 539 540 541
    global API_DEV_SPEC_FN, API_PR_SPEC_FN, API_DIFF_SPEC_FN  ## readonly
    dev_api = get_api_md5(API_DEV_SPEC_FN)
    pr_api = get_api_md5(API_PR_SPEC_FN)
    with open(API_DIFF_SPEC_FN, 'w') as f:
542 543 544
        for key in pr_api:
            if key in dev_api:
                if dev_api[key] != pr_api[key]:
545 546
                    logger.debug("%s in dev is %s, different from pr's %s", key,
                                 dev_api[key], pr_api[key])
547 548 549
                    f.write(key)
                    f.write('\n')
            else:
550
                logger.debug("%s is not in dev", key)
551 552 553 554
                f.write(key)
                f.write('\n')


555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
def exec_gen_doc():
    result = True
    cmd = ["bash", "document_preview.sh"]
    logger.info("----exec gen_doc----")
    start_time = time.time()
    subprc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, error = subprc.communicate()
    msg = "".join(output.decode(encoding='utf-8'))
    err = "".join(error.decode(encoding='utf-8'))
    end_time = time.time()

    if subprc.returncode != 0:
        logger.info("----gen_doc msg----")
        logger.info(msg)
        logger.error("----gen_doc error msg----")
        logger.error(err)
        logger.error("----exec gen_doc failed----")
        result = False
    else:
        logger.info("----gen_doc msg----")
        logger.info(msg)
        logger.info("----exec gen_doc success----")

    for fn in [
            '/docs/en/develop/index_en.html', '/docs/zh/develop/index_cn.html'
    ]:
        if os.path.exists(fn):
            logger.info('%s exists.', fn)
        else:
            logger.error('%s not exists.', fn)

    # msg is the returned code execution report
    return result, msg, end_time - start_time


591 592 593 594 595 596
arguments = [
    # flags, dest, type, default, help
    ['--gpu_id', 'gpu_id', int, 0, 'GPU device id to use [0]'],
    ['--logf', 'logf', str, None, 'file for logging'],
    ['--threads', 'threads', int, 0, 'sub processes number'],
]
597

598 599 600 601 602 603 604 605 606 607 608

def parse_args():
    """
    Parse input arguments
    """
    global arguments
    parser = argparse.ArgumentParser(description='run Sample Code Test')
    # parser.add_argument('--cpu', dest='cpu_mode', action="store_true",
    #                     help='Use CPU mode (overrides --gpu)')
    # parser.add_argument('--gpu', dest='gpu_mode', action="store_true")
    parser.add_argument('--debug', dest='debug', action="store_true")
609
    parser.add_argument('--full-test', dest='full_test', action="store_true")
610
    parser.add_argument('mode', type=str, help='run on device', default='cpu')
611 612 613 614 615
    parser.add_argument(
        '--build-doc',
        dest='build_doc',
        action='store_true',
        help='build doc if need.')
616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
    for item in arguments:
        parser.add_argument(
            item[0], dest=item[1], help=item[4], type=item[2], default=item[3])

    if len(sys.argv) == 1:
        args = parser.parse_args(['cpu'])
        return args
    #    parser.print_help()
    #    sys.exit(1)

    args = parser.parse_args()
    return args


if __name__ == '__main__':
    args = parse_args()
    if args.debug:
        logger.setLevel(logging.DEBUG)
634 635
    else:
        logger.setLevel(logging.INFO)
636 637 638 639 640 641 642 643 644 645 646 647 648 649
    if args.logf:
        logfHandler = logging.FileHandler(args.logf)
        logfHandler.setFormatter(
            logging.Formatter(
                "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s"
            ))
        logger.addHandler(logfHandler)

    if args.mode == "gpu":
        GPU_ID = args.gpu_id
        logger.info("using GPU_ID %d", GPU_ID)
    elif args.mode != "cpu":
        logger.error("Unrecognized argument:%s, 'cpu' or 'gpu' is desired.",
                     args.mode)
650
        sys.exit("Invalid arguments")
651
    RUN_ON_DEVICE = args.mode
652
    get_test_capacity()
653 654 655 656 657 658 659 660 661 662 663
    logger.info("API check -- Example Code")
    logger.info("sample_test running under python %s",
                platform.python_version())

    if os.path.exists(SAMPLECODE_TEMPDIR):
        if not os.path.isdir(SAMPLECODE_TEMPDIR):
            os.remove(SAMPLECODE_TEMPDIR)
            os.mkdir(SAMPLECODE_TEMPDIR)
    else:
        os.mkdir(SAMPLECODE_TEMPDIR)

664
    filenames = get_filenames(args.full_test)
665
    if len(filenames) == 0 and len(whl_error) == 0:
666
        logger.info("-----API_PR.spec is the same as API_DEV.spec-----")
667
        exit(0)
668 669 670 671 672 673
    logger.info("API_PR is diff from API_DEV: %s", filenames)

    threads = multiprocessing.cpu_count()
    if args.threads:
        threads = args.threads
    po = multiprocessing.Pool(threads)
674
    results = po.map_async(execute_samplecode, filenames.keys())
675 676
    po.close()
    po.join()
677

678
    result = results.get()
679

680
    # delete temp files
681 682
    if not args.debug:
        shutil.rmtree(SAMPLECODE_TEMPDIR)
683

684 685
    stdout_handler = logging.StreamHandler(stream=sys.stdout)
    logger.addHandler(stdout_handler)
686
    logger.info("----------------End of the Check--------------------")
687
    if len(whl_error) != 0:
688 689 690 691 692 693 694
        logger.info("%s is not in whl.", whl_error)
        logger.info("")
        logger.info("Please check the whl package and API_PR.spec!")
        logger.info("You can follow these steps in order to generate API.spec:")
        logger.info("1. cd ${paddle_path}, compile paddle;")
        logger.info("2. pip install build/python/dist/(build whl package);")
        logger.info(
695 696 697
            "3. run 'python tools/print_signatures.py paddle > paddle/fluid/API.spec'."
        )
        for temp in result:
698 699 700 701
            if not temp[0]:
                logger.info("In addition, mistakes found in sample codes: %s",
                            temp[1])
        logger.info("----------------------------------------------------")
702 703
        exit(1)
    else:
704
        timeovered_test = {}
705
        for temp in result:
706 707 708
            if not temp[0]:
                logger.info("In addition, mistakes found in sample codes: %s",
                            temp[1])
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
                SUMMARY_INFO['failed'].append(temp[1])
            else:
                SUMMARY_INFO['success'].append(temp[1])
            if temp[3] > 10:
                timeovered_test[temp[1]] = temp[3]

        if len(timeovered_test):
            logger.info("%d sample codes ran time over 10s",
                        len(timeovered_test))
            if args.debug:
                for k, v in timeovered_test.items():
                    logger.info('{} - {}s'.format(k, v))
        if len(SUMMARY_INFO['success']):
            logger.info("%d sample codes ran success",
                        len(SUMMARY_INFO['success']))
        for k, v in SUMMARY_INFO.items():
            if k not in ['success', 'failed', 'skiptest', 'nocodes']:
                logger.info("%d sample codes required not match for %s",
                            len(v), k)
        if len(SUMMARY_INFO['skiptest']):
            logger.info("%d sample codes skipped",
                        len(SUMMARY_INFO['skiptest']))
            if args.debug:
                logger.info('\n'.join(SUMMARY_INFO['skiptest']))
        if len(SUMMARY_INFO['nocodes']):
            logger.info("%d apis don't have sample codes",
                        len(SUMMARY_INFO['nocodes']))
            if args.debug:
                logger.info('\n'.join(SUMMARY_INFO['nocodes']))
        if len(SUMMARY_INFO['failed']):
            logger.info("%d sample codes ran failed",
                        len(SUMMARY_INFO['failed']))
            logger.info('\n'.join(SUMMARY_INFO['failed']))
            logger.info(
                "Mistakes found in sample codes. Please recheck the sample codes."
            )
745
            exit(1)
746

747
    logger.info("Sample code check is successful!")
748 749 750 751

    if args.mode == "cpu":
        # As cpu mode is also run with the GPU whl, so skip it in gpu mode.
        exec_gen_doc()