gen_ut_cmakelists.py 20.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
16 17 18 19
import os
import argparse

# port range (21200, 23000) is reserved for dist-ops
20 21 22 23 24 25 26 27 28


# function to process pythonpath env
# append "${PADDLE_BINARY_DIR}/python" to PYTHONPATH
def _process_PYTHONPATH(pythonpath_option):
    pythonpath_option += ":${PADDLE_BINARY_DIR}/python"
    return pythonpath_option


R
Roc 已提交
29
def _process_envs(envs):
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
    """
    Desc:
        Input a str and output a str with the same function to specify some environment variables.
    Here we can give a specital process for some variable if needed.
    Example 1:
        Input: "http_proxy=;PYTHONPATH=.."
        Output: "http_proxy=;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python"
    Example 2:
        Input: "http_proxy=;https_proxy=123.123.123.123:1230"
        Output: "http_proxy=;https_proxy=123.123.123.123:1230"
    """
    envs = envs.strip()

    envs_parts = envs.split(";")
    processed_envs = []

    for p in envs_parts:
        assert " " not in p and \
            re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None, \
            f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_',
and the var can not contain space in either env names or values.
However the var's format is '{p}'."""

R
Roc 已提交
53
        # if p starts with "PYTHONPATH=", then process python path
54 55 56 57 58 59 60 61
        if re.compile("^PYTHONPATH=").search(p):
            p = _process_PYTHONPATH(p)

        processed_envs.append(p)

    return ";".join(processed_envs)


R
Roc 已提交
62
def _process_conditions(conditions):
63 64 65 66 67 68 69 70 71 72 73 74
    """
    Desc:
        Input condition expression in cmake grammer and return a string warpped by 'AND ()'.
        If the conditions string is empty, return an empty string.
    Example 1:
        Input: "LINUX"
        Output: "AND (LINUX)"
    Example 2:
        Input: ""
        Output: ""
    """
    if len(conditions.strip()) == 0:
75
        conditions = []
76
    else:
77 78
        conditions = conditions.strip().split(";")
    return [c.strip() for c in conditions]
79 80


R
Roc 已提交
81
def _proccess_archs(arch):
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
    """
    desc:
        Input archs options and warp it with 'WITH_', 'OR' and '()' in cmakelist grammer.
        The case is ignored.
        If the input is empty, return "LOCAL_ALL_ARCH".
    Example 1:
        Input: 'gpu'
        Output: '(WITH_GPU)'
    Example 2:
        Input: 'gpu;ROCM'
        Output: '(WITH_GPU OR WITH_ROCM)'
    """
    archs = ""
    arch = arch.upper().strip()
    if len(arch) > 0:
        for a in arch.split(";"):
98 99
            assert a in ["GPU", "ROCM", "ASCEND", "ASCEND_CL", "XPU"], \
                f"""Supported arhc options are "GPU", "ROCM", "ASCEND" and "ASCEND_CL", "XPU", but the options is {a}"""
100 101 102 103 104 105 106
            archs += "WITH_" + a.upper() + " OR "
        arch = "(" + archs[:-4] + ")"
    else:
        arch = "LOCAL_ALL_ARCH"
    return arch


R
Roc 已提交
107
def _process_os(os_):
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
    """
    Desc:
        Input os options and output warpped options with 'OR' and '()'
        If the input is empty, return "LOCAL_ALL_PLAT"
    Example 1:
        Input: "WIN32"
        Output: "(WIN32)"
    Example 2:
        Input: "WIN32;linux"
        Output: "(WIN32 OR LINUX)"
    """
    os_ = os_.strip()
    if len(os_) > 0:
        os_ = os_.upper()
        for p in os_.split(';'):
            assert p in [
                "WIN32", "APPLE", "LINUX"
            ], f"""Supported os options are 'WIN32', 'APPLE' and 'LINUX', but the options is {p}"""
        os_ = os_.replace(";", " OR ")
        os_ = "(" + os_ + ")"
    else:
        os_ = "LOCAL_ALL_PLAT"
    return os_


# check whether run_serial is 0, 1 or empty
R
Roc 已提交
134
def _process_run_serial(run_serial):
135 136 137 138
    rs = run_serial.strip()
    assert rs in ["1", "0", ""], \
        f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}"""
    if rs == "":
R
Roc 已提交
139
        return ""
140 141 142
    return rs


R
Roc 已提交
143
def _file_with_extension(prefix, suffixes):
144 145 146 147 148 149 150 151 152 153
    """
    Desc:
        check whether test file exists. 
    """
    for ext in suffixes:
        if os.path.isfile(prefix + ext):
            return True
    return False


R
Roc 已提交
154
def _process_name(name, curdir):
155 156 157 158 159 160 161 162 163 164
    """
    Desc:
        check whether name is with a legal format and check whther the test file exists.
    """
    name = name.strip()
    assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \
        f"""If line is not the header of table, the test name must begin with "test_" """ \
        f"""and the following substring must include at least one char of "0-9", "a-z", "A-Z" or "_"."""
    filepath_prefix = os.path.join(curdir, name)
    suffix = [".py", ".sh"]
R
Roc 已提交
165
    assert _file_with_extension(filepath_prefix, suffix), \
166 167 168 169 170
        f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'"""

    return name


R
Roc 已提交
171 172 173 174 175 176 177 178 179 180 181
def _norm_dirs(dirs):
    # reform all dirs' path as normal absolute path
    # abspath() can automatically normalize the path format
    norm_dirs = []
    for d in dirs:
        d = os.path.abspath(d)
        if d not in norm_dirs:
            norm_dirs.append(d)
    return norm_dirs


R
Roc 已提交
182
def _process_run_type(run_type):
183
    rt = run_type.strip()
R
Roc 已提交
184
    # completely match one of the strings: 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'
185 186 187 188 189 190
    assert re.compile("^(NIGHTLY|EXCLUSIVE|CINN|DIST|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$").search(rt), \
        f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'""" \
        f"""but the run_type is {rt}"""
    return rt


R
Roc 已提交
191 192
class DistUTPortManager():

R
Roc 已提交
193
    def __init__(self, ignore_dirs=[]):
R
Roc 已提交
194 195 196 197 198 199
        self.dist_ut_port = 21200
        self.assigned_ports = dict()
        self.last_test_name = ""
        self.last_test_cmake_file = ""
        self.no_cmake_dirs = []
        self.processed_dirs = set()
R
Roc 已提交
200
        self.ignore_dirs = _norm_dirs(ignore_dirs)
R
Roc 已提交
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265

    def reset_current_port(self, port=None):
        self.dist_ut_port = 21200 if port is None else port

    def get_currnt_port(self):
        return self.dist_ut_port

    def gset_port(self, test_name, port):
        '''
        Get and set a port for unit test named test_name. If the test has been already holding a port, return the port it holds.
        Else assign the input port as a new port to the test.
        '''
        if test_name not in self.assigned_ports:
            self.assigned_ports[test_name] = port
        self.dist_ut_port = max(self.dist_ut_port,
                                self.assigned_ports[test_name])
        return self.assigned_ports[test_name]

    def process_dist_port_num(self, port_num):
        assert re.compile("^[0-9]+$").search(port_num) and int(port_num) > 0 or port_num.strip()=="", \
            f"""port_num must be foramt as a positive integer or empty, but this port_num is '{port_num}'"""
        port_num = port_num.strip()
        if len(port_num) == 0:
            return 0
        port = self.dist_ut_port
        assert port < 23000, "dist port is exhausted"
        self.dist_ut_port += int(port_num)
        return port

    def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name):
        '''
        Desc:
            Find all signed ut ports in cmake_file and update the ASSIGNED_PORTS
            and keep the DIST_UT_PORT max of all assigned ports
        '''
        with open(cmake_file_name) as cmake_file:
            # match lines including 'PADDLE_DIST_UT_PORT=' followed by a number
            port_reg = re.compile("PADDLE_DIST_UT_PORT=[0-9]+")
            lines = cmake_file.readlines()
            for idx, line in enumerate(lines):
                matched = port_reg.search(line)
                if matched is None:
                    continue
                p = matched.span()
                port = int(line[p[0]:p[1]].split("=")[-1])

                # find the test name which the port belongs to
                for k in range(idx, 0, -1):
                    if lines[k].strip() == "START_BASH":
                        break
                name = lines[k - 1].strip()

                # matcg right tets name format, the name must start with 'test_' follwed bu at least one cahr of
                # '0-9'. 'a-z'. 'A-Z' or '_'
                assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \
                    f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong
                    at line {k-1}, in file {cmake_file_name}
                    '''
                self.gset_port(name, port)

                # get the test_name which latest assigned port belongs to
                if self.assigned_ports[name] == self.dist_ut_port:
                    self.last_test_name = name
                    self.last_test_cmake_file = cmake_file_name

R
Roc 已提交
266
    def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0):
R
Roc 已提交
267 268 269 270 271 272
        '''
        Desc:
            get all assigned dist ports to keep port of unmodified test fixed.
        '''
        if current_work_dir in self.processed_dirs:
            return
273

R
Roc 已提交
274
        # if root(depth==0)
R
Roc 已提交
275 276 277 278 279 280 281 282 283
        if depth == 0:
            self.processed_dirs.clear()

        self.processed_dirs.add(current_work_dir)
        contents = os.listdir(current_work_dir)
        cmake_file = os.path.join(current_work_dir, "CMakeLists.txt")
        csv = cmake_file.replace("CMakeLists.txt", 'testslist.csv')

        if os.path.isfile(csv) or os.path.isfile(cmake_file):
R
Roc 已提交
284
            if current_work_dir not in self.ignore_dirs:
R
Roc 已提交
285 286 287 288 289 290 291 292 293 294
                if os.path.isfile(cmake_file) and os.path.isfile(csv):
                    self._init_dist_ut_ports_from_cmakefile(cmake_file)
                elif not os.path.isfile(cmake_file):
                    # put the directory which has csv but no cmake into NO_CMAKE_DIR_WARNING
                    self.no_cmake_dirs.append(current_work_dir)

            # recursively process the subdirectories
            for c in contents:
                c_path = os.path.join(current_work_dir, c)
                if os.path.isdir(c_path):
R
Roc 已提交
295
                    self.parse_assigned_dist_ut_ports(c_path, depth + 1)
R
Roc 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323

        if depth == 0:
            # After all directories are scanned and processed
            # 1. Get the num_port of last added test and set DIST_UT_PORT+=num_port
            #    to guarantee the DIST_UT_PORT is not assined
            # 2. Summary all the directories which include csv but no cmake and show an error
            #    if such a drectory exists

            # step 1
            if len(self.last_test_name) > 0 and len(
                    self.last_test_cmake_file) > 0:
                with open(
                        self.last_test_cmake_file.replace(
                            "CMakeLists.txt", "testslist.csv")) as csv_file:
                    found = False
                    for line in csv_file.readlines():
                        name, _, _, _, _, launcher, num_port, _, _, _ = line.strip(
                        ).split(",")
                        if name == self.last_test_name:
                            found = True
                            break
                assert found, f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'"
                if launcher[-2:] == ".sh":
                    self.process_dist_port_num(num_port)

            # step 2
            err_msg = f"""==================[No Old CMakeLists.txt Error]==================================
        Following directories has no CmakeLists.txt files:
324
    """
R
Roc 已提交
325 326 327 328 329 330 331 332 333 334
            for c in self.no_cmake_dirs:
                err_msg += "   " + c + "\n"
            err_msg += """
        This may cause the dist ports different with the old version.
        If the directories are newly created or there is no CMakeLists.txt before, or ignore this error, you
        must specify the directories using the args option --ignore-cmake-dirs/-i.
        If you want to keep the dist ports of old tests unchanged, please ensure the old
        verson CMakeLists.txt file existing before using the gen_ut_cmakelists tool to 
        generate new CmakeLists.txt files.
    ====================================================================================
335
    """
R
Roc 已提交
336 337 338 339 340
            assert len(self.no_cmake_dirs) == 0, err_msg


class CMakeGenerator():

R
Roc 已提交
341
    def __init__(self, current_dirs, ignore_dirs):
R
Roc 已提交
342
        self.processed_dirs = set()
R
Roc 已提交
343 344
        self.port_manager = DistUTPortManager(ignore_dirs)
        self.current_dirs = _norm_dirs(current_dirs)
R
Roc 已提交
345 346 347

    def prepare_dist_ut_port(self):
        for c in self._find_root_dirs():
R
Roc 已提交
348
            self.port_manager.parse_assigned_dist_ut_ports(c, depth=0)
R
Roc 已提交
349 350 351 352 353 354 355 356 357 358 359 360

    def parse_csvs(self):
        for c in self.current_dirs:
            self._gen_cmakelists(c)

    def _find_root_dirs(self):
        root_dirs = []
        # for each current directory, find its highest ancient directory (at least itself)
        # which includes CMakeLists.txt or testslist.csv.txt in the filesys tree
        for c in self.current_dirs:
            while True:
                ppath = os.path.dirname(c)
R
Roc 已提交
361
                if ppath == c:
R
Roc 已提交
362 363 364 365 366
                    break
                cmake = os.path.join(ppath, "CMakeLists.txt")
                csv = os.path.join(ppath, "testslist.csv.txt")
                if not (os.path.isfile(cmake) or os.path.isfile(csv)):
                    break
R
Roc 已提交
367
                c = ppath
R
Roc 已提交
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
            if c not in root_dirs:
                root_dirs.append(c)
        return root_dirs

    def _parse_line(self, line, curdir):
        """
        Desc:
            Input a line in csv file and output a string in cmake grammer, adding the specified test and setting its properties.
        Example:
            Input: "test_allreduce,linux,gpu;rocm,120,DIST,test_runner.py,20071,1,PYTHONPATH=..;http_proxy=;https_proxy=,"
            Output:
                "if((WITH_GPU OR WITH_ROCM) AND (LINUX) )
                    py_test_modules(
                    test_allreduce
                    MODULES
                    test_allreduce
                    ENVS
                    "PADDLE_DIST_UT_PORT=20071;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python;http_proxy=;https_proxy=")
                    set_tests_properties(test_allreduce PROPERTIES  TIMEOUT "120" RUN_SERIAL 1)
                endif()"
        """

        name, os_, archs, timeout, run_type, launcher, num_port, run_serial, envs, conditions = line.strip(
        ).split(",")

        # name == "name" means the line being parsed is the header of the table
        # we should skip this line and return empty here.
        if name == "name":
            return ""
        name = _process_name(name, curdir)

        envs = _process_envs(envs)
        conditions = _process_conditions(conditions)
        archs = _proccess_archs(archs)
        os_ = _process_os(os_)
        run_serial = _process_run_serial(run_serial)
        run_type = _process_run_type(run_type)

        cmd = ""

        for c in conditions:
            cmd += f"if ({c})\n"

        if launcher[-3:] == ".sh":
            dist_ut_port = self.port_manager.process_dist_port_num(num_port)
            dist_ut_port = self.port_manager.gset_port(name, dist_ut_port)
            cmd += f'''if({archs} AND {os_})
        bash_test_modules(
        {name}
        START_BASH
        {launcher}
        LABELS
        "RUN_TYPE={run_type}"
        ENVS
        "PADDLE_DIST_UT_PORT={dist_ut_port};{envs}")%s
    endif()
    '''
        else:
            cmd += f'''if({archs} AND {os_})
        py_test_modules(
        {name}
        MODULES
        {name}
        ENVS
        "{envs}")%s
    endif()
    '''
        time_out_str = f' TIMEOUT "{timeout}"' if len(
            timeout.strip()) > 0 else ''
        run_serial_str = f' RUN_SERIAL {run_serial}' if len(
            run_serial) > 0 else ''
        if len(time_out_str) > 0 or len(run_serial_str) > 0:
            set_properties = f'''
        set_tests_properties({name} PROPERTIES{time_out_str}{run_serial_str})'''
        else:
            set_properties = ""
        cmd = cmd % set_properties
        for _ in conditions:
            cmd += f"endif()\n"
        return cmd

    def _gen_cmakelists(self, current_work_dir, depth=0):
        if depth == 0:
            self.processed_dirs.clear()
        print("procfessing dir:", current_work_dir)
        if current_work_dir == "":
            current_work_dir = "."

        contents = os.listdir(current_work_dir)
        contents.sort()
        sub_dirs = []
        for c in contents:
            c_path = os.path.join(current_work_dir, c)
            if c_path in self.processed_dirs:
                return
            if not os.path.isdir(c_path):
                continue
            self.processed_dirs.add(c_path)
466 467
            if os.path.isfile(os.path.join(current_work_dir, c, "testslist.csv")) \
                or os.path.isfile(os.path.join(current_work_dir, c, "CMakeLists.txt")):
R
Roc 已提交
468 469
                self._gen_cmakelists(os.path.join(current_work_dir, c),
                                     depth + 1)
470 471
                sub_dirs.append(c)

R
Roc 已提交
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
        if not os.path.isfile(os.path.join(current_work_dir, "testslist.csv")):
            return
        cmds = """# This file is generated by ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py.
    # Please don't modify this file manually.
    # If you need to change unittests in this file, please modify testslist.csv in the current directory 
    # and then run the command `python3 ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py -f ${CURRENT_DIRECTORY}/testslist.csv`
    set(LOCAL_ALL_ARCH ON)
    set(LOCAL_ALL_PLAT ON)\n"""
        with open(f"{current_work_dir}/testslist.csv") as csv_file:
            for i, line in enumerate(csv_file.readlines()):
                try:
                    cmds += self._parse_line(line, current_work_dir)
                except Exception as e:
                    print("===============PARSE LINE ERRORS OCCUR==========")
                    print(e)
                    print(f"[ERROR FILE]: {current_work_dir}/testslist.csv")
                    print(f"[ERROR LINE {i+1}]: {line.strip()}")
                    exit(1)

        for sub in sub_dirs:
            cmds += f"add_subdirectory({sub})\n"
        print(cmds, end="")
        with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file:
            print(cmds, end="", file=cmake_file)
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--files",
        "-f",
        type=str,
        required=False,
        default=[],
        nargs="+",
        help=
        "Input a list of files named testslist.csv and output files named CmakeLists.txt in the same directories as the csv files respectly"
    )
    parser.add_argument(
        "--dirpaths",
        "-d",
        type=str,
        required=False,
        default=[],
        nargs="+",
        help=
        "Input a list of dir paths including files named testslist.csv and output CmakeLists.txt in these directories respectly"
    )
R
Roc 已提交
520 521 522 523 524 525 526 527 528 529
    parser.add_argument(
        "--ignore-cmake-dirs",
        '-i',
        type=str,
        required=False,
        default=[],
        nargs='*',
        help=
        "To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive."
    )
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
    args = parser.parse_args()

    assert not (len(args.files) == 0 and len(args.dirpaths)
                == 0), "You must provide at leate one file or dirpath"
    current_work_dirs = []
    if len(args.files) >= 1:
        for p in args.files:
            assert os.path.basename(
                p) == "testslist.csv", "you must input file named testslist.csv"
        current_work_dirs = current_work_dirs + [
            os.path.dirname(file) for file in args.files
        ]
    if len(args.dirpaths) >= 1:
        current_work_dirs = current_work_dirs + [d for d in args.dirpaths]

R
Roc 已提交
545
    cmake_generator = CMakeGenerator(current_work_dirs, args.ignore_cmake_dirs)
R
Roc 已提交
546 547
    cmake_generator.prepare_dist_ut_port()
    cmake_generator.parse_csvs()
新手
引导
客服 返回
顶部