gen_doc.py

import paddle
import os
import shutil
import time
import pkgutil
import types
import contextlib
import argparse

en_suffix = "_en.rst"
cn_suffix = "_cn.rst"
file_path_dict = {}


def _str2bool(s):
    if s.lower() in ('yes', 'true'):
        return True
    elif s.lower() in ('no', 'false'):
        return False
    else:
        raise argparse.ArgumentTypeError('Unsupported value encountered.')


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--gen_cn',
        type=_str2bool,
        default=False,
        help='Generate the cn documentations')
    parser.add_argument(
        '--gen_en',
        type=_str2bool,
        default=True,
        help='Generate the en documentations')
    return parser.parse_args()


def gen_doc_dir(root_path='paddle'):
    backup_path = root_path + "_" + str(int(time.time()))
    # move old dirs
    if os.path.isdir(root_path):
        os.rename(root_path, backup_path)

    os.mkdir(root_path)
    for filefiner, name, ispkg in pkgutil.walk_packages(
            path=paddle.__path__, prefix=paddle.__name__ + '.'):
        path = name.replace(".", "/")

        try:
            m = eval(name)
        except AttributeError:
            pass
        else:
            if hasattr(eval(name), "__all__"):
                os.makedirs(path)
                for api in list(set(eval(name).__all__)):
                    os.mknod(path + "/" + api + en_suffix)
                    gen = EnDocGenerator()
                    with gen.guard(path + "/" + api + en_suffix):
                        if api != 'test, get_dict':
                            gen.module_name = name
                            gen.api = api
                            gen.print_header_reminder()
                            gen.print_item()


def clean_en_files(path="./paddle"):
    for root, dirs, files in os.walk(path):
        for file in files:
            if file.endswith(en_suffix):
                os.remove(os.path.join(root, file))


def gen_en_files(root_path='paddle'):
    for filefiner, name, ispkg in pkgutil.walk_packages(
            path=paddle.__path__, prefix=paddle.__name__ + '.'):
        path = name.replace(".", "/")

        try:
            m = eval(name)
        except AttributeError:
            pass
        else:
            if hasattr(eval(name), "__all__"):
                for api in list(set(eval(name).__all__)):
                    gen = EnDocGenerator()
                    with gen.guard(path + "/" + api + en_suffix):
                        if api != 'test, get_dict':
                            gen.module_name = name
                            gen.api = api
                            gen.print_header_reminder()
                            gen.print_item()


def get_cn_files_path_dict(path=None):
    if path == None:
        path = os.getcwd() + "/../../fluid/api_cn"

    for root, dirs, files in os.walk(path):
        for file in files:
            file = file.replace(cn_suffix, "", 1)
            val = str(root + "/" + file)
            if file in file_path_dict:
                file_path_dict[file].append(val)
            else:
                file_path_dict[file] = [val]


def copy_cn_files():
    path = "./paddle"
    for root, dirs, files in os.walk(path):
        for file in files:
            f = file.replace(en_suffix, "", 1)
            if f in file_path_dict:
                key = root[len(path):] + "/" + f
                isfind = False
                for p in file_path_dict[f]:
                    if p.find(key) != -1:
                        src = p + cn_suffix
                        dst = root + "/" + f + cn_suffix
                        shutil.copy(src, dst)
                        isfind = True
                        break
                if not isfind:
                    src = find_max_size_file(file_path_dict[f]) + cn_suffix
                    dst = root + "/" + f + cn_suffix
                    shutil.copy(src, dst)


def check_cn_en_match(path="./paddle", diff_file="en_cn_files_diff"):
    fo = open(diff_file, 'w')
    fo.write("exist\tnot_exits\n")
    for root, dirs, files in os.walk(path):
        for file in files:
            if file.endswith(en_suffix):
                cf = file.replace(en_suffix, cn_suffix)
                if not os.path.exists(root + "/" + cf):
                    fo.write(
                        os.path.join(root, file) + "\t" + os.path.join(
                            root, cf) + "\n")

            elif file.endswith(cn_suffix):
                ef = file.replace(cn_suffix, en_suffix)
                if not os.path.exists(root + "/" + ef):
                    fo.write(
                        os.path.join(root, file) + "\t" + os.path.join(
                            root, ef) + "\n")
    fo.close()


def find_max_size_file(files):
    max_size = 0
    file = ""
    for f in files:
        if os.path.getsize(f + cn_suffix) > max_size:
            max_size = os.path.getsize(f + cn_suffix)
            file = f

    return file


class EnDocGenerator(object):
    def __init__(self, name=None, api=None):
        self.module_name = name
        self.api = api
        self.stream = None

    @contextlib.contextmanager
    def guard(self, filename):
        assert self.stream is None, "stream must be None"
        self.stream = open(filename, 'w')
        yield
        self.stream.close()
        self.stream = None

    def print_item(self):
        try:
            m = eval(self.module_name + "." + self.api)
        except AttributeError:
            #print("attribute error: module_name=" + self.module_name  + ", api=" + self.api)
            pass
        else:
            if isinstance(
                    eval(self.module_name + "." + self.api), types.TypeType):
                self.print_class()
            elif isinstance(
                    eval(self.module_name + "." + self.api),
                    types.FunctionType):
                self.print_function()

    def print_header_reminder(self):
        self.stream.write('''..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
    !DO NOT EDIT THIS FILE MANUALLY!

''')

    def _print_ref_(self):
        self.stream.write(".. _api_{0}_{1}:\n\n".format("_".join(
            self.module_name.split(".")), self.api))

    def _print_header_(self, name, dot, is_title):
        dot_line = dot * len(name)
        if is_title:
            self.stream.write(dot_line)
            self.stream.write('\n')
        self.stream.write(name)
        self.stream.write('\n')
        self.stream.write(dot_line)
        self.stream.write('\n')
        self.stream.write('\n')

    def print_class(self):
        self._print_ref_()
        self._print_header_(self.api, dot='-', is_title=False)
        if "fluid.dygraph" in self.module_name:
            self.stream.write('''..  autoclass:: paddle.{0}.{1}
    :members:
    :noindex:

'''.format(self.module_name, self.api))
        elif "fluid.optimizer" in self.module_name:
            self.stream.write('''..  autoclass:: paddle.{0}.{1}
    :members:
    :inherited-members:
    :exclude-members: apply_gradients, apply_optimize, backward, load
    :noindex:

'''.format(self.module_name, self.api))
        else:
            self.stream.write('''..  autoclass:: paddle.{0}.{1}
    :members:
    :inherited-members:
    :noindex:

'''.format(self.module_name, self.api))

    def print_function(self):
        self._print_ref_()
        self._print_header_(self.api, dot='-', is_title=False)
        self.stream.write('''..  autofunction:: paddle.{0}.{1}
    :noindex:

'''.format(self.module_name, self.api))


if __name__ == "__main__":
    args = parse_args()
    if args.gen_cn:
        gen_doc_dir()
        get_cn_files_path_dict()
        copy_cn_files()

    clean_en_files()
    if args.gen_en:
        gen_en_files()
        check_cn_en_match()