From ec866c5acdd83d0abe97bb151ee0c3aeb8b9a78c Mon Sep 17 00:00:00 2001 From: Andrey Kamaev Date: Mon, 11 Jul 2011 15:03:42 +0000 Subject: [PATCH] Improved rst_parser and javadoc generation scripts --- modules/java/gen_javadoc.py | 313 ++++++++++++++++++++---------------- modules/java/rst_parser.py | 4 +- 2 files changed, 180 insertions(+), 137 deletions(-) diff --git a/modules/java/gen_javadoc.py b/modules/java/gen_javadoc.py index 40f21d5989..cddde1499f 100644 --- a/modules/java/gen_javadoc.py +++ b/modules/java/gen_javadoc.py @@ -1,147 +1,185 @@ import os, sys, re, string, glob - -javadoc_marker = "//javadoc:" - -def parceJavadocMarker(line): - assert line.lstrip().startswith(javadoc_marker) - offset = line[:line.find(javadoc_marker)] - line = line.strip()[len(javadoc_marker):] - args_start = line.rfind("(") - args_end = line.rfind(")") - assert args_start * args_end > 0 - if args_start >= 0: - assert args_start < args_end - return (line[:args_start].strip(), offset, filter(None, list(arg.strip() for arg in line[args_start+1:args_end].split(",")))) - return (line, offset, []) - -def document(infile, outfile, decls): - inf = open(infile, "rt") - outf = open(outfile, "wt") - try: - for l in inf.readlines(): - if l.lstrip().startswith(javadoc_marker): - marker = parceJavadocMarker(l) - decl = decls.get(marker[0],None) - if decl: - for line in makeJavadoc(decl, decls, marker[2]).split("\n"): - outf.write(marker[1] + line + "\n") +allmodules = ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"] +verbose = False +show_warnings = True +show_errors = True + +class JavadocGenerator(object): + def __init__(self, definitions = {}, javadoc_marker = "//javadoc:"): + self.definitions = definitions + self.javadoc_marker = javadoc_marker + self.markers_processed = 0 + self.markers_documented = 0 + self.params_documented = 0 + self.params_undocumented = 0 + + def parceJavadocMarker(self, line): + assert line.lstrip().startswith(self.javadoc_marker) + offset = line[:line.find(self.javadoc_marker)] + line = line.strip()[len(self.javadoc_marker):] + args_start = line.rfind("(") + args_end = line.rfind(")") + assert args_start * args_end > 0 + if args_start >= 0: + assert args_start < args_end + return (line[:args_start].strip(), offset, filter(None, list(arg.strip() for arg in line[args_start+1:args_end].split(",")))) + return (line, offset, []) + + def document(self, infile, outfile): + inf = open(infile, "rt") + outf = open(outfile, "wt") + module = os.path.splitext(os.path.basename(infile))[0] + if module not in allmodules: + module = "unknown" + try: + for l in inf.readlines(): + if l.lstrip().startswith(self.javadoc_marker): + marker = self.parceJavadocMarker(l) + self.markers_processed += 1 + decl = self.definitions.get(marker[0],None) + if decl: + javadoc = self.makeJavadoc(decl, marker[2]) + if verbose: + print + print "Javadoc for \"%s\" File: %s (line %s)" % (decl["name"], decl["file"], decl["line"]) + print javadoc + for line in javadoc.split("\n"): + outf.write(marker[1] + line + "\n") + self.markers_documented += 1 + elif show_errors: + print >> sys.stderr, "gen_javadoc error: could not find documentation for %s (module: %s)" % (l.lstrip()[len(self.javadoc_marker):-1].strip(), module) else: - print "Error: could not find documentation for %s" % l.lstrip()[len(javadoc_marker):-1] - else: - outf.write(l.replace("\t", " ").rstrip()+"\n") - except: - inf.close() - outf.close() - os.remove(outfile) - raise - else: - inf.close() - outf.close() - -def ReformatForJavadoc(s): - out = "" - for term in s.split("\n"): - if term.startswith("*") or term.startswith("#."): - term = " " + term - if not term: - out += " *\n" + outf.write(l.replace("\t", " ").rstrip()+"\n") + except: + inf.close() + outf.close() + os.remove(outfile) + raise else: - pos_start = 0 - pos_end = min(77, len(term)-1) - while pos_start < pos_end: - if pos_end - pos_start == 77: - while pos_end >= pos_start+60: - if not term[pos_end].isspace(): - pos_end -= 1 - else: - break - if pos_end < pos_start+60: - pos_end = min(pos_start + 77, len(term)-1) - while pos_end < len(term): + inf.close() + outf.close() + + def ReformatForJavadoc(self, s): + out = "" + for term in s.split("\n"): + if term.startswith("*") or term.startswith("#."): + term = " " + term + if not term: + out += " *\n" + else: + pos_start = 0 + pos_end = min(77, len(term)-1) + while pos_start < pos_end: + if pos_end - pos_start == 77: + while pos_end >= pos_start+60: if not term[pos_end].isspace(): - pos_end += 1 + pos_end -= 1 else: break - out += " * " + term[pos_start:pos_end+1].rstrip() + "\n" - pos_start = pos_end + 1 - pos_end = min(pos_start + 77, len(term)-1) - return out - -def getJavaName(decl): - name = "org.opencv." - name += decl["module"] - if "class" in decl: - name += "." + decl["class"] - else: - name += "." + decl["module"].capitalize() - if "method" in decl: - name += "." + decl["method"] - return name - -def getDocURL(decl): - url = "http://opencv.itseez.com/modules/" - url += decl["module"] - url += "/doc/" - url += os.path.basename(decl["file"]).replace(".rst",".html") - url += "#" + decl["name"].replace("::","-").replace("()","").replace("=","").strip().rstrip("_").replace(" ","-").replace("_","-").lower() - return url - -def makeJavadoc(decl, decls, args = None): - doc = "" - prefix = "/**\n" - - if decl.get("isclass", False): - decl_type = "class" - elif decl.get("isstruct", False): - decl_type = "struct" - elif "class" in decl: - decl_type = "method" - else: - decl_type = "function" - - # brief goes first - if "brief" in decl: - doc += prefix + ReformatForJavadoc(decl["brief"]) - prefix = " *\n" - elif "long" not in decl: - print "Warning: no description for " + decl_type + " \"%s\" File: %s (line %s)" % (func["name"], func["file"], func["line"]) - doc += prefix + ReformatForJavadoc("This " + decl_type + " is undocumented") - prefix = " *\n" + if pos_end < pos_start+60: + pos_end = min(pos_start + 77, len(term)-1) + while pos_end < len(term): + if not term[pos_end].isspace(): + pos_end += 1 + else: + break + out += " * " + term[pos_start:pos_end+1].rstrip() + "\n" + pos_start = pos_end + 1 + pos_end = min(pos_start + 77, len(term)-1) + return out + + def getJavaName(self, decl): + name = "org.opencv." + name += decl["module"] + if "class" in decl: + name += "." + decl["class"] + else: + name += "." + decl["module"].capitalize() + if "method" in decl: + name += "." + decl["method"] + return name + + def getDocURL(self, decl): + url = "http://opencv.itseez.com/modules/" + url += decl["module"] + url += "/doc/" + url += os.path.basename(decl["file"]).replace(".rst",".html") + url += "#" + decl["name"].replace("::","-").replace("()","").replace("=","").strip().rstrip("_").replace(" ","-").replace("_","-").lower() + return url + + def makeJavadoc(self, decl, args = None): + doc = "" + prefix = "/**\n" + + if decl.get("isclass", False): + decl_type = "class" + elif decl.get("isstruct", False): + decl_type = "struct" + elif "class" in decl: + decl_type = "method" + else: + decl_type = "function" + + # brief goes first + if "brief" in decl: + doc += prefix + self.ReformatForJavadoc(decl["brief"]) + prefix = " *\n" + elif "long" not in decl: + if show_warnings: + print >> sys.stderr, "gen_javadoc warning: no description for " + decl_type + " \"%s\" File: %s (line %s)" % (func["name"], func["file"], func["line"]) + doc += prefix + self.ReformatForJavadoc("This " + decl_type + " is undocumented") + prefix = " *\n" - # long goes after brief - if "long" in decl: - doc += prefix + ReformatForJavadoc(decl["long"]) + # long goes after brief + if "long" in decl: + doc += prefix + self.ReformatForJavadoc(decl["long"]) + prefix = " *\n" + + # @param tags + if args and (decl_type == "method" or decl_type == "function"): + documented_params = decl.get("params",{}) + for arg in args: + arg_doc = documented_params.get(arg, None) + if not arg_doc: + arg_doc = "a " + arg + if show_warnings: + print >> sys.stderr, "gen_javadoc warning: parameter \"%s\" of \"%s\" is undocumented. File: %s (line %s)" % (arg, decl["name"], decl["file"], decl["line"]) + self.params_undocumented += 1 + else: + self.params_documented += 1 + doc += prefix + self.ReformatForJavadoc("@param " + arg + " " + arg_doc) + prefix = "" + prefix = " *\n" + + # @see tags + # always link to documentation + doc += prefix + " * @see " + self.getJavaName(decl) + "\n" + prefix = "" + # other links + if "seealso" in decl: + for see in decl["seealso"]: + seedecl = self.definitions.get(see,None) + if seedecl: + doc += prefix + " * @see " + self.getJavaName(seedecl) + "\n" + else: + doc += prefix + " * @see " + see.replace("::",".") + "\n" prefix = " *\n" - # @param tags - if args and (decl_type == "method" or decl_type == "function"): - documented_params = decl.get("params",{}) - for arg in args: - arg_doc = documented_params.get(arg, None) - if not arg_doc: - arg_doc = "a " + arg - print "Warning: parameter \"%s\" of \"%s\" is undocumented. File: %s (line %s)" % (arg, decl["name"], decl["file"], decl["line"]) - doc += prefix + ReformatForJavadoc("@param " + arg + " " + arg_doc) - prefix = "" - prefix = " *\n" + #doc += prefix + " * File: " + decl["file"] + " (line " + str(decl["line"]) + ")\n" - # @see tags - # always link to documentation - doc += prefix + " * @see " + getJavaName(decl) + "\n" - prefix = "" - # other links - if "seealso" in decl: - for see in decl["seealso"]: - seedecl = decls.get(see,None) - if seedecl: - doc += prefix + " * @see " + getJavaName(seedecl) + "\n" - else: - doc += prefix + " * @see " + see.replace("::",".") + "\n" - prefix = " *\n" + return (doc + " */").replace("::",".") - #doc += prefix + " * File: " + decl["file"] + " (line " + str(decl["line"]) + ")\n" + def printSummary(self): + print + print "Javadoc Generator Summary:" + print " Total markers: %s" % self.markers_processed + print " Undocumented markers: %s" % (self.markers_processed - self.markers_documented) + print " Generated comments: %s" % self.markers_documented - return (doc + " */").replace("::",".") + print + print " Documented params: %s" % self.params_documented + print " Undocumented params: %s" % self.params_undocumented + print if __name__ == "__main__": if len(sys.argv) < 2: @@ -156,16 +194,19 @@ if __name__ == "__main__": import hdr_parser import rst_parser - parser = rst_parser.RstParser(hdr_parser.CppHeaderParser()) - print "Parsing documentation..." - for m in ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]: + parser = rst_parser.RstParser(hdr_parser.CppHeaderParser()) + for m in allmodules: parser.parse(m, os.path.join(selfpath, "../" + m)) parser.printSummary() + print "Generating javadoc comments..." + generator = JavadocGenerator(parser.definitions) for i in range(1, len(sys.argv)): folder = os.path.abspath(sys.argv[i]) for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]: outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java")) - document(jfile, outfile, parser.definitions) + generator.document(jfile, outfile) + + generator.printSummary() diff --git a/modules/java/rst_parser.py b/modules/java/rst_parser.py index d76795eb89..f6c87404d2 100644 --- a/modules/java/rst_parser.py +++ b/modules/java/rst_parser.py @@ -1,4 +1,5 @@ import os, sys, re, string, glob +allmodules = ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"] verbose = False show_warnings = True show_errors = True @@ -564,6 +565,7 @@ class RstParser(object): print " structs documented: %s" % structs for lang in sorted(stat.items()): print " %7s functions documented: %s" % lang + print def mathReplace2(match): m = mathReplace(match) @@ -670,7 +672,7 @@ if __name__ == "__main__": parser = RstParser(hdr_parser.CppHeaderParser()) if module == "all": - for m in ["core", "flann", "imgproc", "ml", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "gpu", "androidcamera", "haartraining", "java", "python", "stitching", "traincascade", "ts"]: + for m in allmodules: parser.parse(m, os.path.join(rst_parser_dir, "../" + m)) else: parser.parse(module, os.path.join(rst_parser_dir, "../" + module)) -- GitLab