qapi: Split up scripts/qapi/common.py

The QAPI code generator clocks in at some 3100 SLOC in 8 source files.
Almost 60% of the code is in qapi/common.py.  Split it into more
focused modules:

* Move QAPISchemaPragma and QAPISourceInfo to qapi/source.py.

* Move QAPIError and its sub-classes to qapi/error.py.

* Move QAPISchemaParser and QAPIDoc to parser.py.  Use the opportunity
  to put QAPISchemaParser first.

* Move check_expr() & friends to qapi/expr.py.  Use the opportunity to
  put the code into a more sensible order.

* Move QAPISchema & friends to qapi/schema.py

* Move QAPIGen and its sub-classes, ifcontext,
  QAPISchemaModularCVisitor, and QAPISchemaModularCVisitor to qapi/gen.py

* Delete camel_case(), it's unused since commit e98859a9 "qapi:
  Clean up after recent conversions to QAPISchemaVisitor"

A number of helper functions remain in qapi/common.py.  I considered
moving the code generator helpers to qapi/gen.py, but decided not to.
Perhaps we should rewrite them as methods of QAPIGen some day.
Signed-off-by: NMarkus Armbruster <armbru@redhat.com>
Reviewed-by: NEric Blake <eblake@redhat.com>
Message-Id: <20191018074345.24034-7-armbru@redhat.com>
[Add "# -*- coding: utf-8 -*-" lines]

Makefile

@@ -582,13 +582,20 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated
 qemu-keymap$(EXESUF): LIBS += $(XKBCOMMON_LIBS)
 qemu-keymap$(EXESUF): QEMU_CFLAGS += $(XKBCOMMON_CFLAGS)
 
-qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \
+qapi-py = $(SRC_PATH)/scripts/qapi/__init__.py \
+$(SRC_PATH)/scripts/qapi/commands.py \
+$(SRC_PATH)/scripts/qapi/common.py \
+$(SRC_PATH)/scripts/qapi/doc.py \
+$(SRC_PATH)/scripts/qapi/error.py \
 $(SRC_PATH)/scripts/qapi/events.py \
+$(SRC_PATH)/scripts/qapi/expr.py \
+$(SRC_PATH)/scripts/qapi/gen.py \
 $(SRC_PATH)/scripts/qapi/introspect.py \
+$(SRC_PATH)/scripts/qapi/parser.py \
+$(SRC_PATH)/scripts/qapi/schema.py \
+$(SRC_PATH)/scripts/qapi/source.py \
 $(SRC_PATH)/scripts/qapi/types.py \
 $(SRC_PATH)/scripts/qapi/visit.py \
-$(SRC_PATH)/scripts/qapi/common.py \
-$(SRC_PATH)/scripts/qapi/doc.py \
 $(SRC_PATH)/scripts/qapi-gen.py
 
 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h \

scripts/qapi-gen.py

@@ -5,16 +5,18 @@
 # See the COPYING file in the top-level directory.
 
 from __future__ import print_function
+
 import argparse
 import re
 import sys
-from qapi.common import QAPIError, QAPISchema
-from qapi.types import gen_types
-from qapi.visit import gen_visit
+
 from qapi.commands import gen_commands
+from qapi.doc import gen_doc
 from qapi.events import gen_events
 from qapi.introspect import gen_introspect
-from qapi.doc import gen_doc
+from qapi.schema import QAPIError, QAPISchema
+from qapi.types import gen_types
+from qapi.visit import gen_visit
 
 
 def main(argv):

scripts/qapi/commands.py

@@ -14,6 +14,7 @@ See the COPYING file in the top-level directory.
 """
 
 from qapi.common import *
+from qapi.gen import QAPIGenCCode, QAPISchemaModularCVisitor, ifcontext
 
 
 def gen_command_decl(name, arg_type, boxed, ret_type):

scripts/qapi/doc.py

@@ -7,7 +7,8 @@
 
 from __future__ import print_function
 import re
-import qapi.common
+from qapi.gen import QAPIGenDoc, QAPISchemaVisitor
+
 
 MSG_FMT = """
 @deftypefn {type} {{}} {name}
@@ -216,10 +217,10 @@ def texi_entity(doc, what, ifcond, base=None, variants=None,
             + texi_sections(doc, ifcond))
 
 
-class QAPISchemaGenDocVisitor(qapi.common.QAPISchemaVisitor):
+class QAPISchemaGenDocVisitor(QAPISchemaVisitor):
     def __init__(self, prefix):
         self._prefix = prefix
-        self._gen = qapi.common.QAPIGenDoc(self._prefix + 'qapi-doc.texi')
+        self._gen = QAPIGenDoc(self._prefix + 'qapi-doc.texi')
         self.cur_doc = None
 
     def write(self, output_dir):

scripts/qapi/error.py

+# -*- coding: utf-8 -*-
+#
+# QAPI error classes
+#
+# Copyright (c) 2017-2019 Red Hat Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#  Marc-André Lureau <marcandre.lureau@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+
+class QAPIError(Exception):
+    def __init__(self, info, col, msg):
+        Exception.__init__(self)
+        self.info = info
+        self.col = col
+        self.msg = msg
+
+    def __str__(self):
+        loc = str(self.info)
+        if self.col is not None:
+            assert self.info.line is not None
+            loc += ':%s' % self.col
+        return loc + ': ' + self.msg
+
+
+class QAPIParseError(QAPIError):
+    def __init__(self, parser, msg):
+        col = 1
+        for ch in parser.src[parser.line_pos:parser.pos]:
+            if ch == '\t':
+                col = (col + 7) % 8 + 1
+            else:
+                col += 1
+        QAPIError.__init__(self, parser.info, col, msg)
+
+
+class QAPISemError(QAPIError):
+    def __init__(self, info, msg):
+        QAPIError.__init__(self, info, None, msg)

scripts/qapi/events.py

@@ -13,6 +13,8 @@ See the COPYING file in the top-level directory.
 """
 
 from qapi.common import *
+from qapi.gen import QAPISchemaModularCVisitor, ifcontext
+from qapi.schema import QAPISchemaEnumMember
 from qapi.types import gen_enum, gen_enum_lookup
 
 

scripts/qapi/expr.py

+# -*- coding: utf-8 -*-
+#
+# Check (context-free) QAPI schema expression structure
+#
+# Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2019 Red Hat Inc.
+#
+# Authors:
+#  Anthony Liguori <aliguori@us.ibm.com>
+#  Markus Armbruster <armbru@redhat.com>
+#  Eric Blake <eblake@redhat.com>
+#  Marc-André Lureau <marcandre.lureau@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+import re
+from collections import OrderedDict
+from qapi.common import c_name
+from qapi.error import QAPISemError
+
+
+# Names must be letters, numbers, -, and _.  They must start with letter,
+# except for downstream extensions which must start with __RFQDN_.
+# Dots are only valid in the downstream extension prefix.
+valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
+                        '[a-zA-Z][a-zA-Z0-9_-]*$')
+
+
+def check_name_is_str(name, info, source):
+    if not isinstance(name, str):
+        raise QAPISemError(info, "%s requires a string name" % source)
+
+
+def check_name_str(name, info, source,
+                   allow_optional=False, enum_member=False,
+                   permit_upper=False):
+    global valid_name
+    membername = name
+
+    if allow_optional and name.startswith('*'):
+        membername = name[1:]
+    # Enum members can start with a digit, because the generated C
+    # code always prefixes it with the enum name
+    if enum_member and membername[0].isdigit():
+        membername = 'D' + membername
+    # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
+    # and 'q_obj_*' implicit type names.
+    if not valid_name.match(membername) or \
+       c_name(membername, False).startswith('q_'):
+        raise QAPISemError(info, "%s has an invalid name" % source)
+    if not permit_upper and name.lower() != name:
+        raise QAPISemError(
+            info, "%s uses uppercase in name" % source)
+    assert not membername.startswith('*')
+
+
+def check_defn_name_str(name, info, meta):
+    check_name_str(name, info, meta, permit_upper=True)
+    if name.endswith('Kind') or name.endswith('List'):
+        raise QAPISemError(
+            info, "%s name should not end in '%s'" % (meta, name[-4:]))
+
+
+def check_keys(value, info, source, required, optional):
+
+    def pprint(elems):
+        return ', '.join("'" + e + "'" for e in sorted(elems))
+
+    missing = set(required) - set(value)
+    if missing:
+        raise QAPISemError(
+            info,
+            "%s misses key%s %s"
+            % (source, 's' if len(missing) > 1 else '',
+               pprint(missing)))
+    allowed = set(required + optional)
+    unknown = set(value) - allowed
+    if unknown:
+        raise QAPISemError(
+            info,
+            "%s has unknown key%s %s\nValid keys are %s."
+            % (source, 's' if len(unknown) > 1 else '',
+               pprint(unknown), pprint(allowed)))
+
+
+def check_flags(expr, info):
+    for key in ['gen', 'success-response']:
+        if key in expr and expr[key] is not False:
+            raise QAPISemError(
+                info, "flag '%s' may only use false value" % key)
+    for key in ['boxed', 'allow-oob', 'allow-preconfig']:
+        if key in expr and expr[key] is not True:
+            raise QAPISemError(
+                info, "flag '%s' may only use true value" % key)
+
+
+def normalize_if(expr):
+    ifcond = expr.get('if')
+    if isinstance(ifcond, str):
+        expr['if'] = [ifcond]
+
+
+def check_if(expr, info, source):
+
+    def check_if_str(ifcond, info):
+        if not isinstance(ifcond, str):
+            raise QAPISemError(
+                info,
+                "'if' condition of %s must be a string or a list of strings"
+                % source)
+        if ifcond.strip() == '':
+            raise QAPISemError(
+                info,
+                "'if' condition '%s' of %s makes no sense"
+                % (ifcond, source))
+
+    ifcond = expr.get('if')
+    if ifcond is None:
+        return
+    if isinstance(ifcond, list):
+        if ifcond == []:
+            raise QAPISemError(
+                info, "'if' condition [] of %s is useless" % source)
+        for elt in ifcond:
+            check_if_str(elt, info)
+    else:
+        check_if_str(ifcond, info)
+
+
+def normalize_members(members):
+    if isinstance(members, OrderedDict):
+        for key, arg in members.items():
+            if isinstance(arg, dict):
+                continue
+            members[key] = {'type': arg}
+
+
+def check_type(value, info, source,
+               allow_array=False, allow_dict=False):
+    if value is None:
+        return
+
+    # Array type
+    if isinstance(value, list):
+        if not allow_array:
+            raise QAPISemError(info, "%s cannot be an array" % source)
+        if len(value) != 1 or not isinstance(value[0], str):
+            raise QAPISemError(info,
+                               "%s: array type must contain single type name" %
+                               source)
+        return
+
+    # Type name
+    if isinstance(value, str):
+        return
+
+    # Anonymous type
+
+    if not allow_dict:
+        raise QAPISemError(info, "%s should be a type name" % source)
+
+    if not isinstance(value, OrderedDict):
+        raise QAPISemError(info,
+                           "%s should be an object or type name" % source)
+
+    permit_upper = allow_dict in info.pragma.name_case_whitelist
+
+    # value is a dictionary, check that each member is okay
+    for (key, arg) in value.items():
+        key_source = "%s member '%s'" % (source, key)
+        check_name_str(key, info, key_source,
+                       allow_optional=True, permit_upper=permit_upper)
+        if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
+            raise QAPISemError(info, "%s uses reserved name" % key_source)
+        check_keys(arg, info, key_source, ['type'], ['if'])
+        check_if(arg, info, key_source)
+        normalize_if(arg)
+        check_type(arg['type'], info, key_source, allow_array=True)
+
+
+def normalize_features(features):
+    if isinstance(features, list):
+        features[:] = [f if isinstance(f, dict) else {'name': f}
+                       for f in features]
+
+
+def normalize_enum(expr):
+    if isinstance(expr['data'], list):
+        expr['data'] = [m if isinstance(m, dict) else {'name': m}
+                        for m in expr['data']]
+
+
+def check_enum(expr, info):
+    name = expr['enum']
+    members = expr['data']
+    prefix = expr.get('prefix')
+
+    if not isinstance(members, list):
+        raise QAPISemError(info, "'data' must be an array")
+    if prefix is not None and not isinstance(prefix, str):
+        raise QAPISemError(info, "'prefix' must be a string")
+
+    permit_upper = name in info.pragma.name_case_whitelist
+
+    for member in members:
+        source = "'data' member"
+        check_keys(member, info, source, ['name'], ['if'])
+        check_name_is_str(member['name'], info, source)
+        source = "%s '%s'" % (source, member['name'])
+        check_name_str(member['name'], info, source,
+                       enum_member=True, permit_upper=permit_upper)
+        check_if(member, info, source)
+        normalize_if(member)
+
+
+def check_struct(expr, info):
+    name = expr['struct']
+    members = expr['data']
+    features = expr.get('features')
+
+    check_type(members, info, "'data'", allow_dict=name)
+    check_type(expr.get('base'), info, "'base'")
+
+    if features:
+        if not isinstance(features, list):
+            raise QAPISemError(info, "'features' must be an array")
+        for f in features:
+            source = "'features' member"
+            assert isinstance(f, dict)
+            check_keys(f, info, source, ['name'], ['if'])
+            check_name_is_str(f['name'], info, source)
+            source = "%s '%s'" % (source, f['name'])
+            check_name_str(f['name'], info, source)
+            check_if(f, info, source)
+            normalize_if(f)
+
+
+def check_union(expr, info):
+    name = expr['union']
+    base = expr.get('base')
+    discriminator = expr.get('discriminator')
+    members = expr['data']
+
+    if discriminator is None:   # simple union
+        if base is not None:
+            raise QAPISemError(info, "'base' requires 'discriminator'")
+    else:                       # flat union
+        check_type(base, info, "'base'", allow_dict=name)
+        if not base:
+            raise QAPISemError(info, "'discriminator' requires 'base'")
+        check_name_is_str(discriminator, info, "'discriminator'")
+
+    for (key, value) in members.items():
+        source = "'data' member '%s'" % key
+        check_name_str(key, info, source)
+        check_keys(value, info, source, ['type'], ['if'])
+        check_if(value, info, source)
+        normalize_if(value)
+        check_type(value['type'], info, source, allow_array=not base)
+
+
+def check_alternate(expr, info):
+    members = expr['data']
+
+    if len(members) == 0:
+        raise QAPISemError(info, "'data' must not be empty")
+    for (key, value) in members.items():
+        source = "'data' member '%s'" % key
+        check_name_str(key, info, source)
+        check_keys(value, info, source, ['type'], ['if'])
+        check_if(value, info, source)
+        normalize_if(value)
+        check_type(value['type'], info, source)
+
+
+def check_command(expr, info):
+    args = expr.get('data')
+    rets = expr.get('returns')
+    boxed = expr.get('boxed', False)
+
+    if boxed and args is None:
+        raise QAPISemError(info, "'boxed': true requires 'data'")
+    check_type(args, info, "'data'", allow_dict=not boxed)
+    check_type(rets, info, "'returns'", allow_array=True)
+
+
+def check_event(expr, info):
+    args = expr.get('data')
+    boxed = expr.get('boxed', False)
+
+    if boxed and args is None:
+        raise QAPISemError(info, "'boxed': true requires 'data'")
+    check_type(args, info, "'data'", allow_dict=not boxed)
+
+
+def check_exprs(exprs):
+    for expr_elem in exprs:
+        expr = expr_elem['expr']
+        info = expr_elem['info']
+        doc = expr_elem.get('doc')
+
+        if 'include' in expr:
+            continue
+
+        if 'enum' in expr:
+            meta = 'enum'
+        elif 'union' in expr:
+            meta = 'union'
+        elif 'alternate' in expr:
+            meta = 'alternate'
+        elif 'struct' in expr:
+            meta = 'struct'
+        elif 'command' in expr:
+            meta = 'command'
+        elif 'event' in expr:
+            meta = 'event'
+        else:
+            raise QAPISemError(info, "expression is missing metatype")
+
+        name = expr[meta]
+        check_name_is_str(name, info, "'%s'" % meta)
+        info.set_defn(meta, name)
+        check_defn_name_str(name, info, meta)
+
+        if doc:
+            if doc.symbol != name:
+                raise QAPISemError(
+                    info, "documentation comment is for '%s'" % doc.symbol)
+            doc.check_expr(expr)
+        elif info.pragma.doc_required:
+            raise QAPISemError(info,
+                               "documentation comment required")
+
+        if meta == 'enum':
+            check_keys(expr, info, meta,
+                       ['enum', 'data'], ['if', 'prefix'])
+            normalize_enum(expr)
+            check_enum(expr, info)
+        elif meta == 'union':
+            check_keys(expr, info, meta,
+                       ['union', 'data'],
+                       ['base', 'discriminator', 'if'])
+            normalize_members(expr.get('base'))
+            normalize_members(expr['data'])
+            check_union(expr, info)
+        elif meta == 'alternate':
+            check_keys(expr, info, meta,
+                       ['alternate', 'data'], ['if'])
+            normalize_members(expr['data'])
+            check_alternate(expr, info)
+        elif meta == 'struct':
+            check_keys(expr, info, meta,
+                       ['struct', 'data'], ['base', 'if', 'features'])
+            normalize_members(expr['data'])
+            normalize_features(expr.get('features'))
+            check_struct(expr, info)
+        elif meta == 'command':
+            check_keys(expr, info, meta,
+                       ['command'],
+                       ['data', 'returns', 'boxed', 'if',
+                        'gen', 'success-response', 'allow-oob',
+                        'allow-preconfig'])
+            normalize_members(expr.get('data'))
+            check_command(expr, info)
+        elif meta == 'event':
+            check_keys(expr, info, meta,
+                       ['event'], ['data', 'boxed', 'if'])
+            normalize_members(expr.get('data'))
+            check_event(expr, info)
+        else:
+            assert False, 'unexpected meta type'
+
+        normalize_if(expr)
+        check_if(expr, info, meta)
+        check_flags(expr, info)
+
+    return exprs

scripts/qapi/gen.py

+# -*- coding: utf-8 -*-
+#
+# QAPI code generation
+#
+# Copyright (c) 2018-2019 Red Hat Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#  Marc-André Lureau <marcandre.lureau@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+
+import errno
+import os
+import re
+import sys
+from contextlib import contextmanager
+
+from qapi.common import *
+from qapi.schema import QAPISchemaVisitor
+
+
+class QAPIGen(object):
+
+    def __init__(self, fname):
+        self.fname = fname
+        self._preamble = ''
+        self._body = ''
+
+    def preamble_add(self, text):
+        self._preamble += text
+
+    def add(self, text):
+        self._body += text
+
+    def get_content(self):
+        return self._top() + self._preamble + self._body + self._bottom()
+
+    def _top(self):
+        return ''
+
+    def _bottom(self):
+        return ''
+
+    def write(self, output_dir):
+        pathname = os.path.join(output_dir, self.fname)
+        dir = os.path.dirname(pathname)
+        if dir:
+            try:
+                os.makedirs(dir)
+            except os.error as e:
+                if e.errno != errno.EEXIST:
+                    raise
+        fd = os.open(pathname, os.O_RDWR | os.O_CREAT, 0o666)
+        if sys.version_info[0] >= 3:
+            f = open(fd, 'r+', encoding='utf-8')
+        else:
+            f = os.fdopen(fd, 'r+')
+        text = self.get_content()
+        oldtext = f.read(len(text) + 1)
+        if text != oldtext:
+            f.seek(0)
+            f.truncate(0)
+            f.write(text)
+        f.close()
+
+
+def _wrap_ifcond(ifcond, before, after):
+    if before == after:
+        return after   # suppress empty #if ... #endif
+
+    assert after.startswith(before)
+    out = before
+    added = after[len(before):]
+    if added[0] == '\n':
+        out += '\n'
+        added = added[1:]
+    out += gen_if(ifcond)
+    out += added
+    out += gen_endif(ifcond)
+    return out
+
+
+class QAPIGenCCode(QAPIGen):
+
+    def __init__(self, fname):
+        QAPIGen.__init__(self, fname)
+        self._start_if = None
+
+    def start_if(self, ifcond):
+        assert self._start_if is None
+        self._start_if = (ifcond, self._body, self._preamble)
+
+    def end_if(self):
+        assert self._start_if
+        self._wrap_ifcond()
+        self._start_if = None
+
+    def _wrap_ifcond(self):
+        self._body = _wrap_ifcond(self._start_if[0],
+                                  self._start_if[1], self._body)
+        self._preamble = _wrap_ifcond(self._start_if[0],
+                                      self._start_if[2], self._preamble)
+
+    def get_content(self):
+        assert self._start_if is None
+        return QAPIGen.get_content(self)
+
+
+class QAPIGenC(QAPIGenCCode):
+
+    def __init__(self, fname, blurb, pydoc):
+        QAPIGenCCode.__init__(self, fname)
+        self._blurb = blurb
+        self._copyright = '\n * '.join(re.findall(r'^Copyright .*', pydoc,
+                                                  re.MULTILINE))
+
+    def _top(self):
+        return mcgen('''
+/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
+
+/*
+%(blurb)s
+ *
+ * %(copyright)s
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+''',
+                     blurb=self._blurb, copyright=self._copyright)
+
+    def _bottom(self):
+        return mcgen('''
+
+/* Dummy declaration to prevent empty .o file */
+char qapi_dummy_%(name)s;
+''',
+                     name=c_fname(self.fname))
+
+
+class QAPIGenH(QAPIGenC):
+
+    def _top(self):
+        return QAPIGenC._top(self) + guardstart(self.fname)
+
+    def _bottom(self):
+        return guardend(self.fname)
+
+
+@contextmanager
+def ifcontext(ifcond, *args):
+    """A 'with' statement context manager to wrap with start_if()/end_if()
+
+    *args: any number of QAPIGenCCode
+
+    Example::
+
+        with ifcontext(ifcond, self._genh, self._genc):
+            modify self._genh and self._genc ...
+
+    Is equivalent to calling::
+
+        self._genh.start_if(ifcond)
+        self._genc.start_if(ifcond)
+        modify self._genh and self._genc ...
+        self._genh.end_if()
+        self._genc.end_if()
+    """
+    for arg in args:
+        arg.start_if(ifcond)
+    yield
+    for arg in args:
+        arg.end_if()
+
+
+class QAPIGenDoc(QAPIGen):
+
+    def _top(self):
+        return (QAPIGen._top(self)
+                + '@c AUTOMATICALLY GENERATED, DO NOT MODIFY\n\n')
+
+
+class QAPISchemaMonolithicCVisitor(QAPISchemaVisitor):
+
+    def __init__(self, prefix, what, blurb, pydoc):
+        self._prefix = prefix
+        self._what = what
+        self._genc = QAPIGenC(self._prefix + self._what + '.c',
+                              blurb, pydoc)
+        self._genh = QAPIGenH(self._prefix + self._what + '.h',
+                              blurb, pydoc)
+
+    def write(self, output_dir):
+        self._genc.write(output_dir)
+        self._genh.write(output_dir)
+
+
+class QAPISchemaModularCVisitor(QAPISchemaVisitor):
+
+    def __init__(self, prefix, what, blurb, pydoc):
+        self._prefix = prefix
+        self._what = what
+        self._blurb = blurb
+        self._pydoc = pydoc
+        self._genc = None
+        self._genh = None
+        self._module = {}
+        self._main_module = None
+
+    @staticmethod
+    def _is_user_module(name):
+        return name and not name.startswith('./')
+
+    @staticmethod
+    def _is_builtin_module(name):
+        return not name
+
+    def _module_dirname(self, what, name):
+        if self._is_user_module(name):
+            return os.path.dirname(name)
+        return ''
+
+    def _module_basename(self, what, name):
+        ret = '' if self._is_builtin_module(name) else self._prefix
+        if self._is_user_module(name):
+            basename = os.path.basename(name)
+            ret += what
+            if name != self._main_module:
+                ret += '-' + os.path.splitext(basename)[0]
+        else:
+            name = name[2:] if name else 'builtin'
+            ret += re.sub(r'-', '-' + name + '-', what)
+        return ret
+
+    def _module_filename(self, what, name):
+        return os.path.join(self._module_dirname(what, name),
+                            self._module_basename(what, name))
+
+    def _add_module(self, name, blurb):
+        basename = self._module_filename(self._what, name)
+        genc = QAPIGenC(basename + '.c', blurb, self._pydoc)
+        genh = QAPIGenH(basename + '.h', blurb, self._pydoc)
+        self._module[name] = (genc, genh)
+        self._set_module(name)
+
+    def _add_user_module(self, name, blurb):
+        assert self._is_user_module(name)
+        if self._main_module is None:
+            self._main_module = name
+        self._add_module(name, blurb)
+
+    def _add_system_module(self, name, blurb):
+        self._add_module(name and './' + name, blurb)
+
+    def _set_module(self, name):
+        self._genc, self._genh = self._module[name]
+
+    def write(self, output_dir, opt_builtins=False):
+        for name in self._module:
+            if self._is_builtin_module(name) and not opt_builtins:
+                continue
+            (genc, genh) = self._module[name]
+            genc.write(output_dir)
+            genh.write(output_dir)
+
+    def _begin_user_module(self, name):
+        pass
+
+    def visit_module(self, name):
+        if name in self._module:
+            self._set_module(name)
+        elif self._is_builtin_module(name):
+            # The built-in module has not been created.  No code may
+            # be generated.
+            self._genc = None
+            self._genh = None
+        else:
+            self._add_user_module(name, self._blurb)
+            self._begin_user_module(name)
+
+    def visit_include(self, name, info):
+        relname = os.path.relpath(self._module_filename(self._what, name),
+                                  os.path.dirname(self._genh.fname))
+        self._genh.preamble_add(mcgen('''
+#include "%(relname)s.h"
+''',
+                                      relname=relname))

scripts/qapi/introspect.py

@@ -10,7 +10,12 @@ This work is licensed under the terms of the GNU GPL, version 2.
 See the COPYING file in the top-level directory.
 """
 
+import string
+
 from qapi.common import *
+from qapi.gen import QAPISchemaMonolithicCVisitor
+from qapi.schema import (QAPISchemaArrayType, QAPISchemaBuiltinType,
+                         QAPISchemaType)
 
 
 def to_qlit(obj, level=0, suppress_first_indent=False):

scripts/qapi/source.py

+#
+# QAPI frontend source file info
+#
+# Copyright (c) 2019 Red Hat Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+import copy
+import sys
+
+
+class QAPISchemaPragma(object):
+    def __init__(self):
+        # Are documentation comments required?
+        self.doc_required = False
+        # Whitelist of commands allowed to return a non-dictionary
+        self.returns_whitelist = []
+        # Whitelist of entities allowed to violate case conventions
+        self.name_case_whitelist = []
+
+
+class QAPISourceInfo(object):
+    def __init__(self, fname, line, parent):
+        self.fname = fname
+        self.line = line
+        self.parent = parent
+        self.pragma = parent.pragma if parent else QAPISchemaPragma()
+        self.defn_meta = None
+        self.defn_name = None
+
+    def set_defn(self, meta, name):
+        self.defn_meta = meta
+        self.defn_name = name
+
+    def next_line(self):
+        info = copy.copy(self)
+        info.line += 1
+        return info
+
+    def loc(self):
+        if self.fname is None:
+            return sys.argv[0]
+        ret = self.fname
+        if self.line is not None:
+            ret += ':%d' % self.line
+        return ret
+
+    def in_defn(self):
+        if self.defn_name:
+            return "%s: In %s '%s':\n" % (self.fname,
+                                          self.defn_meta, self.defn_name)
+        return ''
+
+    def include_path(self):
+        ret = ''
+        parent = self.parent
+        while parent:
+            ret = 'In file included from %s:\n' % parent.loc() + ret
+            parent = parent.parent
+        return ret
+
+    def __str__(self):
+        return self.include_path() + self.in_defn() + self.loc()

scripts/qapi/types.py

@@ -14,6 +14,8 @@ This work is licensed under the terms of the GNU GPL, version 2.
 """
 
 from qapi.common import *
+from qapi.gen import QAPISchemaModularCVisitor, ifcontext
+from qapi.schema import QAPISchemaEnumMember, QAPISchemaObjectType
 
 
 # variants must be emitted before their container; track what has already

scripts/qapi/visit.py

@@ -14,6 +14,8 @@ See the COPYING file in the top-level directory.
 """
 
 from qapi.common import *
+from qapi.gen import QAPISchemaModularCVisitor, ifcontext
+from qapi.schema import QAPISchemaObjectType
 
 
 def gen_visit_decl(name, scalar=False):

tests/Makefile.include

@@ -31,13 +31,20 @@ ifneq ($(wildcard config-host.mak),)
 export SRC_PATH
 
 # TODO don't duplicate $(SRC_PATH)/Makefile's qapi-py here
-qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \
+qapi-py = $(SRC_PATH)/scripts/qapi/__init__.py \
+$(SRC_PATH)/scripts/qapi/commands.py \
+$(SRC_PATH)/scripts/qapi/common.py \
+$(SRC_PATH)/scripts/qapi/doc.py \
+$(SRC_PATH)/scripts/qapi/error.py \
 $(SRC_PATH)/scripts/qapi/events.py \
+$(SRC_PATH)/scripts/qapi/expr.py \
+$(SRC_PATH)/scripts/qapi/gen.py \
 $(SRC_PATH)/scripts/qapi/introspect.py \
+$(SRC_PATH)/scripts/qapi/parser.py \
+$(SRC_PATH)/scripts/qapi/schema.py \
+$(SRC_PATH)/scripts/qapi/source.py \
 $(SRC_PATH)/scripts/qapi/types.py \
 $(SRC_PATH)/scripts/qapi/visit.py \
-$(SRC_PATH)/scripts/qapi/common.py \
-$(SRC_PATH)/scripts/qapi/doc.py \
 $(SRC_PATH)/scripts/qapi-gen.py
 
 # Get the list of all supported sysemu targets

tests/qapi-schema/test-qapi.py

@@ -12,15 +12,19 @@
 #
 
 from __future__ import print_function
+
 import argparse
 import difflib
 import os
 import sys
+
+from qapi.error import QAPIError
+from qapi.schema import QAPISchema, QAPISchemaVisitor
+
 if sys.version_info[0] < 3:
     from cStringIO import StringIO
 else:
     from io import StringIO
-from qapi.common import QAPIError, QAPISchema, QAPISchemaVisitor
 
 
 class QAPISchemaTestVisitor(QAPISchemaVisitor):