graph.py

from google.protobuf import text_format

from .caffe import get_caffe_resolver
from .errors import KaffeError, print_stderr
from .layers import LayerAdapter, LayerType, NodeKind, NodeDispatch
from .shapes import make_tensor


class Node(object):
    def __init__(self, name, kind, layer=None):
        self.name = name
        self.kind = kind
        self.layer = LayerAdapter(layer, kind) if layer else None
        self.parents = []
        self.children = []
        self.data = None  #parameters of this node
        self.output_shape = None  #output shape of this node
        self.metadata = {}

    def add_parent(self, parent_node):
        assert parent_node not in self.parents
        self.parents.append(parent_node)
        if self not in parent_node.children:
            parent_node.children.append(self)

    def add_child(self, child_node):
        assert child_node not in self.children
        self.children.append(child_node)
        if self not in child_node.parents:
            child_node.parents.append(self)

    def get_only_parent(self):
        if len(self.parents) != 1:
            raise KaffeError('Node (%s) expected to have 1 parent. Found %s.' %
                             (self, len(self.parents)))
        return self.parents[0]

    @property
    def parameters(self):
        """ get parameters stored in a protobuf object
        """
        if self.layer is not None:
            return self.layer.parameters
        return None

    @property
    def params(self):
        """ get parameters stored in a dict
        """
        from .protobuf_to_dict import protobuf_to_dict

        p = self.parameters
        if p is not None:
            return protobuf_to_dict(p)
        else:
            return None

    def __str__(self):
        return '[%s] %s' % (self.kind, self.name)

    def __repr__(self):
        return '%s (0x%x)' % (self.name, id(self))


class Graph(object):
    def __init__(self, nodes=None, name=None, trace={}):
        self.nodes = nodes or []
        self.node_lut = {node.name: node for node in self.nodes}
        self.output_trace = trace
        if name is None or name == '':
            self.name = 'MyNet'
        else:
            self.name = name

    def add_node(self, node):
        self.nodes.append(node)
        self.node_lut[node.name] = node

    def get_node(self, name):
        try:
            return self.node_lut[name]
        except KeyError:
            raise KaffeError('Layer not found: %s' % name)

    def add_name_trace(self, trace, which='caffe'):
        self.output_trace[which] = trace

    def get_name_trace(self, which=None):
        if which is not None:
            return self.output_trace[which]
        else:
            return self.output_trace

    def get_input_nodes(self):
        return [node for node in self.nodes if len(node.parents) == 0]

    def get_output_nodes(self):
        return [node for node in self.nodes if len(node.children) == 0]

    def topologically_sorted(self):
        sorted_nodes = []
        unsorted_nodes = list(self.nodes)
        temp_marked = set()
        perm_marked = set()

        def visit(node):
            if node in temp_marked:
                raise KaffeError('Graph is not a DAG.')
            if node in perm_marked:
                return
            temp_marked.add(node)
            for child in node.children:
                visit(child)
            perm_marked.add(node)
            temp_marked.remove(node)
            sorted_nodes.insert(0, node)

        while len(unsorted_nodes):
            visit(unsorted_nodes.pop())
        return sorted_nodes

    def compute_output_shapes(self):
        sorted_nodes = self.topologically_sorted()
        for node in sorted_nodes:
            node.output_shape = make_tensor(
                *NodeKind.compute_output_shape(node))

    def replaced(self, new_nodes):
        return Graph(nodes=new_nodes, name=self.name, trace=self.output_trace)

    def transformed(self, transformers):
        graph = self
        for transformer in transformers:
            graph = transformer(graph)
            if graph is None:
                raise KaffeError('Transformer failed: {}'.format(transformer))
            assert isinstance(graph, Graph)

        return graph

    def __contains__(self, key):
        return key in self.node_lut

    def __str__(self):
        hdr = '{:<20} {:<30} {:>20} {:>20}'.format('Type', 'Name', 'Param',
                                                   'Output')
        s = [hdr, '-' * 94]
        for node in self.topologically_sorted():
            # If the node has learned parameters, display the first one's shape.
            # In case of convolutions, this corresponds to the weights.
            if node.data is None:
                data_shape = '--'
                out_shape = node.output_shape or '--'
                s.append('{:<20} {:<30} {:>20} {:>20}'.format(
                    node.kind, node.name, data_shape, str(tuple(out_shape))))
            else:
                for d in node.data:
                    #data_shape = node.data[0].shape if node.data else '--'
                    data_shape = d.shape
                    out_shape = node.output_shape or '--'
                    s.append('{:<20} {:<30} {:>20} {:>20}'.format(
                        node.kind, node.name, str(data_shape), str(tuple(out_shape))))
        return '\n'.join(s)


class GraphBuilder(object):
    '''Constructs a model graph from a Caffe protocol buffer definition.'''

    def __init__(self, def_path, phase='test'):
        '''
        def_path: Path to the model definition (.prototxt)
        data_path: Path to the model data (.caffemodel)
        phase: Either 'test' or 'train'. Used for filtering phase-specific nodes.
        '''
        self.def_path = def_path
        self.phase = phase
        self.load()

    def load(self):
        '''Load the layer definitions from the prototxt.'''
        self.params = get_caffe_resolver().NetParameter()
        with open(self.def_path, 'rb') as def_file:
            text_format.Merge(def_file.read(), self.params)

    def filter_layers(self, layers):
        '''Filter out layers based on the current phase.'''
        phase_map = {0: 'train', 1: 'test'}
        filtered_layer_names = set()
        filtered_layers = []
        for layer in layers:
            phase = self.phase
            if len(layer.include):
                phase = phase_map[layer.include[0].phase]
            if len(layer.exclude):
                phase = phase_map[1 - layer.include[0].phase]
            exclude = (phase != self.phase)
            # Dropout layers appear in a fair number of Caffe
            # test-time networks. These are just ignored. We'll
            # filter them out here.
            if (not exclude) and (phase == 'test'):
                exclude = (layer.type == LayerType.Dropout)
            if not exclude:
                filtered_layers.append(layer)
                # Guard against dupes.
                assert layer.name not in filtered_layer_names
                filtered_layer_names.add(layer.name)
        return filtered_layers

    def make_node(self, layer):
        '''Create a graph node for the given layer.'''
        kind = NodeKind.map_raw_kind(layer.type)
        if kind is None:
            raise KaffeError('Unknown layer type encountered: %s' % layer.type)

        # We want to use the layer's top names (the "output" names), rather than the
        # name attribute, which is more of readability thing than a functional one.
        # Other layers will refer to a node by its "top name".
        return Node(layer.name, kind, layer=layer)

    def make_input_nodes(self):
        '''
        Create data input nodes.

        This method is for old-style inputs, where the input specification
        was not treated as a first-class layer in the prototext.
        Newer models use the "Input layer" type.
        '''
        nodes = [Node(name, NodeKind.Data) for name in self.params.input]
        inputs_num = len(nodes)
        if inputs_num > 0:
            input_dims_num = len(self.params.input_dim)
            if input_dims_num > 0 and input_dims_num != inputs_num * 4:
                raise KaffeError('invalid input_dim[%d] param in prototxt' %
                                 (input_dims_num))

            input_dims = [[]] * inputs_num
            for i in range(input_dims_num):
                dim = self.params.input_dim[i]
                which = int(i / 4)
                input_dims[which].append(int(dim))

            for i in range(inputs_num):
                if len(self.params.input_shape) == inputs_num:
                    input_dim = map(int, self.params.input_shape[i].dim)
                    input_dims[i] = input_dim

                nodes[i].output_shape = tuple(input_dims[i])
        return nodes

    def build(self):
        '''
        Builds the graph from the Caffe layer definitions.
        '''
        # Get the layers
        layers = self.params.layers or self.params.layer
        # Filter out phase-excluded layers
        layers = self.filter_layers(layers)
        # Get any separately-specified input layers
        nodes = self.make_input_nodes()
        nodes += [self.make_node(layer) for layer in layers]
        # Initialize the graph
        graph = Graph(nodes=nodes, name=self.params.name)
        # Connect the nodes
        #
        # A note on layers and outputs:
        # In Caffe, each layer can produce multiple outputs ("tops") from a set of inputs
        # ("bottoms"). The bottoms refer to other layers' tops. The top can rewrite a bottom
        # (in case of in-place operations). Note that the layer's name is not used for establishing
        # any connectivity. It's only used for data association. By convention, a layer with a
        # single top will often use the same name (although this is not required).
        #
        # The current implementation only supports single-output nodes (note that a node can still
        # have multiple children, since multiple child nodes can refer to the single top's name).
        node_outputs = {}
        output_trace = {}
        for layer in layers:
            node = graph.get_node(layer.name)
            for input_name in layer.bottom:
                assert input_name != layer.name
                parent_node = node_outputs.get(input_name)
                if (parent_node is None) or (parent_node == node):
                    parent_node = graph.get_node(input_name)
                node.add_parent(parent_node)

            if len(layer.top) > 1:
                raise KaffeError('Multiple top nodes are not supported.')

            for output_name in layer.top:
                if output_name == layer.name:
                    # Output is named the same as the node. No further action required.
                    continue
                # There are two possibilities here:
                #
                # Case 1: output_name refers to another node in the graph.
                # This is an "in-place operation" that overwrites an existing node.
                # This would create a cycle in the graph. We'll undo the in-placing
                # by substituting this node wherever the overwritten node is referenced.
                #
                # Case 2: output_name violates the convention layer.name == output_name.
                # Since we are working in the single-output regime, we will can rename it to
                # match the layer name.
                #
                # For both cases, future references to this top re-routes to this node.
                node_outputs[output_name] = node
                if output_name in output_trace:
                    output_trace[output_name].append(node.name)
                else:
                    output_trace[output_name] = [output_name, node.name]

        #build a mapping from real-name to changed-name(for caffe's INPLACE inference)
        real2chg = {}
        deleted = {}
        for k, v in output_trace.items():
            real2chg[v[-1]] = k
            for n in v:
                if n in real2chg:
                    continue
                if n not in deleted:
                    deleted[n] = '%s.%s' % (k, v[-1])

        graph.add_name_trace({
            'real2chg': real2chg,
            'deleted': deleted
        }, 'caffe')
        graph.compute_output_shapes()
        return graph


class NodeMapper(NodeDispatch):
    def __init__(self, graph):
        self.graph = graph

    def map(self):
        nodes = self.graph.topologically_sorted()
        # Remove input nodes - we'll handle them separately.
        input_nodes = self.graph.get_input_nodes()
        nodes = [t for t in nodes if t not in input_nodes]
        # Decompose DAG into chains.
        chains = []
        for node in nodes:
            attach_to_chain = None
            if len(node.parents) == 1:
                parent = node.get_only_parent()
                for chain in chains:
                    if chain[-1] == parent:
                        # Node is part of an existing chain.
                        attach_to_chain = chain
                        break
            if attach_to_chain is None:
                # Start a new chain for this node.
                attach_to_chain = []
                chains.append(attach_to_chain)
            attach_to_chain.append(node)
        # Map each chain.
        mapped_chains = []
        for chain in chains:
            mapped_chains.append(self.map_chain(chain))
        return self.commit(mapped_chains)

    def map_chain(self, chain):
        return [self.map_node(node) for node in chain]

    def map_node(self, node):
        map_func = self.get_handler(node.kind, 'map')
        mapped_node = map_func(node)
        assert mapped_node is not None
        mapped_node.node = node
        return mapped_node

    def commit(self, mapped_chains):
        raise NotImplementedError('Must be implemented by subclass.')