code to compress models using SVD

experiments/compress_model.py

+#!/usr/bin/env python
+
+import fast_rcnn_config as conf
+import caffe
+import argparse
+import sys
+import os
+import numpy as np
+
+def parse_args():
+    """
+    Parse input arguments
+    """
+    parser = argparse.ArgumentParser(description='Compress a fast R-CNN net')
+    parser.add_argument('--def', dest='prototxt',
+                        help='prototxt file defining the uncompressed network',
+                        default=None, type=str)
+    parser.add_argument('--def-svd', dest='prototxt_svd',
+                        help='prototxt file defining the SVD compressed network',
+                        default=None, type=str)
+    parser.add_argument('--net', dest='caffemodel',
+                        help='model to compress',
+                        default=None, type=str)
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+        sys.exit(1)
+
+    args = parser.parse_args()
+    return args
+
+
+
+def compress_weights(W, l):
+    """
+    Compress the weight matrix W of an inner product (fully connected) layer
+    using truncated SVD.
+
+    Parameters:
+    W: N x M weights matrix
+    l: number of singular values to retain
+
+    Returns:
+    Ul, L: matrices such that W \approx Ul*L
+    """
+
+    # numpy doesn't seem to have a fast truncated SVD algorithm...
+    # this could be faster
+    U, s, V = np.linalg.svd(W, full_matrices=False)
+
+    Ul = U[:, :l]
+    sl = s[:l]
+    Vl = V[:l, :]
+
+    L = np.dot(np.diag(sl), Vl)
+    return Ul, L
+
+
+def main():
+    args = parse_args()
+
+    # prototxt = 'models/VGG_16/test.prototxt'
+    # caffemodel = 'snapshots/vgg16_fast_rcnn_iter_40000.caffemodel'
+    net = caffe.Net(args.prototxt, args.caffemodel, caffe.TEST)
+
+    # prototxt_svd = 'models/VGG_16/svd/test_fc6_fc7.prototxt'
+    # caffemodel = 'snapshots/vgg16_fast_rcnn_iter_40000.caffemodel'
+    net_svd = caffe.Net(args.prototxt_svd, args.caffemodel, caffe.TEST)
+
+    print('Uncompressed network {} : {}'.format(args.prototxt, args.caffemodel))
+    print('Compressed network prototxt {}'.format(args.prototxt_svd))
+
+    out = os.path.splitext(os.path.basename(args.caffemodel))[0] + '_svd'
+
+    # Compress fc6
+    if net_svd.params.has_key('fc6_L'):
+        l_fc6 = net_svd.params['fc6_L'][0].data.shape[0]
+        print('  fc6_L bottleneck size: {}'.format(l_fc6))
+
+        # uncompressed weights and biases
+        W_fc6 = net.params['fc6'][0].data
+        B_fc6 = net.params['fc6'][1].data
+
+        print('  compressing fc6...')
+        Ul_fc6, L_fc6 = compress_weights(W_fc6, l_fc6)
+
+        assert(len(net_svd.params['fc6_L']) == 1)
+
+        # install compressed matrix factors (and original biases)
+        net_svd.params['fc6_L'][0].data[...] = L_fc6
+
+        net_svd.params['fc6_U'][0].data[...] = Ul_fc6
+        net_svd.params['fc6_U'][1].data[...] = B_fc6
+
+        out += '_fc6_{}'.format(l_fc6)
+
+    # Compress fc7
+    if net_svd.params.has_key('fc7_L'):
+        l_fc7 = net_svd.params['fc7_L'][0].data.shape[0]
+        print '  fc7_L bottleneck size: {}'.format(l_fc7)
+
+        W_fc7 = net.params['fc7'][0].data
+        B_fc7 = net.params['fc7'][1].data
+
+        print('  compressing fc7...')
+        Ul_fc7, L_fc7 = compress_weights(W_fc7, l_fc7)
+
+        assert(len(net_svd.params['fc7_L']) == 1)
+
+        net_svd.params['fc7_L'][0].data[...] = L_fc7
+
+        net_svd.params['fc7_U'][0].data[...] = Ul_fc7
+        net_svd.params['fc7_U'][1].data[...] = B_fc7
+
+        out += '_fc7_{}'.format(l_fc7)
+
+    filename = 'snapshots/{}.caffemodel'.format(out)
+    net_svd.save(filename)
+    print 'Wrote svd model to: {:s}'.format(filename)
+
+
+if __name__ == '__main__':
+    main()

models/VGG_CNN_M_1024/compressed/test.prototxt

+name: "VGG_CNN_M_1024"
+input: "data"
+input_shape {
+  dim: 1
+  dim: 3
+  dim: 224
+  dim: 224
+}
+input: "rois"
+input_shape {
+  dim: 1 # to be changed on-the-fly to num ROIs
+  dim: 5 # [batch ind, x1, y1, x2, y2] zero-based indexing
+  dim: 1
+  dim: 1
+}
+layer {
+  name: "conv1"
+  type: "Convolution"
+  bottom: "data"
+  top: "conv1"
+  param {
+    lr_mult: 0
+    decay_mult: 0
+  }
+  param {
+    lr_mult: 0
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 96
+    kernel_size: 7
+    stride: 2
+  }
+}
+layer {
+  name: "relu1"
+  type: "ReLU"
+  bottom: "conv1"
+  top: "conv1"
+}
+layer {
+  name: "norm1"
+  type: "LRN"
+  bottom: "conv1"
+  top: "norm1"
+  lrn_param {
+    local_size: 5
+    alpha: 0.0005
+    beta: 0.75
+    k: 2
+  }
+}
+layer {
+  name: "pool1"
+  type: "Pooling"
+  bottom: "norm1"
+  top: "pool1"
+  pooling_param {
+    pool: MAX
+    kernel_size: 3
+    stride: 2
+  }
+}
+layer {
+  name: "conv2"
+  type: "Convolution"
+  bottom: "pool1"
+  top: "conv2"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 256
+    pad: 1
+    kernel_size: 5
+    stride: 2
+  }
+}
+layer {
+  name: "relu2"
+  type: "ReLU"
+  bottom: "conv2"
+  top: "conv2"
+}
+layer {
+  name: "norm2"
+  type: "LRN"
+  bottom: "conv2"
+  top: "norm2"
+  lrn_param {
+    local_size: 5
+    alpha: 0.0005
+    beta: 0.75
+    k: 2
+  }
+}
+layer {
+  name: "pool2"
+  type: "Pooling"
+  bottom: "norm2"
+  top: "pool2"
+  pooling_param {
+    pool: MAX
+    kernel_size: 3
+    stride: 2
+  }
+}
+layer {
+  name: "conv3"
+  type: "Convolution"
+  bottom: "pool2"
+  top: "conv3"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 512
+    pad: 1
+    kernel_size: 3
+  }
+}
+layer {
+  name: "relu3"
+  type: "ReLU"
+  bottom: "conv3"
+  top: "conv3"
+}
+layer {
+  name: "conv4"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 512
+    pad: 1
+    kernel_size: 3
+  }
+}
+layer {
+  name: "relu4"
+  type: "ReLU"
+  bottom: "conv4"
+  top: "conv4"
+}
+layer {
+  name: "conv5"
+  type: "Convolution"
+  bottom: "conv4"
+  top: "conv5"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 512
+    pad: 1
+    kernel_size: 3
+  }
+}
+layer {
+  name: "relu5"
+  type: "ReLU"
+  bottom: "conv5"
+  top: "conv5"
+}
+layer {
+  name: "roi_pool5"
+  type: "ROIPooling"
+  bottom: "conv5"
+  bottom: "rois"
+  top: "pool5"
+  roi_pooling_param {
+    pooled_w: 6
+    pooled_h: 6
+  }
+}
+layer {
+  name: "fc6_L"
+  type: "InnerProduct"
+  bottom: "pool5"
+  top: "fc6_L"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  inner_product_param {
+    num_output: 1024
+    bias_term: false
+  }
+}
+layer {
+  name: "fc6_U"
+  type: "InnerProduct"
+  bottom: "fc6_L"
+  top: "fc6_U"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  inner_product_param {
+    num_output: 4096
+  }
+}
+layer {
+  name: "relu6"
+  type: "ReLU"
+  bottom: "fc6_U"
+  top: "fc6_U"
+}
+layer {
+  name: "drop6"
+  type: "Dropout"
+  bottom: "fc6_U"
+  top: "fc6_U"
+  dropout_param {
+    dropout_ratio: 0.5
+  }
+}
+layer {
+  name: "fc7_L"
+  type: "InnerProduct"
+  bottom: "fc6_U"
+  top: "fc7_L"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  inner_product_param {
+    num_output: 256
+    bias_term: false
+  }
+}
+layer {
+  name: "fc7_U"
+  type: "InnerProduct"
+  bottom: "fc7_L"
+  top: "fc7_U"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  inner_product_param {
+    num_output: 1024
+  }
+}
+layer {
+  name: "relu7"
+  type: "ReLU"
+  bottom: "fc7_U"
+  top: "fc7_U"
+}
+layer {
+  name: "drop7"
+  type: "Dropout"
+  bottom: "fc7_U"
+  top: "fc7_U"
+  dropout_param {
+    dropout_ratio: 0.5
+  }
+}
+layer {
+  name: "cls_score"
+  type: "InnerProduct"
+  bottom: "fc7_U"
+  top: "cls_score"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  inner_product_param {
+    num_output: 21
+    weight_filler {
+      type: "gaussian"
+      std: 0.01
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "bbox_pred"
+  type: "InnerProduct"
+  bottom: "fc7_U"
+  top: "bbox_pred"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  inner_product_param {
+    num_output: 84
+    weight_filler {
+      type: "gaussian"
+      std: 0.001
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "cls_prob"
+  type: "Softmax"
+  bottom: "cls_score"
+  top: "cls_prob"
+}