add alexnet,vgg16,inceptionv3 for e2e

cnn_e2e/alexnet_model.py

+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import oneflow as flow
+from model_util import conv2d_layer
+
+
+def alexnet(images, need_transpose=False):
+
+    if need_transpose:
+        images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
+
+    conv1 = conv2d_layer(
+        "conv1", images, filters=64, kernel_size=11, strides=4, padding="VALID"
+    )
+
+    pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1")
+
+    conv2 = conv2d_layer("conv2", pool1, filters=192, kernel_size=5)
+
+    pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2")
+
+    conv3 = conv2d_layer("conv3", pool2, filters=384)
+
+    conv4 = conv2d_layer("conv4", conv3, filters=384)
+
+    conv5 = conv2d_layer("conv5", conv4, filters=256)
+
+    pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5")
+
+    if len(pool5.shape) > 2:
+        pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1))
+
+    fc1 = flow.layers.dense(
+        inputs=pool5,
+        units=4096,
+        activation=flow.keras.activations.relu,
+        use_bias=False,
+        kernel_initializer=flow.random_uniform_initializer(),
+        bias_initializer=False,
+        name="fc1",
+    )
+
+    dropout1 = flow.nn.dropout(fc1, rate=0.5)
+
+    fc2 = flow.layers.dense(
+        inputs=dropout1,
+        units=4096,
+        activation=flow.keras.activations.relu,
+        use_bias=False,
+        kernel_initializer=flow.random_uniform_initializer(),
+        bias_initializer=False,
+        name="fc2",
+    )
+
+    dropout2 = flow.nn.dropout(fc2, rate=0.5)
+
+    fc3 = flow.layers.dense(
+        inputs=dropout2,
+        units=1001,
+        activation=None,
+        use_bias=False,
+        kernel_initializer=flow.random_uniform_initializer(),
+        bias_initializer=False,
+        name="fc3",
+    )
+
+    return fc3

cnn_e2e/inception_model.py

+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import oneflow as flow
+from model_util import conv2d_layer
+
+
+def InceptionA(in_blob, index):
+    with flow.deprecated.variable_scope("mixed_{}".format(index)):
+        with flow.deprecated.variable_scope("branch1x1"):
+            branch1x1 = conv2d_layer(
+                "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME"
+            )
+        with flow.deprecated.variable_scope("branch5x5"):
+            branch5x5_1 = conv2d_layer(
+                "conv0", in_blob, filters=48, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch5x5_2 = conv2d_layer(
+                "conv1",
+                branch5x5_1,
+                filters=64,
+                kernel_size=5,
+                strides=1,
+                padding="SAME",
+            )
+        with flow.deprecated.variable_scope("branch3x3dbl"):
+            branch3x3dbl_1 = conv2d_layer(
+                "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch3x3dbl_2 = conv2d_layer(
+                "conv1",
+                branch3x3dbl_1,
+                filters=96,
+                kernel_size=3,
+                strides=1,
+                padding="SAME",
+            )
+            branch3x3dbl_3 = conv2d_layer(
+                "conv2",
+                branch3x3dbl_2,
+                filters=96,
+                kernel_size=3,
+                strides=1,
+                padding="SAME",
+            )
+        with flow.deprecated.variable_scope("branch_pool"):
+            branch_pool_1 = flow.nn.avg_pool2d(
+                in_blob,
+                ksize=3,
+                strides=1,
+                padding="SAME",
+                data_format="NCHW",
+                name="pool",
+            )
+            branch_pool_2 = conv2d_layer(
+                "conv",
+                branch_pool_1,
+                filters=32 if index == 0 else 64,
+                kernel_size=1,
+                strides=1,
+                padding="SAME",
+            )
+
+        inceptionA_bn = []
+        inceptionA_bn.append(branch1x1)
+        inceptionA_bn.append(branch5x5_2)
+        inceptionA_bn.append(branch3x3dbl_3)
+        inceptionA_bn.append(branch_pool_2)
+
+        mixed_concat = flow.concat(values=inceptionA_bn, axis=1, name="concat")
+
+    return mixed_concat
+
+
+def InceptionB(in_blob, index):
+    with flow.deprecated.variable_scope("mixed_{}".format(index)):
+        with flow.deprecated.variable_scope("branch3x3"):
+            branch3x3 = conv2d_layer(
+                "conv0", in_blob, filters=384, kernel_size=3, strides=2, padding="VALID"
+            )
+        with flow.deprecated.variable_scope("branch3x3dbl"):
+            branch3x3dbl_1 = conv2d_layer(
+                "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch3x3dbl_2 = conv2d_layer(
+                "conv1",
+                branch3x3dbl_1,
+                filters=96,
+                kernel_size=3,
+                strides=1,
+                padding="SAME",
+            )
+            branch3x3dbl_3 = conv2d_layer(
+                "conv2",
+                branch3x3dbl_2,
+                filters=96,
+                kernel_size=3,
+                strides=2,
+                padding="VALID",
+            )
+        with flow.deprecated.variable_scope("branch_pool"):
+            branch_pool = flow.nn.max_pool2d(
+                in_blob,
+                ksize=3,
+                strides=2,
+                padding="VALID",
+                data_format="NCHW",
+                name="pool0",
+            )
+
+        inceptionB_bn = []
+        inceptionB_bn.append(branch3x3)
+        inceptionB_bn.append(branch3x3dbl_3)
+        inceptionB_bn.append(branch_pool)
+        mixed_concat = flow.concat(values=inceptionB_bn, axis=1, name="concat")
+
+    return mixed_concat
+
+
+def InceptionC(in_blob, index, filters):
+    with flow.deprecated.variable_scope("mixed_{}".format(index)):
+        with flow.deprecated.variable_scope("branch1x1"):
+            branch1x1 = conv2d_layer(
+                "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME"
+            )
+        with flow.deprecated.variable_scope("branch7x7"):
+            branch7x7_1 = conv2d_layer(
+                "conv0",
+                in_blob,
+                filters=filters,
+                kernel_size=1,
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7_2 = conv2d_layer(
+                "conv1",
+                branch7x7_1,
+                filters=filters,
+                kernel_size=[1, 7],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7_3 = conv2d_layer(
+                "conv2",
+                branch7x7_2,
+                filters=192,
+                kernel_size=[7, 1],
+                strides=[1, 1],
+                padding="SAME",
+            )
+        with flow.deprecated.variable_scope("branch7x7dbl"):
+            branch7x7dbl_1 = conv2d_layer(
+                "conv0",
+                in_blob,
+                filters=filters,
+                kernel_size=1,
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7dbl_2 = conv2d_layer(
+                "conv1",
+                branch7x7dbl_1,
+                filters=filters,
+                kernel_size=[7, 1],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7dbl_3 = conv2d_layer(
+                "conv2",
+                branch7x7dbl_2,
+                filters=filters,
+                kernel_size=[1, 7],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7dbl_4 = conv2d_layer(
+                "conv3",
+                branch7x7dbl_3,
+                filters=filters,
+                kernel_size=[7, 1],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7dbl_5 = conv2d_layer(
+                "conv4",
+                branch7x7dbl_4,
+                filters=192,
+                kernel_size=[1, 7],
+                strides=1,
+                padding="SAME",
+            )
+        with flow.deprecated.variable_scope("branch_pool"):
+            branch_pool_1 = flow.nn.avg_pool2d(
+                in_blob,
+                ksize=3,
+                strides=1,
+                padding="SAME",
+                data_format="NCHW",
+                name="pool",
+            )
+            branch_pool_2 = conv2d_layer(
+                "conv",
+                branch_pool_1,
+                filters=192,
+                kernel_size=[1, 1],
+                strides=1,
+                padding="SAME",
+            )
+
+        inceptionC_bn = []
+        inceptionC_bn.append(branch1x1)
+        inceptionC_bn.append(branch7x7_3)
+        inceptionC_bn.append(branch7x7dbl_5)
+        inceptionC_bn.append(branch_pool_2)
+        mixed_concat = flow.concat(values=inceptionC_bn, axis=1, name="concat")
+
+    return mixed_concat
+
+
+def InceptionD(in_blob, index):
+    with flow.deprecated.variable_scope("mixed_{}".format(index)):
+        with flow.deprecated.variable_scope("branch3x3"):
+            branch3x3_1 = conv2d_layer(
+                "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch3x3_2 = conv2d_layer(
+                "conv1",
+                branch3x3_1,
+                filters=320,
+                kernel_size=3,
+                strides=2,
+                padding="VALID",
+            )
+        with flow.deprecated.variable_scope("branch7x7x3"):
+            branch7x7x3_1 = conv2d_layer(
+                "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch7x7x3_2 = conv2d_layer(
+                "conv1",
+                branch7x7x3_1,
+                filters=192,
+                kernel_size=[1, 7],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7x3_3 = conv2d_layer(
+                "conv2",
+                branch7x7x3_2,
+                filters=192,
+                kernel_size=[7, 1],
+                strides=1,
+                padding="SAME",
+            )
+            branch7x7x3_4 = conv2d_layer(
+                "conv3",
+                branch7x7x3_3,
+                filters=192,
+                kernel_size=3,
+                strides=2,
+                padding="VALID",
+            )
+        with flow.deprecated.variable_scope("branch_pool"):
+            branch_pool = flow.nn.max_pool2d(
+                in_blob,
+                ksize=3,
+                strides=2,
+                padding="VALID",
+                data_format="NCHW",
+                name="pool",
+            )
+
+        inceptionD_bn = []
+        inceptionD_bn.append(branch3x3_2)
+        inceptionD_bn.append(branch7x7x3_4)
+        inceptionD_bn.append(branch_pool)
+
+        mixed_concat = flow.concat(values=inceptionD_bn, axis=1, name="concat")
+
+    return mixed_concat
+
+
+def InceptionE(in_blob, index):
+    with flow.deprecated.variable_scope("mixed_{}".format(index)):
+        with flow.deprecated.variable_scope("branch1x1"):
+            branch1x1 = conv2d_layer(
+                "conv0", in_blob, filters=320, kernel_size=1, strides=1, padding="SAME"
+            )
+        with flow.deprecated.variable_scope("branch3x3"):
+            branch3x3_1 = conv2d_layer(
+                "conv0", in_blob, filters=384, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch3x3_2 = conv2d_layer(
+                "conv1",
+                branch3x3_1,
+                filters=384,
+                kernel_size=[1, 3],
+                strides=1,
+                padding="SAME",
+            )
+            branch3x3_3 = conv2d_layer(
+                "conv2",
+                branch3x3_1,
+                filters=384,
+                kernel_size=[3, 1],
+                strides=[1, 1],
+                padding="SAME",
+            )
+            inceptionE_1_bn = []
+            inceptionE_1_bn.append(branch3x3_2)
+            inceptionE_1_bn.append(branch3x3_3)
+            concat_branch3x3 = flow.concat(
+                values=inceptionE_1_bn, axis=1, name="concat"
+            )
+        with flow.deprecated.variable_scope("branch3x3dbl"):
+            branch3x3dbl_1 = conv2d_layer(
+                "conv0", in_blob, filters=448, kernel_size=1, strides=1, padding="SAME"
+            )
+            branch3x3dbl_2 = conv2d_layer(
+                "conv1",
+                branch3x3dbl_1,
+                filters=384,
+                kernel_size=3,
+                strides=1,
+                padding="SAME",
+            )
+            branch3x3dbl_3 = conv2d_layer(
+                "conv2",
+                branch3x3dbl_2,
+                filters=384,
+                kernel_size=[1, 3],
+                strides=1,
+                padding="SAME",
+            )
+            branch3x3dbl_4 = conv2d_layer(
+                "conv3",
+                branch3x3dbl_2,
+                filters=384,
+                kernel_size=[3, 1],
+                strides=1,
+                padding="SAME",
+            )
+            inceptionE_2_bn = []
+            inceptionE_2_bn.append(branch3x3dbl_3)
+            inceptionE_2_bn.append(branch3x3dbl_4)
+            concat_branch3x3dbl = flow.concat(
+                values=inceptionE_2_bn, axis=1, name="concat"
+            )
+        with flow.deprecated.variable_scope("branch_pool"):
+            branch_pool_1 = flow.nn.avg_pool2d(
+                in_blob,
+                ksize=3,
+                strides=1,
+                padding="SAME",
+                data_format="NCHW",
+                name="pool",
+            )
+            branch_pool_2 = conv2d_layer(
+                "conv",
+                branch_pool_1,
+                filters=192,
+                kernel_size=[1, 1],
+                strides=1,
+                padding="SAME",
+            )
+
+        inceptionE_total_bn = []
+        inceptionE_total_bn.append(branch1x1)
+        inceptionE_total_bn.append(concat_branch3x3)
+        inceptionE_total_bn.append(concat_branch3x3dbl)
+        inceptionE_total_bn.append(branch_pool_2)
+
+        concat_total = flow.concat(
+            values=inceptionE_total_bn, axis=1, name="concat")
+
+    return concat_total
+
+
+def inceptionv3(images, trainable=True, need_transpose=False):
+
+    if need_transpose:
+        images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
+
+    with flow.deprecated.variable_scope("InceptionV3"):
+        conv0 = conv2d_layer(
+            "conv0", images, filters=32, kernel_size=3, strides=2, padding="VALID"
+        )
+        conv1 = conv2d_layer(
+            "conv1", conv0, filters=32, kernel_size=3, strides=1, padding="VALID"
+        )
+        conv2 = conv2d_layer(
+            "conv2", conv1, filters=64, kernel_size=3, strides=1, padding="SAME"
+        )
+        pool1 = flow.nn.max_pool2d(
+            conv2, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool1"
+        )
+        conv3 = conv2d_layer(
+            "conv3", pool1, filters=80, kernel_size=1, strides=1, padding="VALID"
+        )
+        conv4 = conv2d_layer(
+            "conv4", conv3, filters=192, kernel_size=3, strides=1, padding="VALID"
+        )
+        pool2 = flow.nn.max_pool2d(
+            conv4, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool2"
+        )
+
+        # mixed_0 ~ mixed_2
+        mixed_0 = InceptionA(pool2, 0)
+        mixed_1 = InceptionA(mixed_0, 1)
+        mixed_2 = InceptionA(mixed_1, 2)
+
+        # mixed_3
+        mixed_3 = InceptionB(mixed_2, 3)
+
+        # mixed_4 ~ mixed_7
+        mixed_4 = InceptionC(mixed_3, 4, 128)
+        mixed_5 = InceptionC(mixed_4, 5, 160)
+        mixed_6 = InceptionC(mixed_5, 6, 160)
+        mixed_7 = InceptionC(mixed_6, 7, 192)
+
+        # mixed_8
+        mixed_8 = InceptionD(mixed_7, 8)
+
+        # mixed_9 ~ mixed_10
+        mixed_9 = InceptionE(mixed_8, 9)
+        mixed_10 = InceptionE(mixed_9, 10)
+
+        # pool3
+        pool3 = flow.nn.avg_pool2d(
+            mixed_10, ksize=8, strides=1, padding="VALID", data_format="NCHW", name="pool3"
+        )
+
+        # TODO: Need to transpose weight when converting model from TF to OF if
+        # you want to use layers.dense interface.
+        fc1 = flow.layers.dense(
+            flow.reshape(pool3, [pool3.shape[0], -1]),
+            1001,
+            activation=None,
+            use_bias=False,
+            kernel_initializer=flow.truncated_normal(0.816496580927726),
+            bias_initializer=flow.constant_initializer(),
+            trainable=trainable,
+            name="fc1",
+        )
+
+    return fc1

cnn_e2e/model_util.py

+from __future__ import absolute_import
+
+import oneflow as flow
+
+
+def conv2d_layer(
+    name,
+    input,
+    filters,
+    kernel_size=3,
+    strides=1,
+    padding="SAME",
+    data_format="NCHW",
+    dilation_rate=1,
+    activation="Relu",
+    use_bias=True,
+    weight_initializer=flow.random_uniform_initializer(),
+    bias_initializer=flow.constant_initializer(),
+):
+    weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
+    weight = flow.get_variable(
+        name + "-weight",
+        shape=weight_shape,
+        dtype=input.dtype,
+        initializer=weight_initializer,
+    )
+    output = flow.nn.conv2d(
+        input, weight, strides, padding, data_format, dilation_rate, name=name
+    )
+    if use_bias:
+        bias = flow.get_variable(
+            name + "-bias",
+            shape=(filters,),
+            dtype=input.dtype,
+            initializer=bias_initializer,
+        )
+        output = flow.nn.bias_add(output, bias, data_format)
+
+    if activation is not None:
+        if activation == "Relu":
+            output = flow.keras.activations.relu(output)
+        else:
+            raise NotImplementedError
+
+    return output

cnn_e2e/of_cnn_train_val.py

@@ -20,6 +20,7 @@ import oneflow as flow
 import alexnet_model
 import vgg_model
 import resnet_model
+import inception_model
 
 
 
@@ -35,6 +36,7 @@ model_dict = {
     "resnet50": resnet_model.resnet50,
     "vgg16": vgg_model.vgg16,
     "alexnet": alexnet_model.alexnet,
+    "inceptionv3":inception_model.inceptionv3,
 }
 
 

cnn_e2e/resnet_model.py

@@ -19,9 +19,8 @@ def _conv2d(
     padding="SAME",
     data_format="NCHW",
     dilations=1,
-    trainable=True,
-    #weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"),
-    weight_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal', data_format="NCHW"),
+    weight_initializer=flow.variance_scaling_initializer(
+        2, 'fan_in', 'random_normal', data_format="NCHW"),
     weight_regularizer=flow.regularizers.l2(1.0/32768),
 ):
     weight = flow.get_variable(
@@ -42,7 +41,7 @@ def _batch_norm(inputs, name=None, trainable=True):
     return flow.layers.batch_normalization(
         inputs=inputs,
         axis=1,
-        momentum=0.9,#97,
+        momentum=0.9,  # 97,
         epsilon=1.001e-5,
         center=True,
         scale=True,
@@ -142,8 +141,9 @@ def resnet50(images, trainable=True, need_transpose=False):
             flow.reshape(pool5, (pool5.shape[0], -1)),
             units=1000,
             use_bias=True,
-            kernel_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal'),
-            #kernel_initializer=flow.xavier_uniform_initializer(),
+            kernel_initializer=flow.variance_scaling_initializer(
+                2, 'fan_in', 'random_normal'),
+            # kernel_initializer=flow.xavier_uniform_initializer(),
             bias_initializer=flow.zeros_initializer(),
             kernel_regularizer=flow.regularizers.l2(1.0/32768),
             trainable=trainable,

cnn_e2e/vgg_model.py

+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import oneflow as flow
+from model_util import conv2d_layer
+
+
+def _conv_block(in_blob, index, filters, conv_times):
+    conv_block = []
+    conv_block.insert(0, in_blob)
+    for i in range(conv_times):
+        conv_i = conv2d_layer(
+            name="conv{}".format(index),
+            input=conv_block[i],
+            filters=filters,
+            kernel_size=3,
+            strides=1,
+        )
+        conv_block.append(conv_i)
+        index += 1
+
+    return conv_block
+
+
+def vgg16(images, need_transpose=False):
+
+    if need_transpose:
+        images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
+
+    conv1 = _conv_block(images, 0, 64, 2)
+    pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", "NCHW", name="pool1")
+
+    conv2 = _conv_block(pool1, 2, 128, 2)
+    pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", "NCHW", name="pool2")
+
+    conv3 = _conv_block(pool2, 4, 256, 3)
+    pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", "NCHW", name="pool3")
+
+    conv4 = _conv_block(pool3, 7, 512, 3)
+    pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2, "VALID", "NCHW", name="pool4")
+
+    conv5 = _conv_block(pool4, 10, 512, 3)
+    pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", "NCHW", name="pool5")
+
+    fc6 = flow.layers.dense(
+        inputs=flow.reshape(pool5, [pool5.shape[0], -1]),
+        units=4096,
+        activation=flow.keras.activations.relu,
+        use_bias=True,
+        kernel_initializer=flow.truncated_normal(0.816496580927726),
+        bias_initializer=flow.constant_initializer(),
+        name="fc1",
+    )
+
+    fc6 = flow.nn.dropout(fc6, rate=0.5)
+
+    fc7 = flow.layers.dense(
+        inputs=fc6,
+        units=4096,
+        activation=flow.keras.activations.relu,
+        use_bias=True,
+        kernel_initializer=flow.truncated_normal(0.816496580927726),
+        bias_initializer=flow.constant_initializer(),
+        name="fc2",
+    )
+    fc7 = flow.nn.dropout(fc7, rate=0.5)
+
+    fc8 = flow.layers.dense(
+        inputs=fc7,
+        units=1001,
+        use_bias=True,
+        kernel_initializer=flow.truncated_normal(0.816496580927726),
+        bias_initializer=flow.constant_initializer(),
+        name="fc_final",
+    )
+
+    return fc8