diff --git a/cnn_e2e/alexnet_model.py b/cnn_e2e/alexnet_model.py new file mode 100755 index 0000000000000000000000000000000000000000..9a559540964a8c158bebb419652baccd74b7aad6 --- /dev/null +++ b/cnn_e2e/alexnet_model.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from model_util import conv2d_layer + + +def alexnet(images, need_transpose=False): + + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + + conv1 = conv2d_layer( + "conv1", images, filters=64, kernel_size=11, strides=4, padding="VALID" + ) + + pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1") + + conv2 = conv2d_layer("conv2", pool1, filters=192, kernel_size=5) + + pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2") + + conv3 = conv2d_layer("conv3", pool2, filters=384) + + conv4 = conv2d_layer("conv4", conv3, filters=384) + + conv5 = conv2d_layer("conv5", conv4, filters=256) + + pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5") + + if len(pool5.shape) > 2: + pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1)) + + fc1 = flow.layers.dense( + inputs=pool5, + units=4096, + activation=flow.keras.activations.relu, + use_bias=False, + kernel_initializer=flow.random_uniform_initializer(), + bias_initializer=False, + name="fc1", + ) + + dropout1 = flow.nn.dropout(fc1, rate=0.5) + + fc2 = flow.layers.dense( + inputs=dropout1, + units=4096, + activation=flow.keras.activations.relu, + use_bias=False, + kernel_initializer=flow.random_uniform_initializer(), + bias_initializer=False, + name="fc2", + ) + + dropout2 = flow.nn.dropout(fc2, rate=0.5) + + fc3 = flow.layers.dense( + inputs=dropout2, + units=1001, + activation=None, + use_bias=False, + kernel_initializer=flow.random_uniform_initializer(), + bias_initializer=False, + name="fc3", + ) + + return fc3 diff --git a/cnn_e2e/inception_model.py b/cnn_e2e/inception_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c427dd6b7eee280451b2aa143e93a35ba2ade890 --- /dev/null +++ b/cnn_e2e/inception_model.py @@ -0,0 +1,446 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from model_util import conv2d_layer + + +def InceptionA(in_blob, index): + with flow.deprecated.variable_scope("mixed_{}".format(index)): + with flow.deprecated.variable_scope("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + with flow.deprecated.variable_scope("branch5x5"): + branch5x5_1 = conv2d_layer( + "conv0", in_blob, filters=48, kernel_size=1, strides=1, padding="SAME" + ) + branch5x5_2 = conv2d_layer( + "conv1", + branch5x5_1, + filters=64, + kernel_size=5, + strides=1, + padding="SAME", + ) + with flow.deprecated.variable_scope("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + with flow.deprecated.variable_scope("branch_pool"): + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=32 if index == 0 else 64, + kernel_size=1, + strides=1, + padding="SAME", + ) + + inceptionA_bn = [] + inceptionA_bn.append(branch1x1) + inceptionA_bn.append(branch5x5_2) + inceptionA_bn.append(branch3x3dbl_3) + inceptionA_bn.append(branch_pool_2) + + mixed_concat = flow.concat(values=inceptionA_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionB(in_blob, index): + with flow.deprecated.variable_scope("mixed_{}".format(index)): + with flow.deprecated.variable_scope("branch3x3"): + branch3x3 = conv2d_layer( + "conv0", in_blob, filters=384, kernel_size=3, strides=2, padding="VALID" + ) + with flow.deprecated.variable_scope("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=96, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.deprecated.variable_scope("branch_pool"): + branch_pool = flow.nn.max_pool2d( + in_blob, + ksize=3, + strides=2, + padding="VALID", + data_format="NCHW", + name="pool0", + ) + + inceptionB_bn = [] + inceptionB_bn.append(branch3x3) + inceptionB_bn.append(branch3x3dbl_3) + inceptionB_bn.append(branch_pool) + mixed_concat = flow.concat(values=inceptionB_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionC(in_blob, index, filters): + with flow.deprecated.variable_scope("mixed_{}".format(index)): + with flow.deprecated.variable_scope("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + with flow.deprecated.variable_scope("branch7x7"): + branch7x7_1 = conv2d_layer( + "conv0", + in_blob, + filters=filters, + kernel_size=1, + strides=1, + padding="SAME", + ) + branch7x7_2 = conv2d_layer( + "conv1", + branch7x7_1, + filters=filters, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7_3 = conv2d_layer( + "conv2", + branch7x7_2, + filters=192, + kernel_size=[7, 1], + strides=[1, 1], + padding="SAME", + ) + with flow.deprecated.variable_scope("branch7x7dbl"): + branch7x7dbl_1 = conv2d_layer( + "conv0", + in_blob, + filters=filters, + kernel_size=1, + strides=1, + padding="SAME", + ) + branch7x7dbl_2 = conv2d_layer( + "conv1", + branch7x7dbl_1, + filters=filters, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7dbl_3 = conv2d_layer( + "conv2", + branch7x7dbl_2, + filters=filters, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7dbl_4 = conv2d_layer( + "conv3", + branch7x7dbl_3, + filters=filters, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7dbl_5 = conv2d_layer( + "conv4", + branch7x7dbl_4, + filters=192, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + with flow.deprecated.variable_scope("branch_pool"): + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=192, + kernel_size=[1, 1], + strides=1, + padding="SAME", + ) + + inceptionC_bn = [] + inceptionC_bn.append(branch1x1) + inceptionC_bn.append(branch7x7_3) + inceptionC_bn.append(branch7x7dbl_5) + inceptionC_bn.append(branch_pool_2) + mixed_concat = flow.concat(values=inceptionC_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionD(in_blob, index): + with flow.deprecated.variable_scope("mixed_{}".format(index)): + with flow.deprecated.variable_scope("branch3x3"): + branch3x3_1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3_2 = conv2d_layer( + "conv1", + branch3x3_1, + filters=320, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.deprecated.variable_scope("branch7x7x3"): + branch7x7x3_1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + branch7x7x3_2 = conv2d_layer( + "conv1", + branch7x7x3_1, + filters=192, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7x3_3 = conv2d_layer( + "conv2", + branch7x7x3_2, + filters=192, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7x3_4 = conv2d_layer( + "conv3", + branch7x7x3_3, + filters=192, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.deprecated.variable_scope("branch_pool"): + branch_pool = flow.nn.max_pool2d( + in_blob, + ksize=3, + strides=2, + padding="VALID", + data_format="NCHW", + name="pool", + ) + + inceptionD_bn = [] + inceptionD_bn.append(branch3x3_2) + inceptionD_bn.append(branch7x7x3_4) + inceptionD_bn.append(branch_pool) + + mixed_concat = flow.concat(values=inceptionD_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionE(in_blob, index): + with flow.deprecated.variable_scope("mixed_{}".format(index)): + with flow.deprecated.variable_scope("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=320, kernel_size=1, strides=1, padding="SAME" + ) + with flow.deprecated.variable_scope("branch3x3"): + branch3x3_1 = conv2d_layer( + "conv0", in_blob, filters=384, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3_2 = conv2d_layer( + "conv1", + branch3x3_1, + filters=384, + kernel_size=[1, 3], + strides=1, + padding="SAME", + ) + branch3x3_3 = conv2d_layer( + "conv2", + branch3x3_1, + filters=384, + kernel_size=[3, 1], + strides=[1, 1], + padding="SAME", + ) + inceptionE_1_bn = [] + inceptionE_1_bn.append(branch3x3_2) + inceptionE_1_bn.append(branch3x3_3) + concat_branch3x3 = flow.concat( + values=inceptionE_1_bn, axis=1, name="concat" + ) + with flow.deprecated.variable_scope("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=448, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=384, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=384, + kernel_size=[1, 3], + strides=1, + padding="SAME", + ) + branch3x3dbl_4 = conv2d_layer( + "conv3", + branch3x3dbl_2, + filters=384, + kernel_size=[3, 1], + strides=1, + padding="SAME", + ) + inceptionE_2_bn = [] + inceptionE_2_bn.append(branch3x3dbl_3) + inceptionE_2_bn.append(branch3x3dbl_4) + concat_branch3x3dbl = flow.concat( + values=inceptionE_2_bn, axis=1, name="concat" + ) + with flow.deprecated.variable_scope("branch_pool"): + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=192, + kernel_size=[1, 1], + strides=1, + padding="SAME", + ) + + inceptionE_total_bn = [] + inceptionE_total_bn.append(branch1x1) + inceptionE_total_bn.append(concat_branch3x3) + inceptionE_total_bn.append(concat_branch3x3dbl) + inceptionE_total_bn.append(branch_pool_2) + + concat_total = flow.concat( + values=inceptionE_total_bn, axis=1, name="concat") + + return concat_total + + +def inceptionv3(images, trainable=True, need_transpose=False): + + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + + with flow.deprecated.variable_scope("InceptionV3"): + conv0 = conv2d_layer( + "conv0", images, filters=32, kernel_size=3, strides=2, padding="VALID" + ) + conv1 = conv2d_layer( + "conv1", conv0, filters=32, kernel_size=3, strides=1, padding="VALID" + ) + conv2 = conv2d_layer( + "conv2", conv1, filters=64, kernel_size=3, strides=1, padding="SAME" + ) + pool1 = flow.nn.max_pool2d( + conv2, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool1" + ) + conv3 = conv2d_layer( + "conv3", pool1, filters=80, kernel_size=1, strides=1, padding="VALID" + ) + conv4 = conv2d_layer( + "conv4", conv3, filters=192, kernel_size=3, strides=1, padding="VALID" + ) + pool2 = flow.nn.max_pool2d( + conv4, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool2" + ) + + # mixed_0 ~ mixed_2 + mixed_0 = InceptionA(pool2, 0) + mixed_1 = InceptionA(mixed_0, 1) + mixed_2 = InceptionA(mixed_1, 2) + + # mixed_3 + mixed_3 = InceptionB(mixed_2, 3) + + # mixed_4 ~ mixed_7 + mixed_4 = InceptionC(mixed_3, 4, 128) + mixed_5 = InceptionC(mixed_4, 5, 160) + mixed_6 = InceptionC(mixed_5, 6, 160) + mixed_7 = InceptionC(mixed_6, 7, 192) + + # mixed_8 + mixed_8 = InceptionD(mixed_7, 8) + + # mixed_9 ~ mixed_10 + mixed_9 = InceptionE(mixed_8, 9) + mixed_10 = InceptionE(mixed_9, 10) + + # pool3 + pool3 = flow.nn.avg_pool2d( + mixed_10, ksize=8, strides=1, padding="VALID", data_format="NCHW", name="pool3" + ) + + # TODO: Need to transpose weight when converting model from TF to OF if + # you want to use layers.dense interface. + fc1 = flow.layers.dense( + flow.reshape(pool3, [pool3.shape[0], -1]), + 1001, + activation=None, + use_bias=False, + kernel_initializer=flow.truncated_normal(0.816496580927726), + bias_initializer=flow.constant_initializer(), + trainable=trainable, + name="fc1", + ) + + return fc1 diff --git a/cnn_e2e/model_util.py b/cnn_e2e/model_util.py new file mode 100644 index 0000000000000000000000000000000000000000..8cc4c917d0efa351ca7e605783743f936abbcd8c --- /dev/null +++ b/cnn_e2e/model_util.py @@ -0,0 +1,45 @@ +from __future__ import absolute_import + +import oneflow as flow + + +def conv2d_layer( + name, + input, + filters, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=flow.random_uniform_initializer(), + bias_initializer=flow.constant_initializer(), +): + weight_shape = (filters, input.shape[1], kernel_size, kernel_size) + weight = flow.get_variable( + name + "-weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "-bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + output = flow.keras.activations.relu(output) + else: + raise NotImplementedError + + return output diff --git a/cnn_e2e/of_cnn_train_val.py b/cnn_e2e/of_cnn_train_val.py index 20b3022c113bd327595d6a4fa05a31631c362b43..67dd2295b3fdb66e04bf31f6a294adc03d925608 100755 --- a/cnn_e2e/of_cnn_train_val.py +++ b/cnn_e2e/of_cnn_train_val.py @@ -20,6 +20,7 @@ import oneflow as flow import alexnet_model import vgg_model import resnet_model +import inception_model @@ -35,6 +36,7 @@ model_dict = { "resnet50": resnet_model.resnet50, "vgg16": vgg_model.vgg16, "alexnet": alexnet_model.alexnet, + "inceptionv3":inception_model.inceptionv3, } diff --git a/cnn_e2e/resnet_model.py b/cnn_e2e/resnet_model.py index 32a2a405131fe2859fb1eb6aa9a044e26e90395c..01e91b18bb977e984b8d73439697c3a6f2d51055 100755 --- a/cnn_e2e/resnet_model.py +++ b/cnn_e2e/resnet_model.py @@ -19,9 +19,8 @@ def _conv2d( padding="SAME", data_format="NCHW", dilations=1, - trainable=True, - #weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"), - weight_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal', data_format="NCHW"), + weight_initializer=flow.variance_scaling_initializer( + 2, 'fan_in', 'random_normal', data_format="NCHW"), weight_regularizer=flow.regularizers.l2(1.0/32768), ): weight = flow.get_variable( @@ -42,7 +41,7 @@ def _batch_norm(inputs, name=None, trainable=True): return flow.layers.batch_normalization( inputs=inputs, axis=1, - momentum=0.9,#97, + momentum=0.9, # 97, epsilon=1.001e-5, center=True, scale=True, @@ -142,8 +141,9 @@ def resnet50(images, trainable=True, need_transpose=False): flow.reshape(pool5, (pool5.shape[0], -1)), units=1000, use_bias=True, - kernel_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal'), - #kernel_initializer=flow.xavier_uniform_initializer(), + kernel_initializer=flow.variance_scaling_initializer( + 2, 'fan_in', 'random_normal'), + # kernel_initializer=flow.xavier_uniform_initializer(), bias_initializer=flow.zeros_initializer(), kernel_regularizer=flow.regularizers.l2(1.0/32768), trainable=trainable, diff --git a/cnn_e2e/vgg_model.py b/cnn_e2e/vgg_model.py new file mode 100755 index 0000000000000000000000000000000000000000..274096ddbe13eb766b1f6dea198e81e72d3fdcd1 --- /dev/null +++ b/cnn_e2e/vgg_model.py @@ -0,0 +1,78 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from model_util import conv2d_layer + + +def _conv_block(in_blob, index, filters, conv_times): + conv_block = [] + conv_block.insert(0, in_blob) + for i in range(conv_times): + conv_i = conv2d_layer( + name="conv{}".format(index), + input=conv_block[i], + filters=filters, + kernel_size=3, + strides=1, + ) + conv_block.append(conv_i) + index += 1 + + return conv_block + + +def vgg16(images, need_transpose=False): + + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + + conv1 = _conv_block(images, 0, 64, 2) + pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", "NCHW", name="pool1") + + conv2 = _conv_block(pool1, 2, 128, 2) + pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", "NCHW", name="pool2") + + conv3 = _conv_block(pool2, 4, 256, 3) + pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", "NCHW", name="pool3") + + conv4 = _conv_block(pool3, 7, 512, 3) + pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2, "VALID", "NCHW", name="pool4") + + conv5 = _conv_block(pool4, 10, 512, 3) + pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", "NCHW", name="pool5") + + fc6 = flow.layers.dense( + inputs=flow.reshape(pool5, [pool5.shape[0], -1]), + units=4096, + activation=flow.keras.activations.relu, + use_bias=True, + kernel_initializer=flow.truncated_normal(0.816496580927726), + bias_initializer=flow.constant_initializer(), + name="fc1", + ) + + fc6 = flow.nn.dropout(fc6, rate=0.5) + + fc7 = flow.layers.dense( + inputs=fc6, + units=4096, + activation=flow.keras.activations.relu, + use_bias=True, + kernel_initializer=flow.truncated_normal(0.816496580927726), + bias_initializer=flow.constant_initializer(), + name="fc2", + ) + fc7 = flow.nn.dropout(fc7, rate=0.5) + + fc8 = flow.layers.dense( + inputs=fc7, + units=1001, + use_bias=True, + kernel_initializer=flow.truncated_normal(0.816496580927726), + bias_initializer=flow.constant_initializer(), + name="fc_final", + ) + + return fc8