!2418 add pretrain for lstm & vgg16 and remove lstm/vgg16/googlenet from...

!2418 add pretrain for lstm & vgg16 and remove lstm/vgg16/googlenet from directory 'mindspore/model_zoo'
Merge pull request !2418 from caojian05/ms_master_dev

mindspore/model_zoo/googlenet.py

-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""GoogleNet"""
-import mindspore.nn as nn
-from mindspore.common.initializer import TruncatedNormal
-from mindspore.ops import operations as P
-
-
-def weight_variable():
-    """Weight variable."""
-    return TruncatedNormal(0.02)
-
-
-class Conv2dBlock(nn.Cell):
-    """
-     Basic convolutional block
-     Args:
-         in_channles (int): Input channel.
-         out_channels (int): Output channel.
-         kernel_size (int): Input kernel size. Default: 1
-         stride (int): Stride size for the first convolutional layer. Default: 1.
-         padding (int): Implicit paddings on both sides of the input. Default: 0.
-         pad_mode (str): Padding mode. Optional values are "same", "valid", "pad". Default: "same".
-      Returns:
-          Tensor, output tensor.
-    """
-
-    def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, pad_mode="same"):
-        super(Conv2dBlock, self).__init__()
-        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride,
-                              padding=padding, pad_mode=pad_mode, weight_init=weight_variable(),
-                              bias_init=False)
-        self.bn = nn.BatchNorm2d(out_channels, eps=0.001)
-        self.relu = nn.ReLU()
-
-    def construct(self, x):
-        x = self.conv(x)
-        x = self.bn(x)
-        x = self.relu(x)
-        return x
-
-
-class Inception(nn.Cell):
-    """
-    Inception Block
-    """
-
-    def __init__(self, in_channels, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes):
-        super(Inception, self).__init__()
-        self.b1 = Conv2dBlock(in_channels, n1x1, kernel_size=1)
-        self.b2 = nn.SequentialCell([Conv2dBlock(in_channels, n3x3red, kernel_size=1),
-                                     Conv2dBlock(n3x3red, n3x3, kernel_size=3, padding=0)])
-        self.b3 = nn.SequentialCell([Conv2dBlock(in_channels, n5x5red, kernel_size=1),
-                                     Conv2dBlock(n5x5red, n5x5, kernel_size=3, padding=0)])
-        self.maxpool = P.MaxPoolWithArgmax(ksize=3, strides=1, padding="same")
-        self.b4 = Conv2dBlock(in_channels, pool_planes, kernel_size=1)
-        self.concat = P.Concat(axis=1)
-
-    def construct(self, x):
-        branch1 = self.b1(x)
-        branch2 = self.b2(x)
-        branch3 = self.b3(x)
-        cell, argmax = self.maxpool(x)
-        branch4 = self.b4(cell)
-        _ = argmax
-        return self.concat((branch1, branch2, branch3, branch4))
-
-
-class GooGLeNet(nn.Cell):
-    """
-    Googlenet architecture
-    """
-
-    def __init__(self, num_classes):
-        super(GooGLeNet, self).__init__()
-        self.conv1 = Conv2dBlock(3, 64, kernel_size=7, stride=2, padding=0)
-        self.maxpool1 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same")
-
-        self.conv2 = Conv2dBlock(64, 64, kernel_size=1)
-        self.conv3 = Conv2dBlock(64, 192, kernel_size=3, padding=0)
-        self.maxpool2 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same")
-
-        self.block3a = Inception(192, 64, 96, 128, 16, 32, 32)
-        self.block3b = Inception(256, 128, 128, 192, 32, 96, 64)
-        self.maxpool3 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same")
-
-        self.block4a = Inception(480, 192, 96, 208, 16, 48, 64)
-        self.block4b = Inception(512, 160, 112, 224, 24, 64, 64)
-        self.block4c = Inception(512, 128, 128, 256, 24, 64, 64)
-        self.block4d = Inception(512, 112, 144, 288, 32, 64, 64)
-        self.block4e = Inception(528, 256, 160, 320, 32, 128, 128)
-        self.maxpool4 = P.MaxPoolWithArgmax(ksize=2, strides=2, padding="same")
-
-        self.block5a = Inception(832, 256, 160, 320, 32, 128, 128)
-        self.block5b = Inception(832, 384, 192, 384, 48, 128, 128)
-
-        self.mean = P.ReduceMean(keep_dims=True)
-        self.dropout = nn.Dropout(keep_prob=0.8)
-        self.flatten = nn.Flatten()
-        self.classifier = nn.Dense(1024, num_classes, weight_init=weight_variable(),
-                                   bias_init=weight_variable())
-
-
-    def construct(self, x):
-        x = self.conv1(x)
-        x, argmax = self.maxpool1(x)
-
-        x = self.conv2(x)
-        x = self.conv3(x)
-        x, argmax = self.maxpool2(x)
-
-        x = self.block3a(x)
-        x = self.block3b(x)
-        x, argmax = self.maxpool3(x)
-
-        x = self.block4a(x)
-        x = self.block4b(x)
-        x = self.block4c(x)
-        x = self.block4d(x)
-        x = self.block4e(x)
-        x, argmax = self.maxpool4(x)
-
-        x = self.block5a(x)
-        x = self.block5b(x)
-
-        x = self.mean(x, (2, 3))
-        x = self.flatten(x)
-        x = self.classifier(x)
-
-        _ = argmax
-        return x

mindspore/model_zoo/lstm.py

-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""LSTM."""
-
-import numpy as np
-
-from mindspore import Tensor, nn, context
-from mindspore.ops import operations as P
-
-# Initialize short-term memory (h) and long-term memory (c) to 0
-def lstm_default_state(batch_size, hidden_size, num_layers, bidirectional):
-    """init default input."""
-    num_directions = 1
-    if bidirectional:
-        num_directions = 2
-
-    if context.get_context("device_target") == "CPU":
-        h_list = []
-        c_list = []
-        i = 0
-        while i < num_layers:
-            hi = Tensor(np.zeros((num_directions, batch_size, hidden_size)).astype(np.float32))
-            h_list.append(hi)
-            ci = Tensor(np.zeros((num_directions, batch_size, hidden_size)).astype(np.float32))
-            c_list.append(ci)
-            i = i + 1
-        h = tuple(h_list)
-        c = tuple(c_list)
-        return h, c
-
-    h = Tensor(
-        np.zeros((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32))
-    c = Tensor(
-        np.zeros((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32))
-    return h, c
-
-
-class SentimentNet(nn.Cell):
-    """Sentiment network structure."""
-
-    def __init__(self,
-                 vocab_size,
-                 embed_size,
-                 num_hiddens,
-                 num_layers,
-                 bidirectional,
-                 num_classes,
-                 weight,
-                 batch_size):
-        super(SentimentNet, self).__init__()
-        # Mapp words to vectors
-        self.embedding = nn.Embedding(vocab_size,
-                                      embed_size,
-                                      embedding_table=weight)
-        self.embedding.embedding_table.requires_grad = False
-        self.trans = P.Transpose()
-        self.perm = (1, 0, 2)
-        self.encoder = nn.LSTM(input_size=embed_size,
-                               hidden_size=num_hiddens,
-                               num_layers=num_layers,
-                               has_bias=True,
-                               bidirectional=bidirectional,
-                               dropout=0.0)
-
-        self.h, self.c = lstm_default_state(batch_size, num_hiddens, num_layers, bidirectional)
-
-        self.concat = P.Concat(1)
-        if bidirectional:
-            self.decoder = nn.Dense(num_hiddens * 4, num_classes)
-        else:
-            self.decoder = nn.Dense(num_hiddens * 2, num_classes)
-
-    def construct(self, inputs):
-        # input：(64,500,300)
-        embeddings = self.embedding(inputs)
-        embeddings = self.trans(embeddings, self.perm)
-        output, _ = self.encoder(embeddings, (self.h, self.c))
-        # states[i] size(64,200)  -> encoding.size(64,400)
-        encoding = self.concat((output[0], output[499]))
-        outputs = self.decoder(encoding)
-        return outputs

mindspore/model_zoo/vgg.py

-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""VGG."""
-import mindspore.nn as nn
-from mindspore.common.initializer import initializer
-import mindspore.common.dtype as mstype
-
-def _make_layer(base, batch_norm):
-    """Make stage network of VGG."""
-    layers = []
-    in_channels = 3
-    for v in base:
-        if v == 'M':
-            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
-        else:
-            weight_shape = (v, in_channels, 3, 3)
-            weight = initializer('XavierUniform', shape=weight_shape, dtype=mstype.float32).to_tensor()
-            conv2d = nn.Conv2d(in_channels=in_channels,
-                               out_channels=v,
-                               kernel_size=3,
-                               padding=0,
-                               pad_mode='same',
-                               weight_init=weight)
-            if batch_norm:
-                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU()]
-            else:
-                layers += [conv2d, nn.ReLU()]
-            in_channels = v
-    return nn.SequentialCell(layers)
-
-
-class Vgg(nn.Cell):
-    """
-    VGG network definition.
-
-    Args:
-        base (list): Configuration for different layers, mainly the channel number of Conv layer.
-        num_classes (int): Class numbers. Default: 1000.
-        batch_norm (bool): Whether to do the batchnorm. Default: False.
-        batch_size (int): Batch size. Default: 1.
-
-    Returns:
-        Tensor, infer output tensor.
-
-    Examples:
-        >>> Vgg([64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
-        >>>     num_classes=1000, batch_norm=False, batch_size=1)
-    """
-
-    def __init__(self, base, num_classes=1000, batch_norm=False, batch_size=1):
-        super(Vgg, self).__init__()
-        _ = batch_size
-        self.layers = _make_layer(base, batch_norm=batch_norm)
-        self.flatten = nn.Flatten()
-        self.classifier = nn.SequentialCell([
-            nn.Dense(512 * 7 * 7, 4096),
-            nn.ReLU(),
-            nn.Dense(4096, 4096),
-            nn.ReLU(),
-            nn.Dense(4096, num_classes)])
-
-    def construct(self, x):
-        x = self.layers(x)
-        x = self.flatten(x)
-        x = self.classifier(x)
-        return x
-
-
-cfg = {
-    '11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
-    '13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
-    '16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
-    '19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
-}
-
-
-def vgg16(num_classes=1000):
-    """
-    Get Vgg16 neural network with batch normalization.
-
-    Args:
-        num_classes (int): Class numbers. Default: 1000.
-
-    Returns:
-        Cell, cell instance of Vgg16 neural network with batch normalization.
-
-    Examples:
-        >>> vgg16(num_classes=1000)
-    """
-
-    net = Vgg(cfg['16'], num_classes=num_classes, batch_norm=True)
-    return net

model_zoo/lstm/README.md

@@ -72,7 +72,8 @@ result: {'acc': 0.83}
 ```
 usage: train.py [--preprocess {true,false}] [--aclimdb_path ACLIMDB_PATH]
                 [--glove_path GLOVE_PATH] [--preprocess_path PREPROCESS_PATH]
-                [--ckpt_path CKPT_PATH] [--device_target {GPU,CPU}]
+                [--ckpt_path CKPT_PATH] [--pre_trained PRE_TRAINED]
+                [--device_target {GPU,CPU}]
 
 parameters/options:
   --preprocess          whether to preprocess data.
@@ -80,6 +81,7 @@ parameters/options:
   --glove_path          path where the GloVe is stored.
   --preprocess_path     path where the pre-process data is stored.
   --ckpt_path           the path to save the checkpoint file.
+  --pre_trained         the pretrained checkpoint file path.
   --device_target       the target device to run, support "GPU", "CPU".
 ```
 

model_zoo/lstm/train.py

@@ -28,6 +28,7 @@ from mindspore import Tensor, nn, Model, context
 from mindspore.model_zoo.lstm import SentimentNet
 from mindspore.nn import Accuracy
 from mindspore.train.callback import LossMonitor, CheckpointConfig, ModelCheckpoint, TimeMonitor
+from mindspore.train.serialization import load_param_into_net, load_checkpoint
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='MindSpore LSTM Example')
@@ -41,6 +42,8 @@ if __name__ == '__main__':
                         help='path where the pre-process data is stored.')
     parser.add_argument('--ckpt_path', type=str, default="./",
                         help='the path to save the checkpoint file.')
+    parser.add_argument('--pre_trained', type=str, default=None,
+                        help='the pretrained checkpoint file path.')
     parser.add_argument('--device_target', type=str, default="GPU", choices=['GPU', 'CPU'],
                         help='the target device to run, support "GPU", "CPU". Default: "GPU".')
     args = parser.parse_args()
@@ -63,6 +66,9 @@ if __name__ == '__main__':
                            num_classes=cfg.num_classes,
                            weight=Tensor(embedding_table),
                            batch_size=cfg.batch_size)
+    # pre_trained
+    if args.pre_trained:
+        load_param_into_net(network, load_checkpoint(args.pre_trained))
 
     loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
     opt = nn.Momentum(network.trainable_params(), cfg.learning_rate, cfg.momentum)

model_zoo/vgg16/README.md

@@ -73,12 +73,13 @@ train_parallel1/log:epcoh: 2 step: 97, loss is 1.7133579
 ### Training
 ```
 usage: train.py [--device_target TARGET][--data_path DATA_PATH]
-                [--device_id DEVICE_ID]
+                [--device_id DEVICE_ID][--pre_trained PRE_TRAINED]
 
 parameters/options:
   --device_target       the training backend type, default is Ascend.
   --data_path           the storage path of dataset
   --device_id           the device which used to train model.
+  --pre_trained         the pretrained checkpoint file path.
 
 ```
 

model_zoo/vgg16/train.py

@@ -29,6 +29,7 @@ from mindspore.communication.management import init
 from mindspore.nn.optim.momentum import Momentum
 from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor
 from mindspore.train.model import Model, ParallelMode
+from mindspore.train.serialization import load_param_into_net, load_checkpoint
 from src.config import cifar_cfg as cfg
 from src.dataset import vgg_create_dataset
 from src.vgg import vgg16
@@ -64,6 +65,7 @@ if __name__ == '__main__':
                         help='device where the code will be implemented. (Default: Ascend)')
     parser.add_argument('--data_path', type=str, default='./cifar', help='path where the dataset is saved')
     parser.add_argument('--device_id', type=int, default=None, help='device id of GPU or Ascend. (Default: None)')
+    parser.add_argument('--pre_trained', type=str, default=None, help='the pretrained checkpoint file path.')
     args_opt = parser.parse_args()
 
     context.set_context(mode=context.GRAPH_MODE, device_target=args_opt.device_target)
@@ -80,6 +82,10 @@ if __name__ == '__main__':
     batch_num = dataset.get_dataset_size()
 
     net = vgg16(num_classes=cfg.num_classes)
+    # pre_trained
+    if args_opt.pre_trained:
+        load_param_into_net(net, load_checkpoint(args_opt.pre_trained))
+
     lr = lr_steps(0, lr_max=cfg.lr_init, total_epochs=cfg.epoch_size, steps_per_epoch=batch_num)
     opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), Tensor(lr), cfg.momentum,
                    weight_decay=cfg.weight_decay)

tests/ut/python/model/test_vgg.py

@@ -17,7 +17,7 @@ import numpy as np
 import pytest
 
 from mindspore import Tensor
-from mindspore.model_zoo.vgg import vgg16
+from model_zoo.vgg16.src.vgg import vgg16
 from ..ut_filter import non_graph_engine