From deb9694f63c296fc5f156f6436ae4b77ca758a7e Mon Sep 17 00:00:00 2001 From: caojian05 Date: Sat, 20 Jun 2020 19:38:39 +0800 Subject: [PATCH] add pretrain for lstm & vgg16 and remove lstm/vgg16/googlenet from directory 'mindspore/model_zoo' --- mindspore/model_zoo/googlenet.py | 143 ------------------------------ mindspore/model_zoo/lstm.py | 93 ------------------- mindspore/model_zoo/vgg.py | 104 ---------------------- model_zoo/lstm/README.md | 4 +- model_zoo/lstm/train.py | 6 ++ model_zoo/vgg16/README.md | 3 +- model_zoo/vgg16/train.py | 6 ++ tests/ut/python/model/test_vgg.py | 2 +- 8 files changed, 18 insertions(+), 343 deletions(-) delete mode 100644 mindspore/model_zoo/googlenet.py delete mode 100644 mindspore/model_zoo/lstm.py delete mode 100644 mindspore/model_zoo/vgg.py diff --git a/mindspore/model_zoo/googlenet.py b/mindspore/model_zoo/googlenet.py deleted file mode 100644 index 4a572828d..000000000 --- a/mindspore/model_zoo/googlenet.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""GoogleNet""" -import mindspore.nn as nn -from mindspore.common.initializer import TruncatedNormal -from mindspore.ops import operations as P - - -def weight_variable(): - """Weight variable.""" - return TruncatedNormal(0.02) - - -class Conv2dBlock(nn.Cell): - """ - Basic convolutional block - Args: - in_channles (int): Input channel. - out_channels (int): Output channel. - kernel_size (int): Input kernel size. Default: 1 - stride (int): Stride size for the first convolutional layer. Default: 1. - padding (int): Implicit paddings on both sides of the input. Default: 0. - pad_mode (str): Padding mode. Optional values are "same", "valid", "pad". Default: "same". - Returns: - Tensor, output tensor. - """ - - def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, pad_mode="same"): - super(Conv2dBlock, self).__init__() - self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, - padding=padding, pad_mode=pad_mode, weight_init=weight_variable(), - bias_init=False) - self.bn = nn.BatchNorm2d(out_channels, eps=0.001) - self.relu = nn.ReLU() - - def construct(self, x): - x = self.conv(x) - x = self.bn(x) - x = self.relu(x) - return x - - -class Inception(nn.Cell): - """ - Inception Block - """ - - def __init__(self, in_channels, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes): - super(Inception, self).__init__() - self.b1 = Conv2dBlock(in_channels, n1x1, kernel_size=1) - self.b2 = nn.SequentialCell([Conv2dBlock(in_channels, n3x3red, kernel_size=1), - Conv2dBlock(n3x3red, n3x3, kernel_size=3, padding=0)]) - self.b3 = nn.SequentialCell([Conv2dBlock(in_channels, n5x5red, kernel_size=1), - Conv2dBlock(n5x5red, n5x5, kernel_size=3, padding=0)]) - self.maxpool = P.MaxPoolWithArgmax(ksize=3, strides=1, padding="same") - self.b4 = Conv2dBlock(in_channels, pool_planes, kernel_size=1) - self.concat = P.Concat(axis=1) - - def construct(self, x): - branch1 = self.b1(x) - branch2 = self.b2(x) - branch3 = self.b3(x) - cell, argmax = self.maxpool(x) - branch4 = self.b4(cell) - _ = argmax - return self.concat((branch1, branch2, branch3, branch4)) - - -class GooGLeNet(nn.Cell): - """ - Googlenet architecture - """ - - def __init__(self, num_classes): - super(GooGLeNet, self).__init__() - self.conv1 = Conv2dBlock(3, 64, kernel_size=7, stride=2, padding=0) - self.maxpool1 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same") - - self.conv2 = Conv2dBlock(64, 64, kernel_size=1) - self.conv3 = Conv2dBlock(64, 192, kernel_size=3, padding=0) - self.maxpool2 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same") - - self.block3a = Inception(192, 64, 96, 128, 16, 32, 32) - self.block3b = Inception(256, 128, 128, 192, 32, 96, 64) - self.maxpool3 = P.MaxPoolWithArgmax(ksize=3, strides=2, padding="same") - - self.block4a = Inception(480, 192, 96, 208, 16, 48, 64) - self.block4b = Inception(512, 160, 112, 224, 24, 64, 64) - self.block4c = Inception(512, 128, 128, 256, 24, 64, 64) - self.block4d = Inception(512, 112, 144, 288, 32, 64, 64) - self.block4e = Inception(528, 256, 160, 320, 32, 128, 128) - self.maxpool4 = P.MaxPoolWithArgmax(ksize=2, strides=2, padding="same") - - self.block5a = Inception(832, 256, 160, 320, 32, 128, 128) - self.block5b = Inception(832, 384, 192, 384, 48, 128, 128) - - self.mean = P.ReduceMean(keep_dims=True) - self.dropout = nn.Dropout(keep_prob=0.8) - self.flatten = nn.Flatten() - self.classifier = nn.Dense(1024, num_classes, weight_init=weight_variable(), - bias_init=weight_variable()) - - - def construct(self, x): - x = self.conv1(x) - x, argmax = self.maxpool1(x) - - x = self.conv2(x) - x = self.conv3(x) - x, argmax = self.maxpool2(x) - - x = self.block3a(x) - x = self.block3b(x) - x, argmax = self.maxpool3(x) - - x = self.block4a(x) - x = self.block4b(x) - x = self.block4c(x) - x = self.block4d(x) - x = self.block4e(x) - x, argmax = self.maxpool4(x) - - x = self.block5a(x) - x = self.block5b(x) - - x = self.mean(x, (2, 3)) - x = self.flatten(x) - x = self.classifier(x) - - _ = argmax - return x diff --git a/mindspore/model_zoo/lstm.py b/mindspore/model_zoo/lstm.py deleted file mode 100644 index c3ca0bbf7..000000000 --- a/mindspore/model_zoo/lstm.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""LSTM.""" - -import numpy as np - -from mindspore import Tensor, nn, context -from mindspore.ops import operations as P - -# Initialize short-term memory (h) and long-term memory (c) to 0 -def lstm_default_state(batch_size, hidden_size, num_layers, bidirectional): - """init default input.""" - num_directions = 1 - if bidirectional: - num_directions = 2 - - if context.get_context("device_target") == "CPU": - h_list = [] - c_list = [] - i = 0 - while i < num_layers: - hi = Tensor(np.zeros((num_directions, batch_size, hidden_size)).astype(np.float32)) - h_list.append(hi) - ci = Tensor(np.zeros((num_directions, batch_size, hidden_size)).astype(np.float32)) - c_list.append(ci) - i = i + 1 - h = tuple(h_list) - c = tuple(c_list) - return h, c - - h = Tensor( - np.zeros((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)) - c = Tensor( - np.zeros((num_layers * num_directions, batch_size, hidden_size)).astype(np.float32)) - return h, c - - -class SentimentNet(nn.Cell): - """Sentiment network structure.""" - - def __init__(self, - vocab_size, - embed_size, - num_hiddens, - num_layers, - bidirectional, - num_classes, - weight, - batch_size): - super(SentimentNet, self).__init__() - # Mapp words to vectors - self.embedding = nn.Embedding(vocab_size, - embed_size, - embedding_table=weight) - self.embedding.embedding_table.requires_grad = False - self.trans = P.Transpose() - self.perm = (1, 0, 2) - self.encoder = nn.LSTM(input_size=embed_size, - hidden_size=num_hiddens, - num_layers=num_layers, - has_bias=True, - bidirectional=bidirectional, - dropout=0.0) - - self.h, self.c = lstm_default_state(batch_size, num_hiddens, num_layers, bidirectional) - - self.concat = P.Concat(1) - if bidirectional: - self.decoder = nn.Dense(num_hiddens * 4, num_classes) - else: - self.decoder = nn.Dense(num_hiddens * 2, num_classes) - - def construct(self, inputs): - # input：(64,500,300) - embeddings = self.embedding(inputs) - embeddings = self.trans(embeddings, self.perm) - output, _ = self.encoder(embeddings, (self.h, self.c)) - # states[i] size(64,200) -> encoding.size(64,400) - encoding = self.concat((output[0], output[499])) - outputs = self.decoder(encoding) - return outputs diff --git a/mindspore/model_zoo/vgg.py b/mindspore/model_zoo/vgg.py deleted file mode 100644 index 55130871c..000000000 --- a/mindspore/model_zoo/vgg.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""VGG.""" -import mindspore.nn as nn -from mindspore.common.initializer import initializer -import mindspore.common.dtype as mstype - -def _make_layer(base, batch_norm): - """Make stage network of VGG.""" - layers = [] - in_channels = 3 - for v in base: - if v == 'M': - layers += [nn.MaxPool2d(kernel_size=2, stride=2)] - else: - weight_shape = (v, in_channels, 3, 3) - weight = initializer('XavierUniform', shape=weight_shape, dtype=mstype.float32).to_tensor() - conv2d = nn.Conv2d(in_channels=in_channels, - out_channels=v, - kernel_size=3, - padding=0, - pad_mode='same', - weight_init=weight) - if batch_norm: - layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU()] - else: - layers += [conv2d, nn.ReLU()] - in_channels = v - return nn.SequentialCell(layers) - - -class Vgg(nn.Cell): - """ - VGG network definition. - - Args: - base (list): Configuration for different layers, mainly the channel number of Conv layer. - num_classes (int): Class numbers. Default: 1000. - batch_norm (bool): Whether to do the batchnorm. Default: False. - batch_size (int): Batch size. Default: 1. - - Returns: - Tensor, infer output tensor. - - Examples: - >>> Vgg([64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], - >>> num_classes=1000, batch_norm=False, batch_size=1) - """ - - def __init__(self, base, num_classes=1000, batch_norm=False, batch_size=1): - super(Vgg, self).__init__() - _ = batch_size - self.layers = _make_layer(base, batch_norm=batch_norm) - self.flatten = nn.Flatten() - self.classifier = nn.SequentialCell([ - nn.Dense(512 * 7 * 7, 4096), - nn.ReLU(), - nn.Dense(4096, 4096), - nn.ReLU(), - nn.Dense(4096, num_classes)]) - - def construct(self, x): - x = self.layers(x) - x = self.flatten(x) - x = self.classifier(x) - return x - - -cfg = { - '11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], - '13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], - '16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], - '19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], -} - - -def vgg16(num_classes=1000): - """ - Get Vgg16 neural network with batch normalization. - - Args: - num_classes (int): Class numbers. Default: 1000. - - Returns: - Cell, cell instance of Vgg16 neural network with batch normalization. - - Examples: - >>> vgg16(num_classes=1000) - """ - - net = Vgg(cfg['16'], num_classes=num_classes, batch_norm=True) - return net diff --git a/model_zoo/lstm/README.md b/model_zoo/lstm/README.md index 95ac30f3d..00a333496 100644 --- a/model_zoo/lstm/README.md +++ b/model_zoo/lstm/README.md @@ -72,7 +72,8 @@ result: {'acc': 0.83} ``` usage: train.py [--preprocess {true,false}] [--aclimdb_path ACLIMDB_PATH] [--glove_path GLOVE_PATH] [--preprocess_path PREPROCESS_PATH] - [--ckpt_path CKPT_PATH] [--device_target {GPU,CPU}] + [--ckpt_path CKPT_PATH] [--pre_trained PRE_TRAINED] + [--device_target {GPU,CPU}] parameters/options: --preprocess whether to preprocess data. @@ -80,6 +81,7 @@ parameters/options: --glove_path path where the GloVe is stored. --preprocess_path path where the pre-process data is stored. --ckpt_path the path to save the checkpoint file. + --pre_trained the pretrained checkpoint file path. --device_target the target device to run, support "GPU", "CPU". ``` diff --git a/model_zoo/lstm/train.py b/model_zoo/lstm/train.py index fd0e7fdd1..732655f1d 100644 --- a/model_zoo/lstm/train.py +++ b/model_zoo/lstm/train.py @@ -28,6 +28,7 @@ from mindspore import Tensor, nn, Model, context from mindspore.model_zoo.lstm import SentimentNet from mindspore.nn import Accuracy from mindspore.train.callback import LossMonitor, CheckpointConfig, ModelCheckpoint, TimeMonitor +from mindspore.train.serialization import load_param_into_net, load_checkpoint if __name__ == '__main__': parser = argparse.ArgumentParser(description='MindSpore LSTM Example') @@ -41,6 +42,8 @@ if __name__ == '__main__': help='path where the pre-process data is stored.') parser.add_argument('--ckpt_path', type=str, default="./", help='the path to save the checkpoint file.') + parser.add_argument('--pre_trained', type=str, default=None, + help='the pretrained checkpoint file path.') parser.add_argument('--device_target', type=str, default="GPU", choices=['GPU', 'CPU'], help='the target device to run, support "GPU", "CPU". Default: "GPU".') args = parser.parse_args() @@ -63,6 +66,9 @@ if __name__ == '__main__': num_classes=cfg.num_classes, weight=Tensor(embedding_table), batch_size=cfg.batch_size) + # pre_trained + if args.pre_trained: + load_param_into_net(network, load_checkpoint(args.pre_trained)) loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) opt = nn.Momentum(network.trainable_params(), cfg.learning_rate, cfg.momentum) diff --git a/model_zoo/vgg16/README.md b/model_zoo/vgg16/README.md index 75f36d288..53eb05f66 100644 --- a/model_zoo/vgg16/README.md +++ b/model_zoo/vgg16/README.md @@ -73,12 +73,13 @@ train_parallel1/log:epcoh: 2 step: 97, loss is 1.7133579 ### Training ``` usage: train.py [--device_target TARGET][--data_path DATA_PATH] - [--device_id DEVICE_ID] + [--device_id DEVICE_ID][--pre_trained PRE_TRAINED] parameters/options: --device_target the training backend type, default is Ascend. --data_path the storage path of dataset --device_id the device which used to train model. + --pre_trained the pretrained checkpoint file path. ``` diff --git a/model_zoo/vgg16/train.py b/model_zoo/vgg16/train.py index 496aedb25..c582cdd67 100644 --- a/model_zoo/vgg16/train.py +++ b/model_zoo/vgg16/train.py @@ -29,6 +29,7 @@ from mindspore.communication.management import init from mindspore.nn.optim.momentum import Momentum from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor from mindspore.train.model import Model, ParallelMode +from mindspore.train.serialization import load_param_into_net, load_checkpoint from src.config import cifar_cfg as cfg from src.dataset import vgg_create_dataset from src.vgg import vgg16 @@ -64,6 +65,7 @@ if __name__ == '__main__': help='device where the code will be implemented. (Default: Ascend)') parser.add_argument('--data_path', type=str, default='./cifar', help='path where the dataset is saved') parser.add_argument('--device_id', type=int, default=None, help='device id of GPU or Ascend. (Default: None)') + parser.add_argument('--pre_trained', type=str, default=None, help='the pretrained checkpoint file path.') args_opt = parser.parse_args() context.set_context(mode=context.GRAPH_MODE, device_target=args_opt.device_target) @@ -80,6 +82,10 @@ if __name__ == '__main__': batch_num = dataset.get_dataset_size() net = vgg16(num_classes=cfg.num_classes) + # pre_trained + if args_opt.pre_trained: + load_param_into_net(net, load_checkpoint(args_opt.pre_trained)) + lr = lr_steps(0, lr_max=cfg.lr_init, total_epochs=cfg.epoch_size, steps_per_epoch=batch_num) opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), Tensor(lr), cfg.momentum, weight_decay=cfg.weight_decay) diff --git a/tests/ut/python/model/test_vgg.py b/tests/ut/python/model/test_vgg.py index 8f05179ee..ed8a217e5 100644 --- a/tests/ut/python/model/test_vgg.py +++ b/tests/ut/python/model/test_vgg.py @@ -17,7 +17,7 @@ import numpy as np import pytest from mindspore import Tensor -from mindspore.model_zoo.vgg import vgg16 +from model_zoo.vgg16.src.vgg import vgg16 from ..ut_filter import non_graph_engine -- GitLab