optimize GetDatasize

tutorials/notebook/data_loading_enhance/data_loading_enhancement.ipynb

@@ -119,7 +119,7 @@
    "source": [
     "- ### repeat\n",
     "\n",
-    "在有限的数据集内，为了优化网络，通常会将一个数据集训练多次。加倍数据集，通常用在多个`epoch`训练中，通过`repeat`来加倍数据量。\n",
+    "在有限的数据集内，为了优化网络，通常会将一个数据集训练多次。加倍数据集，通过`repeat`来加倍数据量。\n",
     "\n",
     "我们可以定义`ds2`数据集，调用`repeat`来加倍数据量。其中，将倍数设为2，故`ds3`数据量为原始数据集`ds2`的2倍。"
    ]

tutorials/notebook/mindinsight/mindinsight_model_lineage_and_data_lineage.ipynb

@@ -425,7 +425,7 @@
     "    mnist_path = \"./MNIST_Data\"\n",
     "    \n",
     "    net_loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')\n",
-    "    repeat_size = epoch_size\n",
+    "    repeat_size = 1\n",
     "    # create the network\n",
     "    network = LeNet5()\n",
     "\n",
@@ -702,4 +702,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}
\ No newline at end of file

tutorials/notebook/nlp_application.ipynb

@@ -567,7 +567,7 @@
     "\n",
     "    return data_set\n",
     "\n",
-    "ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size, cfg.num_epochs)"
+    "ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size)"
    ]
   },
   {
@@ -5143,4 +5143,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}
\ No newline at end of file

tutorials/source_en/advanced_use/computer_vision_application.md

@@ -203,7 +203,7 @@ The trained model file (such as `resnet.ckpt`) can be used to predict the class
 ```python
 param_dict = load_checkpoint(args_opt.checkpoint_path)
 load_param_into_net(net, param_dict)
-eval_dataset = create_dataset(1, training=False)
+eval_dataset = create_dataset(training=False)
 res = model.eval(eval_dataset)
 print("result: ", res)
 ```

tutorials/source_en/advanced_use/differential_privacy.md

@@ -234,8 +234,7 @@ ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet",
 
 # get training dataset
 ds_train = generate_mnist_dataset(os.path.join(cfg.data_path, "train"),
-                                  cfg.batch_size,
-                                  cfg.epoch_size)
+                                  cfg.batch_size)
 ```
 
 ### Introducing the Differential Privacy

tutorials/source_en/advanced_use/distributed_training.md

@@ -247,7 +247,7 @@ context.set_context(device_id=device_id) # set device_id
 def test_train_cifar(epoch_size=10):
     context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, mirror_mean=True)
     loss_cb = LossMonitor()
-    dataset = create_dataset(data_path, epoch_size)
+    dataset = create_dataset(data_path)
     batch_size = 32
     num_classes = 10
     net = resnet50(batch_size, num_classes)

tutorials/source_en/advanced_use/nlp_application.md

@@ -204,7 +204,7 @@ Load the corresponding dataset, configure the CheckPoint generation information,
 model = Model(network, loss, opt, {'acc': Accuracy()})
 
 print("============== Starting Training ==============")
-ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size, cfg.num_epochs)
+ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size)
 config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps,
                              keep_checkpoint_max=cfg.keep_checkpoint_max)
 ckpoint_cb = ModelCheckpoint(prefix="lstm", directory=args.ckpt_path, config=config_ck)

tutorials/source_en/quick_start/quick_start.md

@@ -355,7 +355,7 @@ if __name__ == "__main__":
     
     epoch_size = 1    
     mnist_path = "./MNIST_Data"
-    repeat_size = epoch_size
+    repeat_size = 1
     model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
     train_net(args, model, epoch_size, mnist_path, repeat_size, ckpoint_cb, dataset_sink_mode)
     ...

tutorials/source_en/use/data_preparation/data_processing_and_augmentation.md

@@ -88,7 +88,7 @@ In limited datasets, to optimize the network, a dataset is usually trained for m
 
 > In machine learning, an epoch refers to one cycle through the full training dataset.
 
-During multiple epochs, `repeat` can be used to increase the data size. The definition of `repeat` is as follows:
+During training, `repeat` can be used to increase the data size. The definition of `repeat` is as follows:
 ```python
 def repeat(self, count=None):
 ```

tutorials/source_zh_cn/advanced_use/computer_vision_application.md

@@ -205,7 +205,7 @@ model.train(epoch_size, dataset, callbacks=[ckpoint_cb, loss_cb])
 ```python
 param_dict = load_checkpoint(args_opt.checkpoint_path)
 load_param_into_net(net, param_dict)
-eval_dataset = create_dataset(1, training=False)
+eval_dataset = create_dataset(training=False)
 res = model.eval(eval_dataset)
 print("result: ", res)
 ```

tutorials/source_zh_cn/advanced_use/differential_privacy.md

@@ -220,8 +220,7 @@ ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet",
 
 # get training dataset
 ds_train = generate_mnist_dataset(os.path.join(cfg.data_path, "train"),
-                                  cfg.batch_size,
-                                  cfg.epoch_size)
+                                  cfg.batch_size)
 ```
 
 ### 引入差分隐私

tutorials/source_zh_cn/advanced_use/distributed_training.md

@@ -248,7 +248,7 @@ context.set_context(device_id=device_id) # set device_id
 def test_train_cifar(epoch_size=10):
     context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, mirror_mean=True)
     loss_cb = LossMonitor()
-    dataset = create_dataset(data_path, epoch_size)
+    dataset = create_dataset(data_path)
     batch_size = 32
     num_classes = 10
     net = resnet50(batch_size, num_classes)

tutorials/source_zh_cn/advanced_use/nlp_application.md

@@ -204,7 +204,7 @@ loss_cb = LossMonitor()
 model = Model(network, loss, opt, {'acc': Accuracy()})
 
 print("============== Starting Training ==============")
-ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size, cfg.num_epochs)
+ds_train = lstm_create_dataset(args.preprocess_path, cfg.batch_size)
 config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps,
                              keep_checkpoint_max=cfg.keep_checkpoint_max)
 ckpoint_cb = ModelCheckpoint(prefix="lstm", directory=args.ckpt_path, config=config_ck)

tutorials/source_zh_cn/quick_start/quick_start.md

@@ -357,7 +357,7 @@ if __name__ == "__main__":
     
     epoch_size = 1    
     mnist_path = "./MNIST_Data"
-    repeat_size = epoch_size
+    repeat_size = 1
     model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()})
     train_net(args, model, epoch_size, mnist_path, repeat_size, ckpoint_cb, dataset_sink_mode)
     ...

tutorials/source_zh_cn/use/data_preparation/data_processing_and_augmentation.md

@@ -89,7 +89,7 @@ ds1 = ds1.repeat(10)
 
 > 在机器学习中，每训练完一个完整的数据集，我们称为训练完了一个epoch。
 
-加倍数据集，通常用在多个epoch（迭代）训练中，通过`repeat`来加倍数据量。`repeat`定义如下：
+加倍数据集，通常用在训练中，通过`repeat`来加倍数据量。`repeat`定义如下：
 ```python
 def repeat(self, count=None):
 ```

tutorials/tutorial_code/distributed_training/resnet50_distributed_training.py

@@ -120,7 +120,7 @@ def test_train_cifar(epoch_size=10):
     context.set_auto_parallel_context(parallel_mode=ParallelMode.AUTO_PARALLEL, mirror_mean=True)
     loss_cb = LossMonitor()
     data_path = os.getenv('DATA_PATH')
-    dataset = create_dataset(data_path, epoch_size)
+    dataset = create_dataset(data_path)
     batch_size = 32
     num_classes = 10
     net = resnet50(batch_size, num_classes)

tutorials/tutorial_code/lenet.py

@@ -206,7 +206,7 @@ if __name__ == "__main__":
     mnist_path = "./MNIST_Data"
     # define the loss function
     net_loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')
-    repeat_size = epoch_size
+    repeat_size = 1
     # create the network
     network = LeNet5()
     # define the optimizer

tutorials/tutorial_code/resnet/cifar_resnet50.py

@@ -118,7 +118,7 @@ if __name__ == '__main__':
 
     # as for train, users could use model.train
     if args_opt.do_train:
-        dataset = create_dataset(epoch_size)
+        dataset = create_dataset()
         batch_num = dataset.get_dataset_size()
         config_ck = CheckpointConfig(save_checkpoint_steps=batch_num, keep_checkpoint_max=35)
         ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory="./", config=config_ck)
@@ -130,6 +130,6 @@ if __name__ == '__main__':
         if args_opt.checkpoint_path:
             param_dict = load_checkpoint(args_opt.checkpoint_path)
             load_param_into_net(net, param_dict)
-        eval_dataset = create_dataset(1, training=False)
+        eval_dataset = create_dataset(training=False)
         res = model.eval(eval_dataset)
         print("result: ", res)

tutorials/tutorial_code/sample_for_cloud/resnet50_train.py

@@ -130,7 +130,7 @@ def resnet50_train(args_opt):
     # create dataset
     print('Create train and evaluate dataset.')
     train_dataset = create_dataset(dataset_path=local_data_path, do_train=True,
-                                   repeat_num=epoch_size, batch_size=batch_size)
+                                   repeat_num=1, batch_size=batch_size)
     eval_dataset = create_dataset(dataset_path=local_data_path, do_train=False,
                                    repeat_num=1, batch_size=batch_size)
     train_step_size = train_dataset.get_dataset_size()