Update api in image classification (#3660)

* update new fluid.data and data_loader interface and add check_version

PaddleCV/image_classification/README.md

@@ -32,7 +32,7 @@
 
 ### 安装说明
 
-在当前目录下运行样例代码需要python 2.7及以上版本，PadddlePaddle Fluid v1.5.1或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据 [安装文档](http://paddlepaddle.org/documentation/docs/zh/1.5/beginners_guide/install/index_cn.html) 中的说明来更新PaddlePaddle。
+在当前目录下运行样例代码需要python 2.7及以上版本，PadddlePaddle Fluid v1.6或以上的版本。如果你的运行环境中的PaddlePaddle低于此版本，请根据 [安装文档](http://paddlepaddle.org/documentation/docs/zh/1.6/beginners_guide/install/index_cn.html) 中的说明来更新PaddlePaddle。
 
 #### 环境依赖
 

PaddleCV/image_classification/README_en.md

@@ -33,7 +33,7 @@ We also recommend users to take a look at the  [IPython Notebook demo](https:/
 
 ### Installation
 
-Running samples in this directory requires Python 2.7 and later, CUDA 8.0 and later, CUDNN 7.0 and later, python package: numpy and opencv-python, PaddelPaddle Fluid v1.5 and later, the latest release version is recommended, If the PaddlePaddle on your device is lower than v1.5, please follow the instructions in [installation document](http://paddlepaddle.org/documentation/docs/zh/1.5/beginners_guide/install/index_cn.html) and make an update.
+Running samples in this directory requires Python 2.7 and later, CUDA 8.0 and later, CUDNN 7.0 and later, python package: numpy and opencv-python, PaddelPaddle Fluid v1.6 and later, the latest release version is recommended, If the PaddlePaddle on your device is lower than v1.6, please follow the instructions in [installation document](http://paddlepaddle.org/documentation/docs/zh/1.6/beginners_guide/install/index_cn.html) and make an update.
 
 ### Data preparation
 

PaddleCV/image_classification/build_model.py

@@ -107,7 +107,7 @@ def _mixup_model(data, model, args, is_train):
 def create_model(model, args, is_train):
     """Create model, include basic model, googlenet model and mixup model
     """
-    py_reader, data = utility.create_pyreader(is_train, args)
+    data_loader, data = utility.create_data_loader(is_train, args)
 
     if args.model == "GoogLeNet":
         loss_out = _googlenet_model(data, model, args, is_train)
@@ -116,4 +116,4 @@ def create_model(model, args, is_train):
             loss_out = _mixup_model(data, model, args, is_train)
         else:
             loss_out = _basic_model(data, model, args, is_train)
-    return py_reader, loss_out
+    return data_loader, loss_out

PaddleCV/image_classification/eval.py

@@ -62,12 +62,14 @@ def eval(args):
     ), "{} doesn't exist, please load right pretrained model path for eval".format(
         args.pretrained_model)
 
-    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+    image = fluid.data(
+        name='image', shape=[None] + image_shape, dtype='float32')
+    label = fluid.data(name='label', shape=[None, 1], dtype='int64')
 
     # model definition
     if args.model.startswith('EfficientNet'):
-        model = models.__dict__[args.model](is_test=True, padding_type=args.padding_type)
+        model = models.__dict__[args.model](is_test=True,
+                                            padding_type=args.padding_type)
     else:
         model = models.__dict__[args.model]()
 
@@ -142,6 +144,7 @@ def main():
     args = parser.parse_args()
     print_arguments(args)
     check_gpu()
+    check_version()
     eval(args)
 
 

PaddleCV/image_classification/fast_imagenet/train.py

@@ -112,10 +112,9 @@ def build_program(args,
                             use_double_buffer=True)
                 input, label = fluid.layers.read_file(pyreader)
             else:
-                input = fluid.layers.data(
-                    name="image", shape=[3, 244, 244], dtype="uint8")
-                label = fluid.layers.data(
-                    name="label", shape=[1], dtype="int64")
+                input = fluid.data(
+                    name="image", shape=[None, 3, 244, 244], dtype="uint8")
+                label = fluid.data(name="label", shape=[None, 1], dtype="int64")
             cast_img_type = "float16" if args.fp16 else "float32"
             cast = fluid.layers.cast(input, cast_img_type)
             img_mean = fluid.layers.create_global_var(

PaddleCV/image_classification/infer.py

@@ -59,7 +59,8 @@ def infer(args):
                                                                      model_list)
     assert os.path.isdir(args.pretrained_model
                          ), "please load right pretrained model path for infer"
-    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
+    image = fluid.data(
+        name='image', shape=[None] + image_shape, dtype='float32')
     model = models.__dict__[args.model]()
     if args.model == "GoogLeNet":
         out, _, _ = model.net(input=image, class_dim=args.class_dim)
@@ -120,6 +121,7 @@ def main():
     args = parser.parse_args()
     print_arguments(args)
     check_gpu()
+    check_version()
     infer(args)
 
 

PaddleCV/image_classification/train.py

@@ -21,6 +21,7 @@ import numpy as np
 import time
 import sys
 
+
 def set_paddle_flags(flags):
     for key, value in flags.items():
         if os.environ.get(key, None) is None:
@@ -42,6 +43,7 @@ from utils import *
 import models
 from build_model import create_model
 
+
 def build_program(is_train, main_prog, startup_prog, args):
     """build program, and add grad op in program accroding to different mode
 
@@ -52,14 +54,16 @@ def build_program(is_train, main_prog, startup_prog, args):
         args: arguments
 
     Returns : 
-        train mode: [Loss, global_lr, py_reader]
-        test mode: [Loss, py_reader]
+        train mode: [Loss, global_lr, data_loader]
+        test mode: [Loss, data_loader]
     """
     if args.model.startswith('EfficientNet'):
         is_test = False if is_train else True
         override_params = {"drop_connect_rate": args.drop_connect_rate}
         padding_type = args.padding_type
-        model = models.__dict__[args.model](is_test=is_test, override_params=override_params, padding_type=padding_type)
+        model = models.__dict__[args.model](is_test=is_test,
+                                            override_params=override_params,
+                                            padding_type=padding_type)
     else:
         model = models.__dict__[args.model]()
     with fluid.program_guard(main_prog, startup_prog):
@@ -67,7 +71,7 @@ def build_program(is_train, main_prog, startup_prog, args):
             main_prog.random_seed = args.random_seed
             startup_prog.random_seed = args.random_seed
         with fluid.unique_name.guard():
-            py_reader, loss_out = create_model(model, args, is_train)
+            data_loader, loss_out = create_model(model, args, is_train)
             # add backward op in program
             if is_train:
                 optimizer = create_optimizer(args)
@@ -78,18 +82,22 @@ def build_program(is_train, main_prog, startup_prog, args):
                 global_lr.persistable = True
                 loss_out.append(global_lr)
                 if args.use_ema:
-                    global_steps = fluid.layers.learning_rate_scheduler._decay_step_counter()
-                    ema = ExponentialMovingAverage(args.ema_decay, thres_steps=global_steps)
+                    global_steps = fluid.layers.learning_rate_scheduler._decay_step_counter(
+                    )
+                    ema = ExponentialMovingAverage(
+                        args.ema_decay, thres_steps=global_steps)
                     ema.update()
                     loss_out.append(ema)
-            loss_out.append(py_reader)
+            loss_out.append(data_loader)
     return loss_out
 
-def validate(args, test_py_reader, exe, test_prog, test_fetch_list, pass_id, train_batch_metrics_record):
+
+def validate(args, test_data_loader, exe, test_prog, test_fetch_list, pass_id,
+             train_batch_metrics_record):
     test_batch_time_record = []
     test_batch_metrics_record = []
     test_batch_id = 0
-    test_py_reader.start()
+    test_data_loader.start()
     try:
         while True:
             t1 = time.time()
@@ -109,7 +117,7 @@ def validate(args, test_py_reader, exe, test_prog, test_fetch_list, pass_id, tra
             test_batch_id += 1
 
     except fluid.core.EOFException:
-        test_py_reader.reset()
+        test_data_loader.reset()
     #train_epoch_time_avg = np.mean(np.array(train_batch_time_record))
     train_epoch_metrics_avg = np.mean(
         np.array(train_batch_metrics_record), axis=0)
@@ -122,6 +130,7 @@ def validate(args, test_py_reader, exe, test_prog, test_fetch_list, pass_id, tra
                list(train_epoch_metrics_avg) + list(test_epoch_metrics_avg),
                test_epoch_time_avg, "epoch")
 
+
 def train(args):
     """Train model
     
@@ -137,7 +146,7 @@ def train(args):
         main_prog=train_prog,
         startup_prog=startup_prog,
         args=args)
-    train_py_reader = train_out[-1]
+    train_data_loader = train_out[-1]
     if args.use_ema:
         train_fetch_vars = train_out[:-2]
         ema = train_out[-2]
@@ -151,7 +160,7 @@ def train(args):
         main_prog=test_prog,
         startup_prog=startup_prog,
         args=args)
-    test_py_reader = test_out[-1]
+    test_data_loader = test_out[-1]
     test_fetch_vars = test_out[:-1]
 
     test_fetch_list = [var.name for var in test_fetch_vars]
@@ -171,20 +180,21 @@ def train(args):
     train_reader = imagenet_reader.train(settings=args)
     test_reader = imagenet_reader.val(settings=args)
 
-    train_py_reader.decorate_sample_list_generator(train_reader, place)
-    test_py_reader.decorate_sample_list_generator(test_reader, place)
+    train_data_loader.set_sample_list_generator(train_reader, place)
+    test_data_loader.set_sample_list_generator(test_reader, place)
 
     compiled_train_prog = best_strategy_compiled(args, train_prog,
                                                  train_fetch_vars[0], exe)
     trainer_id = int(os.getenv("PADDLE_TRAINER_ID", 0))
     for pass_id in range(args.num_epochs):
         if num_trainers > 1:
-            imagenet_reader.set_shuffle_seed(pass_id + (args.random_seed if args.random_seed else 0))
+            imagenet_reader.set_shuffle_seed(pass_id + (
+                args.random_seed if args.random_seed else 0))
         train_batch_id = 0
         train_batch_time_record = []
         train_batch_metrics_record = []
 
-        train_py_reader.start()
+        train_data_loader.start()
 
         try:
             while True:
@@ -199,21 +209,25 @@ def train(args):
                 train_batch_metrics_record.append(train_batch_metrics_avg)
                 if trainer_id == 0:
                     print_info(pass_id, train_batch_id, args.print_step,
-                               train_batch_metrics_avg, train_batch_elapse, "batch")
+                               train_batch_metrics_avg, train_batch_elapse,
+                               "batch")
                     sys.stdout.flush()
                 train_batch_id += 1
 
         except fluid.core.EOFException:
-            train_py_reader.reset()
+            train_data_loader.reset()
 
         if trainer_id == 0:
             if args.use_ema:
                 print('ExponentialMovingAverage validate start...')
                 with ema.apply(exe):
-                    validate(args, test_py_reader, exe, test_prog, test_fetch_list, pass_id, train_batch_metrics_record)
+                    validate(args, test_data_loader, exe, test_prog,
+                             test_fetch_list, pass_id,
+                             train_batch_metrics_record)
                 print('ExponentialMovingAverage validate over!')
 
-            validate(args, test_py_reader, exe, test_prog, test_fetch_list, pass_id, train_batch_metrics_record)
+            validate(args, test_data_loader, exe, test_prog, test_fetch_list,
+                     pass_id, train_batch_metrics_record)
             #For now, save model per epoch.
             if pass_id % args.save_step == 0:
                 save_model(args, exe, train_prog, pass_id)

PaddleCV/image_classification/utils/__init__.py

@@ -12,4 +12,4 @@
 #See the License for the specific language governing permissions and
 #limitations under the License.
 from .optimizer import cosine_decay, lr_warmup, cosine_decay_with_warmup, exponential_decay_with_warmup, Optimizer, create_optimizer
-from .utility import add_arguments, print_arguments, parse_args, check_gpu, check_args, init_model, save_model, create_pyreader, print_info, best_strategy_compiled, init_model, save_model, ExponentialMovingAverage
+from .utility import add_arguments, print_arguments, parse_args, check_gpu, check_args, init_model, save_model, create_data_loader, print_info, best_strategy_compiled, init_model, save_model, ExponentialMovingAverage

PaddleCV/image_classification/utils/utility.py

@@ -34,6 +34,7 @@ from paddle.fluid.framework import Program, program_guard, name_scope, default_m
 from paddle.fluid import unique_name, layers
 from utils import dist_utils
 
+
 def print_arguments(args):
     """Print argparse's arguments.
 
@@ -166,6 +167,22 @@ def check_gpu():
         pass
 
 
+def check_version():
+    """
+    Log error and exit when the installed version of paddlepaddle is
+    not satisfied.
+    """
+    err = "PaddlePaddle version 1.6 or higher is required, " \
+          "or a suitable develop version is satisfied as well. \n" \
+          "Please make sure the version is good with your code." \
+
+    try:
+        fluid.require_version('1.6.0')
+    except Exception as e:
+        logger.error(err)
+        sys.exit(1)
+
+
 def check_args(args):
     """check arguments before running
 
@@ -182,7 +199,8 @@ def check_args(args):
 
     # check learning rate strategy
     lr_strategy_list = [
-        "piecewise_decay", "cosine_decay", "linear_decay", "cosine_decay_warmup", "exponential_decay_warmup"
+        "piecewise_decay", "cosine_decay", "linear_decay",
+        "cosine_decay_warmup", "exponential_decay_warmup"
     ]
     if args.lr_strategy not in lr_strategy_list:
         warnings.warn(
@@ -241,6 +259,7 @@ def check_args(args):
     #check gpu
 
     check_gpu()
+    check_version()
 
 
 def init_model(exe, args, program):
@@ -268,49 +287,52 @@ def save_model(args, exe, train_prog, info):
     print("Already save model in %s" % (model_path))
 
 
-def create_pyreader(is_train, args):
-    """create PyReader
+def create_data_loader(is_train, args):
+    """create data_loader
 
     Usage:
-        Using mixup process in training, it will return 5 results, include py_reader, image, y_a(label), y_b(label) and lamda, or it will return 3 results, include py_reader, image, and label.
+        Using mixup process in training, it will return 5 results, include data_loader, image, y_a(label), y_b(label) and lamda, or it will return 3 results, include data_loader, image, and label.
 
     Args: 
         is_train: mode
         args: arguments
 
     Returns:
-        py_reader and the input data of net, 
+        data_loader and the input data of net, 
     """
     image_shape = [int(m) for m in args.image_shape.split(",")]
 
-    feed_image = fluid.layers.data(
-        name="feed_image", shape=image_shape, dtype="float32", lod_level=0)
+    feed_image = fluid.data(
+        name="feed_image",
+        shape=[None] + image_shape,
+        dtype="float32",
+        lod_level=0)
 
-    feed_label = fluid.layers.data(
-        name="feed_label", shape=[1], dtype="int64", lod_level=0)
-    feed_y_a = fluid.layers.data(
-        name="feed_y_a", shape=[1], dtype="int64", lod_level=0)
+    feed_label = fluid.data(
+        name="feed_label", shape=[None, 1], dtype="int64", lod_level=0)
+    feed_y_a = fluid.data(
+        name="feed_y_a", shape=[None, 1], dtype="int64", lod_level=0)
 
     if is_train and args.use_mixup:
-        feed_y_b = fluid.layers.data(
-            name="feed_y_b", shape=[1], dtype="int64", lod_level=0)
-        feed_lam = fluid.layers.data(
-            name="feed_lam", shape=[1], dtype="float32", lod_level=0)
+        feed_y_b = fluid.data(
+            name="feed_y_b", shape=[None, 1], dtype="int64", lod_level=0)
+        feed_lam = fluid.data(
+            name="feed_lam", shape=[None, 1], dtype="float32", lod_level=0)
 
-        py_reader = fluid.io.PyReader(
+        data_loader = fluid.io.DataLoader.from_generator(
             feed_list=[feed_image, feed_y_a, feed_y_b, feed_lam],
             capacity=64,
             use_double_buffer=True,
             iterable=False)
-        return py_reader, [feed_image, feed_y_a, feed_y_b, feed_lam]
+        return data_loader, [feed_image, feed_y_a, feed_y_b, feed_lam]
     else:
-        py_reader = fluid.io.PyReader(
+        data_loader = fluid.io.DataLoader.from_generator(
             feed_list=[feed_image, feed_label],
             capacity=64,
             use_double_buffer=True,
             iterable=False)
 
-        return py_reader, [feed_image, feed_label]
+        return data_loader, [feed_image, feed_label]
 
 
 def print_info(pass_id, batch_id, print_step, metrics, time_info, info_mode):
@@ -408,8 +430,11 @@ def best_strategy_compiled(args, program, loss, exe):
 
 
 class ExponentialMovingAverage(object):
-
-    def __init__(self, decay=0.999, thres_steps=None, zero_debias=False, name=None):
+    def __init__(self,
+                 decay=0.999,
+                 thres_steps=None,
+                 zero_debias=False,
+                 name=None):
         self._decay = decay
         self._thres_steps = thres_steps
         self._name = name if name is not None else ''
@@ -429,7 +454,7 @@ class ExponentialMovingAverage(object):
         self._ema_vars = {}
         for param, tmp in self._params_tmps:
             with param.block.program._optimized_guard(
-                    [param, tmp]), name_scope('moving_average'):
+                [param, tmp]), name_scope('moving_average'):
                 self._ema_vars[param.name] = self._create_ema_vars(param)
 
         self.apply_program = Program()
@@ -499,14 +524,14 @@ class ExponentialMovingAverage(object):
         param_master_emas = []
         for param, tmp in self._params_tmps:
             with param.block.program._optimized_guard(
-                    [param, tmp]), name_scope('moving_average'):
+                [param, tmp]), name_scope('moving_average'):
                 param_ema = self._ema_vars[param.name]
                 if param.name + '.master' in self._ema_vars:
                     master_ema = self._ema_vars[param.name + '.master']
                     param_master_emas.append([param_ema, master_ema])
                 else:
                     ema_t = param_ema * self._decay_var + param * (
-                            1 - self._decay_var)
+                        1 - self._decay_var)
                     layers.assign(input=ema_t, output=param_ema)
 
         # for fp16 params