update according reviewing

dygraph/datasets/dataset.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os.path as osp
+import os
 from threading import Thread
 import multiprocessing
 import collections
@@ -170,8 +170,8 @@ def multiprocess_reader(mapper,
             index = i % num_workers
             total_samples[index].append(sample)
         for i in range(num_workers):
-            p = multiprocessing.Process(
-                target=_read_into_queue, args=(total_samples[i], mapper, queue))
+            p = multiprocessing.Process(target=_read_into_queue,
+                                        args=(total_samples[i], mapper, queue))
             p.start()
 
         finish_num = 0
@@ -230,18 +230,18 @@ class Dataset:
                 items = line.strip().split()
                 if not is_pic(items[0]):
                     continue
-                full_path_im = osp.join(data_dir, items[0])
-                full_path_label = osp.join(data_dir, items[1])
-                if not osp.exists(full_path_im):
+                full_path_im = os.path.join(data_dir, items[0])
+                full_path_label = os.path.join(data_dir, items[1])
+                if not os.path.exists(full_path_im):
                     raise IOError(
                         'The image file {} is not exist!'.format(full_path_im))
-                if not osp.exists(full_path_label):
+                if not os.path.exists(full_path_label):
                     raise IOError('The image file {} is not exist!'.format(
                         full_path_label))
                 self.file_list.append([full_path_im, full_path_label])
         self.num_samples = len(self.file_list)
-        logging.info("{} samples in file {}".format(
-            len(self.file_list), file_list))
+        logging.info("{} samples in file {}".format(len(self.file_list),
+                                                    file_list))
 
     def iterator(self):
         self._epoch += 1
@@ -266,8 +266,7 @@ class Dataset:
                 )
             else:
                 parallel_reader = multiprocess_reader
-        return parallel_reader(
-            self.transforms,
+        return parallel_reader(self.transforms,
                                self.iterator,
                                num_workers=self.num_workers,
                                buffer_size=self.buffer_size,

dygraph/infer.py

@@ -14,7 +14,6 @@
 
 import argparse
 import os
-import os.path as osp
 
 from paddle.fluid.dygraph.base import to_variable
 import numpy as np
@@ -33,54 +32,46 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Model training')
 
     # params of model
-    parser.add_argument(
-        '--model_name',
+    parser.add_argument('--model_name',
                         dest='model_name',
                         help="Model type for traing, which is one of ('UNet')",
                         type=str,
                         default='UNet')
 
     # params of dataset
-    parser.add_argument(
-        '--data_dir',
+    parser.add_argument('--data_dir',
                         dest='data_dir',
                         help='The root directory of dataset',
                         type=str)
-    parser.add_argument(
-        '--test_list',
+    parser.add_argument('--test_list',
                         dest='test_list',
                         help='Val list file of dataset',
                         type=str,
                         default=None)
-    parser.add_argument(
-        '--num_classes',
+    parser.add_argument('--num_classes',
                         dest='num_classes',
                         help='Number of classes',
                         type=int,
                         default=2)
 
     # params of prediction
-    parser.add_argument(
-        "--input_size",
+    parser.add_argument("--input_size",
                         dest="input_size",
                         help="The image size for net inputs.",
                         nargs=2,
                         default=[512, 512],
                         type=int)
-    parser.add_argument(
-        '--batch_size',
+    parser.add_argument('--batch_size',
                         dest='batch_size',
                         help='Mini batch size',
                         type=int,
                         default=2)
-    parser.add_argument(
-        '--model_dir',
+    parser.add_argument('--model_dir',
                         dest='model_dir',
                         help='The path of model for evaluation',
                         type=str,
                         default=None)
-    parser.add_argument(
-        '--save_dir',
+    parser.add_argument('--save_dir',
                         dest='save_dir',
                         help='The directory for saving the inference results',
                         type=str,
@@ -90,27 +81,27 @@ def parse_args():
 
 
 def mkdir(path):
-    sub_dir = osp.dirname(path)
-    if not osp.exists(sub_dir):
+    sub_dir = os.path.dirname(path)
+    if not os.path.exists(sub_dir):
         os.makedirs(sub_dir)
 
 
 def infer(model, data_dir=None, test_list=None, model_dir=None,
           transforms=None):
-    ckpt_path = osp.join(model_dir, 'model')
+    ckpt_path = os.path.join(model_dir, 'model')
     para_state_dict, opti_state_dict = fluid.load_dygraph(ckpt_path)
     model.set_dict(para_state_dict)
     model.eval()
 
-    added_saved_dir = osp.join(args.save_dir, 'added')
-    pred_saved_dir = osp.join(args.save_dir, 'prediction')
+    added_saved_dir = os.path.join(args.save_dir, 'added')
+    pred_saved_dir = os.path.join(args.save_dir, 'prediction')
 
     logging.info("Start to predict...")
     with open(test_list, 'r') as f:
         files = f.readlines()
     for file in tqdm.tqdm(files):
         file = file.strip()
-        im_file = osp.join(data_dir, file)
+        im_file = os.path.join(data_dir, file)
         im, im_info, _ = transforms(im_file)
         im = np.expand_dims(im, axis=0)
         im = to_variable(im)
@@ -129,25 +120,25 @@ def infer(model, data_dir=None, test_list=None, model_dir=None,
 
         # save added image
         added_image = utils.visualize(im_file, pred, weight=0.6)
-        added_image_path = osp.join(added_saved_dir, file)
+        added_image_path = os.path.join(added_saved_dir, file)
         mkdir(added_image_path)
         cv2.imwrite(added_image_path, added_image)
 
         # save prediction
         pred_im = utils.visualize(im_file, pred, weight=0.0)
-        pred_saved_path = osp.join(pred_saved_dir, file)
+        pred_saved_path = os.path.join(pred_saved_dir, file)
         mkdir(pred_saved_path)
         cv2.imwrite(pred_saved_path, pred_im)
 
 
 def main(args):
+    with fluid.dygraph.guard(places):
         test_transforms = T.Compose([T.Resize(args.input_size), T.Normalize()])
 
         if args.model_name == 'UNet':
             model = models.UNet(num_classes=args.num_classes)
 
-    infer(
-        model,
+        infer(model,
               data_dir=args.data_dir,
               test_list=args.test_list,
               model_dir=args.model_dir,
@@ -161,5 +152,4 @@ if __name__ == '__main__':
         places = fluid.CPUPlace()
     else:
         places = fluid.CUDAPlace(0)
-    with fluid.dygraph.guard(places):
     main(args)

dygraph/train.py

@@ -14,7 +14,6 @@
 
 import argparse
 import os
-import os.path as osp
 
 from paddle.fluid.dygraph.base import to_variable
 import numpy as np
@@ -33,84 +32,65 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Model training')
 
     # params of model
-    parser.add_argument(
-        '--model_name',
+    parser.add_argument('--model_name',
                         dest='model_name',
                         help="Model type for traing, which is one of ('UNet')",
                         type=str,
                         default='UNet')
 
     # params of dataset
-    parser.add_argument(
-        '--data_dir',
+    parser.add_argument('--data_dir',
                         dest='data_dir',
                         help='The root directory of dataset',
                         type=str)
-    parser.add_argument(
-        '--train_list',
+    parser.add_argument('--train_list',
                         dest='train_list',
                         help='Train list file of dataset',
                         type=str)
-    parser.add_argument(
-        '--val_list',
+    parser.add_argument('--val_list',
                         dest='val_list',
                         help='Val list file of dataset',
                         type=str,
                         default=None)
-    parser.add_argument(
-        '--num_classes',
+    parser.add_argument('--num_classes',
                         dest='num_classes',
                         help='Number of classes',
                         type=int,
                         default=2)
-    parser.add_argument(
-        '--ingore_index',
-        dest='ignore_index',
-        help=
-        'The pixel equaling ignore_index will not be computed during training',
-        type=int,
-        default=255)
 
     # params of training
-    parser.add_argument(
-        "--input_size",
+    parser.add_argument("--input_size",
                         dest="input_size",
                         help="The image size for net inputs.",
                         nargs=2,
                         default=[512, 512],
                         type=int)
-    parser.add_argument(
-        '--num_epochs',
+    parser.add_argument('--num_epochs',
                         dest='num_epochs',
                         help='Number epochs for training',
                         type=int,
                         default=100)
-    parser.add_argument(
-        '--batch_size',
+    parser.add_argument('--batch_size',
                         dest='batch_size',
                         help='Mini batch size',
                         type=int,
                         default=2)
-    parser.add_argument(
-        '--learning_rate',
+    parser.add_argument('--learning_rate',
                         dest='learning_rate',
                         help='Learning rate',
                         type=float,
                         default=0.01)
-    parser.add_argument(
-        '--pretrained_model',
+    parser.add_argument('--pretrained_model',
                         dest='pretrained_model',
                         help='The path of pretrianed weight',
                         type=str,
                         default=None)
-    parser.add_argument(
-        '--save_interval_epochs',
+    parser.add_argument('--save_interval_epochs',
                         dest='save_interval_epochs',
                         help='The interval epochs for save a model snapshot',
                         type=int,
                         default=5)
-    parser.add_argument(
-        '--save_dir',
+    parser.add_argument('--save_dir',
                         dest='save_dir',
                         help='The directory for saving the model snapshot',
                         type=str,
@@ -129,15 +109,15 @@ def train(model,
           pretrained_model=None,
           save_interval_epochs=1,
           num_classes=None):
-    if not osp.isdir(save_dir):
-        if osp.exists(save_dir):
+    if not os.path.isdir(save_dir):
+        if os.path.exists(save_dir):
             os.remove(save_dir)
         os.makedirs(save_dir)
 
     load_pretrained_model(model, pretrained_model)
 
-    data_generator = train_dataset.generator(
-        batch_size=batch_size, drop_last=True)
+    data_generator = train_dataset.generator(batch_size=batch_size,
+                                             drop_last=True)
     num_steps_each_epoch = train_dataset.num_samples // args.batch_size
 
     for epoch in range(num_epochs):
@@ -156,16 +136,16 @@ def train(model,
         if (
                 epoch + 1
         ) % save_interval_epochs == 0 or num_steps_each_epoch == num_epochs - 1:
-            current_save_dir = osp.join(save_dir, "epoch_{}".format(epoch + 1))
-            if not osp.isdir(current_save_dir):
+            current_save_dir = os.path.join(save_dir,
+                                            "epoch_{}".format(epoch + 1))
+            if not os.path.isdir(current_save_dir):
                 os.makedirs(current_save_dir)
             fluid.save_dygraph(model.state_dict(),
-                               osp.join(current_save_dir, 'model'))
+                               os.path.join(current_save_dir, 'model'))
 
             if eval_dataset is not None:
                 model.eval()
-                evaluate(
-                    model,
+                evaluate(model,
                          eval_dataset,
                          model_dir=current_save_dir,
                          num_classes=num_classes,
@@ -176,13 +156,14 @@ def train(model,
 
 
 def main(args):
+    with fluid.dygraph.guard(places):
         # Creat dataset reader
-    train_transforms = T.Compose(
-        [T.Resize(args.input_size),
+        train_transforms = T.Compose([
+            T.Resize(args.input_size),
             T.RandomHorizontalFlip(),
-         T.Normalize()])
-    train_dataset = Dataset(
-        data_dir=args.data_dir,
+            T.Normalize()
+        ])
+        train_dataset = Dataset(data_dir=args.data_dir,
                                 file_list=args.train_list,
                                 transforms=train_transforms,
                                 num_workers='auto',
@@ -190,9 +171,10 @@ def main(args):
                                 parallel_method='thread',
                                 shuffle=True)
         if args.val_list is not None:
-        eval_transforms = T.Compose([T.Resize(args.input_size), T.Normalize()])
-        eval_dataset = Dataset(
-            data_dir=args.data_dir,
+            eval_transforms = T.Compose(
+                [T.Resize(args.input_size),
+                 T.Normalize()])
+            eval_dataset = Dataset(data_dir=args.data_dir,
                                    file_list=args.val_list,
                                    transforms=eval_transforms,
                                    num_workers='auto',
@@ -201,22 +183,22 @@ def main(args):
                                    shuffle=False)
 
         if args.model_name == 'UNet':
-        model = models.UNet(
-            num_classes=args.num_classes, ignore_index=args.ignore_index)
+            model = models.UNet(num_classes=args.num_classes, ignore_index=255)
 
         # Creat optimizer
         num_steps_each_epoch = train_dataset.num_samples // args.batch_size
         decay_step = args.num_epochs * num_steps_each_epoch
-    lr_decay = fluid.layers.polynomial_decay(
-        args.learning_rate, decay_step, end_learning_rate=0, power=0.9)
+        lr_decay = fluid.layers.polynomial_decay(args.learning_rate,
+                                                 decay_step,
+                                                 end_learning_rate=0,
+                                                 power=0.9)
         optimizer = fluid.optimizer.Momentum(
             lr_decay,
             momentum=0.9,
             parameter_list=model.parameters(),
             regularization=fluid.regularizer.L2Decay(regularization_coeff=4e-5))
 
-    train(
-        model,
+        train(model,
               train_dataset,
               eval_dataset,
               optimizer,
@@ -235,5 +217,4 @@ if __name__ == '__main__':
         places = fluid.CPUPlace()
     else:
         places = fluid.CUDAPlace(0)
-    with fluid.dygraph.guard(places):
     main(args)

dygraph/utils/utils.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-import os.path as osp
 import numpy as np
 import math
 import cv2
@@ -59,8 +58,8 @@ def get_environ_info():
 def load_pretrained_model(model, pretrained_model):
     logging.info('Load pretrained model!')
     if pretrained_model is not None:
-        if osp.exists(pretrained_model):
-            ckpt_path = osp.join(pretrained_model, 'model')
+        if os.path.exists(pretrained_model):
+            ckpt_path = os.path.join(pretrained_model, 'model')
             para_state_dict, _ = fluid.load_dygraph(ckpt_path)
             model_state_dict = model.state_dict()
             keys = model_state_dict.keys()

dygraph/val.py

@@ -14,7 +14,6 @@
 
 import argparse
 import os
-import os.path as osp
 import math
 
 from paddle.fluid.dygraph.base import to_variable
@@ -33,55 +32,41 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Model training')
 
     # params of model
-    parser.add_argument(
-        '--model_name',
+    parser.add_argument('--model_name',
                         dest='model_name',
                         help="Model type for traing, which is one of ('UNet')",
                         type=str,
                         default='UNet')
 
     # params of dataset
-    parser.add_argument(
-        '--data_dir',
+    parser.add_argument('--data_dir',
                         dest='data_dir',
                         help='The root directory of dataset',
                         type=str)
-    parser.add_argument(
-        '--val_list',
+    parser.add_argument('--val_list',
                         dest='val_list',
                         help='Val list file of dataset',
                         type=str,
                         default=None)
-    parser.add_argument(
-        '--num_classes',
+    parser.add_argument('--num_classes',
                         dest='num_classes',
                         help='Number of classes',
                         type=int,
                         default=2)
-    parser.add_argument(
-        '--ingore_index',
-        dest='ignore_index',
-        help=
-        'The pixel equaling ignore_index will not be computed during evaluation',
-        type=int,
-        default=255)
 
     # params of evaluate
-    parser.add_argument(
-        "--input_size",
+    parser.add_argument("--input_size",
                         dest="input_size",
                         help="The image size for net inputs.",
                         nargs=2,
                         default=[512, 512],
                         type=int)
-    parser.add_argument(
-        '--batch_size',
+    parser.add_argument('--batch_size',
                         dest='batch_size',
                         help='Mini batch size',
                         type=int,
                         default=2)
-    parser.add_argument(
-        '--model_dir',
+    parser.add_argument('--model_dir',
                         dest='model_dir',
                         help='The path of model for evaluation',
                         type=str,
@@ -97,13 +82,13 @@ def evaluate(model,
              batch_size=2,
              ignore_index=255,
              epoch_id=None):
-    ckpt_path = osp.join(model_dir, 'model')
+    ckpt_path = os.path.join(model_dir, 'model')
     para_state_dict, opti_state_dict = fluid.load_dygraph(ckpt_path)
     model.set_dict(para_state_dict)
     model.eval()
 
-    data_generator = eval_dataset.generator(
-        batch_size=batch_size, drop_last=True)
+    data_generator = eval_dataset.generator(batch_size=batch_size,
+                                            drop_last=True)
     total_steps = math.ceil(eval_dataset.num_samples * 1.0 / batch_size)
     conf_mat = ConfusionMatrix(num_classes, streaming=True)
 
@@ -135,9 +120,9 @@ def evaluate(model,
 
 
 def main(args):
+    with fluid.dygraph.guard(places):
         eval_transforms = T.Compose([T.Resize(args.input_size), T.Normalize()])
-    eval_dataset = Dataset(
-        data_dir=args.data_dir,
+        eval_dataset = Dataset(data_dir=args.data_dir,
                                file_list=args.val_list,
                                transforms=eval_transforms,
                                num_workers='auto',
@@ -148,14 +133,11 @@ def main(args):
         if args.model_name == 'UNet':
             model = models.UNet(num_classes=args.num_classes)
 
-    evaluate(
-        model,
+        evaluate(model,
                  eval_dataset,
                  model_dir=args.model_dir,
                  num_classes=args.num_classes,
-        batch_size=args.batch_size,
-        ignore_index=args.ignore_index,
-    )
+                 batch_size=args.batch_size)
 
 
 if __name__ == '__main__':
@@ -165,5 +147,4 @@ if __name__ == '__main__':
         places = fluid.CPUPlace()
     else:
         places = fluid.CUDAPlace(0)
-    with fluid.dygraph.guard(places):
     main(args)