polish code

fluid/PaddleCV/image_classification/fast_resnet/torchvision_reader.py

@@ -3,33 +3,30 @@ import os
 import numpy as np
 import math
 import random
-import torchvision
+import torch
+import torch.utils.data
+from torch.utils.data.distributed import DistributedSampler
 import torchvision.transforms as transforms
 import torchvision.datasets as datasets
 
+from torch.utils.data.sampler import Sampler
+import torchvision
 import pickle
 from tqdm import tqdm
 import time
 import multiprocessing
 
-TRAINER_NUMS = int(os.getenv("PADDLE_TRAINERS_NUM", "1"))
-TRAINER_ID = int(os.getenv("PADDLE_TRAINER_ID", "0"))
-
 FINISH_EVENT = "FINISH_EVENT"
 class PaddleDataLoader(object):
-    def __init__(self, torch_dataset, indices=None, concurrent=16, queue_size=3072, shuffle=True, batch_size=224, is_distributed=True):
+    def __init__(self, torch_dataset, indices=None, concurrent=24, queue_size=3072, shuffle=True):
         self.torch_dataset = torch_dataset
         self.data_queue = multiprocessing.Queue(queue_size)
         self.indices = indices
         self.concurrent = concurrent
-        self.shuffle_seed = 0
         self.shuffle = shuffle
-        self.is_distributed = is_distributed
-        self.batch_size = batch_size
 
     def _worker_loop(self, dataset, worker_indices, worker_id):
         cnt = 0
-        print("worker [%d], len: [%d], indices: [%s]"%(worker_id, len(worker_indices), worker_indices[:10]))
         for idx in worker_indices:
             cnt += 1
             img, label = self.torch_dataset[idx]
@@ -43,28 +40,21 @@ class PaddleDataLoader(object):
             worker_processes = []
             total_img = len(self.torch_dataset)
             print("total image: ", total_img)
-            if self.indices is None:
-                self.indices = [i for i in xrange(total_img)]
+            #if self.indices is None:
             if self.shuffle:
-                random.seed(self.shuffle_seed)
+                self.indices = [i for i in xrange(total_img)]
+                random.seed(time.time())
                 random.shuffle(self.indices)
-            worker_indices = self.indices
-            if self.is_distributed:
-                cnt_per_node = len(self.indices) / TRAINER_NUMS
-                offset = TRAINER_ID * cnt_per_node
-                worker_indices = self.indices[offset: (offset + cnt_per_node)]
-            if len(worker_indices) % self.batch_size != 0:
-                worker_indices += worker_indices[-(self.batch_size - (len(worker_indices) % self.batch_size)):]
-            print("shuffle: [%d], shuffle seed: [%d], worker indices len: [%d], %s" % (self.shuffle, self.shuffle_seed, len(worker_indices), worker_indices[:10]))
-
-            cnt_per_thread = int(math.ceil(len(worker_indices) / self.concurrent))
+                print("shuffle indices: %s ..." % self.indices[:10])
+
+            imgs_per_worker = int(math.ceil(total_img / self.concurrent))
             for i in xrange(self.concurrent):
-                offset = i * cnt_per_thread
-                thread_incides = worker_indices[offset: (offset + cnt_per_thread)]
-                print("loader thread: [%d] start idx: [%d], end idx: [%d], len: [%d]" % (i, offset, (offset + cnt_per_thread), len(thread_incides)))
+                start = i * imgs_per_worker
+                end = (i + 1) * imgs_per_worker if i != self.concurrent - 1 else None
+                sliced_indices = self.indices[start:end]
                 w = multiprocessing.Process(
                     target=self._worker_loop,
-                    args=(self.torch_dataset, thread_incides, i)
+                    args=(self.torch_dataset, sliced_indices, i)
                 )
                 w.daemon = True
                 w.start()
@@ -80,13 +70,13 @@ class PaddleDataLoader(object):
 
         return _reader_creator
 
-def train(traindir, bs, sz, min_scale=0.08):
+def train(traindir, sz, min_scale=0.08):
     train_tfms = [
         transforms.RandomResizedCrop(sz, scale=(min_scale, 1.0)),
         transforms.RandomHorizontalFlip()
     ]
     train_dataset = datasets.ImageFolder(traindir, transforms.Compose(train_tfms))
-    return PaddleDataLoader(train_dataset, batch_size=bs)
+    return PaddleDataLoader(train_dataset).reader()
 
 def test(valdir, bs, sz, rect_val=False):
     if rect_val:
@@ -96,12 +86,12 @@ def test(valdir, bs, sz, rect_val=False):
 
         ar_tfms = [transforms.Resize(int(sz* 1.14)), CropArTfm(idx2ar, sz)]
         val_dataset = ValDataset(valdir, transform=ar_tfms)
-        return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False, is_distributed=False)
+        return PaddleDataLoader(val_dataset, concurrent=1, indices=idx_sorted, shuffle=False).reader()
 
     val_tfms = [transforms.Resize(int(sz* 1.14)), transforms.CenterCrop(sz)]
     val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms))
 
-    return PaddleDataLoader(val_dataset, is_distributed=False)
+    return PaddleDataLoader(val_dataset).reader()
 
 
 class ValDataset(datasets.ImageFolder):
@@ -122,6 +112,7 @@ class ValDataset(datasets.ImageFolder):
 
         return sample, target
 
+
 class CropArTfm(object):
     def __init__(self, idx2ar, target_size):
         self.idx2ar, self.target_size = idx2ar, target_size
@@ -134,7 +125,7 @@ class CropArTfm(object):
         else:
             h = int(self.target_size * target_ar)
             size = (self.target_size, h // 8 * 8)
-        return transforms.functional.center_crop(img, size)
+        return torchvision.transforms.functional.center_crop(img, size)
 
 
 def sort_ar(valdir):
@@ -166,5 +157,15 @@ def map_idx2ar(idx_ar_sorted, batch_size):
     return idx2ar
 
 if __name__ == "__main__":
-    reader = test("/work/fast_resnet_data", 64, 128).reader()
-    print(next(reader()))
\ No newline at end of file
+    #ds, sampler = create_validation_set("/data/imagenet/validation", 128, 288, True, True)
+    #for item in sampler:
+    #    for idx in item:
+    #        ds[idx]
+
+    import time
+    test_reader = test(valdir="/data/imagenet/validation", bs=64, sz=288, rect_val=True)
+    start_ts = time.time()
+    for idx, data in enumerate(test_reader()):
+        print(idx, data[0], data[0].shape, data[1])
+        if idx == 2:
+            break
\ No newline at end of file

fluid/PaddleCV/image_classification/models/fast_resnet.py

@@ -30,14 +30,12 @@ import paddle.fluid.core as core
 import paddle.fluid.profiler as profiler
 import utils
 
-## visreader for imagenet
-import torchvision_reader
-
 __all__ = ["FastResNet"]
 
 class FastResNet():
-    def __init__(self, layers=50):
+    def __init__(self, layers=50, is_train=True):
         self.layers = layers
+        self.is_train = is_train
 
     def net(self, input, class_dim=1000, img_size=224, is_train=True):
         layers = self.layers
@@ -54,7 +52,7 @@ class FastResNet():
         num_filters = [64, 128, 256, 512]
 
         conv = self.conv_bn_layer(
-            input=input, num_filters=64, filter_size=7, stride=2, act='relu', is_train=is_train)
+            input=input, num_filters=64, filter_size=7, stride=2, act='relu')
         conv = fluid.layers.pool2d(
             input=conv,
             pool_size=3,
@@ -73,6 +71,7 @@ class FastResNet():
             input=conv, pool_size=pool_size, pool_type='avg', global_pooling=True)
         out = fluid.layers.fc(input=pool,
                               size=class_dim,
+                              act=None,
                               param_attr=fluid.param_attr.ParamAttr(
                                   initializer=fluid.initializer.NormalInitializer(0.0, 0.01),
                                   regularizer=fluid.regularizer.L2Decay(1e-4)),
@@ -87,8 +86,7 @@ class FastResNet():
                       stride=1,
                       groups=1,
                       act=None,
-                      bn_init_value=1.0,
-                      is_train=True):
+                      bn_init_value=1.0):
         conv = fluid.layers.conv2d(
             input=input,
             num_filters=num_filters,
@@ -98,10 +96,8 @@ class FastResNet():
             groups=groups,
             act=None,
             bias_attr=False,
-            param_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.MSRAInitializer(),
-                regularizer=fluid.regularizer.L2Decay(1e-4)))
-        return fluid.layers.batch_norm(input=conv, act=act, is_test=not is_train,
+            param_attr=fluid.ParamAttr(regularizer=fluid.regularizer.L2Decay(1e-4)))
+        return fluid.layers.batch_norm(input=conv, act=act, is_test=not self.is_train,
             param_attr=fluid.param_attr.ParamAttr(
                 initializer=fluid.initializer.Constant(bn_init_value),
                 regularizer=None))
@@ -129,3 +125,67 @@ class FastResNet():
         short = self.shortcut(input, num_filters * 4, stride)
 
         return fluid.layers.elementwise_add(x=short, y=conv2, act='relu')
+
+def lr_decay(lrs, epochs, bs, total_image):
+    boundaries = []
+    values = []
+    for idx, epoch in enumerate(epochs):
+        step = total_image // bs[idx]
+        if step * bs[idx] < total_image:
+            step += 1
+        ratio = (lrs[idx][1] - lrs[idx][0])*1.0 / (epoch[1] - epoch[0])
+        lr_base = lrs[idx][0]
+        for s in xrange(epoch[0], epoch[1]):
+            if boundaries:
+                boundaries.append(boundaries[-1] + step)
+            else:
+                boundaries = [step]
+            lr = lr_base + ratio * (s - epoch[0])
+            values.append(lr)
+            print("epoch: [%d], steps: [%d], lr: [%f]" % (s, boundaries[-1], values[-1])) 
+    values.append(lrs[-1])
+    print("epoch: [%d:], steps: [%d:], lr:[%f]" % (epochs[-1][-1], boundaries[-1], values[-1]))
+    return boundaries, values
+
+def linear_lr_decay_by_epoch(lr_values, epochs, bs_values, total_images):
+    from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
+    import paddle.fluid.layers.tensor as tensor
+    import math
+
+    with paddle.fluid.default_main_program()._lr_schedule_guard():
+        global_step = _decay_step_counter()
+
+        lr = tensor.create_global_var(
+            shape=[1],
+            value=0.0,
+            dtype='float32',
+            persistable=True,
+            name="learning_rate")
+        with fluid.layers.control_flow.Switch() as switch:
+            last_steps = 0
+            for idx, epoch_bound in enumerate(epochs):
+                start_epoch, end_epoch = epoch_bound
+                linear_epoch = end_epoch - start_epoch
+                start_lr, end_lr = lr_values[idx]
+                linear_lr = end_lr - start_lr
+                for epoch_step in xrange(linear_epoch):
+                    steps = last_steps + (1 + epoch_step) * total_images / bs_values[idx] + 1
+                    boundary_val = tensor.fill_constant(
+                        shape=[1],
+                        dtype='float32',
+                        value=float(steps),
+                        force_cpu=True)
+                    decayed_lr = start_lr + epoch_step * linear_lr * 1.0 / linear_epoch
+                    with switch.case(global_step < boundary_val):
+                        value_var = tensor.fill_constant(shape=[1], dtype='float32', value=float(decayed_lr))
+                        print("steps: [%d], epoch : [%d], decayed_lr: [%f]" % (steps, start_epoch + epoch_step, decayed_lr))
+                        fluid.layers.tensor.assign(value_var, lr)
+                last_steps = steps
+            last_value_var = tensor.fill_constant(
+                shape=[1],
+                dtype='float32',
+                value=float(lr_values[-1]))
+            with switch.default():
+                fluid.layers.tensor.assign(last_value_var, lr)
+
+        return lr
\ No newline at end of file