refine code

hapi/model.py

@@ -1097,10 +1097,10 @@ class Model(fluid.dygraph.Layer):
             batch_size (int): Integer number. The batch size of train_data and eval_data. 
                 When train_data and eval_data are both the instance of Dataloader, this 
                 argument will be ignored. Default: 1.
-            num_workers (int): the number of subprocess to load data, 0 for no subprocess 
+            num_workers (int): The number of subprocess to load data, 0 for no subprocess 
                 used and loading data in main process. When train_data and eval_data are
                 both the instance of Dataloader, this argument will be ignored. Default: 0.
-            stack_output (bool): whether stack output field like a batch, as for an output
+            stack_output (bool): Whether stack output field like a batch, as for an output
                 filed of a sample is in shape [X, Y], test_data contains N samples, predict
                 output field will be in shape [N, X, Y] if stack_output is True, and will
                 be a length N list in shape [[X, Y], [X, Y], ....[X, Y]] if stack_outputs

hapi/tests/dist_mnist.py

@@ -48,7 +48,7 @@ class MnistDataset(MNIST):
         return len(self.images)
 
 
-def get_predict_accuracy(pred, gt):
+def compute_accuracy(pred, gt):
     pred = np.argmax(pred, -1)
     gt = np.array(gt)
 
@@ -58,7 +58,7 @@ def get_predict_accuracy(pred, gt):
 
 
 class TestModel(unittest.TestCase):
-    def fit(self, dynamic):
+    def run(self, dynamic):
         device = set_device('gpu')
         fluid.enable_dygraph(device) if dynamic else None
 
@@ -74,7 +74,9 @@ class TestModel(unittest.TestCase):
 
         model = LeNet()
         optim = fluid.optimizer.Momentum(
-            learning_rate=0.01, momentum=.9, parameter_list=model.parameters())
+            learning_rate=0.001,
+            momentum=.9,
+            parameter_list=model.parameters())
         loss = CrossEntropy()
         model.prepare(optim, loss, Accuracy(), inputs, labels, device=device)
         cbk = ProgBarLogger(50)
@@ -92,15 +94,15 @@ class TestModel(unittest.TestCase):
 
         np.testing.assert_equal(output[0].shape[0], len(test_dataset))
 
-        acc = get_predict_accuracy(output[0], val_dataset.labels)
+        acc = compute_accuracy(output[0], val_dataset.labels)
 
         np.testing.assert_allclose(acc, eval_result['acc'])
 
     def test_multiple_gpus_static(self):
-        self.fit(False)
+        self.run(False)
 
     def test_multiple_gpus_dygraph(self):
-        self.fit(True)
+        self.run(True)
 
 
 if __name__ == '__main__':

hapi/tests/test_distributed.py

@@ -30,40 +30,21 @@ import paddle.distributed.cloud_utils as cloud_utils
 def get_cluster_from_args(selected_gpus):
     cluster_node_ips = '127.0.0.1'
     node_ip = '127.0.0.1'
-    use_paddlecloud = False
-    started_port = None
+
     node_ips = [x.strip() for x in cluster_node_ips.split(',')]
 
-    node_rank = node_ips.index(node_ip)
+    node_ips.index(node_ip)
 
     free_ports = None
-    if not use_paddlecloud and len(node_ips) <= 1 and started_port is None:
-        free_ports = find_free_ports(len(selected_gpus))
-        if free_ports is not None:
-            free_ports = list(free_ports)
-    else:
-        started_port = 6070
-
-        free_ports = [
-            x for x in range(started_port, started_port + len(selected_gpus))
-        ]
+
+    free_ports = find_free_ports(len(selected_gpus))
+    if free_ports is not None:
+        free_ports = list(free_ports)
     return get_cluster(node_ips, node_ip, free_ports, selected_gpus)
 
 
 def get_gpus(selected_gpus):
-    cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
-    if cuda_visible_devices is None or cuda_visible_devices == "":
-        selected_gpus = [x.strip() for x in selected_gpus.split(',')]
-    else:
-        cuda_visible_devices_list = cuda_visible_devices.split(',')
-        for x in selected_gpus.split(','):
-            assert x in cuda_visible_devices_list, "Can't find "\
-            "your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]."\
-            % (x, cuda_visible_devices)
-        selected_gpus = [
-            cuda_visible_devices_list.index(x.strip())
-            for x in selected_gpus.split(',')
-        ]
+    selected_gpus = [x.strip() for x in selected_gpus.split(',')]
     return selected_gpus
 
 
@@ -94,7 +75,7 @@ def start_local_trainers(cluster,
 
         print("trainer proc env:{}".format(current_env))
 
-        cmd = "python -m coverage run --branch -p " + training_script
+        cmd = "python -u " + training_script
 
         print("start trainer proc:{} env:{}".format(cmd, proc_env))
 

hapi/vision/models/resnet.py

@@ -36,7 +36,7 @@ model_urls = {
     'resnet34': ('https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams',
                  '46bc9f7c3dd2e55b7866285bee91eff3'),
     'resnet50': ('https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams',
-                 '0884c9087266496c41c60d14a96f8530'),
+                 '5ce890a9ad386df17cf7fe2313dca0a1'),
     'resnet101':
     ('https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams',
      'fb07a451df331e4b0bb861ed97c3a9b9'),

hapi/vision/transforms/functional.py

@@ -35,7 +35,7 @@ def flip(image, code):
 
     Args:
         image: Input image, with (H, W, C) shape
-        code: code that indicates the type of flip.
+        code: Code that indicates the type of flip.
             -1 : Flip horizontally and vertically
             0 : Flip vertically
             1 : Flip horizontally

hapi/vision/transforms/transforms.py

@@ -61,7 +61,7 @@ class Compose(object):
     together for a dataset transform.
 
     Args:
-        transforms (list of ``Transform`` objects): list of transforms to compose.
+        transforms (list): List of transforms to compose.
 
     Returns:
         A compose object which is callable, __call__ for this Compose
@@ -115,8 +115,8 @@ class BatchCompose(object):
     """Composes several batch transforms together
 
     Args:
-        transforms (list of ``Transform`` objects): list of transforms to compose.
-                                            these transforms perform on batch data.
+        transforms (list): List of transforms to compose.
+                           these transforms perform on batch data.
 
     Examples:
     
@@ -209,21 +209,22 @@ class Resize(object):
             smaller edge of the image will be matched to this number.
             i.e, if height > width, then image will be rescaled to
             (size * height / width, size)
-        interpolation (int): interpolation mode of resize. Default: cv2.INTER_LINEAR.
+        interpolation (int): Interpolation mode of resize. Default: cv2.INTER_LINEAR.
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Resize
+            import numpy as np
 
-            transform = Compose([Resize(size=224)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import Resize
 
-            for i in range(10):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = Resize(size=224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, size, interpolation=cv2.INTER_LINEAR):
@@ -251,15 +252,16 @@ class RandomResizedCrop(object):
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Resize, RandomResizedCrop
+            import numpy as np
 
-            transform = Compose([Resize(500), RandomResizedCrop(224)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import RandomResizedCrop
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = RandomResizedCrop(224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self,
@@ -320,22 +322,23 @@ class CenterCropResize(object):
 
     Args:
         size (int|list|tuple): Target size of output image, with (height, width) shape.
-        crop_padding (int): center crop with the padding. Default: 32.
-        interpolation (int): interpolation mode of resize. Default: cv2.INTER_LINEAR.
+        crop_padding (int): Center crop with the padding. Default: 32.
+        interpolation (int): Interpolation mode of resize. Default: cv2.INTER_LINEAR.
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Resize, CenterCropResize
+            import numpy as np
 
-            transform = Compose([Resize(500), CenterCropResize(224)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import CenterCropResize
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = CenterCropResize(224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, size, crop_padding=32, interpolation=cv2.INTER_LINEAR):
@@ -370,15 +373,16 @@ class CenterCrop(object):
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Resize, CenterCrop
+            import numpy as np
 
-            transform = Compose([Resize(500), CenterCrop(224)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import CenterCrop
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = CenterCrop(224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, output_size):
@@ -405,21 +409,22 @@ class RandomHorizontalFlip(object):
     """Horizontally flip the input data randomly with a given probability.
 
     Args:
-        prob (float): probability of the input data being flipped. Default: 0.5
+        prob (float): Probability of the input data being flipped. Default: 0.5
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, RandomHorizontalFlip
+            import numpy as np
 
-            transform = Compose([RandomHorizontalFlip()])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import RandomHorizontalFlip
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = RandomHorizontalFlip(224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, prob=0.5):
@@ -435,21 +440,22 @@ class RandomVerticalFlip(object):
     """Vertically flip the input data randomly with a given probability.
 
     Args:
-        prob (float): probability of the input data being flipped. Default: 0.5
+        prob (float): Probability of the input data being flipped. Default: 0.5
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, RandomVerticalFlip
+            import numpy as np
 
-            transform = Compose([RandomVerticalFlip()])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import RandomVerticalFlip
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = RandomVerticalFlip(224)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, prob=0.5):
@@ -475,18 +481,17 @@ class Normalize(object):
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Normalize, Permute
+            import numpy as np
 
-            normalize = Normalize(mean=[123.675, 116.28, 103.53], 
-                                std=[58.395, 57.120, 57.375])
+            from hapi.vision.transforms import Normalize
 
-            transform = Compose([Permute(), normalize])
-            flowers = Flowers(mode='test', transform=transform)
+            normalize = Normalize(mean=[0.5, 0.5, 0.5], 
+                                std=[0.5, 0.5, 0.5])
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            fake_img = np.random.rand(3, 500, 500).astype('float32')
+
+            fake_img = normalize(fake_img)
+            print(fake_img.shape)
     
     """
 
@@ -511,22 +516,23 @@ class Permute(object):
     Input image should be HWC mode and an instance of numpy.ndarray. 
 
     Args:
-        mode: Output mode of input. Default: "CHW".
-        to_rgb: convert 'bgr' image to 'rgb'. Default: True.
+        mode (str): Output mode of input. Default: "CHW".
+        to_rgb (bool): Convert 'bgr' image to 'rgb'. Default: True.
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, Permute
+            import numpy as np
 
-            transform = Compose([Permute()])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import Permute
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = Permute()
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, mode="CHW", to_rgb=True):
@@ -549,22 +555,23 @@ class GaussianNoise(object):
     Gaussian noise is generated with given mean and std.
 
     Args:
-        mean: Gaussian mean used to generate noise.
-        std: Gaussian standard deviation used to generate noise.
+        mean (float): Gaussian mean used to generate noise.
+        std (float): Gaussian standard deviation used to generate noise.
     
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, GaussianNoise
+            import numpy as np
 
-            transform = Compose([GaussianNoise()])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import GaussianNoise
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = GaussianNoise()
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, mean=0.0, std=1.0):
@@ -582,22 +589,23 @@ class BrightnessTransform(object):
     """Adjust brightness of the image.
 
     Args:
-        value: How much to adjust the brightness. Can be any
+        value (float): How much to adjust the brightness. Can be any
             non negative number. 0 gives the original image
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, BrightnessTransform
+            import numpy as np
 
-            transform = Compose([BrightnessTransform(0.4)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import BrightnessTransform
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = BrightnessTransform(0.4)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, value):
@@ -620,22 +628,23 @@ class ContrastTransform(object):
     """Adjust contrast of the image.
 
     Args:
-        value: How much to adjust the contrast. Can be any
+        value (float): How much to adjust the contrast. Can be any
             non negative number. 0 gives the original image
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, ContrastTransform
+            import numpy as np
 
-            transform = Compose([ContrastTransform(0.4)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import ContrastTransform
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = ContrastTransform(0.4)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, value):
@@ -659,22 +668,23 @@ class SaturationTransform(object):
     """Adjust saturation of the image.
 
     Args:
-        value: How much to adjust the saturation. Can be any
+        value (float): How much to adjust the saturation. Can be any
             non negative number. 0 gives the original image
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, SaturationTransform
+            import numpy as np
 
-            transform = Compose([SaturationTransform(0.4)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import SaturationTransform
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = SaturationTransform(0.4)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+        
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, value):
@@ -699,22 +709,23 @@ class HueTransform(object):
     """Adjust hue of the image.
 
     Args:
-        value: How much to adjust the hue. Can be any number
+        value (float): How much to adjust the hue. Can be any number
             between 0 and 0.5, 0 gives the original image
 
     Examples:
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, HueTransform
+            import numpy as np
 
-            transform = Compose([HueTransform(0.4)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import HueTransform
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = HueTransform(0.4)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, value):
@@ -761,15 +772,16 @@ class ColorJitter(object):
     
         .. code-block:: python
 
-            from hapi.datasets import Flowers
-            from hapi.vision.transforms import Compose, ColorJitter
+            import numpy as np
 
-            transform = Compose([ColorJitter(0.4)])
-            flowers = Flowers(mode='test', transform=transform)
+            from hapi.vision.transforms import ColorJitter
 
-            for i in range(2):
-                sample = flowers[i]
-                print(sample[0].shape, sample[1])
+            transform = ColorJitter(0.4)
+
+            fake_img = np.random.rand(500, 500, 3).astype('float32')
+
+            fake_img = transform(fake_img)
+            print(fake_img.shape)
     """
 
     def __init__(self, brightness=0, contrast=0, saturation=0, hue=0):