diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_dist_save_load.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_dist_save_load.py index 553ce09704565efe6dac82da93bfb2658f70ff08..f443b599cbe5a9f13b3a2fe89791483e00645ac1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_dist_save_load.py @@ -54,14 +54,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -125,18 +129,15 @@ def train_mlp( ) return - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: model.to(device="gpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py index ca4f2fa78436acb9b2c46e11b05fbfc88400d969..3aa33d89d62397bc60b3b63673c027270149a4e7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py @@ -50,14 +50,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_multi_precision, opt_group=False): @@ -92,18 +96,15 @@ def train_mlp( model=model, optimizer=optimizer, level=shard_level, scaler=scaler ) - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): model.train() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py index 3a2851a0c6ed94ba4f4236106640b3344f69bf30..4dcf3697f25c638c4fd8df68986a6b8647c90e83 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py @@ -48,14 +48,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_multi_precision, opt_group=False): @@ -103,18 +107,15 @@ def train_mlp( if shard_level == "os_g": optimizer.set_lr(optimizer.get_lr()) - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): model.train() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py index 57444efb792e76d44112e8afc6f7fa25d8889341..32e1b12e0df1420a90e707b29ab15857d06efbcd 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py @@ -53,14 +53,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -122,18 +126,15 @@ def train_mlp( ) return - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: model.to(device="gpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py index b5907d9e75d9471d11cf35dda9fad1cc7b99cffc..5aa13e7ac6c05caaf63c200f02d626fd951099f3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py @@ -53,14 +53,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -124,18 +128,15 @@ def train_mlp( ) return - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: model.to(device="gpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py index 94373b057112428999184fe55cc95dc63d9e2eba..8b6b9241bdfae24dddc7db9c3d929643592be379 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py @@ -15,11 +15,7 @@ # limitations under the License. import numpy as np -from dygraph_group_sharded_stage2 import ( - MLP, - optimizer_setting, - reader_decorator, -) +from dygraph_group_sharded_stage2 import MLP, RandomDataset, optimizer_setting import paddle from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( @@ -53,18 +49,15 @@ def train_mlp(model, offload=False): ) model = GroupShardedStage2(model, optimizer, buffer_max_size=2**21) - train_reader = paddle.batch( - reader_decorator(linear_size), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): model.train() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py index 84a77855d9464d99a6da1b8197e68ef37f0cac5c..1697655c8bad640a4f3b2682c875da1a3c98378a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py @@ -44,7 +44,7 @@ l2_decay = 1e-4 class MLP(paddle.nn.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): + def __init__(self, linear_size=1024, param_attr=None, bias_attr=None): super().__init__() self._linear1 = Linear(linear_size, linear_size) @@ -102,14 +102,18 @@ class SpecialModel(paddle.nn.Layer): return x -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1024): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -181,21 +185,16 @@ def train_mlp( ) return - train_reader = paddle.batch( - reader_decorator(linear_size=linear_size), + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), batch_size=batch_size, + shuffle=False, drop_last=True, + num_workers=0, ) - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, - ) - train_loader.set_sample_list_generator(train_reader) - for eop in range(epoch): model.train() for batch_id, data in enumerate(train_loader()): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py index 15b02b2cea625190be9e30f4943e4da7c52b1a4f..ad9fb5c86d9e8649ee749f3db44dd0b72dcf73de 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py @@ -52,14 +52,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -103,18 +107,15 @@ def train_mlp( segment_size=2**15, ) - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): model.train() diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py index d08f1727f3ef7bd3064e93fc6637887afabc6c39..a23a3df606ed50b73bf55d4bc57037812353cf9a 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py @@ -63,14 +63,18 @@ class MLP(fluid.Layer): return y -def reader_decorator(linear_size=1000): - def __reader__(): - for _ in range(100): - img = np.random.rand(linear_size).astype('float32') - label = np.ones(1).astype('int64') - yield img, label +class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples=2000, linear_size=1000): + self.num_samples = num_samples + self.linear_size = linear_size - return __reader__ + def __getitem__(self, idx): + img = np.random.rand(self.linear_size).astype('float32') + label = np.ones(1).astype('int64') + return img, label + + def __len__(self): + return self.num_samples def optimizer_setting(model, use_pure_fp16, opt_group=False): @@ -141,18 +145,15 @@ def train_mlp( ) return - train_reader = paddle.batch( - reader_decorator(), batch_size=batch_size, drop_last=True - ) - - train_loader = paddle.io.DataLoader.from_generator( - capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True, + paddle.seed(2023) + np.random.seed(2023) + train_loader = paddle.io.DataLoader( + RandomDataset(), + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, ) - train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): model.train()