From f2c97b6da519138826a3be730f1468c1a5c69389 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Thu, 24 Sep 2020 13:14:27 +0800 Subject: [PATCH] replace dataset with fake data (#27519) --- .../contrib/tests/test_weight_decay_extend.py | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/contrib/tests/test_weight_decay_extend.py b/python/paddle/fluid/contrib/tests/test_weight_decay_extend.py index 906d83fff4f..6000a44ceb6 100644 --- a/python/paddle/fluid/contrib/tests/test_weight_decay_extend.py +++ b/python/paddle/fluid/contrib/tests/test_weight_decay_extend.py @@ -14,6 +14,7 @@ from __future__ import print_function +import six import unittest from functools import partial import numpy as np @@ -24,6 +25,24 @@ import contextlib paddle.enable_static() +def fake_imdb_reader(word_dict_size, + sample_num, + lower_seq_len=100, + upper_seq_len=200, + class_dim=2): + def __reader__(): + for _ in six.moves.range(sample_num): + length = np.random.random_integers( + low=lower_seq_len, high=upper_seq_len, size=[1])[0] + ids = np.random.random_integers( + low=0, high=word_dict_size - 1, size=[length]).astype('int64') + label = np.random.random_integers( + low=0, high=class_dim - 1, size=[1]).astype('int64')[0] + yield ids, label + + return __reader__ + + def get_places(): places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): @@ -68,10 +87,11 @@ def bow_net(data, class TestWeightDecay(unittest.TestCase): def setUp(self): - self.word_dict = paddle.dataset.imdb.word_dict() - reader = paddle.batch( - paddle.dataset.imdb.train(self.word_dict), batch_size=2)() - self.train_data = [next(reader) for _ in range(5)] + self.word_dict_len = 5147 + batch_size = 2 + reader = fake_imdb_reader(self.word_dict_len, batch_size * 100) + reader = paddle.batch(reader, batch_size=batch_size)() + self.train_data = [next(reader) for _ in range(3)] self.learning_rate = .5 def run_program(self, place, feed_list): @@ -103,7 +123,7 @@ class TestWeightDecay(unittest.TestCase): data = fluid.layers.data( name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - avg_cost = model(data, label, len(self.word_dict)) + avg_cost = model(data, label, self.word_dict_len) AdamW = fluid.contrib.extend_with_decoupled_weight_decay( fluid.optimizer.Adam) @@ -127,7 +147,7 @@ class TestWeightDecay(unittest.TestCase): name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - avg_cost = model(data, label, len(self.word_dict)) + avg_cost = model(data, label, self.word_dict_len) param_list = [(var, var * self.learning_rate) for var in main_prog.block(0).all_parameters()] -- GitLab