test_dataset.py 5.6 KB
Newer Older
X
xjqbest 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
X
xjqbest 已提交
14
"""
X
xjqbest 已提交
15 16
TestCases for Dataset,
including create, config, run, etc.
X
xjqbest 已提交
17
"""
X
xjqbest 已提交
18 19 20 21 22 23 24 25 26 27

from __future__ import print_function
import paddle.fluid as fluid
import numpy as np
import os
import shutil
import unittest


class TestDataset(unittest.TestCase):
X
xjqbest 已提交
28
    """  TestCases for Dataset. """
29

X
xjqbest 已提交
30
    def test_dataset_create(self):
X
xjqbest 已提交
31
        """ Testcase for dataset create. """
X
xjqbest 已提交
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48
        try:
            dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
        except:
            self.assertTrue(False)

        try:
            dataset = fluid.DatasetFactory().create_dataset("QueueDataset")
        except:
            self.assertTrue(False)

        try:
            dataset = fluid.DatasetFactory().create_dataset("MyOwnDataset")
            self.assertTrue(False)
        except:
            self.assertTrue(True)

    def test_dataset_config(self):
X
xjqbest 已提交
49
        """ Testcase for dataset configuration. """
X
xjqbest 已提交
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
        dataset = fluid.core.Dataset("MultiSlotDataset")
        dataset.set_thread_num(12)
        dataset.set_filelist(["a.txt", "b.txt", "c.txt"])
        dataset.set_trainer_num(4)
        dataset.set_hdfs_config("my_fs_name", "my_fs_ugi")

        thread_num = dataset.get_thread_num()
        self.assertEqual(thread_num, 12)

        filelist = dataset.get_filelist()
        self.assertEqual(len(filelist), 3)
        self.assertEqual(filelist[0], "a.txt")
        self.assertEqual(filelist[1], "b.txt")
        self.assertEqual(filelist[2], "c.txt")

        trainer_num = dataset.get_trainer_num()
        self.assertEqual(trainer_num, 4)

        name, ugi = dataset.get_hdfs_config()
        self.assertEqual(name, "my_fs_name")
        self.assertEqual(ugi, "my_fs_ugi")

    def test_in_memory_dataset_run(self):
X
xjqbest 已提交
73
        """
X
xjqbest 已提交
74
        Testcase for InMemoryDataset from create to run.
X
xjqbest 已提交
75 76
        """
        with open("test_in_memory_dataset_run_a.txt", "w") as f:
X
xjqbest 已提交
77 78 79 80
            data = "1 1 2 3 3 4 5 5 5 5 1 1\n"
            data += "1 2 2 3 4 4 6 6 6 6 1 2\n"
            data += "1 3 2 3 5 4 7 7 7 7 1 3\n"
            f.write(data)
X
xjqbest 已提交
81
        with open("test_in_memory_dataset_run_b.txt", "w") as f:
X
xjqbest 已提交
82 83 84 85 86 87
            data = "1 4 2 3 3 4 5 5 5 5 1 4\n"
            data += "1 5 2 3 4 4 6 6 6 6 1 5\n"
            data += "1 6 2 3 5 4 7 7 7 7 1 6\n"
            data += "1 7 2 3 6 4 8 8 8 8 1 7\n"
            f.write(data)

88
        slots = ["slot1", "slot2", "slot3", "slot4"]
X
xjqbest 已提交
89 90
        slots_vars = []
        for slot in slots:
91 92
            var = fluid.layers.data(
                name=slot, shape=[1], dtype="int64", lod_level=1)
X
xjqbest 已提交
93 94 95 96 97
            slots_vars.append(var)

        dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
        dataset.set_batch_size(32)
        dataset.set_thread(3)
98 99 100 101
        dataset.set_filelist([
            "test_in_memory_dataset_run_a.txt",
            "test_in_memory_dataset_run_b.txt"
        ])
X
xjqbest 已提交
102 103 104 105 106 107 108 109
        dataset.set_pipe_command("cat")
        dataset.set_use_var(slots_vars)
        dataset.load_into_memory()
        dataset.local_shuffle()

        exe = fluid.Executor(fluid.CPUPlace())
        exe.run(fluid.default_startup_program())
        for i in range(2):
X
xjqbest 已提交
110 111 112 113 114 115
            try:
                exe.train_from_dataset(fluid.default_main_program(), dataset)
            except ImportError as e:
                pass
            except Exception as e:
                self.assertTrue(False)
X
xjqbest 已提交
116

X
xjqbest 已提交
117 118
        os.remove("./test_in_memory_dataset_run_a.txt")
        os.remove("./test_in_memory_dataset_run_b.txt")
X
xjqbest 已提交
119 120

    def test_queue_dataset_run(self):
X
xjqbest 已提交
121
        """
X
xjqbest 已提交
122
        Testcase for QueueDataset from create to run.
X
xjqbest 已提交
123 124
        """
        with open("test_queue_dataset_run_a.txt", "w") as f:
X
xjqbest 已提交
125 126 127 128
            data = "1 1 2 3 3 4 5 5 5 5 1 1\n"
            data += "1 2 2 3 4 4 6 6 6 6 1 2\n"
            data += "1 3 2 3 5 4 7 7 7 7 1 3\n"
            f.write(data)
X
xjqbest 已提交
129
        with open("test_queue_dataset_run_b.txt", "w") as f:
X
xjqbest 已提交
130 131 132 133 134 135
            data = "1 4 2 3 3 4 5 5 5 5 1 4\n"
            data += "1 5 2 3 4 4 6 6 6 6 1 5\n"
            data += "1 6 2 3 5 4 7 7 7 7 1 6\n"
            data += "1 7 2 3 6 4 8 8 8 8 1 7\n"
            f.write(data)

136
        slots = ["slot1", "slot2", "slot3", "slot4"]
X
xjqbest 已提交
137 138
        slots_vars = []
        for slot in slots:
139 140
            var = fluid.layers.data(
                name=slot, shape=[1], dtype="int64", lod_level=1)
X
xjqbest 已提交
141 142 143 144 145
            slots_vars.append(var)

        dataset = fluid.DatasetFactory().create_dataset("QueueDataset")
        dataset.set_batch_size(32)
        dataset.set_thread(3)
146 147
        dataset.set_filelist(
            ["test_queue_dataset_run_a.txt", "test_queue_dataset_run_b.txt"])
X
xjqbest 已提交
148 149 150 151 152 153
        dataset.set_pipe_command("cat")
        dataset.set_use_var(slots_vars)

        exe = fluid.Executor(fluid.CPUPlace())
        exe.run(fluid.default_startup_program())
        for i in range(2):
X
xjqbest 已提交
154 155 156 157 158 159
            try:
                exe.train_from_dataset(fluid.default_main_program(), dataset)
            except ImportError as e:
                pass
            except Exception as e:
                self.assertTrue(False)
X
xjqbest 已提交
160

X
xjqbest 已提交
161 162
        os.remove("./test_queue_dataset_run_a.txt")
        os.remove("./test_queue_dataset_run_b.txt")
X
xjqbest 已提交
163 164 165


if __name__ == '__main__':
X
xjqbest 已提交
166
    unittest.main()