dataset_instance.py 1.6 KB
Newer Older
T
tangwei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
T
tangwei 已提交
14

T
tangwei 已提交
15 16 17
from __future__ import print_function
import sys

18
from paddlerec.core.utils.envs import lazy_instance_by_fliename
X
xujiaqi01 已提交
19
from paddlerec.core.reader import SlotReader
T
tangwei 已提交
20

X
xujiaqi01 已提交
21
if len(sys.argv) < 4:
T
tangwei 已提交
22 23 24
    raise ValueError(
        "reader only accept 3 argument: 1. reader_class 2.train/evaluate/slotreader 3.yaml_abs_path"
    )
T
tangwei 已提交
25 26 27

reader_package = sys.argv[1]

X
xujiaqi01 已提交
28
if sys.argv[2].upper() == "TRAIN":
T
tangwei 已提交
29
    reader_name = "TrainReader"
X
xujiaqi01 已提交
30
elif sys.argv[2].upper() == "EVALUATE":
T
tangwei 已提交
31
    reader_name = "EvaluateReader"
X
xujiaqi01 已提交
32 33 34 35 36 37
else:
    reader_name = "SlotReader"
    namespace = sys.argv[4]
    sparse_slots = sys.argv[5].replace("#", " ")
    dense_slots = sys.argv[6].replace("#", " ")
    padding = int(sys.argv[7])
T
tangwei 已提交
38

T
tangwei 已提交
39
yaml_abs_path = sys.argv[3]
X
xujiaqi01 已提交
40 41 42 43 44 45 46 47 48 49

if reader_name != "SlotReader":
    reader_class = lazy_instance_by_fliename(reader_package, reader_name)
    reader = reader_class(yaml_abs_path)
    reader.init()
    reader.run_from_stdin()
else:
    reader = SlotReader(yaml_abs_path)
    reader.init(sparse_slots, dense_slots, padding)
    reader.run_from_stdin()