diff --git a/core/reader.py b/core/reader.py index a05916a3a1088956554be80798674cbb23f4c32d..6c2af005f23b790e1686f3e2fd894399a3dbffaf 100755 --- a/core/reader.py +++ b/core/reader.py @@ -17,7 +17,6 @@ import abc import os from functools import reduce import paddle.fluid.incubate.data_generator as dg -import yaml from paddlerec.core.utils import envs diff --git a/core/utils/envs.py b/core/utils/envs.py index 45619a2b051b61780316ecd511255c64bde8b8fa..91e97f7a27e7a16914b0656f58fb7f4f80f698c2 100755 --- a/core/utils/envs.py +++ b/core/utils/envs.py @@ -95,6 +95,7 @@ def set_global_envs(envs, adapter): if adapter: workspace_adapter() os_path_adapter() + reader_adapter() def get_global_env(env_name, default_value=None, namespace=None): @@ -141,6 +142,15 @@ def workspace_adapter(): global_envs[name] = value +def reader_adapter(): + if get_platform() != "WINDOWS": + return + + datasets = global_envs.get("dataset") + for dataset in datasets: + dataset["type"] = "DataLoader" + + def pretty_print_envs(envs, header=None): spacing = 5 max_k = 45 diff --git a/models/rank/dataset/Criteo_data/get_slot_data.py b/models/rank/dataset/Criteo_data/get_slot_data.py index 973ff2fd262ee041a98edeb2183ac0d44e131169..f090c666de3709736eda701c7447e58cd9bec2a0 100644 --- a/models/rank/dataset/Criteo_data/get_slot_data.py +++ b/models/rank/dataset/Criteo_data/get_slot_data.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -import os - -from paddlerec.core.reader import ReaderBase from paddlerec.core.utils import envs import paddle.fluid.incubate.data_generator as dg try: @@ -27,12 +23,7 @@ except ImportError: class Reader(dg.MultiSlotDataGenerator): def __init__(self, config): dg.MultiSlotDataGenerator.__init__(self) - - if os.path.isfile(config): - with open(config, 'r') as rb: - _config = yaml.load(rb.read(), Loader=yaml.FullLoader) - else: - raise ValueError("reader config only support yaml") + _config = envs.load_yaml(config) def init(self): self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator): v = i[1] for j in v: s += " " + k + ":" + str(j) - print s.strip() yield None return data_iter diff --git a/models/rank/logistic_regression/data/get_slot_data.py b/models/rank/logistic_regression/data/get_slot_data.py index 8e7b1f9f0879e83f032c0dd43155a3aa26113a58..ea1a96ef778e219de89b99b536c7473419bd26f5 100644 --- a/models/rank/logistic_regression/data/get_slot_data.py +++ b/models/rank/logistic_regression/data/get_slot_data.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -import os - -from paddlerec.core.reader import ReaderBase from paddlerec.core.utils import envs import paddle.fluid.incubate.data_generator as dg @@ -28,12 +24,7 @@ except ImportError: class Reader(dg.MultiSlotDataGenerator): def __init__(self, config): dg.MultiSlotDataGenerator.__init__(self) - - if os.path.isfile(config): - with open(config, 'r') as rb: - _config = yaml.load(rb.read(), Loader=yaml.FullLoader) - else: - raise ValueError("reader config only support yaml") + _config = envs.load_yaml(config) def init(self): self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -88,7 +79,6 @@ class Reader(dg.MultiSlotDataGenerator): v = i[1] for j in v: s += " " + k + ":" + str(j) - print s.strip() yield None return data_iter diff --git a/models/rank/nfm/data/get_slot_data.py b/models/rank/nfm/data/get_slot_data.py index 973ff2fd262ee041a98edeb2183ac0d44e131169..f090c666de3709736eda701c7447e58cd9bec2a0 100644 --- a/models/rank/nfm/data/get_slot_data.py +++ b/models/rank/nfm/data/get_slot_data.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -import os - -from paddlerec.core.reader import ReaderBase from paddlerec.core.utils import envs import paddle.fluid.incubate.data_generator as dg try: @@ -27,12 +23,7 @@ except ImportError: class Reader(dg.MultiSlotDataGenerator): def __init__(self, config): dg.MultiSlotDataGenerator.__init__(self) - - if os.path.isfile(config): - with open(config, 'r') as rb: - _config = yaml.load(rb.read(), Loader=yaml.FullLoader) - else: - raise ValueError("reader config only support yaml") + _config = envs.load_yaml(config) def init(self): self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] @@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator): v = i[1] for j in v: s += " " + k + ":" + str(j) - print s.strip() yield None return data_iter