From fc724787dd06ab2b5914f6bde46cb2262ef1ab5d Mon Sep 17 00:00:00 2001
From: tangwei <tangwei12@baidu.com>
Date: Wed, 10 Jun 2020 18:05:02 +0800
Subject: [PATCH] fix windows adapter

---
 core/reader.py                                       |  1 -
 core/utils/envs.py                                   | 10 ++++++++++
 models/rank/dataset/Criteo_data/get_slot_data.py     | 12 +-----------
 .../rank/logistic_regression/data/get_slot_data.py   | 12 +-----------
 models/rank/nfm/data/get_slot_data.py                | 12 +-----------
 5 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/core/reader.py b/core/reader.py
index a05916a3..6c2af005 100755
--- a/core/reader.py
+++ b/core/reader.py
@@ -17,7 +17,6 @@ import abc
 import os
 from functools import reduce
 import paddle.fluid.incubate.data_generator as dg
-import yaml
 from paddlerec.core.utils import envs
 
 
diff --git a/core/utils/envs.py b/core/utils/envs.py
index 45619a2b..91e97f7a 100755
--- a/core/utils/envs.py
+++ b/core/utils/envs.py
@@ -95,6 +95,7 @@ def set_global_envs(envs, adapter):
     if adapter:
         workspace_adapter()
         os_path_adapter()
+        reader_adapter()
 
 
 def get_global_env(env_name, default_value=None, namespace=None):
@@ -141,6 +142,15 @@ def workspace_adapter():
             global_envs[name] = value
 
 
+def reader_adapter():
+    if get_platform() != "WINDOWS":
+        return
+
+    datasets = global_envs.get("dataset")
+    for dataset in datasets:
+        dataset["type"] = "DataLoader"
+
+
 def pretty_print_envs(envs, header=None):
     spacing = 5
     max_k = 45
diff --git a/models/rank/dataset/Criteo_data/get_slot_data.py b/models/rank/dataset/Criteo_data/get_slot_data.py
index 973ff2fd..f090c666 100644
--- a/models/rank/dataset/Criteo_data/get_slot_data.py
+++ b/models/rank/dataset/Criteo_data/get_slot_data.py
@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 try:
@@ -27,12 +23,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter
diff --git a/models/rank/logistic_regression/data/get_slot_data.py b/models/rank/logistic_regression/data/get_slot_data.py
index 8e7b1f9f..ea1a96ef 100644
--- a/models/rank/logistic_regression/data/get_slot_data.py
+++ b/models/rank/logistic_regression/data/get_slot_data.py
@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 
@@ -28,12 +24,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -88,7 +79,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter
diff --git a/models/rank/nfm/data/get_slot_data.py b/models/rank/nfm/data/get_slot_data.py
index 973ff2fd..f090c666 100644
--- a/models/rank/nfm/data/get_slot_data.py
+++ b/models/rank/nfm/data/get_slot_data.py
@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 try:
@@ -27,12 +23,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter
-- 
GitLab