fix windows adapter

core/reader.py

@@ -17,7 +17,6 @@ import abc
 import os
 from functools import reduce
 import paddle.fluid.incubate.data_generator as dg
-import yaml
 from paddlerec.core.utils import envs
 
 

core/utils/envs.py

@@ -95,6 +95,7 @@ def set_global_envs(envs, adapter):
     if adapter:
         workspace_adapter()
         os_path_adapter()
+        reader_adapter()
 
 
 def get_global_env(env_name, default_value=None, namespace=None):
@@ -141,6 +142,15 @@ def workspace_adapter():
             global_envs[name] = value
 
 
+def reader_adapter():
+    if get_platform() != "WINDOWS":
+        return
+
+    datasets = global_envs.get("dataset")
+    for dataset in datasets:
+        dataset["type"] = "DataLoader"
+
+
 def pretty_print_envs(envs, header=None):
     spacing = 5
     max_k = 45

models/rank/dataset/Criteo_data/get_slot_data.py

@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 try:
@@ -27,12 +23,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter

models/rank/logistic_regression/data/get_slot_data.py

@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 
@@ -28,12 +24,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -88,7 +79,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter

models/rank/nfm/data/get_slot_data.py

@@ -12,10 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import yaml
-import os
-
-from paddlerec.core.reader import ReaderBase
 from paddlerec.core.utils import envs
 import paddle.fluid.incubate.data_generator as dg
 try:
@@ -27,12 +23,7 @@ except ImportError:
 class Reader(dg.MultiSlotDataGenerator):
     def __init__(self, config):
         dg.MultiSlotDataGenerator.__init__(self)
-
-        if os.path.isfile(config):
-            with open(config, 'r') as rb:
-                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
-        else:
-            raise ValueError("reader config only support yaml")
+        _config = envs.load_yaml(config)
 
     def init(self):
         self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
@@ -87,7 +78,6 @@ class Reader(dg.MultiSlotDataGenerator):
                 v = i[1]
                 for j in v:
                     s += " " + k + ":" + str(j)
-            print s.strip()
             yield None
 
         return data_iter