add cu yaml

models/contentunderstanding/classification/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/contentunderstanding/classification/config.yaml

@@ -12,28 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-train:
-  trainer:
-    # for cluster training
-    strategy: "async"
+workspace: "paddlerec.models.contentunderstanding.classification"
 
-  epochs: 10
-  workspace: "paddlerec.models.contentunderstanding.classification"
-
-  reader:
+dataset:
+- name: data1
   batch_size: 5
-    class: "{workspace}/reader.py"
-    train_data_path: "{workspace}/train_data"
+  type: DataLoader
+  data_path: "{workspace}/data/train_data"
+  data_converter: "{workspace}/reader.py"
+
+hyper_parameters:
+  optimizer:
+    class: Adagrad
+    learning_rate: 0.001
 
-  model:
-    models: "{workspace}/model.py"
+mode: runner1
+
+runner:
+- name: runner1
+  class: single_train
+  epochs: 10
+  device: cpu
+  save_checkpoint_interval: 2
+  save_inference_interval: 4
+  save_checkpoint_path: "increment"
+  save_inference_path: "inference"
+  save_inference_feed_varnames: []
+  save_inference_fetch_varnames: []
 
-  save:
-    increment:
-      dirname: "increment"
-      epoch_interval: 1
-      save_last: True
-    inference:
-      dirname: "inference"
-      epoch_interval: 100
-      save_last: True
+phase:
+- name: phase1
+  model: "{workspace}/model.py"
+  dataset_name: data1
+  thread_num: 1

models/contentunderstanding/classification/model.py

@@ -28,15 +28,18 @@ class Model(ModelBase):
         self.hid_dim = 128
         self.class_dim = 2
 
-    def train_net(self):
-        """ network definition """
-
+    def input_data(self, is_infer=False, **kwargs):
         data = fluid.data(
             name="input", shape=[None, self.max_len], dtype='int64')
         label = fluid.data(name="label", shape=[None, 1], dtype='int64')
         seq_len = fluid.data(name="seq_len", shape=[None], dtype='int64')
+        return [data, label, seq_len]
 
-        self._data_var = [data, label, seq_len]
+    def net(self, input, is_infer=False):
+        """ network definition """
+        data = input[0]
+        label = input[1]
+        seq_len = input[2]
 
         # embedding layer
         emb = fluid.embedding(input=data, size=[self.dict_dim, self.emb_dim])
@@ -59,19 +62,8 @@ class Model(ModelBase):
         avg_cost = fluid.layers.mean(x=cost)
         acc = fluid.layers.accuracy(input=prediction, label=label)
 
-        self.cost = avg_cost
+        self._cost = avg_cost
+        if is_infer:
+            self._infer_results["acc"] = acc
+        else:
             self._metrics["acc"] = acc
-
-    def get_avg_cost(self):
-        return self.cost
-
-    def get_metrics(self):
-        return self._metrics
-
-    def optimizer(self):
-        learning_rate = 0.01
-        sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=learning_rate)
-        return sgd_optimizer
-
-    def infer_net(self):
-        self.train_net()

models/contentunderstanding/classification/reader.py

@@ -22,7 +22,7 @@ class TrainReader(Reader):
         pass
 
     def _process_line(self, l):
-        l = l.strip().split(" ")
+        l = l.strip().split()
         data = l[0:10]
         seq_len = l[10:11]
         label = l[11:]
@@ -37,8 +37,6 @@ class TrainReader(Reader):
             data = [int(i) for i in data]
             label = [int(i) for i in label]
             seq_len = [int(i) for i in seq_len]
-            print >> sys.stderr, str(
-                [('data', data), ('label', label), ('seq_len', seq_len)])
             yield [('data', data), ('label', label), ('seq_len', seq_len)]
 
         return data_iter

models/contentunderstanding/tagspace/config.yaml

@@ -12,22 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-train:
-  trainer:
-    # for cluster training
-    strategy: "async"
+workspace: "paddlerec.models.contentunderstanding.tagspace"
 
-  epochs: 10
-  workspace: "paddlerec.models.contentunderstanding.tagspace"
-
-  reader:
+dataset:
+- name: sample_1
+  type: QueueDataset
   batch_size: 5
-    class: "{workspace}/reader.py"
-    train_data_path: "{workspace}/train_data"
+  data_path: "{workspace}/data/train_data"
+  data_converter: "{workspace}/reader.py"
 
-  model:
-    models: "{workspace}/model.py"
-    hyper_parameters:
+hyper_parameters:
+  optimizer:
+    class: Adagrad
+    learning_rate: 0.001
   vocab_text_size: 11447
   vocab_tag_size: 4
   emb_dim: 10
@@ -37,13 +34,22 @@ train:
   neg_size: 3
   num_devices: 1
 
+mode: runner1
+
+runner:
+- name: runner1
+  class: single_train
+  epochs: 10
+  device: cpu
+  save_checkpoint_interval: 2
+  save_inference_interval: 4
+  save_checkpoint_path: "increment"
+  save_inference_path: "inference"
+  save_inference_feed_varnames: []
+  save_inference_fetch_varnames: []
 
-  save:
-    increment:
-      dirname: "increment"
-      epoch_interval: 1
-      save_last: True
-    inference:
-      dirname: "inference"
-      epoch_interval: 100
-      save_last: True
+phase:
+- name: phase1
+  model: "{workspace}/model.py"
+  dataset_name: sample_1
+  thread_num: 1

models/contentunderstanding/tagspace/model.py

@@ -26,26 +26,30 @@ class Model(ModelBase):
         ModelBase.__init__(self, config)
         self.cost = None
         self.metrics = {}
-        self.vocab_text_size = envs.get_global_env("vocab_text_size", None,
-                                                   self._namespace)
-        self.vocab_tag_size = envs.get_global_env("vocab_tag_size", None,
-                                                  self._namespace)
-        self.emb_dim = envs.get_global_env("emb_dim", None, self._namespace)
-        self.hid_dim = envs.get_global_env("hid_dim", None, self._namespace)
-        self.win_size = envs.get_global_env("win_size", None, self._namespace)
-        self.margin = envs.get_global_env("margin", None, self._namespace)
-        self.neg_size = envs.get_global_env("neg_size", None, self._namespace)
+        self.vocab_text_size = envs.get_global_env(
+            "hyper_parameters.vocab_text_size")
+        self.vocab_tag_size = envs.get_global_env(
+            "hyper_parameters.vocab_tag_size")
+        self.emb_dim = envs.get_global_env("hyper_parameters.emb_dim")
+        self.hid_dim = envs.get_global_env("hyper_parameters.hid_dim")
+        self.win_size = envs.get_global_env("hyper_parameters.win_size")
+        self.margin = envs.get_global_env("hyper_parameters.margin")
+        self.neg_size = envs.get_global_env("hyper_parameters.neg_size")
 
-    def train_net(self):
-        """ network"""
+    def input_data(self, is_infer=False, **kwargs):
         text = fluid.data(
             name="text", shape=[None, 1], lod_level=1, dtype='int64')
         pos_tag = fluid.data(
             name="pos_tag", shape=[None, 1], lod_level=1, dtype='int64')
         neg_tag = fluid.data(
             name="neg_tag", shape=[None, 1], lod_level=1, dtype='int64')
+        return [text, pos_tag, neg_tag]
 
-        self._data_var = [text, pos_tag, neg_tag]
+    def net(self, input, is_infer=False):
+        """ network"""
+        text = input[0]
+        pos_tag = input[1]
+        neg_tag = input[2]
 
         text_emb = fluid.embedding(
             input=text,
@@ -97,22 +101,11 @@ class Model(ModelBase):
         avg_cost = nn.mean(loss_part3)
         less = tensor.cast(cf.less_than(cos_neg, cos_pos), dtype='float32')
         correct = nn.reduce_sum(less)
-        self.cost = avg_cost
-
-        self.metrics["correct"] = correct
-        self.metrics["cos_pos"] = cos_pos
-
-    def get_avg_cost(self):
-        return self.cost
-
-    def get_metrics(self):
-        return self.metrics
-
-    def optimizer(self):
-        learning_rate = envs.get_global_env("hyper_parameters.base_lr", None,
-                                            self._namespace)
-        sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=learning_rate)
-        return sgd_optimizer
+        self._cost = avg_cost
 
-    def infer_net(self, parameter_list):
-        self.train_net()
+        if is_infer:
+            self._infer_results["correct"] = correct
+            self._infer_results["cos_pos"] = cos_pos
+        else:
+            self._metrics["correct"] = correct
+            self._metrics["cos_pos"] = cos_pos

setup.py

@@ -62,7 +62,8 @@ def build(dirname):
 
     models_copy = [
         'data/*.txt', 'data/*/*.txt', '*.yaml', '*.sh', 'tree/*.npy',
-        'tree/*.txt', 'data/sample_data/*', 'data/sample_data/train/*'
+        'tree/*.txt', 'data/sample_data/*', 'data/sample_data/train/*',
+        'data/*/*.csv'
     ]
 
     engine_copy = ['*/*.sh']