fix share-bottom

models/multitask/share-bottom/census_infer_reader.py

-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-from paddlerec.core.reader import Reader
-
-
-class EvaluateReader(Reader):
-    def init(self):
-        pass
-
-    def generate_sample(self, line):
-        """
-        Read the data line by line and process it as a dictionary
-        """
-
-        def reader():
-            """
-            This function needs to be implemented by the user, based on data format
-            """
-            l = line.strip().split(',')
-            l = list(map(float, l))
-            label_income = []
-            label_marital = []
-            data = l[2:]
-            if int(l[1]) == 0:
-                label_income = [1, 0]
-            elif int(l[1]) == 1:
-                label_income = [0, 1]
-            if int(l[0]) == 0:
-                label_marital = [1, 0]
-            elif int(l[0]) == 1:
-                label_marital = [0, 1]
-            feature_name = ["input", "label_income", "label_marital"]
-            yield zip(feature_name, [data] + [label_income] + [label_marital])
-
-        return reader

models/multitask/share-bottom/config.yaml

@@ -12,42 +12,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-evaluate:
-  reader:
-    batch_size: 1
-    class: "{workspace}/census_infer_reader.py"
-    test_data_path: "{workspace}/data/train"
-
-train:
-  trainer:
-    # for cluster training
-    strategy: "async"
+workspace: "paddlerec.models.multitask.share-bottom"
 
-  epochs: 3
-  workspace: "paddlerec.models.multitask.share-bottom"
-  device: cpu
-
-  reader:
-    batch_size: 2
-    class: "{workspace}/census_reader.py"
-    train_data_path: "{workspace}/data/train"
+dataset:
+- name: dataset_train
+  batch_size: 1
+  type: QueueDataset
+  data_path: "{workspace}/data/train"
+  data_converter: "{workspace}/census_reader.py"
+- name: dataset_infer
+  batch_size: 1
+  type: QueueDataset
+  data_path: "{workspace}/data/train"
+  data_converter: "{workspace}/census_reader.py"
 
-  model:
-    models: "{workspace}/model.py"
-    hyper_parameters:
+hyper_parameters:
   feature_size: 499
   bottom_size: 117
   tower_nums: 2
   tower_size: 8
+  optimizer: 
+    class: adam
     learning_rate: 0.001
-      optimizer: adam
+    strategy: async
+
+#use infer_runner mode and modify 'phase' below if infer
+mode: train_runner
+#mode: infer_runner
+
+runner:
+- name: train_runner
+  class: single_train
+  device: cpu
+  epochs: 3
+  save_checkpoint_interval: 2
+  save_inference_interval: 4
+  save_checkpoint_path: "increment"
+  save_inference_path: "inference"
+  print_interval: 5
+- name: infer_runner
+  class: single_infer
+  init_model_path: "increment/0"
+  device: cpu
+  epochs: 3
 
-  save:
-    increment:
-      dirname: "increment"
-      epoch_interval: 2
-      save_last: True
-    inference:
-      dirname: "inference"
-      epoch_interval: 4
-      save_last: True
+phase:
+- name: train
+  model: "{workspace}/model.py"
+  dataset_name: dataset_train
+  thread_num: 1
+    #- name: infer
+    #  model: "{workspace}/model.py"
+    #  dataset_name: dataset_infer
+    #  thread_num: 1

models/multitask/share-bottom/model.py

@@ -22,46 +22,42 @@ class Model(ModelBase):
     def __init__(self, config):
         ModelBase.__init__(self, config)
 
-    def model(self, is_infer=False):
-
-        feature_size = envs.get_global_env("hyper_parameters.feature_size",
-                                           None, self._namespace)
-        bottom_size = envs.get_global_env("hyper_parameters.bottom_size", None,
-                                          self._namespace)
-        tower_size = envs.get_global_env("hyper_parameters.tower_size", None,
-                                         self._namespace)
-        tower_nums = envs.get_global_env("hyper_parameters.tower_nums", None,
-                                         self._namespace)
-
-        input_data = fluid.data(
-            name="input", shape=[-1, feature_size], dtype="float32")
+    def _init_hyper_parameters(self):
+        self.feature_size = envs.get_global_env(
+            "hyper_parameters.feature_size")
+        self.bottom_size = envs.get_global_env("hyper_parameters.bottom_size")
+        self.tower_size = envs.get_global_env("hyper_parameters.tower_size")
+        self.tower_nums = envs.get_global_env("hyper_parameters.tower_nums")
+
+    def input_data(self, is_infer=False, **kwargs):
+        inputs = fluid.data(
+            name="input", shape=[-1, self.feature_size], dtype="float32")
         label_income = fluid.data(
             name="label_income", shape=[-1, 2], dtype="float32", lod_level=0)
         label_marital = fluid.data(
             name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0)
-
         if is_infer:
-            self._infer_data_var = [input_data, label_income, label_marital]
-            self._infer_data_loader = fluid.io.DataLoader.from_generator(
-                feed_list=self._infer_data_var,
-                capacity=64,
-                use_double_buffer=False,
-                iterable=False)
+            return [inputs, label_income, label_marital]
+        else:
+            return [inputs, label_income, label_marital]
 
-        self._data_var.extend([input_data, label_income, label_marital])
+    def net(self, inputs, is_infer=False):
+        input_data = inputs[0]
+        label_income = inputs[1]
+        label_marital = inputs[2]
 
         bottom_output = fluid.layers.fc(
             input=input_data,
-            size=bottom_size,
+            size=self.bottom_size,
             act='relu',
             bias_attr=fluid.ParamAttr(learning_rate=1.0),
             name='bottom_output')
 
         # Build tower layer from bottom layer
         output_layers = []
-        for index in range(tower_nums):
+        for index in range(self.tower_nums):
             tower_layer = fluid.layers.fc(input=bottom_output,
-                                          size=tower_size,
+                                          size=self.tower_size,
                                           act='relu',
                                           name='task_layer_' + str(index))
             output_layer = fluid.layers.fc(input=tower_layer,
@@ -107,9 +103,3 @@ class Model(ModelBase):
         self._metrics["BATCH_AUC_income"] = batch_auc_1
         self._metrics["AUC_marital"] = auc_marital
         self._metrics["BATCH_AUC_marital"] = batch_auc_2
-
-    def train_net(self):
-        self.model()
-
-    def infer_net(self):
-        self.model(is_infer=True)