提交 189f986f 编写于 作者: Y yaoxuefeng

add lr fix codes

上级 e0f6e95b
...@@ -12,9 +12,11 @@ ...@@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import yaml import yaml, os
from paddlerec.core.reader import Reader from paddlerec.core.reader import Reader
from paddlerec.core.utils import envs from paddlerec.core.utils import envs
import paddle.fluid.incubate.data_generator as dg
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
...@@ -44,7 +46,7 @@ class TrainReader(dg.MultiSlotDataGenerator): ...@@ -44,7 +46,7 @@ class TrainReader(dg.MultiSlotDataGenerator):
self.continuous_range_ = range(1, 14) self.continuous_range_ = range(1, 14)
self.categorical_range_ = range(14, 40) self.categorical_range_ = range(14, 40)
# load preprocessed feature dict # load preprocessed feature dict
self.feat_dict_name = "aid_data/feat_dict_10.pkl2" self.feat_dict_name = "sample_data/feat_dict_10.pkl2"
self.feat_dict_ = pickle.load(open(self.feat_dict_name, 'rb')) self.feat_dict_ = pickle.load(open(self.feat_dict_name, 'rb'))
def _process_line(self, line): def _process_line(self, line):
......
...@@ -13,8 +13,12 @@ ...@@ -13,8 +13,12 @@
# limitations under the License. # limitations under the License.
import yaml import yaml
import os
from paddlerec.core.reader import Reader from paddlerec.core.reader import Reader
from paddlerec.core.utils import envs from paddlerec.core.utils import envs
import paddle.fluid.incubate.data_generator as dg
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
...@@ -44,7 +48,7 @@ class TrainReader(dg.MultiSlotDataGenerator): ...@@ -44,7 +48,7 @@ class TrainReader(dg.MultiSlotDataGenerator):
self.continuous_range_ = range(1, 14) self.continuous_range_ = range(1, 14)
self.categorical_range_ = range(14, 40) self.categorical_range_ = range(14, 40)
# load preprocessed feature dict # load preprocessed feature dict
self.feat_dict_name = "aid_data/feat_dict_10.pkl2" self.feat_dict_name = "sample_data/feat_dict_10.pkl2"
self.feat_dict_ = pickle.load(open(self.feat_dict_name, 'rb')) self.feat_dict_ = pickle.load(open(self.feat_dict_name, 'rb'))
def _process_line(self, line): def _process_line(self, line):
...@@ -90,6 +94,7 @@ class TrainReader(dg.MultiSlotDataGenerator): ...@@ -90,6 +94,7 @@ class TrainReader(dg.MultiSlotDataGenerator):
return data_iter return data_iter
reader = TrainReader("../config.yaml") reader = TrainReader(
"../config.yaml") # run this file in original folder to find config.yaml
reader.init() reader.init()
reader.run_from_stdin() reader.run_from_stdin()
...@@ -66,7 +66,6 @@ def get_feat_dict(): ...@@ -66,7 +66,6 @@ def get_feat_dict():
categorical_range_ = range(14, 40) categorical_range_ = range(14, 40)
if not os.path.exists(dir_feat_dict_): if not os.path.exists(dir_feat_dict_):
# print('generate a feature dict')
# Count the number of occurrences of discrete features # Count the number of occurrences of discrete features
feat_cnt = Counter() feat_cnt = Counter()
with open('train.txt', 'r') as fin: with open('train.txt', 'r') as fin:
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册