bug fix (#70)

demo/data_loader.py

+import numpy as np
+import sys
+import os
+
+word_title_num = 50
+word_cont_num = 1024
+word_att_num = 10
+CLASS_NUM = 1284213
+
+def pad_and_trunk(_list, fix_sz = -1):
+    if len(_list) > 0 and _list[0] == '':
+        _list = []
+    _list = _list[:fix_sz]
+    if len(_list) < fix_sz:
+        pad = ['0' for i in range(fix_sz - len(_list))]
+        _list.extend(pad)
+    return _list
+
+def generate_reader(url2fea, topic2fea, _path, class_num=CLASS_NUM):
+
+    def reader():
+        print 'file open.'
+        trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0"))
+        if os.getenv("PADDLE_TRAINER_ENDPOINTS"):
+            trainer_count = len(os.getenv("PADDLE_TRAINER_ENDPOINTS").split(","))
+        else:
+            trainer_count = int(os.getenv("PADDLE_TRAINERS", "1"))
+        f = open(_path)
+        sample_index = 0
+        for line in f:
+            line = line.strip('\n')
+            if len(line) == 0:
+                continue
+            
+            part = line.split('\t')
+
+            url  = part[0]
+            title_ids = part[1]
+            content_ids = part[2]
+            label = int(part[3])
+
+            if sample_index % trainer_count != trainer_id:
+                sample_index += 1
+                continue
+            sample_index += 1
+
+            title_ids = pad_and_trunk(title_ids.split(','), word_title_num)
+            content_ids = pad_and_trunk(content_ids.split(','), word_cont_num)
+
+            title_input_x_train = np.asarray(title_ids, dtype='int64').reshape( (len(title_ids), 1) )
+            content_input_x_train = np.asarray(content_ids, dtype='int64').reshape( (len(content_ids), 1) )
+
+            label = np.array([label])
+            yield title_input_x_train, content_input_x_train, label
+    
+        f.close()
+        print 'file close.'
+    return reader
+
+if __name__ == '__main__':
+    
+    #load_validation(url2fea, topic2fea, './data_makeup/merge_att_data/format_sample_v1/test.sample.shuffle')
+    
+    '''
+    for (x1, x2, x3, y) in generate_batch_from_file(url2fea, topic2fea, \
+                    './data_makeup/merge_att_data/format_sample_v1/train.sample.shuffle', 50):
+        print x1[0], x2[0], x3[0], y[0]
+        break
+    '''
+
+    for x1, x2, x3, x4 in generate_reader(None, None, './data_makeup/merge_att_data/format_sample_v4/test.10w.sample.shuffle').reader():
+        print x1, x2, x3, x4
+        break
+

demo/demo_noncv.py

+import os
+import sys
+from plsc import Entry
+from plsc.models import BaseModel
+import paddle
+import paddle.fluid as fluid
+from utils import LogUtil
+import numpy as np
+
+CLASS_NUM = 1284213
+
+from data_loader import generate_reader
+
+class UserModel(BaseModel):
+    def __init__(self, emb_dim=512):
+        self.emb_dim = emb_dim
+
+    def build_network(self,
+                      input,
+                      is_train=True):
+        title_ids = input.title_ids
+        content_ids = input.content_ids
+        label = input.label
+        vob_size = 1841178 + 1
+        #embedding layer
+        #current shape is [-1, seq_length, emb_dim]
+        word_title_sequence_input = fluid.layers.embedding(
+            input=title_ids, size=[vob_size, 128], is_sparse=False,
+            param_attr=fluid.ParamAttr(name='word_embedding'))
+        word_cont_sequence_input = fluid.layers.embedding(
+            input=content_ids, size=[vob_size, 128], is_sparse=False,
+            param_attr=fluid.ParamAttr(name='word_embedding'))
+
+        #current shape is [-1, emb_dim, seq_length]
+        word_title_sequence_input = fluid.layers.transpose(word_title_sequence_input, perm=[0, 2, 1], name='title_transpose')
+        word_cont_sequence_input = fluid.layers.transpose(word_cont_sequence_input, perm=[0, 2, 1], name='cont_transpose')
+
+        #current shape is [-1, emb_dim, 1, seq_length], which is NCHW format
+        _shape = word_title_sequence_input.shape
+        word_title_sequence_input = fluid.layers.reshape(x=word_title_sequence_input,
+                    shape=[_shape[0], _shape[1], 1, _shape[2]], inplace=True, name='title_reshape')
+        _shape = word_cont_sequence_input.shape
+        word_cont_sequence_input = fluid.layers.reshape(x=word_cont_sequence_input,
+                    shape=[_shape[0], _shape[1], 1, _shape[2]], inplace=True, name='cont_reshape')
+
+        word_title_win_3 = fluid.layers.conv2d(input=word_title_sequence_input, num_filters=128,
+                                            filter_size=(1,3), stride=(1,1), padding=(0,1), act='relu',
+                                            name='word_title_win_3_conv')
+
+        word_title_x = fluid.layers.pool2d(input=word_title_win_3, pool_size=(1,4),
+                                           pool_type='max', pool_stride=(1,4),
+                                           name='word_title_win_3_pool')
+
+        word_cont_win_3 = fluid.layers.conv2d(input=word_cont_sequence_input, num_filters=128,
+                                            filter_size=(1,3), stride=(1,1), padding=(0,1), act='relu',
+                                            name='word_cont_win_3_conv')
+
+        word_cont_x = fluid.layers.pool2d(input=word_cont_win_3, pool_size=(1,20),
+                                           pool_type='max', pool_stride=(1,20), 
+                                           name='word_cont_win_3_pool')
+
+        print('word_title_x.shape:', word_title_x.shape)
+        print('word_cont_x.shape:', word_cont_x.shape)
+        x_concat = fluid.layers.concat(input=[word_title_x, word_cont_x], axis=3, name='feature_concat')
+        x_flatten = fluid.layers.flatten(x=x_concat, axis=1, name='feature_flatten')
+        x_fc = fluid.layers.fc(input=x_flatten, size=self.emb_dim, act="relu", name='final_fc')
+        return x_fc
+
+
+def train(url2fea_path, topic2fea_path, train_path, val_path, model_save_dir):
+    ins = Entry()
+    ins.set_with_test(False)
+    ins.set_train_epochs(20)
+    
+    #load id features
+    
+    word_title_num = 50
+    word_cont_num = 1024
+    batch_size = int(os.getenv("BATCH_SIZE", "64"))
+
+    input_info = [{'name': 'title_ids',
+                   'shape': [-1, word_title_num, 1],
+                   'dtype': 'int64'},
+                  {'name': 'content_ids',
+                   'shape': [-1, word_cont_num, 1],
+                   'dtype': 'int64'},
+                  {'name': 'label',
+                   'shape': [-1, 1],
+                   'dtype': 'int64'}
+                 ]
+    ins.set_input_info(input_info)
+    ins.set_class_num(CLASS_NUM)
+
+    emb_dim = int(os.getenv("EMB_DIM", "512"))
+    model = UserModel(emb_dim=emb_dim)
+    ins.set_model(model)
+    ins.set_train_batch_size(batch_size)
+
+    sgd_optimizer = fluid.optimizer.Adam(learning_rate=1e-3)
+    ins.set_optimizer(sgd_optimizer)
+
+    train_reader = generate_reader(None, None, train_path)
+    ins.train_reader = train_reader
+    
+    ins.set_train_epochs(20)
+    ins.set_model_save_dir("./saved_model")
+    ins.set_loss_type('dist_softmax')
+    ins.train()
+
+
+
+if __name__ == "__main__":
+    data = './package/'
+    url2fea_path = data + 'click_search_all.url_title_cont.seg.lower.id'
+    topic2fea_path = data + 'click_search_all.att.seg.id'
+    train_path = data +'train.sample.shuffle.label_expand'
+    val_path = data +'test.10w.sample.shuffle.label_expand'
+    model_save_dir = data + 'saved_models'
+    
+    train(url2fea_path, topic2fea_path, train_path, val_path, model_save_dir)
+