diff --git a/models/match/multiview-simnet/data_process.sh b/models/match/multiview-simnet/data_process.sh index 91548c5863063f7a23cb2f713a3754b121b235b5..c8633cc7a41f62a29eee1778251b72a6f3b601eb 100755 --- a/models/match/multiview-simnet/data_process.sh +++ b/models/match/multiview-simnet/data_process.sh @@ -22,4 +22,3 @@ mkdir -p data/train mkdir -p data/test python generate_synthetic_data.py - diff --git a/models/match/multiview-simnet/evaluate_reader.py b/models/match/multiview-simnet/evaluate_reader.py index e0f8f9e43de80d003834056ea417914f1d10e898..d77032f3ca4e07cbbf20874f79023dc4a6fed8b4 100755 --- a/models/match/multiview-simnet/evaluate_reader.py +++ b/models/match/multiview-simnet/evaluate_reader.py @@ -18,8 +18,10 @@ from paddlerec.core.utils import envs class EvaluateReader(Reader): def init(self): - self.query_slots = envs.get_global_env("hyper_parameters.query_slots", None, "train.model") - self.title_slots = envs.get_global_env("hyper_parameters.title_slots", None, "train.model") + self.query_slots = envs.get_global_env("hyper_parameters.query_slots", + None, "train.model") + self.title_slots = envs.get_global_env("hyper_parameters.title_slots", + None, "train.model") self.all_slots = [] for i in range(self.query_slots): diff --git a/models/match/multiview-simnet/generate_synthetic_data.py b/models/match/multiview-simnet/generate_synthetic_data.py index d453e031cdca9be29892b913ea5f2636a6c05f5e..eb60e5c82f9decc2cfcd87da7bc6832ca98ee9d4 100755 --- a/models/match/multiview-simnet/generate_synthetic_data.py +++ b/models/match/multiview-simnet/generate_synthetic_data.py @@ -21,7 +21,11 @@ class Dataset: class SyntheticDataset(Dataset): - def __init__(self, sparse_feature_dim, query_slot_num, title_slot_num, dataset_size=10000): + def __init__(self, + sparse_feature_dim, + query_slot_num, + title_slot_num, + dataset_size=10000): # ids are randomly generated self.ids_per_slot = 10 self.sparse_feature_dim = sparse_feature_dim @@ -46,14 +50,20 @@ class SyntheticDataset(Dataset): for i in range(self.title_slot_num): pt_slot = generate_ids(self.ids_per_slot, self.sparse_feature_dim) - pt_slot = [str(fea) + ':' + str(i + self.query_slot_num) for fea in pt_slot] + pt_slot = [ + str(fea) + ':' + str(i + self.query_slot_num) + for fea in pt_slot + ] pos_title_slots += pt_slot if is_train: for i in range(self.title_slot_num): nt_slot = generate_ids(self.ids_per_slot, self.sparse_feature_dim) - nt_slot = [str(fea) + ':' + str(i + self.query_slot_num + self.title_slot_num) for fea in - nt_slot] + nt_slot = [ + str(fea) + ':' + + str(i + self.query_slot_num + self.title_slot_num) + for fea in nt_slot + ] neg_title_slots += nt_slot yield query_slots + pos_title_slots + neg_title_slots else: @@ -76,7 +86,8 @@ if __name__ == '__main__': query_slots = 1 title_slots = 1 dataset_size = 10 - dataset = SyntheticDataset(sparse_feature_dim, query_slots, title_slots, dataset_size) + dataset = SyntheticDataset(sparse_feature_dim, query_slots, title_slots, + dataset_size) train_reader = dataset.train() test_reader = dataset.test() diff --git a/models/match/multiview-simnet/model.py b/models/match/multiview-simnet/model.py index 5ba9fb5d05b27339d924bfe42c0e6ba0c2c68da3..f80a1cd0390f3c7aafc772ef535eb36b9657b439 100755 --- a/models/match/multiview-simnet/model.py +++ b/models/match/multiview-simnet/model.py @@ -103,12 +103,18 @@ class Model(ModelBase): def init_config(self): self._fetch_interval = 1 - query_encoder = envs.get_global_env("hyper_parameters.query_encoder", None, self._namespace) - title_encoder = envs.get_global_env("hyper_parameters.title_encoder", None, self._namespace) - query_encode_dim = envs.get_global_env("hyper_parameters.query_encode_dim", None, self._namespace) - title_encode_dim = envs.get_global_env("hyper_parameters.title_encode_dim", None, self._namespace) - query_slots = envs.get_global_env("hyper_parameters.query_slots", None, self._namespace) - title_slots = envs.get_global_env("hyper_parameters.title_slots", None, self._namespace) + query_encoder = envs.get_global_env("hyper_parameters.query_encoder", + None, self._namespace) + title_encoder = envs.get_global_env("hyper_parameters.title_encoder", + None, self._namespace) + query_encode_dim = envs.get_global_env( + "hyper_parameters.query_encode_dim", None, self._namespace) + title_encode_dim = envs.get_global_env( + "hyper_parameters.title_encode_dim", None, self._namespace) + query_slots = envs.get_global_env("hyper_parameters.query_slots", None, + self._namespace) + title_slots = envs.get_global_env("hyper_parameters.title_slots", None, + self._namespace) factory = SimpleEncoderFactory() self.query_encoders = [ factory.create(query_encoder, query_encode_dim) @@ -119,10 +125,13 @@ class Model(ModelBase): for i in range(title_slots) ] - self.emb_size = envs.get_global_env("hyper_parameters.sparse_feature_dim", None, self._namespace) - self.emb_dim = envs.get_global_env("hyper_parameters.embedding_dim", None, self._namespace) + self.emb_size = envs.get_global_env( + "hyper_parameters.sparse_feature_dim", None, self._namespace) + self.emb_dim = envs.get_global_env("hyper_parameters.embedding_dim", + None, self._namespace) self.emb_shape = [self.emb_size, self.emb_dim] - self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, self._namespace) + self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size", + None, self._namespace) self.margin = 0.1 def input(self, is_train=True): @@ -133,8 +142,10 @@ class Model(ModelBase): ] self.pt_slots = [ fluid.data( - name="%d" % (i + len(self.query_encoders)), shape=[None, 1], lod_level=1, dtype='int64') - for i in range(len(self.title_encoders)) + name="%d" % (i + len(self.query_encoders)), + shape=[None, 1], + lod_level=1, + dtype='int64') for i in range(len(self.title_encoders)) ] if is_train == False: @@ -142,9 +153,11 @@ class Model(ModelBase): self.nt_slots = [ fluid.data( - name="%d" % (i + len(self.query_encoders) + len(self.title_encoders)), shape=[None, 1], lod_level=1, - dtype='int64') - for i in range(len(self.title_encoders)) + name="%d" % + (i + len(self.query_encoders) + len(self.title_encoders)), + shape=[None, 1], + lod_level=1, + dtype='int64') for i in range(len(self.title_encoders)) ] return self.q_slots + self.pt_slots + self.nt_slots @@ -153,11 +166,15 @@ class Model(ModelBase): res = self.input() self._data_var = res - use_dataloader = envs.get_global_env("hyper_parameters.use_DataLoader", False, self._namespace) + use_dataloader = envs.get_global_env("hyper_parameters.use_DataLoader", + False, self._namespace) if self._platform != "LINUX" or use_dataloader: self._data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._data_var, capacity=256, use_double_buffer=False, iterable=False) + feed_list=self._data_var, + capacity=256, + use_double_buffer=False, + iterable=False) def get_acc(self, x, y): less = tensor.cast(cf.less_than(x, y), dtype='float32') @@ -190,10 +207,12 @@ class Model(ModelBase): self.query_encoders[i].forward(emb) for i, emb in enumerate(q_embs) ] pt_encodes = [ - self.title_encoders[i].forward(emb) for i, emb in enumerate(pt_embs) + self.title_encoders[i].forward(emb) + for i, emb in enumerate(pt_embs) ] nt_encodes = [ - self.title_encoders[i].forward(emb) for i, emb in enumerate(nt_embs) + self.title_encoders[i].forward(emb) + for i, emb in enumerate(nt_embs) ] # concat multi view for query, pos_title, neg_title @@ -252,7 +271,8 @@ class Model(ModelBase): self.metrics() def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate=learning_rate) return optimizer @@ -261,7 +281,10 @@ class Model(ModelBase): self._infer_data_var = res self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def infer_net(self): self.infer_input() @@ -281,7 +304,8 @@ class Model(ModelBase): self.query_encoders[i].forward(emb) for i, emb in enumerate(q_embs) ] pt_encodes = [ - self.title_encoders[i].forward(emb) for i, emb in enumerate(pt_embs) + self.title_encoders[i].forward(emb) + for i, emb in enumerate(pt_embs) ] # concat multi view for query, pos_title, neg_title q_concat = fluid.layers.concat(q_encodes) diff --git a/models/match/multiview-simnet/reader.py b/models/match/multiview-simnet/reader.py index 43cd1a629a7540e727e423a98d497964203134ac..4c0e42a44b0ea05272c832d65a6cfbc0d3f6c495 100755 --- a/models/match/multiview-simnet/reader.py +++ b/models/match/multiview-simnet/reader.py @@ -18,8 +18,10 @@ from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): - self.query_slots = envs.get_global_env("hyper_parameters.query_slots", None, "train.model") - self.title_slots = envs.get_global_env("hyper_parameters.title_slots", None, "train.model") + self.query_slots = envs.get_global_env("hyper_parameters.query_slots", + None, "train.model") + self.title_slots = envs.get_global_env("hyper_parameters.title_slots", + None, "train.model") self.all_slots = [] for i in range(self.query_slots): diff --git a/models/multitask/esmm/esmm_infer_reader.py b/models/multitask/esmm/esmm_infer_reader.py index 8ca9eca67fdbb9e11f39db34b5dd9cfae518773b..70e3e989df611419f378a8920b499e42690d1cae 100644 --- a/models/multitask/esmm/esmm_infer_reader.py +++ b/models/multitask/esmm/esmm_infer_reader.py @@ -20,9 +20,11 @@ from paddlerec.core.reader import Reader class EvaluateReader(Reader): def init(self): - all_field_id = ['101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', '125', '126', '127', '128', - '129', - '205', '206', '207', '210', '216', '508', '509', '702', '853', '301'] + all_field_id = [ + '101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', + '125', '126', '127', '128', '129', '205', '206', '207', '210', + '216', '508', '509', '702', '853', '301' + ] self.all_field_id_dict = defaultdict(int) for i, field_id in enumerate(all_field_id): self.all_field_id_dict[field_id] = [False, i] diff --git a/models/multitask/esmm/esmm_reader.py b/models/multitask/esmm/esmm_reader.py index 3d663038eefb4971b466336601ba436ff884e580..036e146ee923b6feda6398c7dcd49486eac51c50 100644 --- a/models/multitask/esmm/esmm_reader.py +++ b/models/multitask/esmm/esmm_reader.py @@ -21,9 +21,11 @@ from paddlerec.core.reader import Reader class TrainReader(Reader): def init(self): - all_field_id = ['101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', '125', '126', '127', '128', - '129', - '205', '206', '207', '210', '216', '508', '509', '702', '853', '301'] + all_field_id = [ + '101', '109_14', '110_14', '127_14', '150_14', '121', '122', '124', + '125', '126', '127', '128', '129', '205', '206', '207', '210', + '216', '508', '509', '702', '853', '301' + ] self.all_field_id_dict = defaultdict(int) for i, field_id in enumerate(all_field_id): self.all_field_id_dict[field_id] = [False, i] diff --git a/models/multitask/esmm/model.py b/models/multitask/esmm/model.py index 8a8a203a87504cff310c0a799df40e937e2bbde8..71c6539579504407a22f3174407b517f9d9a55b5 100644 --- a/models/multitask/esmm/model.py +++ b/models/multitask/esmm/model.py @@ -28,11 +28,13 @@ class Model(ModelBase): init_stddev = 1.0 scales = 1.0 / np.sqrt(data.shape[1]) - p_attr = fluid.param_attr.ParamAttr(name='%s_weight' % tag, - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=init_stddev * scales)) + p_attr = fluid.param_attr.ParamAttr( + name='%s_weight' % tag, + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=init_stddev * scales)) - b_attr = fluid.ParamAttr(name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1)) + b_attr = fluid.ParamAttr( + name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1)) out = fluid.layers.fc(input=data, size=out_dim, @@ -44,7 +46,11 @@ class Model(ModelBase): def input_data(self): sparse_input_ids = [ - fluid.data(name="field_" + str(i), shape=[-1, 1], dtype="int64", lod_level=1) for i in range(0, 23) + fluid.data( + name="field_" + str(i), + shape=[-1, 1], + dtype="int64", + lod_level=1) for i in range(0, 23) ] label_ctr = fluid.data(name="ctr", shape=[-1, 1], dtype="int64") label_cvr = fluid.data(name="cvr", shape=[-1, 1], dtype="int64") @@ -55,19 +61,23 @@ class Model(ModelBase): def net(self, inputs, is_infer=False): - vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) - embed_size = envs.get_global_env("hyper_parameters.embed_size", None, self._namespace) + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, + self._namespace) + embed_size = envs.get_global_env("hyper_parameters.embed_size", None, + self._namespace) emb = [] for data in inputs[0:-2]: - feat_emb = fluid.embedding(input=data, - size=[vocab_size, embed_size], - param_attr=fluid.ParamAttr(name='dis_emb', - learning_rate=5, - initializer=fluid.initializer.Xavier( - fan_in=embed_size, fan_out=embed_size) - ), - is_sparse=True) - field_emb = fluid.layers.sequence_pool(input=feat_emb, pool_type='sum') + feat_emb = fluid.embedding( + input=data, + size=[vocab_size, embed_size], + param_attr=fluid.ParamAttr( + name='dis_emb', + learning_rate=5, + initializer=fluid.initializer.Xavier( + fan_in=embed_size, fan_out=embed_size)), + is_sparse=True) + field_emb = fluid.layers.sequence_pool( + input=feat_emb, pool_type='sum') emb.append(field_emb) concat_emb = fluid.layers.concat(emb, axis=1) @@ -85,14 +95,20 @@ class Model(ModelBase): ctr_clk = inputs[-2] ctcvr_buy = inputs[-1] - ctr_prop_one = fluid.layers.slice(ctr_out, axes=[1], starts=[1], ends=[2]) - cvr_prop_one = fluid.layers.slice(cvr_out, axes=[1], starts=[1], ends=[2]) + ctr_prop_one = fluid.layers.slice( + ctr_out, axes=[1], starts=[1], ends=[2]) + cvr_prop_one = fluid.layers.slice( + cvr_out, axes=[1], starts=[1], ends=[2]) - ctcvr_prop_one = fluid.layers.elementwise_mul(ctr_prop_one, cvr_prop_one) - ctcvr_prop = fluid.layers.concat(input=[1 - ctcvr_prop_one, ctcvr_prop_one], axis=1) + ctcvr_prop_one = fluid.layers.elementwise_mul(ctr_prop_one, + cvr_prop_one) + ctcvr_prop = fluid.layers.concat( + input=[1 - ctcvr_prop_one, ctcvr_prop_one], axis=1) - auc_ctr, batch_auc_ctr, auc_states_ctr = fluid.layers.auc(input=ctr_out, label=ctr_clk) - auc_ctcvr, batch_auc_ctcvr, auc_states_ctcvr = fluid.layers.auc(input=ctcvr_prop, label=ctcvr_buy) + auc_ctr, batch_auc_ctr, auc_states_ctr = fluid.layers.auc( + input=ctr_out, label=ctr_clk) + auc_ctcvr, batch_auc_ctcvr, auc_states_ctcvr = fluid.layers.auc( + input=ctcvr_prop, label=ctcvr_buy) if is_infer: self._infer_results["AUC_ctr"] = auc_ctr @@ -100,7 +116,8 @@ class Model(ModelBase): return loss_ctr = fluid.layers.cross_entropy(input=ctr_out, label=ctr_clk) - loss_ctcvr = fluid.layers.cross_entropy(input=ctcvr_prop, label=ctcvr_buy) + loss_ctcvr = fluid.layers.cross_entropy( + input=ctcvr_prop, label=ctcvr_buy) cost = loss_ctr + loss_ctcvr avg_cost = fluid.layers.mean(cost) @@ -117,5 +134,8 @@ class Model(ModelBase): def infer_net(self): self._infer_data_var = self.input_data() self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) self.net(self._infer_data_var, is_infer=True) diff --git a/models/multitask/mmoe/census_infer_reader.py b/models/multitask/mmoe/census_infer_reader.py index c62de8e69ce6ccfbb4df1e1252d9630a84fc56b3..fada3990fdcc756a2938c5a4fd763f022dda53c4 100644 --- a/models/multitask/mmoe/census_infer_reader.py +++ b/models/multitask/mmoe/census_infer_reader.py @@ -19,6 +19,7 @@ from paddlerec.core.reader import Reader class EvaluateReader(Reader): def init(self): + pass def generate_sample(self, line): diff --git a/models/multitask/mmoe/census_reader.py b/models/multitask/mmoe/census_reader.py index 211e566882e5d8a7f50f22b0a1628307777099c8..d71133bd91692c8b17e7449aa305e5241db7777a 100644 --- a/models/multitask/mmoe/census_reader.py +++ b/models/multitask/mmoe/census_reader.py @@ -24,6 +24,7 @@ class TrainReader(Reader): def generate_sample(self, line): """ Read the data line by line and process it as a dictionary + """ def reader(): diff --git a/models/multitask/mmoe/model.py b/models/multitask/mmoe/model.py index 525e9d5cc0086757901262253cf0f23ee72f314c..035733690f46960906c902dbe240603acd136565 100644 --- a/models/multitask/mmoe/model.py +++ b/models/multitask/mmoe/model.py @@ -23,44 +23,58 @@ class Model(ModelBase): ModelBase.__init__(self, config) def MMOE(self, is_infer=False): - - feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace) - expert_num = envs.get_global_env("hyper_parameters.expert_num", None, self._namespace) - gate_num = envs.get_global_env("hyper_parameters.gate_num", None, self._namespace) - expert_size = envs.get_global_env("hyper_parameters.expert_size", None, self._namespace) - tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace) - - input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32") - label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) - label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) + feature_size = envs.get_global_env("hyper_parameters.feature_size", + None, self._namespace) + expert_num = envs.get_global_env("hyper_parameters.expert_num", None, + self._namespace) + gate_num = envs.get_global_env("hyper_parameters.gate_num", None, + self._namespace) + expert_size = envs.get_global_env("hyper_parameters.expert_size", None, + self._namespace) + tower_size = envs.get_global_env("hyper_parameters.tower_size", None, + self._namespace) + + input_data = fluid.data( + name="input", shape=[-1, feature_size], dtype="float32") + label_income = fluid.data( + name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) + label_marital = fluid.data( + name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) if is_infer: self._infer_data_var = [input_data, label_income, label_marital] self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) self._data_var.extend([input_data, label_income, label_marital]) # f_{i}(x) = activation(W_{i} * x + b), where activation is ReLU according to the paper expert_outputs = [] for i in range(0, expert_num): - expert_output = fluid.layers.fc(input=input_data, - size=expert_size, - act='relu', - bias_attr=fluid.ParamAttr(learning_rate=1.0), - name='expert_' + str(i)) + expert_output = fluid.layers.fc( + input=input_data, + size=expert_size, + act='relu', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='expert_' + str(i)) expert_outputs.append(expert_output) expert_concat = fluid.layers.concat(expert_outputs, axis=1) - expert_concat = fluid.layers.reshape(expert_concat, [-1, expert_num, expert_size]) + expert_concat = fluid.layers.reshape(expert_concat, + [-1, expert_num, expert_size]) # g^{k}(x) = activation(W_{gk} * x + b), where activation is softmax according to the paper output_layers = [] for i in range(0, gate_num): - cur_gate = fluid.layers.fc(input=input_data, - size=expert_num, - act='softmax', - bias_attr=fluid.ParamAttr(learning_rate=1.0), - name='gate_' + str(i)) + cur_gate = fluid.layers.fc( + input=input_data, + size=expert_num, + act='softmax', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='gate_' + str(i)) # f^{k}(x) = sum_{i=1}^{n}(g^{k}(x)_{i} * f_{i}(x)) - cur_gate_expert = fluid.layers.elementwise_mul(expert_concat, cur_gate, axis=0) + cur_gate_expert = fluid.layers.elementwise_mul( + expert_concat, cur_gate, axis=0) cur_gate_expert = fluid.layers.reduce_sum(cur_gate_expert, dim=1) # Build tower layer cur_tower = fluid.layers.fc(input=cur_gate_expert, @@ -74,25 +88,33 @@ class Model(ModelBase): output_layers.append(out) - pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15) - pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15) - - label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2]) - label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2]) - - auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc(input=pred_income, - label=fluid.layers.cast(x=label_income_1, - dtype='int64')) - auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, - label=fluid.layers.cast(x=label_marital_1, - dtype='int64')) + pred_income = fluid.layers.clip( + output_layers[0], min=1e-15, max=1.0 - 1e-15) + pred_marital = fluid.layers.clip( + output_layers[1], min=1e-15, max=1.0 - 1e-15) + + label_income_1 = fluid.layers.slice( + label_income, axes=[1], starts=[1], ends=[2]) + label_marital_1 = fluid.layers.slice( + label_marital, axes=[1], starts=[1], ends=[2]) + + auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc( + input=pred_income, + label=fluid.layers.cast( + x=label_income_1, dtype='int64')) + auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc( + input=pred_marital, + label=fluid.layers.cast( + x=label_marital_1, dtype='int64')) if is_infer: self._infer_results["AUC_income"] = auc_income self._infer_results["AUC_marital"] = auc_marital return - cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income, soft_label=True) - cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital, soft_label=True) + cost_income = fluid.layers.cross_entropy( + input=pred_income, label=label_income, soft_label=True) + cost_marital = fluid.layers.cross_entropy( + input=pred_marital, label=label_marital, soft_label=True) avg_cost_income = fluid.layers.mean(x=cost_income) avg_cost_marital = fluid.layers.mean(x=cost_marital) diff --git a/models/multitask/share-bottom/model.py b/models/multitask/share-bottom/model.py index d570ba77067985b518247c8f6bba16a6431e1f9c..f19ecbe1c43323e30cb9a44eb281f31c68b69909 100644 --- a/models/multitask/share-bottom/model.py +++ b/models/multitask/share-bottom/model.py @@ -24,27 +24,38 @@ class Model(ModelBase): def model(self, is_infer=False): - feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace) - bottom_size = envs.get_global_env("hyper_parameters.bottom_size", None, self._namespace) - tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace) - tower_nums = envs.get_global_env("hyper_parameters.tower_nums", None, self._namespace) - - input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32") - label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) - label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) + feature_size = envs.get_global_env("hyper_parameters.feature_size", + None, self._namespace) + bottom_size = envs.get_global_env("hyper_parameters.bottom_size", None, + self._namespace) + tower_size = envs.get_global_env("hyper_parameters.tower_size", None, + self._namespace) + tower_nums = envs.get_global_env("hyper_parameters.tower_nums", None, + self._namespace) + + input_data = fluid.data( + name="input", shape=[-1, feature_size], dtype="float32") + label_income = fluid.data( + name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) + label_marital = fluid.data( + name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) if is_infer: self._infer_data_var = [input_data, label_income, label_marital] self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) self._data_var.extend([input_data, label_income, label_marital]) - bottom_output = fluid.layers.fc(input=input_data, - size=bottom_size, - act='relu', - bias_attr=fluid.ParamAttr(learning_rate=1.0), - name='bottom_output') + bottom_output = fluid.layers.fc( + input=input_data, + size=bottom_size, + act='relu', + bias_attr=fluid.ParamAttr(learning_rate=1.0), + name='bottom_output') # Build tower layer from bottom layer output_layers = [] @@ -59,26 +70,34 @@ class Model(ModelBase): name='output_layer_' + str(index)) output_layers.append(output_layer) - pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15) - pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15) - - label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2]) - label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2]) - - auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc(input=pred_income, - label=fluid.layers.cast(x=label_income_1, - dtype='int64')) - auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc(input=pred_marital, - label=fluid.layers.cast(x=label_marital_1, - dtype='int64')) + pred_income = fluid.layers.clip( + output_layers[0], min=1e-15, max=1.0 - 1e-15) + pred_marital = fluid.layers.clip( + output_layers[1], min=1e-15, max=1.0 - 1e-15) + + label_income_1 = fluid.layers.slice( + label_income, axes=[1], starts=[1], ends=[2]) + label_marital_1 = fluid.layers.slice( + label_marital, axes=[1], starts=[1], ends=[2]) + + auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc( + input=pred_income, + label=fluid.layers.cast( + x=label_income_1, dtype='int64')) + auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc( + input=pred_marital, + label=fluid.layers.cast( + x=label_marital_1, dtype='int64')) if is_infer: self._infer_results["AUC_income"] = auc_income self._infer_results["AUC_marital"] = auc_marital return - cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income, soft_label=True) - cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital, soft_label=True) + cost_income = fluid.layers.cross_entropy( + input=pred_income, label=label_income, soft_label=True) + cost_marital = fluid.layers.cross_entropy( + input=pred_marital, label=label_marital, soft_label=True) cost = fluid.layers.elementwise_add(cost_income, cost_marital, axis=1) avg_cost = fluid.layers.mean(x=cost) diff --git a/models/rank/dcn/model.py b/models/rank/dcn/model.py index bf3e3051ca3ac92d1e354c6f59313ce496ff2921..67447fedefd180649bb018a3ea23aea216c9a2b4 100755 --- a/models/rank/dcn/model.py +++ b/models/rank/dcn/model.py @@ -25,12 +25,18 @@ class Model(ModelBase): ModelBase.__init__(self, config) def init_network(self): - self.cross_num = envs.get_global_env("hyper_parameters.cross_num", None, self._namespace) - self.dnn_hidden_units = envs.get_global_env("hyper_parameters.dnn_hidden_units", None, self._namespace) - self.l2_reg_cross = envs.get_global_env("hyper_parameters.l2_reg_cross", None, self._namespace) - self.dnn_use_bn = envs.get_global_env("hyper_parameters.dnn_use_bn", None, self._namespace) - self.clip_by_norm = envs.get_global_env("hyper_parameters.clip_by_norm", None, self._namespace) - cat_feat_num = envs.get_global_env("hyper_parameters.cat_feat_num", None, self._namespace) + self.cross_num = envs.get_global_env("hyper_parameters.cross_num", + None, self._namespace) + self.dnn_hidden_units = envs.get_global_env( + "hyper_parameters.dnn_hidden_units", None, self._namespace) + self.l2_reg_cross = envs.get_global_env( + "hyper_parameters.l2_reg_cross", None, self._namespace) + self.dnn_use_bn = envs.get_global_env("hyper_parameters.dnn_use_bn", + None, self._namespace) + self.clip_by_norm = envs.get_global_env( + "hyper_parameters.clip_by_norm", None, self._namespace) + cat_feat_num = envs.get_global_env("hyper_parameters.cat_feat_num", + None, self._namespace) self.sparse_inputs = self._sparse_data_var[1:] self.dense_inputs = self._dense_data_var @@ -43,7 +49,8 @@ class Model(ModelBase): cat_feat_dims_dict[spls[0]] = int(spls[1]) self.cat_feat_dims_dict = cat_feat_dims_dict if cat_feat_dims_dict else OrderedDict( ) - self.is_sparse = envs.get_global_env("hyper_parameters.is_sparse", None, self._namespace) + self.is_sparse = envs.get_global_env("hyper_parameters.is_sparse", + None, self._namespace) self.dense_feat_names = [i.name for i in self.dense_inputs] self.sparse_feat_names = [i.name for i in self.sparse_inputs] @@ -55,16 +62,19 @@ class Model(ModelBase): self.net_input = None self.loss = None - + def _create_embedding_input(self): # sparse embedding sparse_emb_dict = OrderedDict() for var in self.sparse_inputs: - sparse_emb_dict[var.name] = fluid.embedding(input=var, - size=[self.feat_dims_dict[var.name] + 1, - 6 * int(pow(self.feat_dims_dict[var.name], 0.25)) - ],is_sparse=self.is_sparse) - + sparse_emb_dict[var.name] = fluid.embedding( + input=var, + size=[ + self.feat_dims_dict[var.name] + 1, + 6 * int(pow(self.feat_dims_dict[var.name], 0.25)) + ], + is_sparse=self.is_sparse) + # combine dense and sparse_emb dense_input_list = self.dense_inputs sparse_emb_list = list(sparse_emb_dict.values()) @@ -114,10 +124,11 @@ class Model(ModelBase): def train_net(self): self.model._init_slots() self.init_network() - + self.net_input = self._create_embedding_input() - - deep_out = self._deep_net(self.net_input, self.dnn_hidden_units, self.dnn_use_bn, False) + + deep_out = self._deep_net(self.net_input, self.dnn_hidden_units, + self.dnn_use_bn, False) cross_out, l2_reg_cross_loss = self._cross_net(self.net_input, self.cross_num) @@ -134,9 +145,11 @@ class Model(ModelBase): input=prob_2d, label=label_int, slide_steps=0) self._metrics["AUC"] = auc_var self._metrics["BATCH_AUC"] = batch_auc_var - + # logloss - logloss = fluid.layers.log_loss(self.prob, fluid.layers.cast(self.target_input, dtype='float32')) + logloss = fluid.layers.log_loss( + self.prob, fluid.layers.cast( + self.target_input, dtype='float32')) self.avg_logloss = fluid.layers.reduce_mean(logloss) # reg_coeff * l2_reg_cross @@ -145,7 +158,8 @@ class Model(ModelBase): self._cost = self.loss def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/rank/deepfm/model.py b/models/rank/deepfm/model.py index bfda02a21dcc7949b487ef074a783d2f24bcd1f5..0c87b8c869db27b055038e00fe9f6a2efbeb1e29 100755 --- a/models/rank/deepfm/model.py +++ b/models/rank/deepfm/model.py @@ -27,21 +27,26 @@ class Model(ModelBase): def deepfm_net(self): init_value_ = 0.1 is_distributed = True if envs.get_trainer() == "CtrTrainer" else False - sparse_feature_number = envs.get_global_env("hyper_parameters.sparse_feature_number", None, self._namespace) - sparse_feature_dim = envs.get_global_env("hyper_parameters.sparse_feature_dim", None, self._namespace) + sparse_feature_number = envs.get_global_env( + "hyper_parameters.sparse_feature_number", None, self._namespace) + sparse_feature_dim = envs.get_global_env( + "hyper_parameters.sparse_feature_dim", None, self._namespace) # ------------------------- network input -------------------------- - num_field = envs.get_global_env("hyper_parameters.num_field", None, self._namespace) - + num_field = envs.get_global_env("hyper_parameters.num_field", None, + self._namespace) + raw_feat_idx = self._sparse_data_var[1] raw_feat_value = self._dense_data_var[0] self.label = self._sparse_data_var[0] - + feat_idx = raw_feat_idx - feat_value = fluid.layers.reshape(raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 - - reg = envs.get_global_env("hyper_parameters.reg", 1e-4, self._namespace) + feat_value = fluid.layers.reshape( + raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 + + reg = envs.get_global_env("hyper_parameters.reg", 1e-4, + self._namespace) first_weights_re = fluid.embedding( input=feat_idx, is_sparse=True, @@ -55,7 +60,8 @@ class Model(ModelBase): regularizer=fluid.regularizer.L1DecayRegularizer(reg))) first_weights = fluid.layers.reshape( first_weights_re, shape=[-1, num_field, 1]) # None * num_field * 1 - y_first_order = fluid.layers.reduce_sum((first_weights * feat_value), 1) + y_first_order = fluid.layers.reduce_sum((first_weights * feat_value), + 1) # ------------------------- second order term -------------------------- @@ -68,7 +74,8 @@ class Model(ModelBase): padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( - loc=0.0, scale=init_value_ / math.sqrt(float(sparse_feature_dim))))) + loc=0.0, + scale=init_value_ / math.sqrt(float(sparse_feature_dim))))) feat_embeddings = fluid.layers.reshape( feat_embeddings_re, shape=[-1, num_field, @@ -76,8 +83,8 @@ class Model(ModelBase): feat_embeddings = feat_embeddings * feat_value # None * num_field * embedding_size # sum_square part - summed_features_emb = fluid.layers.reduce_sum(feat_embeddings, - 1) # None * embedding_size + summed_features_emb = fluid.layers.reduce_sum( + feat_embeddings, 1) # None * embedding_size summed_features_emb_square = fluid.layers.square( summed_features_emb) # None * embedding_size @@ -88,13 +95,16 @@ class Model(ModelBase): squared_features_emb, 1) # None * embedding_size y_second_order = 0.5 * fluid.layers.reduce_sum( - summed_features_emb_square - squared_sum_features_emb, 1, + summed_features_emb_square - squared_sum_features_emb, + 1, keep_dim=True) # None * 1 # ------------------------- DNN -------------------------- - layer_sizes = envs.get_global_env("hyper_parameters.fc_sizes", None, self._namespace) - act = envs.get_global_env("hyper_parameters.act", None, self._namespace) + layer_sizes = envs.get_global_env("hyper_parameters.fc_sizes", None, + self._namespace) + act = envs.get_global_env("hyper_parameters.act", None, + self._namespace) y_dnn = fluid.layers.reshape(feat_embeddings, [-1, num_field * sparse_feature_dim]) for s in layer_sizes: @@ -121,7 +131,8 @@ class Model(ModelBase): # ------------------------- DeepFM -------------------------- - self.predict = fluid.layers.sigmoid(y_first_order + y_second_order + y_dnn) + self.predict = fluid.layers.sigmoid(y_first_order + y_second_order + + y_dnn) def train_net(self): self.model._init_slots() @@ -129,7 +140,8 @@ class Model(ModelBase): # ------------------------- Cost(logloss) -------------------------- - cost = fluid.layers.log_loss(input=self.predict, label=fluid.layers.cast(self.label, "float32")) + cost = fluid.layers.log_loss( + input=self.predict, label=fluid.layers.cast(self.label, "float32")) avg_cost = fluid.layers.reduce_sum(cost) self._cost = avg_cost @@ -145,7 +157,8 @@ class Model(ModelBase): self._metrics["BATCH_AUC"] = batch_auc_var def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/rank/din/model.py b/models/rank/din/model.py index 2abc658b6d5cb58aaff222e1121d2c4282bcd65f..c2acbe66b6c704655cf9a5aff86d583233672f6c 100755 --- a/models/rank/din/model.py +++ b/models/rank/din/model.py @@ -21,14 +21,14 @@ from paddlerec.core.model import Model as ModelBase class Model(ModelBase): def __init__(self, config): ModelBase.__init__(self, config) - + def config_read(self, config_path): with open(config_path, "r") as fin: user_count = int(fin.readline().strip()) item_count = int(fin.readline().strip()) cat_count = int(fin.readline().strip()) return user_count, item_count, cat_count - + def din_attention(self, hist, target_expand, mask): """activation weight""" @@ -58,56 +58,66 @@ class Model(ModelBase): out = fluid.layers.matmul(weight, hist) out = fluid.layers.reshape(x=out, shape=[0, hidden_size]) return out - + def train_net(self): seq_len = -1 - self.item_emb_size = envs.get_global_env("hyper_parameters.item_emb_size", 64, self._namespace) - self.cat_emb_size = envs.get_global_env("hyper_parameters.cat_emb_size", 64, self._namespace) - self.act = envs.get_global_env("hyper_parameters.act", "sigmoid", self._namespace) + self.item_emb_size = envs.get_global_env( + "hyper_parameters.item_emb_size", 64, self._namespace) + self.cat_emb_size = envs.get_global_env( + "hyper_parameters.cat_emb_size", 64, self._namespace) + self.act = envs.get_global_env("hyper_parameters.act", "sigmoid", + self._namespace) #item_emb_size = 64 #cat_emb_size = 64 - self.is_sparse = envs.get_global_env("hyper_parameters.is_sparse", False, self._namespace) + self.is_sparse = envs.get_global_env("hyper_parameters.is_sparse", + False, self._namespace) #significant for speeding up the training process - self.config_path = envs.get_global_env("hyper_parameters.config_path", "data/config.txt", self._namespace) - self.use_DataLoader = envs.get_global_env("hyper_parameters.use_DataLoader", False, self._namespace) + self.config_path = envs.get_global_env( + "hyper_parameters.config_path", "data/config.txt", self._namespace) + self.use_DataLoader = envs.get_global_env( + "hyper_parameters.use_DataLoader", False, self._namespace) user_count, item_count, cat_count = self.config_read(self.config_path) - item_emb_attr = fluid.ParamAttr(name="item_emb") cat_emb_attr = fluid.ParamAttr(name="cat_emb") hist_item_seq = fluid.data( name="hist_item_seq", shape=[None, seq_len], dtype="int64") self._data_var.append(hist_item_seq) - + hist_cat_seq = fluid.data( name="hist_cat_seq", shape=[None, seq_len], dtype="int64") self._data_var.append(hist_cat_seq) - - target_item = fluid.data(name="target_item", shape=[None], dtype="int64") + + target_item = fluid.data( + name="target_item", shape=[None], dtype="int64") self._data_var.append(target_item) - + target_cat = fluid.data(name="target_cat", shape=[None], dtype="int64") self._data_var.append(target_cat) - + label = fluid.data(name="label", shape=[None, 1], dtype="float32") self._data_var.append(label) - - mask = fluid.data(name="mask", shape=[None, seq_len, 1], dtype="float32") + + mask = fluid.data( + name="mask", shape=[None, seq_len, 1], dtype="float32") self._data_var.append(mask) - + target_item_seq = fluid.data( name="target_item_seq", shape=[None, seq_len], dtype="int64") self._data_var.append(target_item_seq) - + target_cat_seq = fluid.data( name="target_cat_seq", shape=[None, seq_len], dtype="int64") self._data_var.append(target_cat_seq) if self.use_DataLoader: self._data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._data_var, capacity=10000, use_double_buffer=False, iterable=False) - + feed_list=self._data_var, + capacity=10000, + use_double_buffer=False, + iterable=False) + hist_item_emb = fluid.embedding( input=hist_item_seq, size=[item_count, self.item_emb_size], @@ -149,7 +159,8 @@ class Model(ModelBase): size=[item_count, 1], param_attr=fluid.initializer.Constant(value=0.0)) - hist_seq_concat = fluid.layers.concat([hist_item_emb, hist_cat_emb], axis=2) + hist_seq_concat = fluid.layers.concat( + [hist_item_emb, hist_cat_emb], axis=2) target_seq_concat = fluid.layers.concat( [target_item_seq_emb, target_cat_seq_emb], axis=2) target_concat = fluid.layers.concat( @@ -157,21 +168,22 @@ class Model(ModelBase): out = self.din_attention(hist_seq_concat, target_seq_concat, mask) out_fc = fluid.layers.fc(name="out_fc", - input=out, - size=self.item_emb_size + self.cat_emb_size, - num_flatten_dims=1) + input=out, + size=self.item_emb_size + self.cat_emb_size, + num_flatten_dims=1) embedding_concat = fluid.layers.concat([out_fc, target_concat], axis=1) fc1 = fluid.layers.fc(name="fc1", - input=embedding_concat, - size=80, - act=self.act) + input=embedding_concat, + size=80, + act=self.act) fc2 = fluid.layers.fc(name="fc2", input=fc1, size=40, act=self.act) fc3 = fluid.layers.fc(name="fc3", input=fc2, size=1) logit = fc3 + item_b - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logit, label=label) - + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logit, label=label) + avg_loss = fluid.layers.mean(loss) self._cost = avg_loss @@ -179,14 +191,14 @@ class Model(ModelBase): predict_2d = fluid.layers.concat([1 - self.predict, self.predict], 1) label_int = fluid.layers.cast(label, 'int64') auc_var, batch_auc_var, _ = fluid.layers.auc(input=predict_2d, - label=label_int, - slide_steps=0) + label=label_int, + slide_steps=0) self._metrics["AUC"] = auc_var self._metrics["BATCH_AUC"] = batch_auc_var - def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/rank/din/reader.py b/models/rank/din/reader.py index 39ed690fdc6fd35d50ebdcb46b5becc5ae399b62..aba06141da6c60beb81ea446d3c7e7dc8a731df9 100755 --- a/models/rank/din/reader.py +++ b/models/rank/din/reader.py @@ -29,13 +29,15 @@ from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): - self.train_data_path = envs.get_global_env("train_data_path", None, "train.reader") + self.train_data_path = envs.get_global_env("train_data_path", None, + "train.reader") self.res = [] self.max_len = 0 data_file_list = os.listdir(self.train_data_path) for i in range(0, len(data_file_list)): - train_data_file = os.path.join(self.train_data_path, data_file_list[i]) + train_data_file = os.path.join(self.train_data_path, + data_file_list[i]) with open(train_data_file, "r") as fin: for line in fin: line = line.strip().split(';') @@ -78,11 +80,13 @@ class TrainReader(Reader): len_array = [len(x[0]) for x in b] mask = np.array( [[0] * x + [-1e9] * (max_len - x) for x in len_array]).reshape( - [-1, max_len, 1]) + [-1, max_len, 1]) target_item_seq = np.array( - [[x[2]] * max_len for x in b]).astype("int64").reshape([-1, max_len]) + [[x[2]] * max_len for x in b]).astype("int64").reshape( + [-1, max_len]) target_cat_seq = np.array( - [[x[3]] * max_len for x in b]).astype("int64").reshape([-1, max_len]) + [[x[3]] * max_len for x in b]).astype("int64").reshape( + [-1, max_len]) res = [] for i in range(len(b)): res.append([ @@ -127,4 +131,5 @@ class TrainReader(Reader): def generate_batch_from_trainfiles(self, files): data_set = self.base_read(files) random.shuffle(data_set) - return self.batch_reader(data_set, self.batch_size, self.batch_size * 20) + return self.batch_reader(data_set, self.batch_size, + self.batch_size * 20) diff --git a/models/rank/dnn/model.py b/models/rank/dnn/model.py index 3a61d288b40545619a49e81df1f6160670a6a0c1..d7ab801f38fdffbdeb0ca5259abaec37136d3fc9 100755 --- a/models/rank/dnn/model.py +++ b/models/rank/dnn/model.py @@ -31,8 +31,10 @@ class Model(ModelBase): def net(self): is_distributed = True if envs.get_trainer() == "CtrTrainer" else False - sparse_feature_number = envs.get_global_env("hyper_parameters.sparse_feature_number", None, self._namespace) - sparse_feature_dim = envs.get_global_env("hyper_parameters.sparse_feature_dim", None, self._namespace) + sparse_feature_number = envs.get_global_env( + "hyper_parameters.sparse_feature_number", None, self._namespace) + sparse_feature_dim = envs.get_global_env( + "hyper_parameters.sparse_feature_dim", None, self._namespace) def embedding_layer(input): emb = fluid.layers.embedding( @@ -42,25 +44,27 @@ class Model(ModelBase): size=[sparse_feature_number, sparse_feature_dim], param_attr=fluid.ParamAttr( name="SparseFeatFactors", - initializer=fluid.initializer.Uniform()), - ) - emb_sum = fluid.layers.sequence_pool( - input=emb, pool_type='sum') + initializer=fluid.initializer.Uniform()), ) + emb_sum = fluid.layers.sequence_pool(input=emb, pool_type='sum') return emb_sum def fc(input, output_size): output = fluid.layers.fc( - input=input, size=output_size, - act='relu', param_attr=fluid.ParamAttr( + input=input, + size=output_size, + act='relu', + param_attr=fluid.ParamAttr( initializer=fluid.initializer.Normal( scale=1.0 / math.sqrt(input.shape[1])))) return output sparse_embed_seq = list(map(embedding_layer, self.sparse_inputs)) - concated = fluid.layers.concat(sparse_embed_seq + [self.dense_input], axis=1) + concated = fluid.layers.concat( + sparse_embed_seq + [self.dense_input], axis=1) fcs = [concated] - hidden_layers = envs.get_global_env("hyper_parameters.fc_sizes", None, self._namespace) + hidden_layers = envs.get_global_env("hyper_parameters.fc_sizes", None, + self._namespace) for size in hidden_layers: fcs.append(fc(fcs[-1], size)) @@ -75,14 +79,15 @@ class Model(ModelBase): self.predict = predict def avg_loss(self): - cost = fluid.layers.cross_entropy(input=self.predict, label=self.label_input) + cost = fluid.layers.cross_entropy( + input=self.predict, label=self.label_input) avg_cost = fluid.layers.reduce_mean(cost) self._cost = avg_cost def metrics(self): auc, batch_auc, _ = fluid.layers.auc(input=self.predict, label=self.label_input, - num_thresholds=2 ** 12, + num_thresholds=2**12, slide_steps=20) self._metrics["AUC"] = auc self._metrics["BATCH_AUC"] = batch_auc @@ -95,7 +100,8 @@ class Model(ModelBase): self.metrics() def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/rank/wide_deep/model.py b/models/rank/wide_deep/model.py index 27eb5e1f0c1588d7634407a3dcd250726dea28bb..a7d51d958c55f0368cdd7f9ff7baa51dd25a6f76 100755 --- a/models/rank/wide_deep/model.py +++ b/models/rank/wide_deep/model.py @@ -25,27 +25,27 @@ class Model(ModelBase): ModelBase.__init__(self, config) def wide_part(self, data): - out = fluid.layers.fc(input=data, - size=1, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.TruncatedNormal(loc=0.0, - scale=1.0 / math.sqrt( - data.shape[ - 1])), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4)), - act=None, - name='wide') + out = fluid.layers.fc( + input=data, + size=1, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.TruncatedNormal( + loc=0.0, scale=1.0 / math.sqrt(data.shape[1])), + regularizer=fluid.regularizer.L2DecayRegularizer( + regularization_coeff=1e-4)), + act=None, + name='wide') return out def fc(self, data, hidden_units, active, tag): - output = fluid.layers.fc(input=data, - size=hidden_units, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.TruncatedNormal(loc=0.0, - scale=1.0 / math.sqrt( - data.shape[ - 1]))), - act=active, - name=tag) + output = fluid.layers.fc( + input=data, + size=hidden_units, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.TruncatedNormal( + loc=0.0, scale=1.0 / math.sqrt(data.shape[1]))), + act=active, + name=tag) return output @@ -62,43 +62,63 @@ class Model(ModelBase): deep_input = self._dense_data_var[1] label = self._sparse_data_var[0] - hidden1_units = envs.get_global_env("hyper_parameters.hidden1_units", 75, self._namespace) - hidden2_units = envs.get_global_env("hyper_parameters.hidden2_units", 50, self._namespace) - hidden3_units = envs.get_global_env("hyper_parameters.hidden3_units", 25, self._namespace) + hidden1_units = envs.get_global_env("hyper_parameters.hidden1_units", + 75, self._namespace) + hidden2_units = envs.get_global_env("hyper_parameters.hidden2_units", + 50, self._namespace) + hidden3_units = envs.get_global_env("hyper_parameters.hidden3_units", + 25, self._namespace) wide_output = self.wide_part(wide_input) - deep_output = self.deep_part(deep_input, hidden1_units, hidden2_units, hidden3_units) - - wide_model = fluid.layers.fc(input=wide_output, - size=1, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.TruncatedNormal(loc=0.0, scale=1.0)), - act=None, - name='w_wide') - - deep_model = fluid.layers.fc(input=deep_output, - size=1, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.TruncatedNormal(loc=0.0, scale=1.0)), - act=None, - name='w_deep') + deep_output = self.deep_part(deep_input, hidden1_units, hidden2_units, + hidden3_units) + + wide_model = fluid.layers.fc( + input=wide_output, + size=1, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.TruncatedNormal( + loc=0.0, scale=1.0)), + act=None, + name='w_wide') + + deep_model = fluid.layers.fc( + input=deep_output, + size=1, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.TruncatedNormal( + loc=0.0, scale=1.0)), + act=None, + name='w_deep') prediction = fluid.layers.elementwise_add(wide_model, deep_model) - pred = fluid.layers.sigmoid(fluid.layers.clip(prediction, min=-15.0, max=15.0), name="prediction") + pred = fluid.layers.sigmoid( + fluid.layers.clip( + prediction, min=-15.0, max=15.0), + name="prediction") num_seqs = fluid.layers.create_tensor(dtype='int64') - acc = fluid.layers.accuracy(input=pred, label=fluid.layers.cast(x=label, dtype='int64'), total=num_seqs) - auc_var, batch_auc, auc_states = fluid.layers.auc(input=pred, label=fluid.layers.cast(x=label, dtype='int64')) + acc = fluid.layers.accuracy( + input=pred, + label=fluid.layers.cast( + x=label, dtype='int64'), + total=num_seqs) + auc_var, batch_auc, auc_states = fluid.layers.auc( + input=pred, label=fluid.layers.cast( + x=label, dtype='int64')) self._metrics["AUC"] = auc_var self._metrics["BATCH_AUC"] = batch_auc self._metrics["ACC"] = acc - cost = fluid.layers.sigmoid_cross_entropy_with_logits(x=prediction, label=fluid.layers.cast(label, dtype='float32')) + cost = fluid.layers.sigmoid_cross_entropy_with_logits( + x=prediction, label=fluid.layers.cast( + label, dtype='float32')) avg_cost = fluid.layers.mean(cost) self._cost = avg_cost def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/rank/xdeepfm/model.py b/models/rank/xdeepfm/model.py index 059e83d40290d713e9bc5f25f59fbb285e5a855a..d1045897d9cb4ca5b7018a1dcb9da726829c4744 100755 --- a/models/rank/xdeepfm/model.py +++ b/models/rank/xdeepfm/model.py @@ -28,18 +28,22 @@ class Model(ModelBase): loc=0.0, scale=init_value_) is_distributed = True if envs.get_trainer() == "CtrTrainer" else False - sparse_feature_number = envs.get_global_env("hyper_parameters.sparse_feature_number", None, self._namespace) - sparse_feature_dim = envs.get_global_env("hyper_parameters.sparse_feature_dim", None, self._namespace) + sparse_feature_number = envs.get_global_env( + "hyper_parameters.sparse_feature_number", None, self._namespace) + sparse_feature_dim = envs.get_global_env( + "hyper_parameters.sparse_feature_dim", None, self._namespace) # ------------------------- network input -------------------------- - num_field = envs.get_global_env("hyper_parameters.num_field", None, self._namespace) + num_field = envs.get_global_env("hyper_parameters.num_field", None, + self._namespace) raw_feat_idx = self._sparse_data_var[1] raw_feat_value = self._dense_data_var[0] self.label = self._sparse_data_var[0] feat_idx = raw_feat_idx - feat_value = fluid.layers.reshape(raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 + feat_value = fluid.layers.reshape( + raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 feat_embeddings = fluid.embedding( input=feat_idx, @@ -48,9 +52,9 @@ class Model(ModelBase): size=[sparse_feature_number + 1, sparse_feature_dim], padding_idx=0, param_attr=fluid.ParamAttr(initializer=initer)) - feat_embeddings = fluid.layers.reshape( - feat_embeddings, - [-1, num_field, sparse_feature_dim]) # None * num_field * embedding_size + feat_embeddings = fluid.layers.reshape(feat_embeddings, [ + -1, num_field, sparse_feature_dim + ]) # None * num_field * embedding_size feat_embeddings = feat_embeddings * feat_value # None * num_field * embedding_size # -------------------- linear -------------------- @@ -73,7 +77,8 @@ class Model(ModelBase): # -------------------- CIN -------------------- - layer_sizes_cin = envs.get_global_env("hyper_parameters.layer_sizes_cin", None, self._namespace) + layer_sizes_cin = envs.get_global_env( + "hyper_parameters.layer_sizes_cin", None, self._namespace) Xs = [feat_embeddings] last_s = num_field for s in layer_sizes_cin: @@ -84,7 +89,8 @@ class Model(ModelBase): 1]) # None, embedding_size, num_field, 1 X_k = fluid.layers.reshape( fluid.layers.transpose(Xs[-1], [0, 2, 1]), - [-1, sparse_feature_dim, 1, last_s]) # None, embedding_size, 1, last_s + [-1, sparse_feature_dim, 1, + last_s]) # None, embedding_size, 1, last_s Z_k_1 = fluid.layers.matmul( X_0, X_k) # None, embedding_size, num_field, last_s @@ -124,16 +130,19 @@ class Model(ModelBase): # -------------------- DNN -------------------- - layer_sizes_dnn = envs.get_global_env("hyper_parameters.layer_sizes_dnn", None, self._namespace) - act = envs.get_global_env("hyper_parameters.act", None, self._namespace) + layer_sizes_dnn = envs.get_global_env( + "hyper_parameters.layer_sizes_dnn", None, self._namespace) + act = envs.get_global_env("hyper_parameters.act", None, + self._namespace) y_dnn = fluid.layers.reshape(feat_embeddings, [-1, num_field * sparse_feature_dim]) for s in layer_sizes_dnn: - y_dnn = fluid.layers.fc(input=y_dnn, - size=s, - act=act, - param_attr=fluid.ParamAttr(initializer=initer), - bias_attr=None) + y_dnn = fluid.layers.fc( + input=y_dnn, + size=s, + act=act, + param_attr=fluid.ParamAttr(initializer=initer), + bias_attr=None) y_dnn = fluid.layers.fc(input=y_dnn, size=1, act=None, @@ -148,7 +157,10 @@ class Model(ModelBase): self.model._init_slots() self.xdeepfm_net() - cost = fluid.layers.log_loss(input=self.predict, label=fluid.layers.cast(self.label, "float32"), epsilon=0.0000001) + cost = fluid.layers.log_loss( + input=self.predict, + label=fluid.layers.cast(self.label, "float32"), + epsilon=0.0000001) batch_cost = fluid.layers.reduce_mean(cost) self._cost = batch_cost @@ -162,7 +174,8 @@ class Model(ModelBase): self._metrics["BATCH_AUC"] = batch_auc_var def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer diff --git a/models/recall/gnn/evaluate_reader.py b/models/recall/gnn/evaluate_reader.py index 904140c2febf5164592348d0b4e8f90f197bbf06..b26ea8fa9fc347ce402575104dcfa6de23aa80fc 100755 --- a/models/recall/gnn/evaluate_reader.py +++ b/models/recall/gnn/evaluate_reader.py @@ -23,7 +23,8 @@ from paddlerec.core.utils import envs class EvaluateReader(Reader): def init(self): - self.batch_size = envs.get_global_env("batch_size", None, "evaluate.reader") + self.batch_size = envs.get_global_env("batch_size", None, + "evaluate.reader") self.input = [] self.length = None @@ -34,7 +35,8 @@ class EvaluateReader(Reader): with open(f, "r") as fin: for line in fin: line = line.strip().split('\t') - res.append(tuple([map(int, line[0].split(',')), int(line[1])])) + res.append( + tuple([map(int, line[0].split(',')), int(line[1])])) return res def make_data(self, cur_batch, batch_size): @@ -75,10 +77,8 @@ class EvaluateReader(Reader): u_deg_out[np.where(u_deg_out == 0)] = 1 adj_out.append(np.divide(adj.transpose(), u_deg_out).transpose()) - seq_index.append( - [[id, np.where(node == i)[0][0]] for i in e[0]]) - last_index.append( - [id, np.where(node == e[0][last_id[id]])[0][0]]) + seq_index.append([[id, np.where(node == i)[0][0]] for i in e[0]]) + last_index.append([id, np.where(node == e[0][last_id[id]])[0][0]]) label.append(e[1] - 1) mask.append([[1] * (last_id[id] + 1) + [0] * (max_seq_len - last_id[id] - 1)]) @@ -101,10 +101,13 @@ class EvaluateReader(Reader): def _reader(): random.shuffle(self.input) group_remain = self.length % batch_group_size - for bg_id in range(0, self.length - group_remain, batch_group_size): - cur_bg = copy.deepcopy(self.input[bg_id:bg_id + batch_group_size]) + for bg_id in range(0, self.length - group_remain, + batch_group_size): + cur_bg = copy.deepcopy(self.input[bg_id:bg_id + + batch_group_size]) if train: - cur_bg = sorted(cur_bg, key=lambda x: len(x[0]), reverse=True) + cur_bg = sorted( + cur_bg, key=lambda x: len(x[0]), reverse=True) for i in range(0, batch_group_size, batch_size): cur_batch = cur_bg[i:i + batch_size] yield self.make_data(cur_batch, batch_size) diff --git a/models/recall/gnn/model.py b/models/recall/gnn/model.py index b98625a6afc094e106b26d1e2b31a8712a9d7b94..027fbb721131e203ed22485b4d8f9bd96b8ed3a3 100755 --- a/models/recall/gnn/model.py +++ b/models/recall/gnn/model.py @@ -30,15 +30,21 @@ class Model(ModelBase): def init_config(self): self._fetch_interval = 1 self.items_num, self.ins_num = self.config_read( - envs.get_global_env("hyper_parameters.config_path", None, self._namespace)) - self.train_batch_size = envs.get_global_env("batch_size", None, "train.reader") - self.evaluate_batch_size = envs.get_global_env("batch_size", None, "evaluate.reader") - self.hidden_size = envs.get_global_env("hyper_parameters.sparse_feature_dim", None, self._namespace) - self.step = envs.get_global_env("hyper_parameters.gnn_propogation_steps", None, self._namespace) + envs.get_global_env("hyper_parameters.config_path", None, + self._namespace)) + self.train_batch_size = envs.get_global_env("batch_size", None, + "train.reader") + self.evaluate_batch_size = envs.get_global_env("batch_size", None, + "evaluate.reader") + self.hidden_size = envs.get_global_env( + "hyper_parameters.sparse_feature_dim", None, self._namespace) + self.step = envs.get_global_env( + "hyper_parameters.gnn_propogation_steps", None, self._namespace) def config_read(self, config_path=None): if config_path is None: - raise ValueError("please set train.model.hyper_parameters.config_path at first") + raise ValueError( + "please set train.model.hyper_parameters.config_path at first") with open(config_path, "r") as fin: item_nums = int(fin.readline().strip()) ins_nums = int(fin.readline().strip()) @@ -46,100 +52,108 @@ class Model(ModelBase): def input(self, bs): self.items = fluid.data( - name="items", - shape=[bs, -1], + name="items", shape=[bs, -1], dtype="int64") # [batch_size, uniq_max] self.seq_index = fluid.data( - name="seq_index", - shape=[bs, -1, 2], + name="seq_index", shape=[bs, -1, 2], dtype="int32") # [batch_size, seq_max, 2] self.last_index = fluid.data( - name="last_index", - shape=[bs, 2], - dtype="int32") # [batch_size, 2] + name="last_index", shape=[bs, 2], dtype="int32") # [batch_size, 2] self.adj_in = fluid.data( - name="adj_in", - shape=[bs, -1, -1], + name="adj_in", shape=[bs, -1, -1], dtype="float32") # [batch_size, seq_max, seq_max] self.adj_out = fluid.data( - name="adj_out", - shape=[bs, -1, -1], + name="adj_out", shape=[bs, -1, -1], dtype="float32") # [batch_size, seq_max, seq_max] self.mask = fluid.data( - name="mask", - shape=[bs, -1, 1], + name="mask", shape=[bs, -1, 1], dtype="float32") # [batch_size, seq_max, 1] self.label = fluid.data( - name="label", - shape=[bs, 1], - dtype="int64") # [batch_size, 1] + name="label", shape=[bs, 1], dtype="int64") # [batch_size, 1] - res = [self.items, self.seq_index, self.last_index, self.adj_in, self.adj_out, self.mask, self.label] + res = [ + self.items, self.seq_index, self.last_index, self.adj_in, + self.adj_out, self.mask, self.label + ] return res def train_input(self): res = self.input(self.train_batch_size) self._data_var = res - use_dataloader = envs.get_global_env("hyper_parameters.use_DataLoader", False, self._namespace) + use_dataloader = envs.get_global_env("hyper_parameters.use_DataLoader", + False, self._namespace) if self._platform != "LINUX" or use_dataloader: self._data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._data_var, capacity=256, use_double_buffer=False, iterable=False) + feed_list=self._data_var, + capacity=256, + use_double_buffer=False, + iterable=False) def net(self, items_num, hidden_size, step, bs): stdv = 1.0 / math.sqrt(hidden_size) - def embedding_layer(input, table_name, emb_dim, initializer_instance=None): + def embedding_layer(input, + table_name, + emb_dim, + initializer_instance=None): emb = fluid.embedding( input=input, size=[items_num, emb_dim], param_attr=fluid.ParamAttr( - name=table_name, - initializer=initializer_instance), - ) + name=table_name, initializer=initializer_instance), ) return emb sparse_initializer = fluid.initializer.Uniform(low=-stdv, high=stdv) - items_emb = embedding_layer(self.items, "emb", hidden_size, sparse_initializer) + items_emb = embedding_layer(self.items, "emb", hidden_size, + sparse_initializer) pre_state = items_emb for i in range(step): - pre_state = layers.reshape(x=pre_state, shape=[bs, -1, hidden_size]) + pre_state = layers.reshape( + x=pre_state, shape=[bs, -1, hidden_size]) state_in = layers.fc( input=pre_state, name="state_in", size=hidden_size, act=None, num_flatten_dims=2, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv))) # [batch_size, uniq_max, h] + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv)), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv))) # [batch_size, uniq_max, h] state_out = layers.fc( input=pre_state, name="state_out", size=hidden_size, act=None, num_flatten_dims=2, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv))) # [batch_size, uniq_max, h] + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv)), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv))) # [batch_size, uniq_max, h] - state_adj_in = layers.matmul(self.adj_in, state_in) # [batch_size, uniq_max, h] - state_adj_out = layers.matmul(self.adj_out, state_out) # [batch_size, uniq_max, h] + state_adj_in = layers.matmul(self.adj_in, + state_in) # [batch_size, uniq_max, h] + state_adj_out = layers.matmul( + self.adj_out, state_out) # [batch_size, uniq_max, h] gru_input = layers.concat([state_adj_in, state_adj_out], axis=2) - gru_input = layers.reshape(x=gru_input, shape=[-1, hidden_size * 2]) - gru_fc = layers.fc( - input=gru_input, - name="gru_fc", - size=3 * hidden_size, - bias_attr=False) + gru_input = layers.reshape( + x=gru_input, shape=[-1, hidden_size * 2]) + gru_fc = layers.fc(input=gru_input, + name="gru_fc", + size=3 * hidden_size, + bias_attr=False) pre_state, _, _ = fluid.layers.gru_unit( input=gru_fc, - hidden=layers.reshape(x=pre_state, shape=[-1, hidden_size]), + hidden=layers.reshape( + x=pre_state, shape=[-1, hidden_size]), size=3 * hidden_size) final_state = layers.reshape(pre_state, shape=[bs, -1, hidden_size]) @@ -153,24 +167,22 @@ class Model(ModelBase): bias_attr=False, act=None, num_flatten_dims=2, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv))) # [batch_size, seq_max, h] - last_fc = layers.fc( - input=last, - name="last_fc", - size=hidden_size, - bias_attr=False, - act=None, - num_flatten_dims=1, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv))) # [bathc_size, h] + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv))) # [batch_size, seq_max, h] + last_fc = layers.fc(input=last, + name="last_fc", + size=hidden_size, + bias_attr=False, + act=None, + num_flatten_dims=1, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv))) # [bathc_size, h] seq_fc_t = layers.transpose( seq_fc, perm=[1, 0, 2]) # [seq_max, batch_size, h] - add = layers.elementwise_add( - seq_fc_t, last_fc) # [seq_max, batch_size, h] + add = layers.elementwise_add(seq_fc_t, + last_fc) # [seq_max, batch_size, h] b = layers.create_parameter( shape=[hidden_size], dtype='float32', @@ -188,12 +200,13 @@ class Model(ModelBase): act=None, num_flatten_dims=2, bias_attr=False, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-stdv, high=stdv))) # [batch_size, seq_max, 1] + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform( + low=-stdv, high=stdv))) # [batch_size, seq_max, 1] weight *= self.mask - weight_mask = layers.elementwise_mul(seq, weight, axis=0) # [batch_size, seq_max, h] - global_attention = layers.reduce_sum(weight_mask, dim=1) # [batch_size, h] + weight_mask = layers.elementwise_mul( + seq, weight, axis=0) # [batch_size, seq_max, h] + global_attention = layers.reduce_sum( + weight_mask, dim=1) # [batch_size, h] final_attention = layers.concat( [global_attention, last], axis=1) # [batch_size, 2*h] @@ -213,7 +226,8 @@ class Model(ModelBase): # persistable=True, # name="all_vocab") all_vocab = np.arange(1, items_num).reshape((-1)).astype('int32') - all_vocab = fluid.layers.cast(x=fluid.layers.assign(all_vocab), dtype='int64') + all_vocab = fluid.layers.cast( + x=fluid.layers.assign(all_vocab), dtype='int64') all_emb = fluid.embedding( input=all_vocab, @@ -240,15 +254,19 @@ class Model(ModelBase): def train_net(self): self.train_input() - self.net(self.items_num, self.hidden_size, self.step, self.train_batch_size) + self.net(self.items_num, self.hidden_size, self.step, + self.train_batch_size) self.avg_loss() self.metrics() def optimizer(self): - learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) step_per_epoch = self.ins_num // self.train_batch_size - decay_steps = envs.get_global_env("hyper_parameters.decay_steps", None, self._namespace) - decay_rate = envs.get_global_env("hyper_parameters.decay_rate", None, self._namespace) + decay_steps = envs.get_global_env("hyper_parameters.decay_steps", None, + self._namespace) + decay_rate = envs.get_global_env("hyper_parameters.decay_rate", None, + self._namespace) l2 = envs.get_global_env("hyper_parameters.l2", None, self._namespace) optimizer = fluid.optimizer.Adam( learning_rate=fluid.layers.exponential_decay( @@ -266,10 +284,14 @@ class Model(ModelBase): self._infer_data_var = res self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def infer_net(self): self.infer_input() - self.net(self.items_num, self.hidden_size, self.step, self.evaluate_batch_size) + self.net(self.items_num, self.hidden_size, self.step, + self.evaluate_batch_size) self._infer_results['acc'] = self.acc self._infer_results['loss'] = self.loss diff --git a/models/recall/gnn/reader.py b/models/recall/gnn/reader.py index cffb45115ed6a3dd6232b34db8758ad6a20447e2..68170f09a7a7c84547a67f970b6e127de40b0ccc 100755 --- a/models/recall/gnn/reader.py +++ b/models/recall/gnn/reader.py @@ -23,7 +23,8 @@ from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): - self.batch_size = envs.get_global_env("batch_size", None, "train.reader") + self.batch_size = envs.get_global_env("batch_size", None, + "train.reader") self.input = [] self.length = None @@ -34,7 +35,8 @@ class TrainReader(Reader): with open(f, "r") as fin: for line in fin: line = line.strip().split('\t') - res.append(tuple([map(int, line[0].split(',')), int(line[1])])) + res.append( + tuple([map(int, line[0].split(',')), int(line[1])])) return res def make_data(self, cur_batch, batch_size): @@ -75,10 +77,8 @@ class TrainReader(Reader): u_deg_out[np.where(u_deg_out == 0)] = 1 adj_out.append(np.divide(adj.transpose(), u_deg_out).transpose()) - seq_index.append( - [[id, np.where(node == i)[0][0]] for i in e[0]]) - last_index.append( - [id, np.where(node == e[0][last_id[id]])[0][0]]) + seq_index.append([[id, np.where(node == i)[0][0]] for i in e[0]]) + last_index.append([id, np.where(node == e[0][last_id[id]])[0][0]]) label.append(e[1] - 1) mask.append([[1] * (last_id[id] + 1) + [0] * (max_seq_len - last_id[id] - 1)]) @@ -101,10 +101,13 @@ class TrainReader(Reader): def _reader(): random.shuffle(self.input) group_remain = self.length % batch_group_size - for bg_id in range(0, self.length - group_remain, batch_group_size): - cur_bg = copy.deepcopy(self.input[bg_id:bg_id + batch_group_size]) + for bg_id in range(0, self.length - group_remain, + batch_group_size): + cur_bg = copy.deepcopy(self.input[bg_id:bg_id + + batch_group_size]) if train: - cur_bg = sorted(cur_bg, key=lambda x: len(x[0]), reverse=True) + cur_bg = sorted( + cur_bg, key=lambda x: len(x[0]), reverse=True) for i in range(0, batch_group_size, batch_size): cur_batch = cur_bg[i:i + batch_size] yield self.make_data(cur_batch, batch_size) diff --git a/models/recall/gru4rec/model.py b/models/recall/gru4rec/model.py index b79c7642201990efae56a640954154404bf2e606..6848f1e65d51c9d5e3f9890b3f3f148ef68829fc 100644 --- a/models/recall/gru4rec/model.py +++ b/models/recall/gru4rec/model.py @@ -24,14 +24,22 @@ class Model(ModelBase): def all_vocab_network(self, is_infer=False): """ network definition """ - recall_k = envs.get_global_env("hyper_parameters.recall_k", None, self._namespace) - vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) - hid_size = envs.get_global_env("hyper_parameters.hid_size", None, self._namespace) - init_low_bound = envs.get_global_env("hyper_parameters.init_low_bound", None, self._namespace) - init_high_bound = envs.get_global_env("hyper_parameters.init_high_bound", None, self._namespace) - emb_lr_x = envs.get_global_env("hyper_parameters.emb_lr_x", None, self._namespace) - gru_lr_x = envs.get_global_env("hyper_parameters.gru_lr_x", None, self._namespace) - fc_lr_x = envs.get_global_env("hyper_parameters.fc_lr_x", None, self._namespace) + recall_k = envs.get_global_env("hyper_parameters.recall_k", None, + self._namespace) + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, + self._namespace) + hid_size = envs.get_global_env("hyper_parameters.hid_size", None, + self._namespace) + init_low_bound = envs.get_global_env("hyper_parameters.init_low_bound", + None, self._namespace) + init_high_bound = envs.get_global_env( + "hyper_parameters.init_high_bound", None, self._namespace) + emb_lr_x = envs.get_global_env("hyper_parameters.emb_lr_x", None, + self._namespace) + gru_lr_x = envs.get_global_env("hyper_parameters.gru_lr_x", None, + self._namespace) + fc_lr_x = envs.get_global_env("hyper_parameters.fc_lr_x", None, + self._namespace) # Input data src_wordseq = fluid.data( name="src_wordseq", shape=[None, 1], dtype="int64", lod_level=1) @@ -41,7 +49,10 @@ class Model(ModelBase): if is_infer: self._infer_data_var = [src_wordseq, dst_wordseq] self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) emb = fluid.embedding( input=src_wordseq, @@ -56,7 +67,8 @@ class Model(ModelBase): size=hid_size * 3, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Uniform( - low=init_low_bound, high=init_high_bound), + low=init_low_bound, + high=init_high_bound), learning_rate=gru_lr_x)) gru_h0 = fluid.layers.dynamic_gru( input=fc0, diff --git a/models/recall/ncf/model.py b/models/recall/ncf/model.py index be7c465dc75d7186f6d63a6d1fbf604f84945891..d2b7fa371be8f068e11e1dd37a63a90b55e96e65 100644 --- a/models/recall/ncf/model.py +++ b/models/recall/ncf/model.py @@ -25,9 +25,12 @@ class Model(ModelBase): ModelBase.__init__(self, config) def input_data(self, is_infer=False): - user_input = fluid.data(name="user_input", shape=[-1, 1], dtype="int64", lod_level=0) - item_input = fluid.data(name="item_input", shape=[-1, 1], dtype="int64", lod_level=0) - label = fluid.data(name="label", shape=[-1, 1], dtype="int64", lod_level=0) + user_input = fluid.data( + name="user_input", shape=[-1, 1], dtype="int64", lod_level=0) + item_input = fluid.data( + name="item_input", shape=[-1, 1], dtype="int64", lod_level=0) + label = fluid.data( + name="label", shape=[-1, 1], dtype="int64", lod_level=0) if is_infer: inputs = [user_input] + [item_input] else: @@ -35,81 +38,104 @@ class Model(ModelBase): self._data_var = inputs return inputs - + def net(self, inputs, is_infer=False): - num_users = envs.get_global_env("hyper_parameters.num_users", None, self._namespace) - num_items = envs.get_global_env("hyper_parameters.num_items", None, self._namespace) - latent_dim = envs.get_global_env("hyper_parameters.latent_dim", None, self._namespace) - layers = envs.get_global_env("hyper_parameters.layers", None, self._namespace) - - num_layer = len(layers) #Number of layers in the MLP - - MF_Embedding_User = fluid.embedding(input=inputs[0], - size=[num_users, latent_dim], - param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), - is_sparse=True) - MF_Embedding_Item = fluid.embedding(input=inputs[1], - size=[num_items, latent_dim], - param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), - is_sparse=True) - - MLP_Embedding_User = fluid.embedding(input=inputs[0], - size=[num_users, int(layers[0] / 2)], - param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), - is_sparse=True) - MLP_Embedding_Item = fluid.embedding(input=inputs[1], - size=[num_items, int(layers[0] / 2)], - param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), - is_sparse=True) - + num_users = envs.get_global_env("hyper_parameters.num_users", None, + self._namespace) + num_items = envs.get_global_env("hyper_parameters.num_items", None, + self._namespace) + latent_dim = envs.get_global_env("hyper_parameters.latent_dim", None, + self._namespace) + layers = envs.get_global_env("hyper_parameters.layers", None, + self._namespace) + + num_layer = len(layers) #Number of layers in the MLP + + MF_Embedding_User = fluid.embedding( + input=inputs[0], + size=[num_users, latent_dim], + param_attr=fluid.initializer.Normal( + loc=0.0, scale=0.01), + is_sparse=True) + MF_Embedding_Item = fluid.embedding( + input=inputs[1], + size=[num_items, latent_dim], + param_attr=fluid.initializer.Normal( + loc=0.0, scale=0.01), + is_sparse=True) + + MLP_Embedding_User = fluid.embedding( + input=inputs[0], + size=[num_users, int(layers[0] / 2)], + param_attr=fluid.initializer.Normal( + loc=0.0, scale=0.01), + is_sparse=True) + MLP_Embedding_Item = fluid.embedding( + input=inputs[1], + size=[num_items, int(layers[0] / 2)], + param_attr=fluid.initializer.Normal( + loc=0.0, scale=0.01), + is_sparse=True) + # MF part mf_user_latent = fluid.layers.flatten(x=MF_Embedding_User, axis=1) mf_item_latent = fluid.layers.flatten(x=MF_Embedding_Item, axis=1) - mf_vector = fluid.layers.elementwise_mul(mf_user_latent, mf_item_latent) - + mf_vector = fluid.layers.elementwise_mul(mf_user_latent, + mf_item_latent) + # MLP part # The 0-th layer is the concatenation of embedding layers mlp_user_latent = fluid.layers.flatten(x=MLP_Embedding_User, axis=1) mlp_item_latent = fluid.layers.flatten(x=MLP_Embedding_Item, axis=1) - mlp_vector = fluid.layers.concat(input=[mlp_user_latent, mlp_item_latent], axis=-1) - + mlp_vector = fluid.layers.concat( + input=[mlp_user_latent, mlp_item_latent], axis=-1) + for i in range(1, num_layer): - mlp_vector = fluid.layers.fc(input=mlp_vector, - size=layers[i], - act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.TruncatedNormal(loc=0.0, scale=1.0 / math.sqrt(mlp_vector.shape[1])), - regularizer=fluid.regularizer.L2DecayRegularizer(regularization_coeff=1e-4)), - name='layer_' + str(i)) - + mlp_vector = fluid.layers.fc( + input=mlp_vector, + size=layers[i], + act='relu', + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.TruncatedNormal( + loc=0.0, scale=1.0 / math.sqrt(mlp_vector.shape[1])), + regularizer=fluid.regularizer.L2DecayRegularizer( + regularization_coeff=1e-4)), + name='layer_' + str(i)) + # Concatenate MF and MLP parts - predict_vector = fluid.layers.concat(input=[mf_vector, mlp_vector], axis=-1) + predict_vector = fluid.layers.concat( + input=[mf_vector, mlp_vector], axis=-1) # Final prediction layer - prediction = fluid.layers.fc(input=predict_vector, - size=1, - act='sigmoid', - param_attr=fluid.initializer.MSRAInitializer(uniform=True), - name='prediction') + prediction = fluid.layers.fc( + input=predict_vector, + size=1, + act='sigmoid', + param_attr=fluid.initializer.MSRAInitializer(uniform=True), + name='prediction') if is_infer: self._infer_results["prediction"] = prediction return - - cost = fluid.layers.log_loss(input=prediction, label=fluid.layers.cast(x=inputs[2], dtype='float32')) + + cost = fluid.layers.log_loss( + input=prediction, + label=fluid.layers.cast( + x=inputs[2], dtype='float32')) avg_cost = fluid.layers.mean(cost) - self._cost = avg_cost self._metrics["cost"] = avg_cost - def train_net(self): input_data = self.input_data() self.net(input_data) - def infer_net(self): self._infer_data_var = self.input_data(is_infer=True) self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) self.net(self._infer_data_var, is_infer=True) diff --git a/models/recall/ncf/movielens_infer_reader.py b/models/recall/ncf/movielens_infer_reader.py index 04f159962e89b28d0e044cfbbc1fcae5a15f3f0d..dc737aed2b8f93a5d4274938cf468e8d9240be04 100644 --- a/models/recall/ncf/movielens_infer_reader.py +++ b/models/recall/ncf/movielens_infer_reader.py @@ -33,7 +33,9 @@ class EvaluateReader(Reader): This function needs to be implemented by the user, based on data format """ features = line.strip().split(',') - + feature_name = ["user_input", "item_input"] - yield zip(feature_name, [[int(features[0])]] + [[int(features[1])]]) + yield zip(feature_name, + [[int(features[0])]] + [[int(features[1])]]) + return reader diff --git a/models/recall/ncf/movielens_reader.py b/models/recall/ncf/movielens_reader.py index 789a71add824e9759734be3bc571ec2152e9f50c..add9b6397cef93f3a8f416f19c6847c41537fb5f 100644 --- a/models/recall/ncf/movielens_reader.py +++ b/models/recall/ncf/movielens_reader.py @@ -33,10 +33,9 @@ class TrainReader(Reader): This function needs to be implemented by the user, based on data format """ features = line.strip().split(',') - + feature_name = ["user_input", "item_input", "label"] - yield zip(feature_name, [[int(features[0])]] + [[int(features[1])]] + [[int(features[2])]]) + yield zip(feature_name, [[int(features[0])]] + + [[int(features[1])]] + [[int(features[2])]]) return reader - - diff --git a/models/recall/ssr/model.py b/models/recall/ssr/model.py index 2c4b7f190088cd7681720f83e3a53730b790d462..3abe3ae41514d97d46d86b52680076cf5932386c 100644 --- a/models/recall/ssr/model.py +++ b/models/recall/ssr/model.py @@ -79,9 +79,12 @@ class Model(ModelBase): return correct def train(self): - vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) - emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, self._namespace) - hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, self._namespace) + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, + self._namespace) + emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, + self._namespace) + hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, + self._namespace) emb_shape = [vocab_size, emb_dim] self.user_encoder = GrnnEncoder() @@ -131,24 +134,34 @@ class Model(ModelBase): self.train() def infer(self): - vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) - emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, self._namespace) - hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, self._namespace) + vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, + self._namespace) + emb_dim = envs.get_global_env("hyper_parameters.emb_dim", None, + self._namespace) + hidden_size = envs.get_global_env("hyper_parameters.hidden_size", None, + self._namespace) user_data = fluid.data( name="user", shape=[None, 1], dtype="int64", lod_level=1) all_item_data = fluid.data( name="all_item", shape=[None, vocab_size], dtype="int64") - pos_label = fluid.data(name="pos_label", shape=[None, 1], dtype="int64") + pos_label = fluid.data( + name="pos_label", shape=[None, 1], dtype="int64") self._infer_data_var = [user_data, all_item_data, pos_label] self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) user_emb = fluid.embedding( input=user_data, size=[vocab_size, emb_dim], param_attr="emb.item") all_item_emb = fluid.embedding( - input=all_item_data, size=[vocab_size, emb_dim], param_attr="emb.item") - all_item_emb_re = fluid.layers.reshape(x=all_item_emb, shape=[-1, emb_dim]) + input=all_item_data, + size=[vocab_size, emb_dim], + param_attr="emb.item") + all_item_emb_re = fluid.layers.reshape( + x=all_item_emb, shape=[-1, emb_dim]) user_encoder = GrnnEncoder() user_enc = user_encoder.forward(user_emb) @@ -156,7 +169,8 @@ class Model(ModelBase): size=hidden_size, param_attr='user.w', bias_attr="user.b") - user_exp = fluid.layers.expand(x=user_hid, expand_times=[1, vocab_size]) + user_exp = fluid.layers.expand( + x=user_hid, expand_times=[1, vocab_size]) user_re = fluid.layers.reshape(x=user_exp, shape=[-1, hidden_size]) all_item_hid = fluid.layers.fc(input=all_item_emb_re, diff --git a/models/recall/ssr/ssr_infer_reader.py b/models/recall/ssr/ssr_infer_reader.py index 18f3fc2f37236907801fb00047fd3b6da5b5fa8c..1f94b1d21fbd428282d3e9faecd09a590588fbc9 100644 --- a/models/recall/ssr/ssr_infer_reader.py +++ b/models/recall/ssr/ssr_infer_reader.py @@ -22,7 +22,8 @@ from paddlerec.core.utils import envs class EvaluateReader(Reader): def init(self): - self.vocab_size = envs.get_global_env("vocab_size", 10, "train.model.hyper_parameters") + self.vocab_size = envs.get_global_env("vocab_size", 10, + "train.model.hyper_parameters") def generate_sample(self, line): """ @@ -39,6 +40,9 @@ class EvaluateReader(Reader): src = conv_ids[:boundary] pos_tgt = [conv_ids[boundary]] feature_name = ["user", "all_item", "p_item"] - yield zip(feature_name, [src] + [np.arange(self.vocab_size).astype("int64").tolist()] + [pos_tgt]) + yield zip( + feature_name, + [src] + [np.arange(self.vocab_size).astype("int64").tolist()] + + [pos_tgt]) return reader diff --git a/models/recall/word2vec/model.py b/models/recall/word2vec/model.py index bf09a04648a71a6618b99ef7de7d7244aaecbdba..fefc89043c2f926f37318e1094b9cdf98dd6235a 100755 --- a/models/recall/word2vec/model.py +++ b/models/recall/word2vec/model.py @@ -24,46 +24,57 @@ class Model(ModelBase): ModelBase.__init__(self, config) def input(self): - neg_num = int(envs.get_global_env( - "hyper_parameters.neg_num", None, self._namespace)) - self.input_word = fluid.data(name="input_word", shape=[ - None, 1], dtype='int64') - self.true_word = fluid.data(name='true_label', shape=[ - None, 1], dtype='int64') + neg_num = int( + envs.get_global_env("hyper_parameters.neg_num", None, + self._namespace)) + self.input_word = fluid.data( + name="input_word", shape=[None, 1], dtype='int64') + self.true_word = fluid.data( + name='true_label', shape=[None, 1], dtype='int64') self._data_var.append(self.input_word) self._data_var.append(self.true_word) - with_shuffle_batch = bool(int(envs.get_global_env( - "hyper_parameters.with_shuffle_batch", None, self._namespace))) + with_shuffle_batch = bool( + int( + envs.get_global_env("hyper_parameters.with_shuffle_batch", + None, self._namespace))) if not with_shuffle_batch: - self.neg_word = fluid.data(name="neg_label", shape=[ - None, neg_num], dtype='int64') + self.neg_word = fluid.data( + name="neg_label", shape=[None, neg_num], dtype='int64') self._data_var.append(self.neg_word) if self._platform != "LINUX": self._data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def net(self): is_distributed = True if envs.get_trainer() == "CtrTrainer" else False - neg_num = int(envs.get_global_env( - "hyper_parameters.neg_num", None, self._namespace)) + neg_num = int( + envs.get_global_env("hyper_parameters.neg_num", None, + self._namespace)) sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None, self._namespace) sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None, self._namespace) - with_shuffle_batch = bool(int(envs.get_global_env( - "hyper_parameters.with_shuffle_batch", None, self._namespace))) + with_shuffle_batch = bool( + int( + envs.get_global_env("hyper_parameters.with_shuffle_batch", + None, self._namespace))) - def embedding_layer(input, table_name, emb_dim, initializer_instance=None, squeeze=False): + def embedding_layer(input, + table_name, + emb_dim, + initializer_instance=None, + squeeze=False): emb = fluid.embedding( input=input, is_sparse=True, is_distributed=is_distributed, size=[sparse_feature_number, emb_dim], param_attr=fluid.ParamAttr( - name=table_name, - initializer=initializer_instance), - ) + name=table_name, initializer=initializer_instance), ) if squeeze: return fluid.layers.squeeze(input=emb, axes=[1]) else: @@ -73,35 +84,38 @@ class Model(ModelBase): emb_initializer = fluid.initializer.Uniform(-init_width, init_width) emb_w_initializer = fluid.initializer.Constant(value=0.0) - input_emb = embedding_layer( - self.input_word, "emb", sparse_feature_dim, emb_initializer, True) - true_emb_w = embedding_layer( - self.true_word, "emb_w", sparse_feature_dim, emb_w_initializer, True) - true_emb_b = embedding_layer( - self.true_word, "emb_b", 1, emb_w_initializer, True) + input_emb = embedding_layer(self.input_word, "emb", sparse_feature_dim, + emb_initializer, True) + true_emb_w = embedding_layer(self.true_word, "emb_w", + sparse_feature_dim, emb_w_initializer, + True) + true_emb_b = embedding_layer(self.true_word, "emb_b", 1, + emb_w_initializer, True) if with_shuffle_batch: neg_emb_w_list = [] for i in range(neg_num): - neg_emb_w_list.append(fluid.contrib.layers.shuffle_batch( - true_emb_w)) # shuffle true_word + neg_emb_w_list.append( + fluid.contrib.layers.shuffle_batch( + true_emb_w)) # shuffle true_word neg_emb_w_concat = fluid.layers.concat(neg_emb_w_list, axis=0) neg_emb_w = fluid.layers.reshape( neg_emb_w_concat, shape=[-1, neg_num, sparse_feature_dim]) neg_emb_b_list = [] for i in range(neg_num): - neg_emb_b_list.append(fluid.contrib.layers.shuffle_batch( - true_emb_b)) # shuffle true_word + neg_emb_b_list.append( + fluid.contrib.layers.shuffle_batch( + true_emb_b)) # shuffle true_word neg_emb_b = fluid.layers.concat(neg_emb_b_list, axis=0) neg_emb_b_vec = fluid.layers.reshape( neg_emb_b, shape=[-1, neg_num]) else: - neg_emb_w = embedding_layer( - self.neg_word, "emb_w", sparse_feature_dim, emb_w_initializer) - neg_emb_b = embedding_layer( - self.neg_word, "emb_b", 1, emb_w_initializer) + neg_emb_w = embedding_layer(self.neg_word, "emb_w", + sparse_feature_dim, emb_w_initializer) + neg_emb_b = embedding_layer(self.neg_word, "emb_b", 1, + emb_w_initializer) neg_emb_b_vec = fluid.layers.reshape( neg_emb_b, shape=[-1, neg_num]) @@ -117,7 +131,8 @@ class Model(ModelBase): neg_matmul = fluid.layers.matmul( input_emb_re, neg_emb_w, transpose_y=True) neg_logits = fluid.layers.elementwise_add( - fluid.layers.reshape(neg_matmul, shape=[-1, neg_num]), + fluid.layers.reshape( + neg_matmul, shape=[-1, neg_num]), neg_emb_b_vec) label_ones = fluid.layers.fill_constant_batch_size_like( @@ -136,9 +151,17 @@ class Model(ModelBase): neg_xent, dim=1)) self.avg_cost = fluid.layers.reduce_mean(cost) global_right_cnt = fluid.layers.create_global_var( - name="global_right_cnt", persistable=True, dtype='float32', shape=[1], value=0) + name="global_right_cnt", + persistable=True, + dtype='float32', + shape=[1], + value=0) global_total_cnt = fluid.layers.create_global_var( - name="global_total_cnt", persistable=True, dtype='float32', shape=[1], value=0) + name="global_total_cnt", + persistable=True, + dtype='float32', + shape=[1], + value=0) global_right_cnt.stop_gradient = True global_total_cnt.stop_gradient = True @@ -155,12 +178,12 @@ class Model(ModelBase): self.metrics() def optimizer(self): - learning_rate = envs.get_global_env( - "hyper_parameters.learning_rate", None, self._namespace) - decay_steps = envs.get_global_env( - "hyper_parameters.decay_steps", None, self._namespace) - decay_rate = envs.get_global_env( - "hyper_parameters.decay_rate", None, self._namespace) + learning_rate = envs.get_global_env("hyper_parameters.learning_rate", + None, self._namespace) + decay_steps = envs.get_global_env("hyper_parameters.decay_steps", None, + self._namespace) + decay_rate = envs.get_global_env("hyper_parameters.decay_rate", None, + self._namespace) optimizer = fluid.optimizer.SGD( learning_rate=fluid.layers.exponential_decay( learning_rate=learning_rate, @@ -180,11 +203,15 @@ class Model(ModelBase): name="analogy_c", shape=[None], dtype='int64') self.analogy_d = fluid.data( name="analogy_d", shape=[None], dtype='int64') - self._infer_data_var = [self.analogy_a, - self.analogy_b, self.analogy_c, self.analogy_d] + self._infer_data_var = [ + self.analogy_a, self.analogy_b, self.analogy_c, self.analogy_d + ] self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def infer_net(self): sparse_feature_dim = envs.get_global_env( @@ -216,18 +243,28 @@ class Model(ModelBase): dist = fluid.layers.matmul( x=target, y=emb_all_label_l2, transpose_y=True) values, pred_idx = fluid.layers.topk(input=dist, k=4) - label = fluid.layers.expand(fluid.layers.unsqueeze( - self.analogy_d, axes=[1]), expand_times=[1, 4]) + label = fluid.layers.expand( + fluid.layers.unsqueeze( + self.analogy_d, axes=[1]), + expand_times=[1, 4]) label_ones = fluid.layers.fill_constant_batch_size_like( label, shape=[-1, 1], value=1.0, dtype='float32') - right_cnt = fluid.layers.reduce_sum( - input=fluid.layers.cast(fluid.layers.equal(pred_idx, label), dtype='float32')) + right_cnt = fluid.layers.reduce_sum(input=fluid.layers.cast( + fluid.layers.equal(pred_idx, label), dtype='float32')) total_cnt = fluid.layers.reduce_sum(label_ones) global_right_cnt = fluid.layers.create_global_var( - name="global_right_cnt", persistable=True, dtype='float32', shape=[1], value=0) + name="global_right_cnt", + persistable=True, + dtype='float32', + shape=[1], + value=0) global_total_cnt = fluid.layers.create_global_var( - name="global_total_cnt", persistable=True, dtype='float32', shape=[1], value=0) + name="global_total_cnt", + persistable=True, + dtype='float32', + shape=[1], + value=0) global_right_cnt.stop_gradient = True global_total_cnt.stop_gradient = True diff --git a/models/recall/word2vec/preprocess.py b/models/recall/word2vec/preprocess.py index 9c9934e40589bdc700b7df5dc432d9b6dc92a8cc..6c9ee16cd2d136006dc10e7ce0c970974e8bf2b5 100755 --- a/models/recall/word2vec/preprocess.py +++ b/models/recall/word2vec/preprocess.py @@ -49,8 +49,7 @@ def parse_args(): '--file_nums', type=int, default=1024, - help="re-split input corpus file nums" - ) + help="re-split input corpus file nums") parser.add_argument( '--downsample', type=float, @@ -137,9 +136,11 @@ def filter_corpus(args): if not os.path.exists(args.output_corpus_dir): os.makedirs(args.output_corpus_dir) for file in os.listdir(args.input_corpus_dir): - with io.open(args.output_corpus_dir + '/convert_' + file + '.csv', "w") as wf: + with io.open(args.output_corpus_dir + '/convert_' + file + '.csv', + "w") as wf: with io.open( - args.input_corpus_dir + '/' + file, encoding='utf-8') as rf: + args.input_corpus_dir + '/' + file, + encoding='utf-8') as rf: print(args.input_corpus_dir + '/' + file) for line in rf: signal = False @@ -154,9 +155,9 @@ def filter_corpus(args): count_w = id_counts[idx] corpus_size = word_all_count keep_prob = ( - math.sqrt(count_w / - (args.downsample * corpus_size)) + 1 - ) * (args.downsample * corpus_size) / count_w + math.sqrt(count_w / + (args.downsample * corpus_size)) + 1 + ) * (args.downsample * corpus_size) / count_w r_value = random.random() if r_value > keep_prob: continue @@ -182,7 +183,8 @@ def build_dict(args): for file in os.listdir(args.build_dict_corpus_dir): with io.open( - args.build_dict_corpus_dir + "/" + file, encoding='utf-8') as f: + args.build_dict_corpus_dir + "/" + file, + encoding='utf-8') as f: print("build dict : ", args.build_dict_corpus_dir + "/" + file) for line in f: line = text_strip(line) @@ -232,7 +234,8 @@ def data_split(args): for i in range(1, num + 1): with open(os.path.join(new_data_dir, "part_" + str(i)), 'w') as fout: - data = contents[(i - 1) * lines_per_file:min(i * lines_per_file, len(contents))] + data = contents[(i - 1) * lines_per_file:min(i * lines_per_file, + len(contents))] for line in data: fout.write(line) diff --git a/models/recall/word2vec/w2v_evaluate_reader.py b/models/recall/word2vec/w2v_evaluate_reader.py index 04be9d41b2cd1ec51768696817a57c38dd958a44..6350c960e61d8ef3580cc4cc605ba24cb5623b0b 100755 --- a/models/recall/word2vec/w2v_evaluate_reader.py +++ b/models/recall/word2vec/w2v_evaluate_reader.py @@ -22,7 +22,8 @@ from paddlerec.core.utils import envs class EvaluateReader(Reader): def init(self): - dict_path = envs.get_global_env("word_id_dict_path", None, "evaluate.reader") + dict_path = envs.get_global_env("word_id_dict_path", None, + "evaluate.reader") self.word_to_id = dict() self.id_to_word = dict() with io.open(dict_path, 'r', encoding='utf-8') as f: @@ -68,14 +69,17 @@ class EvaluateReader(Reader): a unicode string - a space-delimited sequence of words. """ return u" ".join([ - word if word in original_vocab else u"" for word in line.split() + word if word in original_vocab else u"" + for word in line.split() ]) def generate_sample(self, line): def reader(): features = self.strip_lines(line.lower(), self.word_to_id) features = features.split() - yield [('analogy_a', [self.word_to_id[features[0]]]), ('analogy_b', [self.word_to_id[features[1]]]), - ('analogy_c', [self.word_to_id[features[2]]]), ('analogy_d', [self.word_to_id[features[3]]])] + yield [('analogy_a', [self.word_to_id[features[0]]]), + ('analogy_b', [self.word_to_id[features[1]]]), + ('analogy_c', [self.word_to_id[features[2]]]), + ('analogy_d', [self.word_to_id[features[3]]])] return reader diff --git a/models/recall/word2vec/w2v_reader.py b/models/recall/word2vec/w2v_reader.py index 88e52b47692778feef8396dd037448a8053aa958..9b3e69127055118bbc16b30eaac63f9a282bd1eb 100755 --- a/models/recall/word2vec/w2v_reader.py +++ b/models/recall/word2vec/w2v_reader.py @@ -40,10 +40,14 @@ class NumpyRandomInt(object): class TrainReader(Reader): def init(self): - dict_path = envs.get_global_env("word_count_dict_path", None, "train.reader") - self.window_size = envs.get_global_env("hyper_parameters.window_size", None, "train.model") - self.neg_num = envs.get_global_env("hyper_parameters.neg_num", None, "train.model") - self.with_shuffle_batch = envs.get_global_env("hyper_parameters.with_shuffle_batch", None, "train.model") + dict_path = envs.get_global_env("word_count_dict_path", None, + "train.reader") + self.window_size = envs.get_global_env("hyper_parameters.window_size", + None, "train.model") + self.neg_num = envs.get_global_env("hyper_parameters.neg_num", None, + "train.model") + self.with_shuffle_batch = envs.get_global_env( + "hyper_parameters.with_shuffle_batch", None, "train.model") self.random_generator = NumpyRandomInt(1, self.window_size + 1) self.cs = None @@ -81,13 +85,15 @@ class TrainReader(Reader): def reader(): word_ids = [w for w in line.split()] for idx, target_id in enumerate(word_ids): - context_word_ids = self.get_context_words( - word_ids, idx) + context_word_ids = self.get_context_words(word_ids, idx) for context_id in context_word_ids: - output = [('input_word', [int(target_id)]), ('true_label', [int(context_id)])] + output = [('input_word', [int(target_id)]), + ('true_label', [int(context_id)])] if not self.with_shuffle_batch: - neg_array = self.cs.searchsorted(np.random.sample(self.neg_num)) - output += [('neg_label', [int(str(i)) for i in neg_array])] + neg_array = self.cs.searchsorted( + np.random.sample(self.neg_num)) + output += [('neg_label', + [int(str(i)) for i in neg_array])] yield output return reader diff --git a/models/recall/youtube_dnn/model.py b/models/recall/youtube_dnn/model.py index 63d1fd2f49aad3c59272e560ed64442ab5f2f41e..22953764d1f81218b2f3d4c232392fe741043fa3 100644 --- a/models/recall/youtube_dnn/model.py +++ b/models/recall/youtube_dnn/model.py @@ -25,14 +25,20 @@ class Model(ModelBase): ModelBase.__init__(self, config) def input_data(self, is_infer=False): - - watch_vec_size = envs.get_global_env("hyper_parameters.watch_vec_size", None, self._namespace) - search_vec_size = envs.get_global_env("hyper_parameters.search_vec_size", None, self._namespace) - other_feat_size = envs.get_global_env("hyper_parameters.other_feat_size", None, self._namespace) - - watch_vec = fluid.data(name="watch_vec", shape=[None, watch_vec_size], dtype="float32") - search_vec = fluid.data(name="search_vec", shape=[None, search_vec_size], dtype="float32") - other_feat = fluid.data(name="other_feat", shape=[None, other_feat_size], dtype="float32") + + watch_vec_size = envs.get_global_env("hyper_parameters.watch_vec_size", + None, self._namespace) + search_vec_size = envs.get_global_env( + "hyper_parameters.search_vec_size", None, self._namespace) + other_feat_size = envs.get_global_env( + "hyper_parameters.other_feat_size", None, self._namespace) + + watch_vec = fluid.data( + name="watch_vec", shape=[None, watch_vec_size], dtype="float32") + search_vec = fluid.data( + name="search_vec", shape=[None, search_vec_size], dtype="float32") + other_feat = fluid.data( + name="other_feat", shape=[None, other_feat_size], dtype="float32") label = fluid.data(name="label", shape=[None, 1], dtype="int64") inputs = [watch_vec] + [search_vec] + [other_feat] + [label] self._data_var = inputs @@ -41,27 +47,32 @@ class Model(ModelBase): def fc(self, tag, data, out_dim, active='relu'): init_stddev = 1.0 - scales = 1.0 / np.sqrt(data.shape[1]) - + scales = 1.0 / np.sqrt(data.shape[1]) + if tag == 'l4': - p_attr = fluid.param_attr.ParamAttr(name='%s_weight' % tag, - initializer=fluid.initializer.NormalInitializer(loc=0.0, scale=init_stddev * scales)) + p_attr = fluid.param_attr.ParamAttr( + name='%s_weight' % tag, + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=init_stddev * scales)) else: p_attr = None - - b_attr = fluid.ParamAttr(name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1)) + + b_attr = fluid.ParamAttr( + name='%s_bias' % tag, initializer=fluid.initializer.Constant(0.1)) out = fluid.layers.fc(input=data, - size=out_dim, - act=active, - param_attr=p_attr, - bias_attr =b_attr, - name=tag) + size=out_dim, + act=active, + param_attr=p_attr, + bias_attr=b_attr, + name=tag) return out def net(self, inputs): - output_size = envs.get_global_env("hyper_parameters.output_size", None, self._namespace) - layers = envs.get_global_env("hyper_parameters.layers", None, self._namespace) + output_size = envs.get_global_env("hyper_parameters.output_size", None, + self._namespace) + layers = envs.get_global_env("hyper_parameters.layers", None, + self._namespace) concat_feats = fluid.layers.concat(input=inputs[:-1], axis=-1) l1 = self.fc('l1', concat_feats, layers[0], 'relu') diff --git a/models/recall/youtube_dnn/random_reader.py b/models/recall/youtube_dnn/random_reader.py index 723c66f9c0fe94d8fe0d36e6a3e75e9945768d40..30df6d1d29cfdf75c7e7cf9b68643af582c9f49f 100644 --- a/models/recall/youtube_dnn/random_reader.py +++ b/models/recall/youtube_dnn/random_reader.py @@ -21,10 +21,14 @@ import numpy as np class TrainReader(Reader): def init(self): - self.watch_vec_size = envs.get_global_env("hyper_parameters.watch_vec_size", None, "train.model") - self.search_vec_size = envs.get_global_env("hyper_parameters.search_vec_size", None, "train.model") - self.other_feat_size = envs.get_global_env("hyper_parameters.other_feat_size", None, "train.model") - self.output_size = envs.get_global_env("hyper_parameters.output_size", None, "train.model") + self.watch_vec_size = envs.get_global_env( + "hyper_parameters.watch_vec_size", None, "train.model") + self.search_vec_size = envs.get_global_env( + "hyper_parameters.search_vec_size", None, "train.model") + self.other_feat_size = envs.get_global_env( + "hyper_parameters.other_feat_size", None, "train.model") + self.output_size = envs.get_global_env("hyper_parameters.output_size", + None, "train.model") def generate_sample(self, line): """ @@ -35,13 +39,12 @@ class TrainReader(Reader): """ This function needs to be implemented by the user, based on data format """ - + feature_name = ["watch_vec", "search_vec", "other_feat", "label"] - yield zip(feature_name, [np.random.rand(self.watch_vec_size).tolist()] + - [np.random.rand(self.search_vec_size).tolist()] + - [np.random.rand(self.other_feat_size).tolist()] + - [[np.random.randint(self.output_size)]] ) + yield zip(feature_name, + [np.random.rand(self.watch_vec_size).tolist()] + + [np.random.rand(self.search_vec_size).tolist()] + + [np.random.rand(self.other_feat_size).tolist()] + + [[np.random.randint(self.output_size)]]) return reader - - diff --git a/models/treebased/tdm/model.py b/models/treebased/tdm/model.py index fa5f225f68068f826df6fc9ef0c7c9d35dbd9b89..319a7b4f9a3695537b43c8f1078dc4e1b73549fb 100755 --- a/models/treebased/tdm/model.py +++ b/models/treebased/tdm/model.py @@ -25,38 +25,38 @@ class Model(ModelBase): def __init__(self, config): ModelBase.__init__(self, config) # tree meta hyper parameters - self.max_layers = envs.get_global_env( - "tree_parameters.max_layers", 4, self._namespace) - self.node_nums = envs.get_global_env( - "tree_parameters.node_nums", 26, self._namespace) + self.max_layers = envs.get_global_env("tree_parameters.max_layers", 4, + self._namespace) + self.node_nums = envs.get_global_env("tree_parameters.node_nums", 26, + self._namespace) self.leaf_node_nums = envs.get_global_env( "tree_parameters.leaf_node_nums", 13, self._namespace) self.output_positive = envs.get_global_env( "tree_parameters.output_positive", True, self._namespace) self.layer_node_num_list = envs.get_global_env( - "tree_parameters.layer_node_num_list", [ - 2, 4, 7, 12], self._namespace) - self.child_nums = envs.get_global_env( - "tree_parameters.child_nums", 2, self._namespace) - self.tree_layer_path = envs.get_global_env( - "tree.tree_layer_path", None, "train.startup") + "tree_parameters.layer_node_num_list", [2, 4, 7, + 12], self._namespace) + self.child_nums = envs.get_global_env("tree_parameters.child_nums", 2, + self._namespace) + self.tree_layer_path = envs.get_global_env("tree.tree_layer_path", + None, "train.startup") # model training hyper parameter self.node_emb_size = envs.get_global_env( "hyper_parameters.node_emb_size", 64, self._namespace) self.input_emb_size = envs.get_global_env( "hyper_parameters.input_emb_size", 768, self._namespace) - self.act = envs.get_global_env( - "hyper_parameters.act", "tanh", self._namespace) + self.act = envs.get_global_env("hyper_parameters.act", "tanh", + self._namespace) self.neg_sampling_list = envs.get_global_env( - "hyper_parameters.neg_sampling_list", [ - 1, 2, 3, 4], self._namespace) + "hyper_parameters.neg_sampling_list", [1, 2, 3, + 4], self._namespace) # model infer hyper parameter - self.topK = envs.get_global_env( - "hyper_parameters.node_nums", 1, self._namespace) - self.batch_size = envs.get_global_env( - "batch_size", 1, "evaluate.reader") + self.topK = envs.get_global_env("hyper_parameters.node_nums", 1, + self._namespace) + self.batch_size = envs.get_global_env("batch_size", 1, + "evaluate.reader") def train_net(self): self.train_input() @@ -76,21 +76,22 @@ class Model(ModelBase): input_emb = fluid.data( name="input_emb", shape=[None, self.input_emb_size], - dtype="float32", - ) + dtype="float32", ) self._data_var.append(input_emb) item_label = fluid.data( name="item_label", shape=[None, 1], - dtype="int64", - ) + dtype="int64", ) self._data_var.append(item_label) if self._platform != "LINUX": self._data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def tdm_net(self): """ @@ -116,8 +117,7 @@ class Model(ModelBase): output_list=True, seed=0, tree_dtype='int64', - dtype='int64' - ) + dtype='int64') # 查表得到每个节点的Embedding sample_nodes_emb = [ @@ -125,35 +125,34 @@ class Model(ModelBase): input=sample_nodes[i], is_sparse=True, size=[self.node_nums, self.node_emb_size], - param_attr=fluid.ParamAttr( - name="TDM_Tree_Emb") - ) for i in range(self.max_layers) + param_attr=fluid.ParamAttr(name="TDM_Tree_Emb")) + for i in range(self.max_layers) ] # 此处进行Reshape是为了之后层次化的分类器训练 sample_nodes_emb = [ - fluid.layers.reshape(sample_nodes_emb[i], - [-1, self.neg_sampling_list[i] + - self.output_positive, self.node_emb_size] - ) for i in range(self.max_layers) + fluid.layers.reshape(sample_nodes_emb[i], [ + -1, self.neg_sampling_list[i] + self.output_positive, + self.node_emb_size + ]) for i in range(self.max_layers) ] # 对输入的input_emb进行转换，使其维度与node_emb维度一致 input_trans_emb = self.input_trans_layer(input_emb) # 分类器的主体网络，分别训练不同层次的分类器 - layer_classifier_res = self.classifier_layer( - input_trans_emb, sample_nodes_emb) + layer_classifier_res = self.classifier_layer(input_trans_emb, + sample_nodes_emb) # 最后的概率判别FC，将所有层次的node分类结果放到一起以相同的标准进行判别 # 考虑到树极大可能不平衡，有些item不在最后一层，所以需要这样的机制保证每个item都有机会被召回 - tdm_fc = fluid.layers.fc(input=layer_classifier_res, - size=2, - act=None, - num_flatten_dims=2, - param_attr=fluid.ParamAttr( - name="tdm.cls_fc.weight"), - bias_attr=fluid.ParamAttr(name="tdm.cls_fc.bias")) + tdm_fc = fluid.layers.fc( + input=layer_classifier_res, + size=2, + act=None, + num_flatten_dims=2, + param_attr=fluid.ParamAttr(name="tdm.cls_fc.weight"), + bias_attr=fluid.ParamAttr(name="tdm.cls_fc.bias")) # 将loss打平，放到一起计算整体网络的loss tdm_fc_re = fluid.layers.reshape(tdm_fc, [-1, 2]) @@ -202,7 +201,7 @@ class Model(ModelBase): def metrics(self): auc, batch_auc, _ = fluid.layers.auc(input=self._predict, label=self.mask_label, - num_thresholds=2 ** 12, + num_thresholds=2**12, slide_steps=20) self._metrics["AUC"] = auc self._metrics["BATCH_AUC"] = batch_auc @@ -218,8 +217,7 @@ class Model(ModelBase): size=self.node_emb_size, act=None, param_attr=fluid.ParamAttr(name="trans.input_fc.weight"), - bias_attr=fluid.ParamAttr(name="trans.input_fc.bias"), - ) + bias_attr=fluid.ParamAttr(name="trans.input_fc.bias"), ) # 将input_emb映射到各个不同层次的向量表示空间 input_layer_fc_out = [ @@ -229,8 +227,9 @@ class Model(ModelBase): act=self.act, param_attr=fluid.ParamAttr( name="trans.layer_fc.weight." + str(i)), - bias_attr=fluid.ParamAttr(name="trans.layer_fc.bias." + str(i)), - ) for i in range(self.max_layers) + bias_attr=fluid.ParamAttr( + name="trans.layer_fc.bias." + str(i)), ) + for i in range(self.max_layers) ] return input_layer_fc_out @@ -246,20 +245,22 @@ class Model(ModelBase): input_layer_unsequeeze, expand_times=[1, node.shape[1], 1]) else: input_layer_expand = fluid.layers.expand( - input_layer_unsequeeze, expand_times=[1, node[layer_idx].shape[1], 1]) + input_layer_unsequeeze, + expand_times=[1, node[layer_idx].shape[1], 1]) return input_layer_expand def classifier_layer(self, input, node): # 扩展input，使维度与node匹配 input_expand = [ - self._expand_layer(input[i], node, i) for i in range(self.max_layers) + self._expand_layer(input[i], node, i) + for i in range(self.max_layers) ] # 将input_emb与node_emb concat到一起过分类器FC input_node_concat = [ fluid.layers.concat( - input=[input_expand[i], node[i]], - axis=2) for i in range(self.max_layers) + input=[input_expand[i], node[i]], axis=2) + for i in range(self.max_layers) ] hidden_states_fc = [ fluid.layers.fc( @@ -269,8 +270,8 @@ class Model(ModelBase): act=self.act, param_attr=fluid.ParamAttr( name="cls.concat_fc.weight." + str(i)), - bias_attr=fluid.ParamAttr(name="cls.concat_fc.bias." + str(i)) - ) for i in range(self.max_layers) + bias_attr=fluid.ParamAttr(name="cls.concat_fc.bias." + str(i))) + for i in range(self.max_layers) ] # 如果将所有层次的node放到一起计算loss，则需要在此处concat @@ -285,12 +286,14 @@ class Model(ModelBase): input_emb = fluid.layers.data( name="input_emb", shape=[self.input_emb_size], - dtype="float32", - ) + dtype="float32", ) self._infer_data_var.append(input_emb) self._infer_data_loader = fluid.io.DataLoader.from_generator( - feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) + feed_list=self._infer_data_var, + capacity=64, + use_double_buffer=False, + iterable=False) def get_layer_list(self): """get layer list from layer_list.txt""" @@ -318,10 +321,12 @@ class Model(ModelBase): node_list = [] mask_list = [] for id in first_layer_node: - node_list.append(fluid.layers.fill_constant( - [self.batch_size, 1], value=int(id), dtype='int64')) - mask_list.append(fluid.layers.fill_constant( - [self.batch_size, 1], value=0, dtype='int64')) + node_list.append( + fluid.layers.fill_constant( + [self.batch_size, 1], value=int(id), dtype='int64')) + mask_list.append( + fluid.layers.fill_constant( + [self.batch_size, 1], value=0, dtype='int64')) self.first_layer_node = fluid.layers.concat(node_list, axis=1) self.first_layer_node_mask = fluid.layers.concat(mask_list, axis=1) @@ -359,28 +364,26 @@ class Model(ModelBase): size=[self.node_nums, self.node_emb_size], param_attr=fluid.ParamAttr(name="TDM_Tree_Emb")) - input_fc_out = self.layer_fc_infer( - input_trans_emb, layer_idx) + input_fc_out = self.layer_fc_infer(input_trans_emb, layer_idx) # 过每一层的分类器 - layer_classifier_res = self.classifier_layer_infer(input_fc_out, - node_emb, - layer_idx) + layer_classifier_res = self.classifier_layer_infer( + input_fc_out, node_emb, layer_idx) # 过最终的判别分类器 - tdm_fc = fluid.layers.fc(input=layer_classifier_res, - size=2, - act=None, - num_flatten_dims=2, - param_attr=fluid.ParamAttr( - name="tdm.cls_fc.weight"), - bias_attr=fluid.ParamAttr(name="tdm.cls_fc.bias")) + tdm_fc = fluid.layers.fc( + input=layer_classifier_res, + size=2, + act=None, + num_flatten_dims=2, + param_attr=fluid.ParamAttr(name="tdm.cls_fc.weight"), + bias_attr=fluid.ParamAttr(name="tdm.cls_fc.bias")) prob = fluid.layers.softmax(tdm_fc) positive_prob = fluid.layers.slice( prob, axes=[2], starts=[1], ends=[2]) - prob_re = fluid.layers.reshape( - positive_prob, [-1, current_layer_node_num]) + prob_re = fluid.layers.reshape(positive_prob, + [-1, current_layer_node_num]) # 过滤掉padding产生的无效节点（node_id=0） node_zero_mask = fluid.layers.cast(current_layer_node, 'bool') @@ -395,11 +398,11 @@ class Model(ModelBase): # index_sample op根据下标索引tensor对应位置的值 # 若paddle版本>2.0，调用方式为paddle.index_sample - top_node = fluid.contrib.layers.index_sample( - current_layer_node, topk_i) + top_node = fluid.contrib.layers.index_sample(current_layer_node, + topk_i) prob_re_mask = prob_re * current_layer_node_mask # 过滤掉非叶子节点 - topk_value = fluid.contrib.layers.index_sample( - prob_re_mask, topk_i) + topk_value = fluid.contrib.layers.index_sample(prob_re_mask, + topk_i) node_score.append(topk_value) node_list.append(top_node) @@ -424,7 +427,8 @@ class Model(ModelBase): res_node = fluid.layers.reshape(res_layer_node, [-1, self.topK, 1]) # 利用Tree_info信息，将node_id转换为item_id - tree_info = fluid.default_main_program().global_block().var("TDM_Tree_Info") + tree_info = fluid.default_main_program().global_block().var( + "TDM_Tree_Info") res_node_emb = fluid.layers.gather_nd(tree_info, res_node) res_item = fluid.layers.slice( @@ -442,8 +446,7 @@ class Model(ModelBase): size=self.node_emb_size, act=None, param_attr=fluid.ParamAttr(name="trans.input_fc.weight"), - bias_attr=fluid.ParamAttr(name="trans.input_fc.bias"), - ) + bias_attr=fluid.ParamAttr(name="trans.input_fc.bias"), ) return input_fc_out def layer_fc_infer(self, input_fc_out, layer_idx): @@ -458,8 +461,7 @@ class Model(ModelBase): param_attr=fluid.ParamAttr( name="trans.layer_fc.weight." + str(layer_idx)), bias_attr=fluid.ParamAttr( - name="trans.layer_fc.bias." + str(layer_idx)), - ) + name="trans.layer_fc.bias." + str(layer_idx)), ) return input_layer_fc_out def classifier_layer_infer(self, input, node, layer_idx): @@ -480,5 +482,6 @@ class Model(ModelBase): act=self.act, param_attr=fluid.ParamAttr( name="cls.concat_fc.weight." + str(layer_idx)), - bias_attr=fluid.ParamAttr(name="cls.concat_fc.bias." + str(layer_idx))) + bias_attr=fluid.ParamAttr( + name="cls.concat_fc.bias." + str(layer_idx))) return hidden_states_fc