model.py 10.2 KB
Newer Older
M
add gnn  
malin10 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math
T
tangwei 已提交
16 17
import numpy as np

M
add gnn  
malin10 已提交
18 19 20
import paddle.fluid as fluid
import paddle.fluid.layers as layers

21 22
from paddlerec.core.utils import envs
from paddlerec.core.model import Model as ModelBase
M
add gnn  
malin10 已提交
23 24 25 26 27


class Model(ModelBase):
    def __init__(self, config):
        ModelBase.__init__(self, config)
T
for mat  
tangwei 已提交
28

M
malin10 已提交
29
    def _init_hyper_parameters(self):
M
malin10 已提交
30 31 32 33 34 35 36 37 38
        self.learning_rate = envs.get_global_env(
            "hyper_parameters.optimizer.learning_rate")
        self.decay_steps = envs.get_global_env(
            "hyper_parameters.optimizer.decay_steps")
        self.decay_rate = envs.get_global_env(
            "hyper_parameters.optimizer.decay_rate")
        self.l2 = envs.get_global_env("hyper_parameters.optimizer.l2")

        self.dict_size = envs.get_global_env(
M
malin10 已提交
39
            "hyper_parameters.sparse_feature_number")
M
malin10 已提交
40 41
        self.corpus_size = envs.get_global_env("hyper_parameters.corpus_size")

M
malin10 已提交
42 43 44 45
        self.train_batch_size = envs.get_global_env(
            "dataset.dataset_train.batch_size")
        self.evaluate_batch_size = envs.get_global_env(
            "dataset.dataset_infer.batch_size")
M
malin10 已提交
46

M
malin10 已提交
47 48 49 50
        self.hidden_size = envs.get_global_env(
            "hyper_parameters.sparse_feature_dim")
        self.step = envs.get_global_env(
            "hyper_parameters.gnn_propogation_steps")
M
malin10 已提交
51 52 53 54 55 56 57

    def input_data(self, is_infer=False, **kwargs):
        if is_infer:
            bs = self.evaluate_batch_size
        else:
            bs = self.train_batch_size
        items = fluid.data(
T
tangwei 已提交
58
            name="items", shape=[bs, -1],
T
for mat  
tangwei 已提交
59
            dtype="int64")  # [batch_size, uniq_max]
M
malin10 已提交
60
        seq_index = fluid.data(
T
tangwei 已提交
61
            name="seq_index", shape=[bs, -1, 2],
T
for mat  
tangwei 已提交
62
            dtype="int32")  # [batch_size, seq_max, 2]
M
malin10 已提交
63
        last_index = fluid.data(
T
tangwei 已提交
64
            name="last_index", shape=[bs, 2], dtype="int32")  # [batch_size, 2]
M
malin10 已提交
65
        adj_in = fluid.data(
T
tangwei 已提交
66
            name="adj_in", shape=[bs, -1, -1],
T
for mat  
tangwei 已提交
67
            dtype="float32")  # [batch_size, seq_max, seq_max]
M
malin10 已提交
68
        adj_out = fluid.data(
T
tangwei 已提交
69
            name="adj_out", shape=[bs, -1, -1],
T
for mat  
tangwei 已提交
70
            dtype="float32")  # [batch_size, seq_max, seq_max]
M
malin10 已提交
71
        mask = fluid.data(
T
tangwei 已提交
72
            name="mask", shape=[bs, -1, 1],
T
for mat  
tangwei 已提交
73
            dtype="float32")  # [batch_size, seq_max, 1]
M
malin10 已提交
74
        label = fluid.data(
T
tangwei 已提交
75
            name="label", shape=[bs, 1], dtype="int64")  # [batch_size, 1]
M
add gnn  
malin10 已提交
76

M
malin10 已提交
77
        res = [items, seq_index, last_index, adj_in, adj_out, mask, label]
M
add gnn  
malin10 已提交
78
        return res
T
for mat  
tangwei 已提交
79

M
malin10 已提交
80 81 82 83 84
    def net(self, inputs, is_infer=False):
        if is_infer:
            bs = self.evaluate_batch_size
        else:
            bs = self.train_batch_size
M
malin10 已提交
85

M
malin10 已提交
86
        stdv = 1.0 / math.sqrt(self.hidden_size)
M
add gnn  
malin10 已提交
87

T
tangwei 已提交
88 89 90 91
        def embedding_layer(input,
                            table_name,
                            emb_dim,
                            initializer_instance=None):
M
add gnn  
malin10 已提交
92 93
            emb = fluid.embedding(
                input=input,
M
malin10 已提交
94
                size=[self.dict_size, emb_dim],
M
add gnn  
malin10 已提交
95
                param_attr=fluid.ParamAttr(
M
malin10 已提交
96
                    name=table_name, initializer=initializer_instance))
T
for mat  
tangwei 已提交
97 98 99
            return emb

        sparse_initializer = fluid.initializer.Uniform(low=-stdv, high=stdv)
M
malin10 已提交
100
        items_emb = embedding_layer(inputs[0], "emb", self.hidden_size,
T
tangwei 已提交
101
                                    sparse_initializer)
M
add gnn  
malin10 已提交
102
        pre_state = items_emb
M
malin10 已提交
103
        for i in range(self.step):
T
tangwei 已提交
104
            pre_state = layers.reshape(
M
malin10 已提交
105
                x=pre_state, shape=[bs, -1, self.hidden_size])
M
add gnn  
malin10 已提交
106 107 108
            state_in = layers.fc(
                input=pre_state,
                name="state_in",
M
malin10 已提交
109
                size=self.hidden_size,
M
add gnn  
malin10 已提交
110 111
                act=None,
                num_flatten_dims=2,
T
tangwei 已提交
112 113 114 115 116 117
                param_attr=fluid.ParamAttr(
                    initializer=fluid.initializer.Uniform(
                        low=-stdv, high=stdv)),
                bias_attr=fluid.ParamAttr(
                    initializer=fluid.initializer.Uniform(
                        low=-stdv, high=stdv)))  # [batch_size, uniq_max, h]
M
add gnn  
malin10 已提交
118 119 120
            state_out = layers.fc(
                input=pre_state,
                name="state_out",
M
malin10 已提交
121
                size=self.hidden_size,
M
add gnn  
malin10 已提交
122 123
                act=None,
                num_flatten_dims=2,
T
tangwei 已提交
124 125 126 127 128 129
                param_attr=fluid.ParamAttr(
                    initializer=fluid.initializer.Uniform(
                        low=-stdv, high=stdv)),
                bias_attr=fluid.ParamAttr(
                    initializer=fluid.initializer.Uniform(
                        low=-stdv, high=stdv)))  # [batch_size, uniq_max, h]
T
for mat  
tangwei 已提交
130

M
malin10 已提交
131
            state_adj_in = layers.matmul(inputs[3],
T
tangwei 已提交
132
                                         state_in)  # [batch_size, uniq_max, h]
M
malin10 已提交
133 134
            state_adj_out = layers.matmul(
                inputs[4], state_out)  # [batch_size, uniq_max, h]
T
for mat  
tangwei 已提交
135

M
add gnn  
malin10 已提交
136
            gru_input = layers.concat([state_adj_in, state_adj_out], axis=2)
T
for mat  
tangwei 已提交
137

T
tangwei 已提交
138
            gru_input = layers.reshape(
M
malin10 已提交
139
                x=gru_input, shape=[-1, self.hidden_size * 2])
T
tangwei 已提交
140 141
            gru_fc = layers.fc(input=gru_input,
                               name="gru_fc",
M
malin10 已提交
142
                               size=3 * self.hidden_size,
T
tangwei 已提交
143
                               bias_attr=False)
M
add gnn  
malin10 已提交
144 145
            pre_state, _, _ = fluid.layers.gru_unit(
                input=gru_fc,
T
tangwei 已提交
146
                hidden=layers.reshape(
M
malin10 已提交
147 148
                    x=pre_state, shape=[-1, self.hidden_size]),
                size=3 * self.hidden_size)
T
for mat  
tangwei 已提交
149

M
malin10 已提交
150 151
        final_state = layers.reshape(
            pre_state, shape=[bs, -1, self.hidden_size])
M
malin10 已提交
152 153
        seq = layers.gather_nd(final_state, inputs[1])
        last = layers.gather_nd(final_state, inputs[2])
T
for mat  
tangwei 已提交
154

M
add gnn  
malin10 已提交
155 156 157
        seq_fc = layers.fc(
            input=seq,
            name="seq_fc",
M
malin10 已提交
158
            size=self.hidden_size,
M
add gnn  
malin10 已提交
159 160 161
            bias_attr=False,
            act=None,
            num_flatten_dims=2,
T
tangwei 已提交
162 163 164 165
            param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform(
                low=-stdv, high=stdv)))  # [batch_size, seq_max, h]
        last_fc = layers.fc(input=last,
                            name="last_fc",
M
malin10 已提交
166
                            size=self.hidden_size,
T
tangwei 已提交
167 168 169 170 171 172
                            bias_attr=False,
                            act=None,
                            num_flatten_dims=1,
                            param_attr=fluid.ParamAttr(
                                initializer=fluid.initializer.Uniform(
                                    low=-stdv, high=stdv)))  # [bathc_size, h]
T
for mat  
tangwei 已提交
173

M
add gnn  
malin10 已提交
174
        seq_fc_t = layers.transpose(
T
for mat  
tangwei 已提交
175
            seq_fc, perm=[1, 0, 2])  # [seq_max, batch_size, h]
T
tangwei 已提交
176 177
        add = layers.elementwise_add(seq_fc_t,
                                     last_fc)  # [seq_max, batch_size, h]
M
add gnn  
malin10 已提交
178
        b = layers.create_parameter(
M
malin10 已提交
179
            shape=[self.hidden_size],
M
add gnn  
malin10 已提交
180
            dtype='float32',
T
for mat  
tangwei 已提交
181 182 183 184
            default_initializer=fluid.initializer.Constant(value=0.0))  # [h]
        add = layers.elementwise_add(add, b)  # [seq_max, batch_size, h]

        add_sigmoid = layers.sigmoid(add)  # [seq_max, batch_size, h]
M
add gnn  
malin10 已提交
185
        add_sigmoid = layers.transpose(
T
for mat  
tangwei 已提交
186 187
            add_sigmoid, perm=[1, 0, 2])  # [batch_size, seq_max, h]

M
add gnn  
malin10 已提交
188 189 190 191 192 193 194
        weight = layers.fc(
            input=add_sigmoid,
            name="weight_fc",
            size=1,
            act=None,
            num_flatten_dims=2,
            bias_attr=False,
T
tangwei 已提交
195 196
            param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform(
                low=-stdv, high=stdv)))  # [batch_size, seq_max, 1]
M
malin10 已提交
197
        weight *= inputs[5]
T
tangwei 已提交
198 199 200 201
        weight_mask = layers.elementwise_mul(
            seq, weight, axis=0)  # [batch_size, seq_max, h]
        global_attention = layers.reduce_sum(
            weight_mask, dim=1)  # [batch_size, h]
T
for mat  
tangwei 已提交
202

M
add gnn  
malin10 已提交
203
        final_attention = layers.concat(
T
for mat  
tangwei 已提交
204
            [global_attention, last], axis=1)  # [batch_size, 2*h]
M
add gnn  
malin10 已提交
205 206 207
        final_attention_fc = layers.fc(
            input=final_attention,
            name="final_attention_fc",
M
malin10 已提交
208
            size=self.hidden_size,
M
add gnn  
malin10 已提交
209 210 211
            bias_attr=False,
            act=None,
            param_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform(
T
for mat  
tangwei 已提交
212 213 214 215 216 217 218 219
                low=-stdv, high=stdv)))  # [batch_size, h]

        # all_vocab = layers.create_global_var(
        #     shape=[items_num - 1],
        #     value=0,
        #     dtype="int64",
        #     persistable=True,
        #     name="all_vocab")
M
malin10 已提交
220
        all_vocab = np.arange(1, self.dict_size).reshape((-1)).astype('int32')
T
tangwei 已提交
221 222
        all_vocab = fluid.layers.cast(
            x=fluid.layers.assign(all_vocab), dtype='int64')
M
add gnn  
malin10 已提交
223 224 225 226 227 228 229

        all_emb = fluid.embedding(
            input=all_vocab,
            param_attr=fluid.ParamAttr(
                name="emb",
                initializer=fluid.initializer.Uniform(
                    low=-stdv, high=stdv)),
M
malin10 已提交
230
            size=[self.dict_size, self.hidden_size])  # [all_vocab, h]
T
for mat  
tangwei 已提交
231

M
add gnn  
malin10 已提交
232 233
        logits = layers.matmul(
            x=final_attention_fc, y=all_emb,
T
for mat  
tangwei 已提交
234
            transpose_y=True)  # [batch_size, all_vocab]
M
add gnn  
malin10 已提交
235
        softmax = layers.softmax_with_cross_entropy(
M
malin10 已提交
236
            logits=logits, label=inputs[6])  # [batch_size, 1]
M
add gnn  
malin10 已提交
237
        self.loss = layers.reduce_mean(softmax)  # [1]
M
malin10 已提交
238
        self.acc = layers.accuracy(input=logits, label=inputs[6], k=20)
M
add gnn  
malin10 已提交
239 240

        self._cost = self.loss
M
malin10 已提交
241
        if is_infer:
M
malin10 已提交
242
            self._infer_results['acc'] = self.acc
M
malin10 已提交
243 244
            self._infer_results['loss'] = self.loss
            return
M
add gnn  
malin10 已提交
245

M
malin10 已提交
246
        self._metrics["LOSS"] = self.loss
M
add gnn  
malin10 已提交
247 248 249
        self._metrics["train_acc"] = self.acc

    def optimizer(self):
M
malin10 已提交
250
        step_per_epoch = self.corpus_size // self.train_batch_size
T
for mat  
tangwei 已提交
251
        optimizer = fluid.optimizer.Adam(
M
add gnn  
malin10 已提交
252
            learning_rate=fluid.layers.exponential_decay(
M
malin10 已提交
253 254 255
                learning_rate=self.learning_rate,
                decay_steps=self.decay_steps * step_per_epoch,
                decay_rate=self.decay_rate),
M
add gnn  
malin10 已提交
256
            regularization=fluid.regularizer.L2DecayRegularizer(
M
malin10 已提交
257
                regularization_coeff=self.l2))
T
for mat  
tangwei 已提交
258
        return optimizer