# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ DGI Pretrain """ import os import pgl from pgl import data_loader from pgl.utils.logger import log import paddle.fluid as fluid import numpy as np import time import argparse def load(name): """Load dataset""" if name == 'cora': dataset = data_loader.CoraDataset() elif name == "pubmed": dataset = data_loader.CitationDataset("pubmed", symmetry_edges=False) elif name == "citeseer": dataset = data_loader.CitationDataset("citeseer", symmetry_edges=False) else: raise ValueError(name + " dataset doesn't exists") return dataset def save_param(dirname, var_name_list): """save_param""" for var_name in var_name_list: var = fluid.global_scope().find_var(var_name) var_tensor = var.get_tensor() np.save(os.path.join(dirname, var_name + '.npy'), np.array(var_tensor)) def main(args): """main""" dataset = load(args.dataset) # normalize indegree = dataset.graph.indegree() norm = np.zeros_like(indegree, dtype="float32") norm[indegree > 0] = np.power(indegree[indegree > 0], -0.5) dataset.graph.node_feat["norm"] = np.expand_dims(norm, -1) place = fluid.CUDAPlace(0) if args.use_cuda else fluid.CPUPlace() train_program = fluid.Program() startup_program = fluid.Program() hidden_size = 512 with fluid.program_guard(train_program, startup_program): pos_gw = pgl.graph_wrapper.GraphWrapper( name="pos_graph", place=place, node_feat=dataset.graph.node_feat_info()) neg_gw = pgl.graph_wrapper.GraphWrapper( name="neg_graph", place=place, node_feat=dataset.graph.node_feat_info()) positive_feat = pgl.layers.gcn(pos_gw, pos_gw.node_feat["words"], hidden_size, activation="relu", norm=pos_gw.node_feat['norm'], name="gcn_layer_1") negative_feat = pgl.layers.gcn(neg_gw, neg_gw.node_feat["words"], hidden_size, activation="relu", norm=neg_gw.node_feat['norm'], name="gcn_layer_1") summary_feat = fluid.layers.sigmoid( fluid.layers.reduce_mean( positive_feat, [0], keep_dim=True)) summary_feat = fluid.layers.fc(summary_feat, hidden_size, bias_attr=False, name="discriminator") pos_logits = fluid.layers.matmul( positive_feat, summary_feat, transpose_y=True) neg_logits = fluid.layers.matmul( negative_feat, summary_feat, transpose_y=True) pos_loss = fluid.layers.sigmoid_cross_entropy_with_logits( x=pos_logits, label=fluid.layers.ones( shape=[dataset.graph.num_nodes, 1], dtype="float32")) neg_loss = fluid.layers.sigmoid_cross_entropy_with_logits( x=neg_logits, label=fluid.layers.zeros( shape=[dataset.graph.num_nodes, 1], dtype="float32")) loss = fluid.layers.reduce_mean(pos_loss) + fluid.layers.reduce_mean( neg_loss) adam = fluid.optimizer.Adam(learning_rate=1e-3) adam.minimize(loss) exe = fluid.Executor(place) exe.run(startup_program) best_loss = 1e9 dur = [] for epoch in range(args.epoch): feed_dict = pos_gw.to_feed(dataset.graph) node_feat = dataset.graph.node_feat["words"].copy() perm = np.arange(0, dataset.graph.num_nodes) np.random.shuffle(perm) dataset.graph.node_feat["words"] = dataset.graph.node_feat["words"][ perm] feed_dict.update(neg_gw.to_feed(dataset.graph)) dataset.graph.node_feat["words"] = node_feat if epoch >= 3: t0 = time.time() train_loss = exe.run(train_program, feed=feed_dict, fetch_list=[loss], return_numpy=True) if train_loss[0] < best_loss: best_loss = train_loss[0] save_param(args.checkpoint, ["gcn_layer_1", "gcn_layer_1_bias"]) if epoch >= 3: time_per_epoch = 1.0 * (time.time() - t0) dur.append(time_per_epoch) log.info("Epoch %d " % epoch + "(%.5lf sec) " % np.mean(dur) + "Train Loss: %f " % train_loss[0]) if __name__ == '__main__': parser = argparse.ArgumentParser(description='DGI pretrain') parser.add_argument( "--dataset", type=str, default="cora", help="dataset (cora, pubmed)") parser.add_argument( "--checkpoint", type=str, default="best_model", help="checkpoint") parser.add_argument( "--epoch", type=int, default=200, help="pretrain epochs") parser.add_argument("--use_cuda", action='store_true', help="use_cuda") args = parser.parse_args() log.info(args) main(args)