Created by: luolimingfighting

import paddle.fluid as fluid import paddle import shutil import six import os import time import numpy as np from paddle.fluid import compiler

define reader

def reader_creator(label_dic, col_dic, max_value, file_list, is_train): def reader(): for file in file_list: files = os.listdir(file) for data in files: with open(file+data, "r") as f: for line in f: fea_index = [] fea_value = [] features = line.rstrip("\n").split(",") features.pop(0) if is_train: label = int(label_dic[features.pop(0)]) else: label = 0 for ele in features: col_name, value = ele.split(" ")[0], ele.split(" ")[1] if col_name in col_dic: index = col_dic[col_name] fea_index.append(index) fea_value.append(float(value))

                    yield fea_index, fea_value, [label]
return reader

def run_iterable(program, exe, loss, py_reader): for data in py_reader: loss_value = exe.run(program=program, feed=data, fetch_list=[loss]) print("loss is {}".format(loss_value))

train_file = "./train_data/" file_list = [train_file] is_train=True

#col_dic, max_value, label_dic, label_index = init_dic(train_file) col_dic = np.load("./thirdparty/col-file.npy", allow_pickle=True).item() max_value = len(col_dic) label_dic = np.load("./thirdparty/label-file.npy", allow_pickle=True).item() label_index = len(label_dic) print("feature_num_is:{}".format(max_value)) print("label_num_is:{}".format(label_index)) sparse_feature_dim = max_value

ITERABLE = True USE_CUDA = True USE_DATA_PARALLEL = False

if ITERABLE: if USE_DATA_PARALLEL: places = fluid.cuda_places() if USE_CUDA else fluid.cpu_places(8) else: places = fluid.cuda_places() if USE_CUDA else fluid.cpu_places(1) else: places = None

#define data input type sparse_input_feature_idx = fluid.layers.data(name="feature_idx", shape=[1], lod_level=1, dtype="int64") sparse_input_feature_value_float = fluid.layers.data(name="feature_value", shape=[1], lod_level=1, dtype="float32") sparse_input_emb_weight = fluid.layers.embedding( input=sparse_input_feature_idx, is_sparse=True, # you need to patch https://github.com/PaddlePaddle/Paddle/pull/14190 # if you want to set is_distributed to True is_distributed=False, size=[sparse_feature_dim, 1],)

#x = fluid.layers.data(name="x", shape=[max_value+1], dtype="float32") y = fluid.layers.data(name="y", shape=[1], dtype="int64") w_mat_x_with_lod = sparse_input_feature_value_float * sparse_input_emb_weight w_mat_x = fluid.layers.sequence_pool(input=w_mat_x_with_lod, pool_type='sum') mean_w_x = fluid.layers.mean(w_mat_x)

##define network def network(input): fc = fluid.layers.fc(input=input, size=label_index+1, act="softmax") return fc

model = network(w_mat_x) cost = fluid.layers.cross_entropy(input=model, label=y) avg_cost = fluid.layers.mean(cost) acc = fluid.layers.accuracy(input=model, label=y)

obtain main program

main_program = fluid.default_main_program()

start_program = fluid.default_startup_program()

define optimizer

optimizer = fluid.optimizer.Adam() opts = optimizer.minimize(avg_cost)

read train data

train_reader = paddle.batch(reader_creator(label_dic, col_dic, max_value, file_list, is_train), batch_size=128)

run

place = fluid.CUDAPlace(0) parallel_places = [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2), fluid.CUDAPlace(3)] if USE_CUDA else [fluid.CPUPlace()] * 4 compiled_program = compiler.CompiledProgram(main_program) compiled_program.with_data_parallel(loss_name=avg_cost.name, places=parallel_places)

exe = fluid.Executor(place) exe.run(start_program)

param_path = "./my_paddle_model" param_path1 = "./params_model"

save model

feeder = fluid.DataFeeder(place=place, feed_list=[sparse_input_feature_idx, sparse_input_feature_value_float, y])

for pass_id in range(20): # for batch_id, data in enumerate(train_reader()): train_cost, train_acc = exe.run(program=compiled_program, feed=feeder.feed(data), fetch_list=[avg_cost, acc]) print_data = [pass_id, batch_id, train_cost[0], train_acc[0]] print('Pass:{}, Batch:{}, Cost:{}, Accuracy:{}'.format(*print_data)) #time.sleep(2) # #print("first epoch complete and start to save model") #fluid.io.save_persistables(executor=exe, dirname=param_path, main_program=compiled_program) #fluid.io.save_params(executor=exe, dirname=param_path, main_program=compiled_program) #print("model save success and start to next epoch") # # save_path = './output/' #print(save_path) #if not os.path.exists(save_path): #os.makedirs(save_path) # #shutil.rmtree(save_path, ignore_errors=True) # #os.makedirs(save_path) # fluid.io.save_inference_model(dirname=save_path, feeded_var_names=[w_mat_x.name], target_vars=[model], executor=exe)

这是我训练的base,我的数据形式是特征索引加特征权重，模型只需一层FC实现LR，不知是不是组网的逻辑问题还是其它原因，acc多个epoch依然很低很低。