diff --git a/ernie/ernie_encoder.py b/ernie/ernie_encoder.py index 8b413746a80b86ce9a5c369807b3b8c1e87360e4..51f2726e3360d736dcebbc023f3d2a7f49fe2ea8 100644 --- a/ernie/ernie_encoder.py +++ b/ernie/ernie_encoder.py @@ -22,13 +22,16 @@ import argparse import numpy as np import multiprocessing +import logging import paddle.fluid as fluid import reader.task_reader as task_reader -from model.ernie import ErnieConfig, ErnieModel -from utils.args import ArgumentGroup, print_arguments +from model.ernie_v1 import ErnieConfig, ErnieModel +from utils.args import ArgumentGroup, print_arguments, prepare_logger from utils.init import init_pretraining_params +log = logging.getLogger() + # yapf: disable parser = argparse.ArgumentParser(__doc__) model_g = ArgumentGroup(parser, "model", "model configuration and paths.") @@ -67,7 +70,6 @@ def create_model(args, pyreader_name, ernie_config): src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, - task_ids=task_ids, input_mask=input_mask, config=ernie_config) @@ -179,6 +181,7 @@ def main(args): if __name__ == '__main__': + prepare_logger(log) args = parser.parse_args() print_arguments(args) diff --git a/ernie/finetune/classifier.py b/ernie/finetune/classifier.py index 285cce2f5a2716fe169d2b4b6a747f728353d15c..6e7d9f098653dd8799bb242c168cb90f824d54a5 100644 --- a/ernie/finetune/classifier.py +++ b/ernie/finetune/classifier.py @@ -52,7 +52,7 @@ def create_model(args, elif is_regression: labels = fluid.layers.data(name='6', shape=[-1, 1], dtype='float32') - pyreader = fluid.io.DataLoader.from_generator(feed_list=[src_ids, sent_ids, pos_ids, task_ids, input_mask, qids], + pyreader = fluid.io.DataLoader.from_generator(feed_list=[src_ids, sent_ids, pos_ids, task_ids, input_mask, labels, qids], capacity=70, iterable=False) diff --git a/ernie/infer_classifyer.py b/ernie/infer_classifyer.py index 20ac683f0bc95b7a44f7a22e4ccbb143808383fd..3a68e926d9f894460d946963984d396c1f223358 100644 --- a/ernie/infer_classifyer.py +++ b/ernie/infer_classifyer.py @@ -130,6 +130,7 @@ def main(args): if not args.use_cuda: log.info("disable gpu") config.disable_gpu() + config.switch_ir_optim(True) else: log.info("using gpu") config.enable_use_gpu(1024) @@ -162,8 +163,7 @@ def main(args): # parse outputs output = outputs[0] - output_data = output.data.float_data() - batch_result = np.array(output_data).reshape(output.shape) + batch_result = output.as_ndarray() for single_example_probs in batch_result: print('\t'.join(map(str, single_example_probs.tolist()))) index += 1 @@ -173,13 +173,7 @@ def main(args): def array2tensor(ndarray): """ convert numpy array to PaddleTensor""" assert isinstance(ndarray, np.ndarray), "input type must be np.ndarray" - if ndarray.dtype == np.float32: - dtype = PaddleDType.FLOAT32 - elif ndarray.dtype == np.int64: - dtype = PaddleDType.INT64 - else: - raise ValueError("{} type ndarray is unsupported".format(ndarray.dtype)) - tensor = PaddleTensor(data=ndarray, name="data") + tensor = PaddleTensor(data=ndarray) return tensor if __name__ == '__main__': diff --git a/propeller/service/utils.py b/propeller/service/utils.py index 2885f8a81c5980b2f7dcd50c5e6020c36cd9a387..6a9a15a239980fab6c41b0075022f9c3505d3be8 100644 --- a/propeller/service/utils.py +++ b/propeller/service/utils.py @@ -73,7 +73,7 @@ def slot_to_paddlearray(slot): raise RuntimeError('know type %s' % slot.type) num = len(slot.data) // struct.calcsize(type_str) arr = struct.unpack('%d%s' % (num, type_str), slot.data) - ret = core.PaddleTensor(name='1', data=np.array(arr, dtype=dtype).reshape(slot.dims)) + ret = core.PaddleTensor(np.array(arr, dtype=dtype).reshape(slot.dims)) return ret