diff --git a/fluid/PaddleNLP/machine_reading_comprehension/dataset.py b/fluid/PaddleNLP/machine_reading_comprehension/dataset.py index 3aaf87be9a7b0659fa9e79eb8329911cbea73c55..c732ce041c5e82ea5e1471ba422f5b056a7cba8f 100644 --- a/fluid/PaddleNLP/machine_reading_comprehension/dataset.py +++ b/fluid/PaddleNLP/machine_reading_comprehension/dataset.py @@ -23,6 +23,7 @@ import json import logging import numpy as np from collections import Counter +import io class BRCDataset(object): @@ -67,7 +68,7 @@ class BRCDataset(object): Args: data_path: the data file to load """ - with open(data_path) as fin: + with io.open(data_path, 'r', encoding='utf-8') as fin: data_set = [] for lidx, line in enumerate(fin): sample = json.loads(line.strip()) diff --git a/fluid/PaddleNLP/machine_reading_comprehension/run.py b/fluid/PaddleNLP/machine_reading_comprehension/run.py index dbe3a4b9a296fdaf089d55be3f0c9845422f0ce5..74561297f003faa4b3d871c0f327b65da63e81e7 100644 --- a/fluid/PaddleNLP/machine_reading_comprehension/run.py +++ b/fluid/PaddleNLP/machine_reading_comprehension/run.py @@ -22,6 +22,7 @@ import os import random import json import six +import multiprocessing import paddle import paddle.fluid as fluid