Merge pull request #26 from Steffy-zxf/fix-py2-py3-compatible

Fix the bug that read_csv compatibility between py2 and py3

demo/text-classification/run_classifier.sh

-export CUDA_VISIBLE_DEVICES=3
+export CUDA_VISIBLE_DEVICES=0
 
 # User can select chnsenticorp, nlpcc_dbqa, lcqmc for different task
 DATASET="chnsenticorp"

paddlehub/__init__.py

@@ -11,6 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+#coding:utf-8
+import six
+
 from . import module
 from . import common
 from . import io
@@ -41,3 +45,8 @@ from .finetune.finetune import finetune_and_eval
 from .finetune.config import RunConfig
 from .finetune.strategy import AdamWeightDecayStrategy
 from .finetune.strategy import DefaultStrategy
+
+if six.PY2:
+    import sys
+    reload(sys)
+    sys.setdefaultencoding("UTF-8")
\ No newline at end of file

paddlehub/dataset/chnsenticorp.py

@@ -17,6 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 from collections import namedtuple
+import codecs
 import os
 import csv
 
@@ -79,7 +80,7 @@ class ChnSentiCorp(HubDataset):
 
     def _read_tsv(self, input_file, quotechar=None):
         """Reads a tab separated value file."""
-        with open(input_file, "r") as f:
+        with codecs.open(input_file, "r", encoding="UTF-8") as f:
             reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
             examples = []
             seq_id = 0

paddlehub/dataset/lcqmc.py

@@ -17,6 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 from collections import namedtuple
+import codecs
 import os
 import csv
 
@@ -75,7 +76,7 @@ class LCQMC(HubDataset):
 
     def _read_tsv(self, input_file, quotechar=None):
         """Reads a tab separated value file."""
-        with open(input_file, "r") as f:
+        with codecs.open(input_file, "r", encoding="UTF-8") as f:
             reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
             examples = []
             seq_id = 0

paddlehub/dataset/msra_ner.py

@@ -17,8 +17,10 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import codecs
 import csv
 import json
+import six
 from collections import namedtuple
 
 from paddlehub.dataset import InputExample, HubDataset
@@ -85,7 +87,7 @@ class MSRA_NER(HubDataset):
 
     def _read_tsv(self, input_file, quotechar=None):
         """Reads a tab separated value file."""
-        with open(input_file, "r") as f:
+        with codecs.open(input_file, "r", encoding="UTF-8") as f:
             reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
             examples = []
             seq_id = 0

paddlehub/dataset/nlpcc_dbqa.py

@@ -17,6 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 from collections import namedtuple
+import codecs
 import os
 import csv
 
@@ -81,7 +82,7 @@ class NLPCC_DBQA(HubDataset):
 
     def _read_tsv(self, input_file, quotechar=None):
         """Reads a tab separated value file."""
-        with open(input_file, "r") as f:
+        with codecs.open(input_file, "r", encoding="UTF-8") as f:
             reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
             examples = []
             seq_id = 0