diff --git a/demo/elmo/elmo_finetune.py b/demo/elmo/elmo_finetune.py index cd4143de4b1edef19c048c02f3bcb925f0199c2f..9ae3b9395364fd10e2391c081ab9d9c1b6474a67 100644 --- a/demo/elmo/elmo_finetune.py +++ b/demo/elmo/elmo_finetune.py @@ -1,3 +1,4 @@ +#coding:utf-8 import argparse import ast import io diff --git a/demo/image-classification/img_classifier.py b/demo/image-classification/img_classifier.py index acef9d224ab75cb5062c645856b4217f707c7ae2..37281ed0857c36d804725a4759ea8c10bbc8180c 100644 --- a/demo/image-classification/img_classifier.py +++ b/demo/image-classification/img_classifier.py @@ -1,3 +1,4 @@ +#coding:utf-8 import argparse import os diff --git a/demo/image-classification/predict.py b/demo/image-classification/predict.py index 32cdc1f1a040c1b176c5d9985f864919d7da3c68..89e28306be4ac4bd2b65f428345bb20a5ca6021b 100644 --- a/demo/image-classification/predict.py +++ b/demo/image-classification/predict.py @@ -1,3 +1,4 @@ +#coding:utf-8 import argparse import os diff --git a/demo/lac/lac_demo.py b/demo/lac/lac_demo.py index 883d94eebef60bdc59306f34386870c41edbccd4..d923d5ecb8786312ea107eba61820aa77a3c022b 100644 --- a/demo/lac/lac_demo.py +++ b/demo/lac/lac_demo.py @@ -1,4 +1,10 @@ +#coding:utf-8 +from __future__ import print_function + +import json import os +import six + import paddlehub as hub if __name__ == "__main__": @@ -12,5 +18,11 @@ if __name__ == "__main__": # execute predict and print the result results = lac.lexical_analysis(data=inputs) for result in results: - print(result['word']) - print(result['tag']) + if six.PY2: + print(json.dumps( + result['word'], encoding="utf8", ensure_ascii=False)) + print(json.dumps( + result['tag'], encoding="utf8", ensure_ascii=False)) + else: + print(result['word']) + print(result['tag']) diff --git a/demo/senta/predict.py b/demo/senta/predict.py index e11070e8ae09a779e0fa5bf6bb1f1cbfc3216a94..a30245c8e63a66af27e19fc17ee4942b50b2ead1 100644 --- a/demo/senta/predict.py +++ b/demo/senta/predict.py @@ -1,3 +1,4 @@ +#coding:utf-8 from __future__ import absolute_import from __future__ import division from __future__ import print_function diff --git a/demo/senta/senta_demo.py b/demo/senta/senta_demo.py index 2ebcc850d48a957faac8754fbc9516c29470589d..07446a86d1d02d77c41ae76434a8574ef1ffd7ca 100644 --- a/demo/senta/senta_demo.py +++ b/demo/senta/senta_demo.py @@ -1,5 +1,10 @@ -# coding: utf-8 +#coding:utf-8 +from __future__ import print_function + +import json import os +import six + import paddlehub as hub if __name__ == "__main__": @@ -11,5 +16,12 @@ if __name__ == "__main__": input_dict = {"text": test_text} results = senta.sentiment_classify(data=input_dict) + + for index, text in enumerate(test_text): + results[index]["text"] = text for index, result in enumerate(results): - print(test_text[index], result['sentiment_key']) + if six.PY2: + print(json.dumps( + results[index], encoding="utf8", ensure_ascii=False)) + else: + print(results[index]) diff --git a/demo/senta/senta_finetune.py b/demo/senta/senta_finetune.py index d87fc8428bd75fc3b7cb7c3484ba2aacd508eee8..b1d5e0c88120a444c57904ee5316eb2c200854ff 100644 --- a/demo/senta/senta_finetune.py +++ b/demo/senta/senta_finetune.py @@ -1,3 +1,4 @@ +#coding:utf-8 import argparse import ast diff --git a/demo/sequence-labeling/predict.py b/demo/sequence-labeling/predict.py index 6465eb0c2f2871ff2996cf2a7d651b67bbb2551b..0c3cf5138ad2e205543749b70ff96d2dd8563cff 100644 --- a/demo/sequence-labeling/predict.py +++ b/demo/sequence-labeling/predict.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/demo/sequence-labeling/sequence_label.py b/demo/sequence-labeling/sequence_label.py index ebbf281cd5f65341eafd22ffed3eb1d0d3693aae..31a3444eea446944a8e5820603b68fab099cfe14 100644 --- a/demo/sequence-labeling/sequence_label.py +++ b/demo/sequence-labeling/sequence_label.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/demo/ssd/ssd_demo.py b/demo/ssd/ssd_demo.py index 4a24751f9829d285c8242f4e4606144183eedfb2..3d4b376984f877dbd1fe25585d9b71b826e2ef20 100644 --- a/demo/ssd/ssd_demo.py +++ b/demo/ssd/ssd_demo.py @@ -1,3 +1,4 @@ +#coding:utf-8 import os import paddlehub as hub diff --git a/demo/text-classification/predict.py b/demo/text-classification/predict.py index 5159f9c5293bd2cc71048fac82856abee85714fc..ff21c51310907a0e7bb1cefbf277d561c2bd616c 100644 --- a/demo/text-classification/predict.py +++ b/demo/text-classification/predict.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/demo/text-classification/simple_demo.py b/demo/text-classification/simple_demo.py index 8527adfe9339d16d180a03d8571df7256a773a1a..645478aafd977e07349da98ab325f651af7da99b 100644 --- a/demo/text-classification/simple_demo.py +++ b/demo/text-classification/simple_demo.py @@ -1,3 +1,4 @@ +#coding:utf-8 import paddle.fluid as fluid import paddlehub as hub diff --git a/demo/text-classification/text_classifier.py b/demo/text-classification/text_classifier.py index 0e379d663cec8ade11c2fd3fded8fd1eccd2317d..1411e86a82474f28bf8e2ad7e126fa786097f4c5 100644 --- a/demo/text-classification/text_classifier.py +++ b/demo/text-classification/text_classifier.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddlehub/__init__.py b/paddlehub/__init__.py index cc52cfa42e82fad437a171a8e170d4cc46ce620a..09cf2bfbb14ede0aae7e75e644a69f04f2a823c4 100644 --- a/paddlehub/__init__.py +++ b/paddlehub/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" @@ -12,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -#coding:utf-8 - import six +if six.PY2: + import sys + reload(sys) + sys.setdefaultencoding("UTF-8") + from . import module from . import common from . import io @@ -47,8 +51,3 @@ from .finetune.config import RunConfig from .finetune.strategy import AdamWeightDecayStrategy from .finetune.strategy import DefaultStrategy from .finetune.strategy import DefaultFinetuneStrategy - -if six.PY2: - import sys - reload(sys) - sys.setdefaultencoding("UTF-8") diff --git a/paddlehub/commands/__init__.py b/paddlehub/commands/__init__.py index 09886b8103b5f30d99acb788001ee95c674270e9..1690a06fe79454b8e89b36a6b9f21e87995be187 100644 --- a/paddlehub/commands/__init__.py +++ b/paddlehub/commands/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/base_command.py b/paddlehub/commands/base_command.py index 96707cdb772c4cf4b1cc2ec6371f42692e4a5d6d..627ef470bcae228243ce39160d4c9de09cb6c6a8 100644 --- a/paddlehub/commands/base_command.py +++ b/paddlehub/commands/base_command.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/clear.py b/paddlehub/commands/clear.py index a2103d850b412f5b4222832ef2009d80237b543c..0b5624c357f7b67bb7ce656b264c65502d99fc17 100644 --- a/paddlehub/commands/clear.py +++ b/paddlehub/commands/clear.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/cml_utils.py b/paddlehub/commands/cml_utils.py index 74dcc382ecfe84c30c1fd64c8cd6684b6616e22b..6936bdbf6b89365c9e880d64bd9a77ca7f3acb1d 100644 --- a/paddlehub/commands/cml_utils.py +++ b/paddlehub/commands/cml_utils.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/download.py b/paddlehub/commands/download.py index d2291fc8b53907ea5e70cc2fa01f083fb2e7bfe0..c70123aef5a921cda0360c1fb339aa5c9f51fb0d 100644 --- a/paddlehub/commands/download.py +++ b/paddlehub/commands/download.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/help.py b/paddlehub/commands/help.py index 8b7e5f1d6f7793c19d93c9cf997c83daf8a9182a..11c2666e649d895903b37145b02f76e8e3b057e0 100644 --- a/paddlehub/commands/help.py +++ b/paddlehub/commands/help.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/hub.py b/paddlehub/commands/hub.py index d2eef0bd45ed3f9235d98f8bdb1964dc1ac16e00..c5f901cc83421fb8bb3e6bb0f32138516a77146a 100644 --- a/paddlehub/commands/hub.py +++ b/paddlehub/commands/hub.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" @@ -16,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import six import sys from paddlehub.common.logger import logger @@ -60,4 +62,10 @@ def main(): if __name__ == "__main__": - command.execute(sys.argv[1:]) + argv = [] + for item in sys.argv: + if six.PY2: + argv.append(item.decode(sys.stdin.encoding).decode("utf8")) + else: + argv.append(item) + command.execute(argv[1:]) diff --git a/paddlehub/commands/install.py b/paddlehub/commands/install.py index c181f3982d2b845cef9b050be2d2822e47236b7c..fc444b19ada6ab39d4c449b4a63535c972962a66 100644 --- a/paddlehub/commands/install.py +++ b/paddlehub/commands/install.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/list.py b/paddlehub/commands/list.py index 44421b519be1e6992671a4be5e233b2417d4a48b..8830eb6a9b137b65451c2c395dccc07182d8da9f 100644 --- a/paddlehub/commands/list.py +++ b/paddlehub/commands/list.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/run.py b/paddlehub/commands/run.py index 4a18210b40d2d8fa022753fbc56a8d7669eebbf7..d1b837aab4e656284ff20ae7a4e54207de740e59 100644 --- a/paddlehub/commands/run.py +++ b/paddlehub/commands/run.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" @@ -17,6 +18,7 @@ from __future__ import division from __future__ import print_function import argparse +import json import os import sys @@ -172,7 +174,7 @@ class RunCommand(BaseCommand): results = module( sign_name=self.args.signature, data=input_data, **config) if six.PY2: - print(repr(results).decode('string_escape')) + print(json.dumps(results, encoding="utf8", ensure_ascii=False)) else: print(results) diff --git a/paddlehub/commands/search.py b/paddlehub/commands/search.py index 73f5988669f57ae5b3e627ed194cf47e7b2693fc..99710ea6f3f1d800f85a57f83e64abab9cc57b79 100644 --- a/paddlehub/commands/search.py +++ b/paddlehub/commands/search.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/show.py b/paddlehub/commands/show.py index 990ce2e0a3a8c94456e029fb964c63a856f6c343..ebd3b8cb879181799565501f09442cbefedbacb6 100644 --- a/paddlehub/commands/show.py +++ b/paddlehub/commands/show.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/uninstall.py b/paddlehub/commands/uninstall.py index c00b9acd4f5e619ae001034abb201187aba23381..ac50029f8135d5c64db21cdf02bbebfb68b8ba17 100644 --- a/paddlehub/commands/uninstall.py +++ b/paddlehub/commands/uninstall.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/commands/version.py b/paddlehub/commands/version.py index 47e959501d29927a3b67fc79769f19593cbfed61..c763044c32e2499197ff18e7faf4355497246bba 100644 --- a/paddlehub/commands/version.py +++ b/paddlehub/commands/version.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/__init__.py b/paddlehub/common/__init__.py index 93814bf48f343540e891b02c6c72e7b9f69a8d0e..17a2fc955052462d2b587bbc330cffcaf6b4c001 100644 --- a/paddlehub/common/__init__.py +++ b/paddlehub/common/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/arg_helper.py b/paddlehub/common/arg_helper.py index 761709a6331c4b9d868b9417a7e84c5e787b7b86..9d8008b522cb010c1d555dbbe6a9fd0347c7d7c5 100644 --- a/paddlehub/common/arg_helper.py +++ b/paddlehub/common/arg_helper.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/dir.py b/paddlehub/common/dir.py index 550ce1fdb0120b35d711fde1e3073c246bf4361c..9baa21f088f51d34dd7b4e2a11ff630b7dea34bc 100644 --- a/paddlehub/common/dir.py +++ b/paddlehub/common/dir.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/downloader.py b/paddlehub/common/downloader.py index 10091e1bcbc2b8ec94607f38d26049597c0a32e1..984bedc0e22084d80a6b0962acf900056fd9d9ee 100644 --- a/paddlehub/common/downloader.py +++ b/paddlehub/common/downloader.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/hub_server.py b/paddlehub/common/hub_server.py index 0029a64919fb65d3dbc6116e6ea58b54af4347bc..1b7f08ced5a9a153aec6c3fbc59a6ecd89bbe25e 100644 --- a/paddlehub/common/hub_server.py +++ b/paddlehub/common/hub_server.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/logger.py b/paddlehub/common/logger.py index d04913a8af4c8bc73098e078171b5e5a13a2f858..fb61fb39ffc7d2ecd47c2718c383a5eecd6415d3 100644 --- a/paddlehub/common/logger.py +++ b/paddlehub/common/logger.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/paddle_helper.py b/paddlehub/common/paddle_helper.py index 955ae80979a1bcbacc4f1d8b17bc98f5783509f7..5bb0fae1045670b6ff28b2ce74c2ab0f9ce169d3 100644 --- a/paddlehub/common/paddle_helper.py +++ b/paddlehub/common/paddle_helper.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/common/utils.py b/paddlehub/common/utils.py index db57a4aad0ce32798cebbde2c7fb6f5a511f8b90..0406de26d6e2d86399fe9ae79fb5411311e65c36 100644 --- a/paddlehub/common/utils.py +++ b/paddlehub/common/utils.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/__init__.py b/paddlehub/dataset/__init__.py index 2239623b526e9038b292ea881c891ae5d0f8d375..f73375e2d22feb89687d73110e4f45b687f41676 100644 --- a/paddlehub/dataset/__init__.py +++ b/paddlehub/dataset/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddlehub/dataset/base_cv_dataset.py b/paddlehub/dataset/base_cv_dataset.py index 9eb41c1044d83baa642e916579d6867b8bc7f316..1606b874976f42b9719ec30fed4ebd952c4137e6 100644 --- a/paddlehub/dataset/base_cv_dataset.py +++ b/paddlehub/dataset/base_cv_dataset.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/chnsenticorp.py b/paddlehub/dataset/chnsenticorp.py index b59a7a393372c1587c7cf555e56217a9941458a1..76b2548512f88be502b3da6ad99ffca005e2646f 100644 --- a/paddlehub/dataset/chnsenticorp.py +++ b/paddlehub/dataset/chnsenticorp.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/dataset.py b/paddlehub/dataset/dataset.py index aca777476628acfd126194e53d34bed34ab19a19..57f02e5dfe49dd74623171b5e0ea47a6f07158df 100644 --- a/paddlehub/dataset/dataset.py +++ b/paddlehub/dataset/dataset.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/dogcat.py b/paddlehub/dataset/dogcat.py index 70600b34e18640bf8b7b10502e7233524ae330bb..6ebbec886d92e0c999d57d6d7394c63f19bcfb95 100644 --- a/paddlehub/dataset/dogcat.py +++ b/paddlehub/dataset/dogcat.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/flowers.py b/paddlehub/dataset/flowers.py index adef50aea3f7d248668342b8de5cf495d80f911f..3b46790dce55a1e77802e36304c1188956add5ff 100644 --- a/paddlehub/dataset/flowers.py +++ b/paddlehub/dataset/flowers.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/food101.py b/paddlehub/dataset/food101.py index de9c8679053493747d46309fe3a8c2d7c0abc2f6..03622ebb3a9a80dbae2177e8325dd7320953473b 100644 --- a/paddlehub/dataset/food101.py +++ b/paddlehub/dataset/food101.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/indoor67.py b/paddlehub/dataset/indoor67.py index 59ebc3e7b603004156e300dcef251d025d8a9ef5..37d014a02a6b67d0f0a24db5e1324ccac78cfd48 100644 --- a/paddlehub/dataset/indoor67.py +++ b/paddlehub/dataset/indoor67.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/lcqmc.py b/paddlehub/dataset/lcqmc.py index 550128755e3460f00769aa7aaf2c617e2ab6b48e..99a94de6bb5c4cc3935beda0211125d2070ddc4c 100644 --- a/paddlehub/dataset/lcqmc.py +++ b/paddlehub/dataset/lcqmc.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/msra_ner.py b/paddlehub/dataset/msra_ner.py index 81c31a960d0bf71c0dad5d1d4a2087c427a78931..a2ac96a2b87823536b98f6d1ed5a393f3c7bc0eb 100644 --- a/paddlehub/dataset/msra_ner.py +++ b/paddlehub/dataset/msra_ner.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" @@ -20,7 +21,6 @@ import os import codecs import csv import json -import six from collections import namedtuple from paddlehub.dataset import InputExample, HubDataset diff --git a/paddlehub/dataset/nlpcc_dbqa.py b/paddlehub/dataset/nlpcc_dbqa.py index 09a64439295aee6d8de93f7b6884bb246cbe40b7..1440e7ad7234254b169ee1ad9455079a575eaca8 100644 --- a/paddlehub/dataset/nlpcc_dbqa.py +++ b/paddlehub/dataset/nlpcc_dbqa.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/dataset/stanford_dogs.py b/paddlehub/dataset/stanford_dogs.py index 9081027e548f4d05aaf45ee60a76c080b49b360b..c399ad357b1898145e871e69559da509fec10575 100644 --- a/paddlehub/dataset/stanford_dogs.py +++ b/paddlehub/dataset/stanford_dogs.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/__init__.py b/paddlehub/finetune/__init__.py index a028662d70d6814266a52ccc6826e640c7671b7e..309e3855eb34f06f4dfc50304465742828071db9 100644 --- a/paddlehub/finetune/__init__.py +++ b/paddlehub/finetune/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/checkpoint.py b/paddlehub/finetune/checkpoint.py index 4fa987df6d7256db7224742f6b43aa8138fc581c..43272685721b29881f057837ca13cb5fb80f66c6 100644 --- a/paddlehub/finetune/checkpoint.py +++ b/paddlehub/finetune/checkpoint.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/checkpoint_pb2.py b/paddlehub/finetune/checkpoint_pb2.py index 39ca52384e8b64f5abfc1d23d7935f3bada717a3..e03832241452af82aecc2430fd968ba1c9542610 100644 --- a/paddlehub/finetune/checkpoint_pb2.py +++ b/paddlehub/finetune/checkpoint_pb2.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Generated by the protocol buffer compiler. DO NOT EDIT! # source: checkpoint.proto diff --git a/paddlehub/finetune/config.py b/paddlehub/finetune/config.py index 4b3d8f71b1c62a9b37b504c9f8359987001991a8..89a0ebb108678ece3a263307e8efef30b6cc2a6e 100644 --- a/paddlehub/finetune/config.py +++ b/paddlehub/finetune/config.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/evaluate.py b/paddlehub/finetune/evaluate.py index 4563e39498186089fa94d919e121173fefc2207f..38a05c56a05d7da6e0b9ce9b0fdc831153013120 100644 --- a/paddlehub/finetune/evaluate.py +++ b/paddlehub/finetune/evaluate.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/finetune.py b/paddlehub/finetune/finetune.py index 0d63cfd50f1465e273fb7c1c8b103ca6939e1153..7965b28c5cd876063b1b49c236e4da2a918e9713 100644 --- a/paddlehub/finetune/finetune.py +++ b/paddlehub/finetune/finetune.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/optimization.py b/paddlehub/finetune/optimization.py index 5b7363c0ba765c421b3d30fd9db9334500a2f833..00658b9dcb9a4b9f16afc8c473a1bdb58083e4fb 100644 --- a/paddlehub/finetune/optimization.py +++ b/paddlehub/finetune/optimization.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddlehub/finetune/regularizer.py b/paddlehub/finetune/regularizer.py index 72fc399f87ada1e2f3061c3c4277ffd05e0c00b9..441b05a3fc5c250b00f8101346e5fbea9a0129d7 100644 --- a/paddlehub/finetune/regularizer.py +++ b/paddlehub/finetune/regularizer.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/strategy.py b/paddlehub/finetune/strategy.py index 31c0baa3c347976f66b2f177352dff753db7840e..f7fa6f59026ee5727b770205e00237f3e95f7d88 100644 --- a/paddlehub/finetune/strategy.py +++ b/paddlehub/finetune/strategy.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/finetune/task.py b/paddlehub/finetune/task.py index bc2b8ac8efd3b35f3ed49cfc8e188edb97aba9ea..cac00ca1e27bbb33c70e5d2b34a35bfd03f1c19c 100644 --- a/paddlehub/finetune/task.py +++ b/paddlehub/finetune/task.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/io/__init__.py b/paddlehub/io/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1ecc0e802314d71062c88952d7e8681540ee921a 100644 --- a/paddlehub/io/__init__.py +++ b/paddlehub/io/__init__.py @@ -0,0 +1 @@ +#coding:utf-8 diff --git a/paddlehub/io/augmentation.py b/paddlehub/io/augmentation.py index ec5e8a6c909c8f0602d9ba6e426cb5d36cb270a5..4710fc1e278c3420128e6b23ae013a7f84efb870 100644 --- a/paddlehub/io/augmentation.py +++ b/paddlehub/io/augmentation.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/io/parser.py b/paddlehub/io/parser.py index d332d7f1fa30a5aa91553f8f4175d4d612da3574..4a60105aefd90b4c709bef3b25720543c3997984 100644 --- a/paddlehub/io/parser.py +++ b/paddlehub/io/parser.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/io/type.py b/paddlehub/io/type.py index 074b5e05da0f0c5680d58481882e27e094d5eacf..4a68a918005406c7ef2e7a458459ff266bd26484 100644 --- a/paddlehub/io/type.py +++ b/paddlehub/io/type.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/__init__.py b/paddlehub/module/__init__.py index b03a52d9b68d392b9b78ea1dd9b2e9a939a93101..0f2925dad95c29cc4bd2cd21639c7d83793cd968 100644 --- a/paddlehub/module/__init__.py +++ b/paddlehub/module/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/base_processor.py b/paddlehub/module/base_processor.py index 1d6b7fb3c2e9a293c3cb17644ef991f9d6d5d740..d5a57d9f7bf0f50f1e53409355001c01a09fabbb 100644 --- a/paddlehub/module/base_processor.py +++ b/paddlehub/module/base_processor.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/check_info_pb2.py b/paddlehub/module/check_info_pb2.py index 95d3e4112d2e4076143e0780afd9739596efdc25..78f5546c49c417508d26fa0f809340459987fc66 100644 --- a/paddlehub/module/check_info_pb2.py +++ b/paddlehub/module/check_info_pb2.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Generated by the protocol buffer compiler. DO NOT EDIT! # source: check_info.proto diff --git a/paddlehub/module/checker.py b/paddlehub/module/checker.py index 13398c2b694d6d23c36dec340eea1f50518aacd8..d76ca6bd74ab4d354ba5fa72e8f1e7215c0ed6f0 100644 --- a/paddlehub/module/checker.py +++ b/paddlehub/module/checker.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/manager.py b/paddlehub/module/manager.py index 460275741db3f2b3529b8bc94badeaf1ec6d4c44..645c43482196dca6c0af35607d1cdaf7e1a1e9fe 100644 --- a/paddlehub/module/manager.py +++ b/paddlehub/module/manager.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index 1c0ddc150b833e791bd5601977ec1dee521e4f88..f8a956045eded88e9e93410f26e9af5d97b94681 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/module/module_desc_pb2.py b/paddlehub/module/module_desc_pb2.py index 001052e4d139299d044584b4afbcc3f0ac7d6be0..4dd6c8120cb69984c3f77774b62d8d660118681b 100644 --- a/paddlehub/module/module_desc_pb2.py +++ b/paddlehub/module/module_desc_pb2.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Generated by the protocol buffer compiler. DO NOT EDIT! # source: module_desc.proto diff --git a/paddlehub/module/signature.py b/paddlehub/module/signature.py index 93150c0a679806cf6b633581243b4431bffcb275..1baba84c681bfb7b550a30f9c6bae96db277c243 100644 --- a/paddlehub/module/signature.py +++ b/paddlehub/module/signature.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/reader/__init__.py b/paddlehub/reader/__init__.py index a0e119df90869fe3c46b8cec70cf3c089d1ebccc..bc0fa0252a08445d5be287dfa8025f945bc12391 100644 --- a/paddlehub/reader/__init__.py +++ b/paddlehub/reader/__init__.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/reader/batching.py b/paddlehub/reader/batching.py index e33b0082627f46bd430c53b06db9cbb2d996edec..5ec5f320cf5ec7bd0ab4624d9b39ef936553c774 100644 --- a/paddlehub/reader/batching.py +++ b/paddlehub/reader/batching.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddlehub/reader/cv_reader.py b/paddlehub/reader/cv_reader.py index 9e06a5a52d218b971d4fade331c962e207d58139..6aba2f8dfcaac839444fc27aa146bd03e5dfd5a0 100644 --- a/paddlehub/reader/cv_reader.py +++ b/paddlehub/reader/cv_reader.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/paddlehub/reader/nlp_reader.py b/paddlehub/reader/nlp_reader.py index 0ecd43b29d46a385b681ad9995f49525ec37ebd7..7d2b80942b197c6dc220706f87874f426efab549 100644 --- a/paddlehub/reader/nlp_reader.py +++ b/paddlehub/reader/nlp_reader.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +21,7 @@ import csv import json import platform import six +import sys from collections import namedtuple import paddle @@ -31,12 +33,6 @@ from .batching import pad_batch_data import paddlehub as hub -def get_encoding(): - if platform.platform().lower().startswith("windows"): - return "gbk" - return "utf8" - - class BaseReader(object): def __init__(self, dataset, @@ -426,15 +422,13 @@ class LACClassifyReader(object): def preprocess(text): data_dict = {self.feed_key: [text]} processed = self.lac.lexical_analysis(data=data_dict) - for data in processed: - for index, word in enumerate(data['word']): - if six.PY2 and type(word) == str: - data['word'][index] = word.decode(get_encoding()) processed = [ self.vocab[word] for word in processed[0]['word'] if word in self.vocab ] if len(processed) == 0: + if six.PY2: + text = text.encode(sys.stdout.encoding) logger.warning( "The words in text %s can't be found in the vocabulary." % (text)) diff --git a/paddlehub/version.py b/paddlehub/version.py index 71a25047160fea7f542b95d2c16e79751b8839e9..669680c5bd00b20b8e4f6cc8615783c19f7ec156 100644 --- a/paddlehub/version.py +++ b/paddlehub/version.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" diff --git a/requirements.txt b/requirements.txt index a9e5ee21e9711e301cab1faab263048213871c55..10dc248598b5917be0b01e09b9dbb86bb546aaeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,3 @@ pyyaml numpy >= 1.12.0 Pillow six >= 1.10.0 -chardet == 3.0.4 diff --git a/setup.py b/setup.py index c6baa6c3e1db5b13f8eaa7bc9ceddaa0220e1445..17c469eb82786f1eb0f9210dfb5822b5ca1e6f2b 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#coding:utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" @@ -30,8 +31,12 @@ def python_version(): max_version, mid_version, min_version = python_version() REQUIRED_PACKAGES = [ - 'numpy >= 1.12.0', 'six >= 1.10.0', 'protobuf >= 3.1.0', 'pyyaml', 'Pillow', - "visualdl >= 1.3.0", "chardet == 3.0.4" + 'numpy >= 1.12.0', + 'six >= 1.10.0', + 'protobuf >= 3.1.0', + 'pyyaml', + 'Pillow', + "visualdl >= 1.3.0", ] if max_version < 3: