diff --git a/python/examples/bert/bert_web_service.py b/python/examples/bert/bert_web_service.py index f72694c0e8c5bb7ab2778278d3fc79f13516dc12..8db64e5eb792a7365ed739bbfb05bf38fd8a0da1 100644 --- a/python/examples/bert/bert_web_service.py +++ b/python/examples/bert/bert_web_service.py @@ -24,7 +24,9 @@ class BertService(WebService): self.reader = BertReader(vocab_file="vocab.txt", max_seq_len=128) def preprocess(self, feed={}, fetch=[]): - feed_res = self.reader.process(feed["words"].encode("utf-8")) + feed_res = [{ + "words": self.reader.process(ins["words"].encode("utf-8")) + } for ins in feed] return feed_res, fetch diff --git a/python/examples/imagenet/image_classification_service.py b/python/examples/imagenet/image_classification_service.py index a041d9d9447184512f010251c85c919985f02c7c..81169d6bdafa7024f2b997c48c0abdc04411e391 100644 --- a/python/examples/imagenet/image_classification_service.py +++ b/python/examples/imagenet/image_classification_service.py @@ -23,23 +23,14 @@ from paddle_serving_app import ImageReader class ImageService(WebService): def preprocess(self, feed={}, fetch=[]): reader = ImageReader() - if "image" not in feed: - raise ("feed data error!") - if isinstance(feed["image"], list): - feed_batch = [] - for image in feed["image"]: - sample = base64.b64decode(image) - img = reader.process_image(sample) - res_feed = {} - res_feed["image"] = img - feed_batch.append(res_feed) - return feed_batch, fetch - else: - sample = base64.b64decode(feed["image"]) + feed_batch = [] + for ins in feed: + if "image" not in ins: + raise ("feed data error!") + sample = base64.b64decode(ins["image"]) img = reader.process_image(sample) - res_feed = {} - res_feed["image"] = img - return res_feed, fetch + feed_batch.append({"image": img}) + return feed_batch, fetch image_service = ImageService(name="image") diff --git a/python/examples/imagenet/image_classification_service_gpu.py b/python/examples/imagenet/image_classification_service_gpu.py index ccb2adac26bb0fd16165dd2dfb3c9507b1bf193f..7cb973547982877a50e86062fe187233a32065e6 100644 --- a/python/examples/imagenet/image_classification_service_gpu.py +++ b/python/examples/imagenet/image_classification_service_gpu.py @@ -23,24 +23,14 @@ from paddle_serving_server_gpu.web_service import WebService class ImageService(WebService): def preprocess(self, feed={}, fetch=[]): reader = ImageReader() - if "image" not in feed: - raise ("feed data error!") - print(type(feed["image"]), isinstance(feed["image"], list)) - if isinstance(feed["image"], list): - feed_batch = [] - for image in feed["image"]: - sample = base64.b64decode(image) - img = reader.process_image(sample) - res_feed = {} - res_feed["image"] = img - feed_batch.append(res_feed) - return feed_batch, fetch - else: - sample = base64.b64decode(feed["image"]) + feed_batch = [] + for ins in feed: + if "image" not in ins: + raise ("feed data error!") + sample = base64.b64decode(ins["image"]) img = reader.process_image(sample) - res_feed = {} - res_feed["image"] = img - return res_feed, fetch + feed_batch.append({"image": img}) + return feed_batch, fetch image_service = ImageService(name="image") diff --git a/python/examples/imdb/benchmark_batch.py b/python/examples/imdb/benchmark_batch.py index 57ee6816989d4a807a328342f188a7298b7772de..5891970b5decc34f35723187e44b166e0482c6e9 100644 --- a/python/examples/imdb/benchmark_batch.py +++ b/python/examples/imdb/benchmark_batch.py @@ -40,21 +40,29 @@ def single_func(idx, resource): if args.batch_size >= 1: feed_batch = [] for bi in range(args.batch_size): - word_ids, label = imdb_dataset.get_words_and_label(line) + word_ids, label = imdb_dataset.get_words_and_label(dataset[ + bi]) feed_batch.append({"words": word_ids}) result = client.predict(feed=feed_batch, fetch=["prediction"]) + if result is None: + raise ("predict failed.") else: print("unsupport batch size {}".format(args.batch_size)) elif args.request == "http": - for fn in filelist: - fin = open(fn) - for line in fin: - word_ids, label = imdb_dataset.get_words_and_label(line) - r = requests.post( - "http://{}/imdb/prediction".format(args.endpoint), - data={"words": word_ids, - "fetch": ["prediction"]}) + if args.batch_size >= 1: + feed_batch = [] + for bi in range(args.batch_size): + feed_batch.append({"words": dataset[bi]}) + r = requests.post( + "http://{}/imdb/prediction".format(args.endpoint), + json={"feed": feed_batch, + "fetch": ["prediction"]}) + if r.status_code != 200: + print('HTTP status code -ne 200') + raise ("predict failed.") + else: + print("unsupport batch size {}".format(args.batch_size)) end = time.time() return [[end - start]] diff --git a/python/examples/imdb/benchmark_batch.sh b/python/examples/imdb/benchmark_batch.sh index 322aaafffb9b252a307fcb63dc7b910e33e5e002..15b65338b21675fd89056cf32f9a247b385a6a36 100644 --- a/python/examples/imdb/benchmark_batch.sh +++ b/python/examples/imdb/benchmark_batch.sh @@ -3,7 +3,7 @@ for thread_num in 1 2 4 8 16 do for batch_size in 1 2 4 8 16 32 64 128 256 512 do - $PYTHONROOT/bin/python benchmark_batch.py --thread $thread_num --batch_size $batch_size --model imdbo_bow_client_conf/serving_client_conf.prototxt --request rpc > profile 2>&1 + $PYTHONROOT/bin/python benchmark_batch.py --thread $thread_num --batch_size $batch_size --model imdb_bow_client_conf/serving_client_conf.prototxt --request rpc > profile 2>&1 echo "========================================" echo "batch size : $batch_size" >> profile_log $PYTHONROOT/bin/python ../util/show_profile.py profile $thread_num >> profile_log diff --git a/python/examples/imdb/text_classify_service.py b/python/examples/imdb/text_classify_service.py index 5ff919ebb44b9a2590b148e4ccf8b91ce85f3f53..4420a99facc7bd3db1c8bf1df0c58765467517de 100755 --- a/python/examples/imdb/text_classify_service.py +++ b/python/examples/imdb/text_classify_service.py @@ -26,10 +26,9 @@ class IMDBService(WebService): self.dataset.load_resource(args["dict_file_path"]) def preprocess(self, feed={}, fetch=[]): - if "words" not in feed: - exit(-1) - res_feed = {} - res_feed["words"] = self.dataset.get_words_only(feed["words"]) + res_feed = [{ + "words": self.dataset.get_words_only(ins["words"]) + } for ins in feed] return res_feed, fetch diff --git a/python/examples/lac/lac_reader.py b/python/examples/lac/lac_reader.py index 3895277dbbf98a9b4ff2d6592d82fe02fe9a4c12..c9f31c148123e1975c0102903abf2f2b3b15d3f6 100644 --- a/python/examples/lac/lac_reader.py +++ b/python/examples/lac/lac_reader.py @@ -108,15 +108,15 @@ class LACReader(object): partial_word = "" for ind, tag in enumerate(tags): if partial_word == "": - partial_word = words[ind] + partial_word = self.id2word_dict[str(words[ind])] tags_out.append(tag.split('-')[0]) continue if tag.endswith("-B") or (tag == "O" and tag[ind - 1] != "O"): sent_out.append(partial_word) tags_out.append(tag.split('-')[0]) - partial_word = words[ind] + partial_word = self.id2word_dict[str(words[ind])] continue - partial_word += words[ind] + partial_word += self.id2word_dict[str(words[ind])] if len(sent_out) < len(tags_out): sent_out.append(partial_word) diff --git a/python/examples/lac/lac_web_service.py b/python/examples/lac/lac_web_service.py index 186d8badf8806606998466e3d1bb4047bf51b5d8..c9bd00986c62abde3ee24ddddbf08dda45bbed05 100644 --- a/python/examples/lac/lac_web_service.py +++ b/python/examples/lac/lac_web_service.py @@ -22,15 +22,24 @@ class LACService(WebService): self.reader = LACReader("lac_dict") def preprocess(self, feed={}, fetch=[]): - if "words" not in feed: - raise ("feed data error!") - feed_data = self.reader.process(feed["words"]) + feed_batch = [] + for ins in feed: + if "words" not in ins: + raise ("feed data error!") + feed_data = self.reader.process(ins["words"]) + feed_batch.append({"words": feed_data}) fetch = ["crf_decode"] - return {"words": feed_data}, fetch + return feed_batch, fetch def postprocess(self, feed={}, fetch=[], fetch_map={}): - segs = self.reader.parse_result(feed["words"], fetch_map["crf_decode"]) - return {"word_seg": "|".join(segs)} + batch_ret = [] + for idx, ins in enumerate(feed): + begin = fetch_map['crf_decode.lod'][idx] + end = fetch_map['crf_decode.lod'][idx + 1] + segs = self.reader.parse_result(ins["words"], + fetch_map["crf_decode"][begin:end]) + batch_ret.append({"word_seg": "|".join(segs)}) + return batch_ret lac_service = LACService(name="lac") @@ -39,3 +48,4 @@ lac_service.load_reader() lac_service.prepare_server( workdir=sys.argv[2], port=int(sys.argv[3]), device="cpu") lac_service.run_server() +lac_service.run_flask() diff --git a/python/examples/senta/senta_web_service.py b/python/examples/senta/senta_web_service.py index 35dab1911ec18af55ef19750b1b239b5aba2c8a9..9a59d9d030ae55584f1b9939a02c782bae3c5011 100644 --- a/python/examples/senta/senta_web_service.py +++ b/python/examples/senta/senta_web_service.py @@ -87,7 +87,7 @@ class SentaService(WebService): if self.show: print("---- senta reader ----") print("feed_data", feed_data) - return {"words": feed_data}, fetch + return [{"words": feed_data}], fetch senta_service = SentaService(name="senta") diff --git a/tools/serving_build.sh b/tools/serving_build.sh index a39e5a63f91ba694a20973bba9e3fe440446be20..a522efe19cb9f4170341f291d8c30db0e6749ad1 100644 --- a/tools/serving_build.sh +++ b/tools/serving_build.sh @@ -157,12 +157,20 @@ function python_test_fit_a_line() { check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" # check http code http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` - setproxy # recover proxy state - kill_server_process if [ ${http_code} -ne 200 ]; then echo "HTTP status code -ne 200" exit 1 fi + # test web batch + check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" + # check http code + http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` + if [ ${http_code} -ne 200 ]; then + echo "HTTP status code -ne 200" + exit 1 + fi + setproxy # recover proxy state + kill_server_process ;; GPU) export CUDA_VISIBLE_DEVICES=0 @@ -179,12 +187,20 @@ function python_test_fit_a_line() { check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" # check http code http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` - setproxy # recover proxy state - kill_server_process if [ ${http_code} -ne 200 ]; then echo "HTTP status code -ne 200" exit 1 fi + # test web batch + check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" + # check http code + http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` + if [ ${http_code} -ne 200 ]; then + echo "HTTP status code -ne 200" + exit 1 + fi + setproxy # recover proxy state + kill_server_process ;; *) echo "error type" @@ -288,15 +304,6 @@ function python_test_bert() { pip install paddle_serving_app check_cmd "head -n 10 data-c.txt | python bert_client.py --model bert_chinese_L-12_H-768_A-12_client/serving_client_conf.prototxt" kill_server_process - # python prepare_model.py 20 - # sh get_data.sh - # check_cmd "python -m paddle_serving_server.serve --model bert_seq20_model/ --port 9292 &" - # sleep 5 - # pip install paddle_serving_app - # check_cmd "head -n 10 data-c.txt | python bert_client.py --model bert_seq20_client/serving_client_conf.prototxt" - # kill_server_process - # ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill - # ps -ef | grep "serving" | grep -v grep | awk '{print $2}' | xargs kill echo "bert RPC inference pass" ;; GPU) @@ -312,14 +319,6 @@ function python_test_bert() { pip install paddle_serving_app check_cmd "head -n 10 data-c.txt | python bert_client.py --model bert_chinese_L-12_H-768_A-12_client/serving_client_conf.prototxt" kill_server_process - # python prepare_model.py 20 - # sh get_data.sh - # check_cmd "python -m paddle_serving_server_gpu.serve --model bert_seq20_model/ --port 9292 --gpu_ids 0 &" - # sleep 5 - # pip install paddle_serving_app - # check_cmd "head -n 10 data-c.txt | python bert_client.py --model bert_seq20_client/serving_client_conf.prototxt" - # kill_server_process - # ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill echo "bert RPC inference pass" ;; *) @@ -340,19 +339,29 @@ function python_test_imdb() { case $TYPE in CPU) sh get_data.sh - sleep 5 check_cmd "python -m paddle_serving_server.serve --model imdb_cnn_model/ --port 9292 &" + sleep 5 check_cmd "head test_data/part-0 | python test_client.py imdb_cnn_client_conf/serving_client_conf.prototxt imdb.vocab" + # test batch predict + check_cmd "python benchmark_batch.py --thread 4 --batch_size 8 --model imdb_bow_client_conf/serving_client_conf.prototxt --request rpc --endpoint 127.0.0.1:9292" echo "imdb CPU RPC inference pass" - ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill + kill_server_process rm -rf work_dir1 sleep 5 - check_cmd "python text_classify_service.py imdb_cnn_model/workdir/9292 imdb.vocab &" + unsetproxy # maybe the proxy is used on iPipe, which makes web-test failed. + check_cmd "python text_classify_service.py imdb_cnn_model/ workdir/ 9292 imdb.vocab &" sleep 5 check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"words\": \"i am very sad | 0\"}], \"fetch\":[\"prediction\"]}' http://127.0.0.1:9292/imdb/prediction" + http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"words": "i am very sad | 0"}], "fetch":["prediction"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9292/imdb/prediction` + if [ ${http_code} -ne 200 ]; then + echo "HTTP status code -ne 200" + exit 1 + fi + # test batch predict + check_cmd "python benchmark_batch.py --thread 4 --batch_size 8 --model imdb_bow_client_conf/serving_client_conf.prototxt --request http --endpoint 127.0.0.1:9292" + setproxy # recover proxy state kill_server_process - ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill ps -ef | grep "text_classify_service.py" | grep -v grep | awk '{print $2}' | xargs kill echo "imdb CPU HTTP inference pass" ;; @@ -379,15 +388,30 @@ function python_test_lac() { sh get_data.sh check_cmd "python -m paddle_serving_server.serve --model jieba_server_model/ --port 9292 &" sleep 5 - check_cmd "echo "我爱北京天安门" | python lac_client.py jieba_client_conf/serving_client_conf.prototxt lac_dict/" + check_cmd "echo \"我爱北京天安门\" | python lac_client.py jieba_client_conf/serving_client_conf.prototxt lac_dict/" echo "lac CPU RPC inference pass" - ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill + kill_server_process + unsetproxy # maybe the proxy is used on iPipe, which makes web-test failed. check_cmd "python lac_web_service.py jieba_server_model/ lac_workdir 9292 &" sleep 5 - check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"words\": \"i am very sad | 0\"}], \"fetch\":[\"prediction\"]}' http://127.0.0.1:9292/imdb/prediction" + check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"words\": \"我爱北京天安门\"}], \"fetch\":[\"word_seg\"]}' http://127.0.0.1:9292/lac/prediction" + # check http code + http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"words": "我爱北京天安门"}], "fetch":["word_seg"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9292/lac/prediction` + if [ ${http_code} -ne 200 ]; then + echo "HTTP status code -ne 200" + exit 1 + fi + # http batch + check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"words\": \"我爱北京天安门\"}, {\"words\": \"我爱北京天安门\"}], \"fetch\":[\"word_seg\"]}' http://127.0.0.1:9292/lac/prediction" + # check http code + http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"words": "我爱北京天安门"}, {"words": "我爱北京天安门"}], "fetch":["word_seg"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9292/lac/prediction` + if [ ${http_code} -ne 200 ]; then + echo "HTTP status code -ne 200" + exit 1 + fi + setproxy # recover proxy state kill_server_process - ps -ef | grep "paddle_serving_server" | grep -v grep | awk '{print $2}' | xargs kill ps -ef | grep "lac_web_service" | grep -v grep | awk '{print $2}' | xargs kill echo "lac CPU HTTP inference pass" ;;