benchmark.py 5.2 KB
Newer Older
1 2
# -*- coding: utf-8 -*-
#
M
MRXLT 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
B
barrierye 已提交
16
# pylint: disable=doc-string-missing
M
MRXLT 已提交
17

18 19
from __future__ import unicode_literals, absolute_import
import os
M
MRXLT 已提交
20
import sys
21
import time
M
MRXLT 已提交
22 23
import json
import requests
H
HexToString 已提交
24
import numpy as np
M
MRXLT 已提交
25 26
from paddle_serving_client import Client
from paddle_serving_client.utils import MultiThreadRunner
M
MRXLT 已提交
27
from paddle_serving_client.utils import benchmark_args, show_latency
M
MRXLT 已提交
28
from paddle_serving_app.reader import ChineseBertReader
M
MRXLT 已提交
29

30
args = benchmark_args()
M
MRXLT 已提交
31

B
barrierye 已提交
32

33 34
def single_func(idx, resource):
    fin = open("data-c.txt")
M
MRXLT 已提交
35 36 37
    dataset = []
    for line in fin:
        dataset.append(line.strip())
M
MRXLT 已提交
38

M
MRXLT 已提交
39
    profile_flags = False
M
MRXLT 已提交
40
    latency_flags = False
M
MRXLT 已提交
41 42
    if os.getenv("FLAGS_profile_client"):
        profile_flags = True
M
MRXLT 已提交
43 44 45 46
    if os.getenv("FLAGS_serving_latency"):
        latency_flags = True
        latency_list = []

47
    if args.request == "rpc":
M
MRXLT 已提交
48
        reader = ChineseBertReader({"max_seq_len": 128})
49 50 51
        fetch = ["pooled_output"]
        client = Client()
        client.load_client_config(args.model)
M
MRXLT 已提交
52
        client.connect([resource["endpoint"][idx % len(resource["endpoint"])]])
53
        start = time.time()
M
MRXLT 已提交
54 55
        for i in range(turns):
            if args.batch_size >= 1:
M
MRXLT 已提交
56
                l_start = time.time()
M
MRXLT 已提交
57 58 59
                feed_batch = []
                b_start = time.time()
                for bi in range(args.batch_size):
H
HexToString 已提交
60 61 62 63 64
                    feed_dict = reader.process(dataset[bi])
                    for key in feed_dict.keys():
                        feed_dict[key] = np.array(feed_dict[key]).reshape(
                            (1, 128, 1))
                    feed_batch.append(feed_dict)
M
MRXLT 已提交
65
                b_end = time.time()
M
MRXLT 已提交
66

M
MRXLT 已提交
67 68 69 70 71 72
                if profile_flags:
                    sys.stderr.write(
                        "PROFILE\tpid:{}\tbert_pre_0:{} bert_pre_1:{}\n".format(
                            os.getpid(),
                            int(round(b_start * 1000000)),
                            int(round(b_end * 1000000))))
H
HexToString 已提交
73 74
                result = client.predict(
                    feed=feed_batch, fetch=fetch, batch=True)
M
MRXLT 已提交
75 76 77 78

                l_end = time.time()
                if latency_flags:
                    latency_list.append(l_end * 1000 - l_start * 1000)
M
MRXLT 已提交
79 80 81
            else:
                print("unsupport batch size {}".format(args.batch_size))

82
    elif args.request == "http":
M
MRXLT 已提交
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
        reader = ChineseBertReader({"max_seq_len": 128})
        fetch = ["pooled_output"]
        server = "http://" + resource["endpoint"][idx % len(resource[
            "endpoint"])] + "/bert/prediction"
        start = time.time()
        for i in range(turns):
            if args.batch_size >= 1:
                l_start = time.time()
                feed_batch = []
                b_start = time.time()
                for bi in range(args.batch_size):
                    feed_batch.append({"words": dataset[bi]})
                req = json.dumps({"feed": feed_batch, "fetch": fetch})
                b_end = time.time()

                if profile_flags:
                    sys.stderr.write(
                        "PROFILE\tpid:{}\tbert_pre_0:{} bert_pre_1:{}\n".format(
                            os.getpid(),
                            int(round(b_start * 1000000)),
                            int(round(b_end * 1000000))))
                result = requests.post(
                    server,
                    data=req,
                    headers={"Content-Type": "application/json"})
                l_end = time.time()
                if latency_flags:
                    latency_list.append(l_end * 1000 - l_start * 1000)
            else:
                print("unsupport batch size {}".format(args.batch_size))

    else:
        raise ValueError("not implemented {} request".format(args.request))
M
MRXLT 已提交
116
    end = time.time()
M
MRXLT 已提交
117 118 119 120
    if latency_flags:
        return [[end - start], latency_list]
    else:
        return [[end - start]]
M
MRXLT 已提交
121

B
barrierye 已提交
122

M
MRXLT 已提交
123
if __name__ == '__main__':
124
    multi_thread_runner = MultiThreadRunner()
H
HexToString 已提交
125
    endpoint_list = ["127.0.0.1:9292", "127.0.0.1:9293"]
G
gentelyang 已提交
126
    turns = 100
M
MRXLT 已提交
127 128 129 130
    start = time.time()
    result = multi_thread_runner.run(
        single_func, args.thread, {"endpoint": endpoint_list,
                                   "turns": turns})
M
MRXLT 已提交
131 132 133
    end = time.time()
    total_cost = end - start

M
MRXLT 已提交
134 135 136 137
    avg_cost = 0
    for i in range(args.thread):
        avg_cost += result[0][i]
    avg_cost = avg_cost / args.thread
M
MRXLT 已提交
138

G
gentelyang 已提交
139 140 141 142
    print("total cost: {}s".format(total_cost))
    print("each thread cost: {}s. ".format(avg_cost))
    print("qps: {}samples/s".format(args.batch_size * args.thread * turns /
                                    total_cost))
M
MRXLT 已提交
143 144
    if os.getenv("FLAGS_serving_latency"):
        show_latency(result[1])