Merge pull request #1248 from HexToString/develop-p

add benchmark gpu

doc/BENCHMARKING_GPU.md

+本次提测的Serving版本，支持GPU预测，希望以此任务为例，对Paddle Serving支持GPU预测的性能给出测试数据。
+
+# 1. 测试环境说明
+
+|          | GPU | 显存 | CPU | 内存 |
+|----------|---------|----------|----------------------------------------------|------|
+| Serving端 | 4x Tesla P4-8GB | 7611MiB | Intel(R) Xeon(R) Gold 5117 CPU @ 2.00GHz 48核 | 216G |
+| Client端  | 4x Tesla P4-8GB | 7611MiB | Intel(R) Xeon(R) Gold 5117 CPU @ 2.00GHz 48核 | 216G |
+
+模型：ResNet_v2_50
+
+# 2. PaddleServing-PipeLine方式
+|model_name |thread_num |batch_size |CPU_util(%) |GPU_memory(mb) |GPU_util(%) |qps(samples/s) |total count |mean(ms) |median(ms) |80 percent(ms) |90 percent(ms) |99 percent(ms) |total cost(s) |each cost(s)|
+|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--
+|ResNet_v2_50 |1 |1 |2.2 |3327 |17.25 |17.633658869240787 |355 |56.428481238996476 |38.646728515625 |39.496826171875 |39.98369140625 |1273.1911083984373 |20.131953477859497 |20.033540725708008|
+|ResNet_v2_50 |1 |4 |2.7 |3617 |28.122 |53.50748430453522 |268 |74.71539215543378 |74.6181640625 |75.3138671875 |75.6051025390625 |77.85322998046874 |20.03458046913147 |20.024930953979492|
+|ResNet_v2_50 |1 |8 |1.7 |3877 |25.7869 |59.60582783086999 |150 |133.5897119140625 |132.7503662109375 |134.968310546875 |136.470703125 |140.79039062499996 |20.132259607315063 |20.03933620452881|
+|ResNet_v2_50 |1 |16 |7.0 |4777 |27.0175 |63.2627646819339 |80 |252.30162048339844 |251.8448486328125 |253.046630859375 |253.91142578125 |263.361640625 |20.233070850372314 |20.18476152420044|
+|ResNet_v2_50 |1 |32 |7.5 |6567 |38.532 |62.945314687348024 |40 |506.8969482421875 |507.3531494140625 |510.562353515625 |511.421240234375 |536.8068920898437 |20.335111618041992 |20.276386737823486|
+|ResNet_v2_50 |2 |1 |4.7 |6567 |49.4828 |50.40600094376044 |1010 |39.63352195815285 |39.5345458984375 |40.452880859375 |41.1375 |42.940522460937494 |20.037296772003174 |20.01696753501892|
+|ResNet_v2_50 |2 |4 |2.7 |6567 |44.4744 |83.4255836891382 |420 |95.38548002697172 |95.7069091796875 |97.599951171875 |98.098583984375 |102.39680908203125 |20.137707471847534 |20.03199553489685|
+|ResNet_v2_50 |2 |8 |2.2 |6567 |42.898 |91.3727510505176 |230 |174.89108568274457 |175.0452880859375 |175.82001953125 |176.7634033203125 |178.64064453125002 |20.13729453086853 |20.1132071018219|
+|ResNet_v2_50 |2 |16 |2.2 |6567 |45 |97.5591285698611 |124 |327.16720088835683 |328.6126708984375 |329.75185546875 |330.386962890625 |336.86397460937496 |20.336385011672974 |20.284939169883728|
+|ResNet_v2_50 |2 |32 |3.2 |6567 |59.5714 |100.70765418116333 |64 |633.9812698364258 |637.8568115234375 |648.103515625 |650.7439697265625 |659.2212915039062 |20.336090803146362 |20.28787398338318|
+|ResNet_v2_50 |4 |1 |3.1 |6567 |64.3333 |80.27845081929433 |1617 |49.56464230756223 |49.4873046875 |51.5537109375 |52.693408203125 |55.207568359374996 |20.142391681671143 |20.038144528865814|
+|ResNet_v2_50 |4 |4 |3.3 |6567 |70.4563 |136.62061939701394 |688 |116.51574919944586 |121.8629150390625 |129.8181640625 |133.384423828125 |142.69500732421875 |20.143372297286987 |20.041599333286285|
+|ResNet_v2_50 |4 |8 |3.0 |6567 |70.896 |158.46554975132275 |399 |201.30669079926378 |210.69775390625 |228.51748046875 |236.427294921875 |252.24822753906233 |20.143179416656494 |20.081032752990723|
+|ResNet_v2_50 |4 |16 |3.2 |6567 |66.3832 |156.4935247130092 |197 |407.6668608224937 |423.974609375 |450.368212890625 |464.45986328125 |482.93658203125 |20.141408443450928 |20.078101694583893|
+|ResNet_v2_50 |4 |32 |3.3 |6567 |72.4791 |162.01742190796557 |104 |785.5079204852765 |813.0341796875 |887.107958984375 |909.6556640625 |935.3334838867188 |20.541000843048096 |20.423666059970856|
+|ResNet_v2_50 |8 |1 |3.5 |6567 |93.977 |115.9749228558386 |2337 |68.5580409078145 |65.45849609375 |76.13930664062501 |83.542041015625 |91.45666015624998 |20.15090799331665 |20.028797417879105|
+|ResNet_v2_50 |8 |4 |4.2 |6567 |90.0952 |175.58748591910316 |889 |180.7330482920592 |170.5810546875 |218.99931640625 |240.06337890625002 |254.413759765625 |20.252012729644775 |20.084695398807526|
+|ResNet_v2_50 |8 |8 |2.6 |6567 |93.8693 |206.76595246418208 |526 |306.52158695119414 |303.043212890625 |321.0791015625 |350.5477294921875 |400.32452392578125 |20.351513147354126 |20.15437400341034|
+|ResNet_v2_50 |8 |16 |3.2 |6567 |85.7273 |205.31850043117367 |265 |614.1745522553066 |552.372314453125 |775.89169921875 |802.022607421875 |902.2763183593761 |20.650842428207397 |20.345011442899704|
+|ResNet_v2_50 |8 |32 |5.0 |6567 |89.8717 |219.8410273718835 |146 |1138.4533474020761 |1039.640869140625 |1364.289794921875 |1474.6744384765625 |1788.2614379882834 |21.251720190048218 |20.777225106954575|
+|ResNet_v2_50 |12 |1 |5.0 |6567 |89.4762 |110.00858327847862 |2218 |108.50048552943953 |103.015625 |121.09404296875003 |137.1392333984375 |151.80401123046872 |20.162063121795654 |20.055511037508648|
+|ResNet_v2_50 |12 |4 |4.1 |6567 |77.7619 |153.7824464757549 |779 |309.68895575507463 |285.585205078125 |378.07421875 |413.481640625 |424.70853515625 |20.262390613555908 |20.104551911354065|
+|ResNet_v2_50 |12 |8 |3.6 |6567 |72.6977 |165.36021780846013 |425 |571.1991590073529 |510.995849609375 |731.9383300781251 |747.6568359375 |757.304716796875 |20.56117272377014 |20.230452219645183|
+|ResNet_v2_50 |12 |16 |1.5 |6567 |76.2222 |189.6414991568285 |252 |987.7153136238219 |926.00390625 |1080.99130859375 |1249.4956298828126 |1434.4802392578124 |21.26116919517517 |20.74245794614156|
+|ResNet_v2_50 |12 |32 |2.8 |6567 |84.25 |203.868228281784 |138 |1811.640237559443 |1764.2760009765625 |1855.28046875 |2023.56826171875 |2586.8038134765625 |21.66105055809021 |20.834286351998646|
+|ResNet_v2_50 |16 |1 |4.8 |6567 |94.3333 |116.34927733312234 |2347 |136.7957122373642 |135.959716796875 |144.1568359375 |146.105517578125 |175.05707519531248 |20.172020435333252 |20.067057371139526|
+|ResNet_v2_50 |16 |4 |15.4 |6567 |83.6364 |160.59012047270738 |822 |393.3079394412447 |396.446533203125 |426.272216796875 |429.777734375 |564.1119360351562 |20.47448492050171 |20.206754431128502|
+|ResNet_v2_50 |16 |8 |6.8 |6567 |81.0233 |169.95774070621547 |437 |741.5512622684854 |751.521484375 |763.199169921875 |948.8041992187501 |1001.156142578125 |20.56981921195984 |20.254074171185493|
+|ResNet_v2_50 |16 |16 |3.5 |6567 |77.8706 |186.56600081516 |248 |1332.1007946383568 |1365.2745361328125 |1399.212255859375 |1432.4037353515625 |1771.4374853515626 |21.26861262321472 |20.64799252152443|
+|ResNet_v2_50 |16 |32 |4.3 |6567 |83.6371 |201.1293408638195 |140 |2419.3400198800223 |2561.09228515625 |2616.081103515625 |2642.0835205078124 |2883.8197412109366 |22.274224042892456 |21.169659316539764|
+
+# 3. 竞品TensorFlow-Serving方式
+me|thread_num|batch_size|CPU_util(%)|GPU_memory(mb)|GPU_util(%)|qps(samples/s)|total count|mean(ms)|median(ms)|80 percent(ms)|90 percent(ms)|99 percent(ms)|total cost(s)|each cost(s)|
+|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|
+|ResNet_v2_50|1|1|2.1|6315|54|43.75570770301271|NaN|15.5063232421875|15.239013671875|15.387646484374999|15.971313476562498|19.846301269531253|0.22854161262512207|0.15510153770446777|
+|ResNet_v2_50|1|4|0.9|6315|89|48.117446702088664|NaN|73.0424560546875|35.518310546875|37.1490234375|74.91518554687487|379.05396972656257|0.8312993049621582|0.7305266857147217|
+|ResNet_v2_50|1|8|1.3|6315|93|77.63418007038673|NaN|93.7514892578125|55.6685791015625|60.045849609375004|97.18415527343737|395.9098217773438|1.0304739475250244|0.9376020431518555|
+|ResNet_v2_50|1|16|1.6|6315|94|97.71724800506956|NaN|147.370068359375|100.296630859375|108.24970703125001|156.34389648437485|513.4688623046875|1.6373772621154785|1.473797082901001|
+|ResNet_v2_50|1|32|1.5|6315|96|116.93800891155549|NaN|260.0307373046875|197.95849609375|213.491015625|280.7433593749998|757.5216992187501|2.73649263381958|2.600431203842163|
+|ResNet_v2_50|2|1|2.4|6315|89|58.92241881896788|NaN|21.91448974609375|21.2923583984375|21.8095703125|22.869189453125006|30.533295898437494|0.3394293785095215|0.21921205520629883|
+|ResNet_v2_50|2|4|1.0|6315|93|148.46502572906388|NaN|42.67369384765625|40.7852783203125|44.94863281250001|52.541333007812504|56.0185498046875|0.5388474464416504|0.4268152713775635|
+|ResNet_v2_50|2|8|1.0|6315|96|190.71839615362427|NaN|69.8621337890625|65.124755859375|73.44399414062501|92.5853759765625|94.67222900390624|0.8389332294464111|0.6987596750259399|
+|ResNet_v2_50|2|16|1.3|6315|97|238.01022776804624|NaN|113.18492431640625|108.5338134765625|109.2408203125|137.240283203125|149.5225146484375|1.344480037689209|1.1319530010223389|
+|ResNet_v2_50|2|32|1.3|6315|100|251.44343658668842|NaN|231.26734619140626|205.041748046875|294.332861328125|309.7132812500001|334.83323486328123|2.5453040599823|2.3127869367599487|
+|ResNet_v2_50|4|1|1.1|6315|96|41.54583069914317|NaN|72.4408447265625|38.391357421875|71.24418945312502|92.31708984375003|514.4540087890625|0.962792158126831|0.7244616746902466|
+|ResNet_v2_50|4|4|1.8|6315|98|166.99795746917292|NaN|77.47444458007813|77.7755126953125|80.92041015625|88.3317138671875|101.21800781249999|0.9580955505371094|0.7748416662216187|
+|ResNet_v2_50|4|8|0.9|6315|99|219.2091124069199|NaN|127.08803100585938|127.197509765625|136.592333984375|152.405126953125|162.7202685546875|1.4597933292388916|1.270979881286621|
+|ResNet_v2_50|4|16|1.6|6315|99|271.2776591628147|NaN|214.3982727050781|213.990966796875|226.74565429687502|233.62548828125|259.1772998046875|2.359206438064575|2.14408540725708|
+|ResNet_v2_50|4|32|1.1|6315|100|286.8647521044879|NaN|413.7820617675781|408.006591796875|465.45664062500003|515.0534423828126|592.00384765625|4.462033033370972|4.137937664985657|
+|ResNet_v2_50|8|1|1.6|6315|98|80.81183612994465|NaN|83.23367309570312|83.27734375|92.96806640625|96.58520507812501|110.42044189453125|0.9899539947509766|0.8324028849601746|
+|ResNet_v2_50|8|4|0.9|6315|99|188.87964973785503|NaN|153.83353576660156|153.7049560546875|166.887939453125|172.56806640625|201.40339599609368|1.6942005157470703|1.5384117662906647|
+|ResNet_v2_50|8|8|1.0|6315|100|237.95324127248458|NaN|251.08477172851562|250.9620361328125|268.9197265625|287.1752197265626|325.4354296875|2.6896040439605713|2.5109518468379974|
+|ResNet_v2_50|8|16|0.9|6315|100|284.54893581995265|NaN|420.44964599609375|435.37158203125|468.86137695312505|515.74599609375|550.1150952148437|4.498347520828247|4.204609364271164|
+|ResNet_v2_50|8|32|4.4|6315|100|284.7884200208036|NaN|797.1125793457031|819.591552734375|920.166796875|978.4201171875001|1046.6215258789061|4.494564771652222|3.9856424927711487|
+|ResNet_v2_50|12|1|4.1|6315|93|73.94944392074078|NaN|141.00926310221354|147.0828857421875|162.398779296875|166.519921875|178.78006591796876|1.6227302551269531|1.4101588129997253|
+|ResNet_v2_50|12|4|3.4|6315|100|188.89136353200647|NaN|232.45508422851563|231.9566650390625|248.440771484375|264.4835693359375|289.8395068359375|2.5411431789398193|2.324637313683828|
+|ResNet_v2_50|12|8|3.3|6315|100|237.90598070167323|NaN|376.739902750651|376.1767578125|400.2998046875|420.91645507812495|461.4720776367187|4.035207509994507|3.7674967845280967|
+|ResNet_v2_50|12|16|5.3|6315|100|270.91933476759425|NaN|642.7119547526041|639.72265625|703.9482421875|776.6265136718749|809.430361328125|3.543490171432495|3.2136237621307373|
+|ResNet_v2_50|16|1|1.7|6315|93|77.94299484331752|NaN|178.47059936523436|178.8289794921875|201.5099609375|214.1606201171875|235.76847412109373|2.0527822971343994|1.7847793251276016|
+|ResNet_v2_50|16|4|1.1|6315|100|190.8966798676615|NaN|307.3357818603516|304.0648193359375|333.9826171875|355.899462890625|406.84044677734374|3.3525989055633545|3.073450207710266|