From aaa0d19b39a3096616978741e71d16c634a51b69 Mon Sep 17 00:00:00 2001 From: Wanli Date: Tue, 6 Jun 2023 16:07:37 +0800 Subject: [PATCH] update generate table script (#164) --- benchmark/README.md | 26 - benchmark/color_table.svg | 3897 ++++++++++++++++++++--------------- benchmark/generate_table.py | 293 ++- benchmark/table_config.yaml | 219 ++ 4 files changed, 2714 insertions(+), 1721 deletions(-) create mode 100644 benchmark/table_config.yaml diff --git a/benchmark/README.md b/benchmark/README.md index fef7ad2..efb9a95 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -57,32 +57,6 @@ python benchmark.py --all --cfg_overwrite_backend_target 1 Benchmark is done with latest `opencv-python==4.7.0.72` and `opencv-contrib-python==4.7.0.72` on the following platforms. Some models are excluded because of support issues. - -| Model | Task | Input Size | CPU-INTEL (ms) | CPU-RPI (ms) | CPU-RV1126 (ms) | CPU-KVE2 (ms) | CPU-HSX3 (ms) | CPU-AXP (ms) | GPU-JETSON (ms) | NPU-KV3 (ms) | NPU-Ascend310 (ms) | CPU-D1 (ms) | -| -------------------------------------------------------- | ----------------------------- | ---------- | -------------- | ------------ | --------------- | ------------- | ------------- | ------------ | --------------- | ------------ | ------------------ | ----------- | -| [YuNet](../models/face_detection_yunet) | Face Detection | 160x120 | 0.72 | 5.43 | 68.89 | 2.47 | 11.04 | 98.16 | 12.18 | 4.04 | 2.24 | 86.69 | -| [SFace](../models/face_recognition_sface) | Face Recognition | 112x112 | 6.04 | 78.83 | 1550.71 | 33.79 | 140.83 | 2093.12 | 24.88 | 46.25 | 2.66 | --- | -| [FER](../models/facial_expression_recognition/) | Facial Expression Recognition | 112x112 | 3.16 | 32.53 | 604.36 | 15.99 | 64.96 | 811.32 | 31.07 | 29.80 | 2.19 | --- | -| [LPD-YuNet](../models/license_plate_detection_yunet/) | License Plate Detection | 320x240 | 8.63 | 167.70 | 3222.92 | 57.57 | 283.75 | 4300.13 | 56.12 | 29.53 | 7.63 | --- | -| [YOLOX](../models/object_detection_yolox/) | Object Detection | 640x640 | 141.20 | 1805.87 | 38359.93 | 577.93 | 2749.22 | 49994.75 | 388.95 | 420.98 | 28.59 | --- | -| [NanoDet](../models/object_detection_nanodet/) | Object Detection | 416x416 | 66.03 | 225.10 | 2303.55 | 118.38 | 408.16 | 3360.20 | 64.94 | 116.64 | 20.62 | --- | -| [DB-IC15](../models/text_detection_db) (EN) | Text Detection | 640x480 | 71.03 | 1862.75 | 49065.03 | 394.77 | 1908.87 | 65681.91 | 208.41 | --- | 17.15 | --- | -| [DB-TD500](../models/text_detection_db) (EN&CN) | Text Detection | 640x480 | 72.31 | 1878.45 | 49052.24 | 392.52 | 1922.34 | 65630.56 | 210.51 | --- | 17.95 | --- | -| [CRNN-EN](../models/text_recognition_crnn) | Text Recognition | 100x32 | 20.16 | 278.11 | 2230.12 | 77.51 | 464.58 | 3277.07 | 196.15 | 125.30 | --- | --- | -| [CRNN-CN](../models/text_recognition_crnn) | Text Recognition | 100x32 | 23.07 | 297.48 | 2244.03 | 82.93 | 495.94 | 3330.69 | 239.76 | 166.79 | --- | --- | -| [PP-ResNet](../models/image_classification_ppresnet) | Image Classification | 224x224 | 34.71 | 463.93 | 11793.09 | 178.87 | 759.81 | 15753.56 | 98.64 | 75.45 | 6.99 | --- | -| [MobileNet-V1](../models/image_classification_mobilenet) | Image Classification | 224x224 | 5.90 | 72.33 | 1546.16 | 32.78 | 140.60 | 2091.13 | 33.18 | 145.66\* | 5.15 | --- | -| [MobileNet-V2](../models/image_classification_mobilenet) | Image Classification | 224x224 | 5.97 | 66.56 | 1166.56 | 28.38 | 122.53 | 1583.25 | 31.92 | 146.31\* | 5.41 | --- | -| [PP-HumanSeg](../models/human_segmentation_pphumanseg) | Human Segmentation | 192x192 | 8.81 | 73.13 | 1610.78 | 34.58 | 144.23 | 2157.86 | 67.97 | 74.77 | 6.94 | --- | -| [WeChatQRCode](../models/qrcode_wechatqrcode) | QR Code Detection and Parsing | 100x100 | 1.29 | 5.71 | --- | --- | --- | --- | --- | --- | --- | --- | -| [DaSiamRPN](../models/object_tracking_dasiamrpn) | Object Tracking | 1280x720 | 29.05 | 712.94 | 14738.64 | 152.78 | 929.63 | 19800.14 | 76.82 | --- | --- | --- | -| [YoutuReID](../models/person_reid_youtureid) | Person Re-Identification | 128x256 | 30.39 | 625.56 | 11117.07 | 195.67 | 898.23 | 14886.02 | 90.07 | 44.61 | 5.58 | --- | -| [MP-PalmDet](../models/palm_detection_mediapipe) | Palm Detection | 192x192 | 6.29 | 86.83 | 872.09 | 38.03 | 142.23 | 1191.81 | 83.20 | 33.81 | 5.17 | --- | -| [MP-HandPose](../models/handpose_estimation_mediapipe) | Hand Pose Estimation | 224x224 | 4.68 | 43.57 | 460.56 | 20.27 | 80.67 | 636.22 | 40.10 | 19.47 | 6.27 | --- | -| [MP-PersonDet](../models/person_detection_mediapipe) | Person Detection | 224x224 | 13.88 | 98.52 | 1326.56 | 46.07 | 191.41 | 1835.97 | 56.69 | --- | 16.45 | --- | - -\*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU. - ### Intel 12700K Specs: [details](https://www.intel.com/content/www/us/en/products/sku/134594/intel-core-i712700k-processor-25m-cache-up-to-5-00-ghz/specifications.html) diff --git a/benchmark/color_table.svg b/benchmark/color_table.svg index 953d357..c4bf741 100644 --- a/benchmark/color_table.svg +++ b/benchmark/color_table.svg @@ -1,7 +1,7 @@ - + @@ -15,3041 +15,3718 @@ - - + +iVBORw0KGgoAAAANSUhEUgAAAwgAAAATCAYAAAA0/CrSAAABSklEQVR4nO3XS27DMAwFQMk5VK/U++8jdVOwNGC2ChIbLTqzIgiKVn4IXm/vb7N96r2FvvXX1/2Enam++nlb2n9L/Vu60r5f1Fsr+ifPP3P/Yn71TP7Y8vl9/4T54t71/hfNF5/B468r19thv7XWtvbz3MquXf1XdqaZ3lKdz1b94uyuv7DnimecsX9njq96jOP+Sn3l2Tx/8V3nKPaMWfR/28wJ+7+Zm7tdqb7PNDMO+/uZ437eOe/H95vF/JV3eHTPuB+/h7Paszg3iv4sXsPSzoWZsl88d2lnUZdf2WfqhZ/E1c97pp5pf/HPAAAA/EcCAgAAEAQEAAAgCAgAAEAQEAAAgCAgAAAAQUAAAACCgAAAAAQBAQAACAICAAAQBAQAACAICAAAQBAQAACAICAAAABBQAAAAIKAAAAABAEBAAAIAgIAABA+AEq1I/9cZQrJAAAAAElFTkSuQmCC" id="imaged38e91eac8" transform="scale(1 -1) translate(0 -13.68)" x="544.784516" y="0.635294" width="558.72" height="13.68"/> - Faster + Faster - Slower + Slower - + - Model + + Model + - + - Task + + Task + - + - Input Size + + Input Size + - + + - CPU-INTEL (ms) + + Intel + + + 12700K + + + CPU + - + + - CPU-RPI (ms) + + Rasberry Pi 4B + + + BCM2711 + + + CPU + - + + - CPU-RV1126 (ms) + + Toybrick + + + RV1126 + + + CPU + - + + - CPU-KVE2 (ms) + + Khadas Edge2 + + + RK3588S + + + CPU + - + + - CPU-HSX3 (ms) + + Horizon Sunrise Pi + + + X3 + + + CPU + - + + - CPU-AXP (ms) + + MAIX-III AX-Pi + + + AX620A + + + CPU + - + + - GPU-JETSON (ms) + + Jetson Nano + + + B01 + + + CPU + - + + - NPU-KV3 (ms) + + Khadas VIM3 + + + A311D + + + CPU + - + + - NPU-Ascend310 (ms) + + Atlas 200 DK + + + Ascend 310 + + + CPU + - + + - CPU-D1 (ms) + + Jetson Nano + + + B01 + + + GPU + - + + - YuNet + + Khadas VIM3 + + + A311D + + + NPU + - + + - Face Detection + + Atlas 200 DK + + + Ascend 310 + + + NPU + - + + - 160x120 + + YuNet + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 0.72 + Face Detection - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 5.43 + 160x120 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 68.89 + 0.58 - +" style="fill: #addc6f; stroke: #000000; stroke-linejoin: miter"/> - 2.47 + 5.45 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 11.04 + 68.89 - +" style="fill: #63bc62; stroke: #000000; stroke-linejoin: miter"/> - 98.16 + 2.47 - +" style="fill: #eff8aa; stroke: #000000; stroke-linejoin: miter"/> - 12.18 + 11.04 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 4.04 + 98.16 - +" style="fill: #abdb6d; stroke: #000000; stroke-linejoin: miter"/> - 2.24 + 5.37 - +" style="fill: #a2d76a; stroke: #000000; stroke-linejoin: miter"/> - 86.69 + 4.93 - +" style="fill: #d3ec87; stroke: #000000; stroke-linejoin: miter"/> - SFace + 8.02 - +" style="fill: #f1f9ac; stroke: #000000; stroke-linejoin: miter"/> - Face Recognition + 11.22 - +" style="fill: #b1de71; stroke: #000000; stroke-linejoin: miter"/> - 112x112 + 5.67 - +" style="fill: #57b65f; stroke: #000000; stroke-linejoin: miter"/> - 6.04 + 2.24 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 78.83 + + SFace + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 1550.71 + Face Recognition - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 33.79 + 112x112 - +" style="fill: #33a456; stroke: #000000; stroke-linejoin: miter"/> - 140.83 + 6.18 - +" style="fill: #fed683; stroke: #000000; stroke-linejoin: miter"/> - 2093.12 + 78.04 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 24.88 + 1550.71 - +" style="fill: #d7ee8a; stroke: #000000; stroke-linejoin: miter"/> - 46.25 + 33.79 - +" style="fill: #e95538; stroke: #000000; stroke-linejoin: miter"/> - 2.66 + 140.83 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 2093.12 - +" style="fill: #feeb9d; stroke: #000000; stroke-linejoin: miter"/> - FER + 65.14 - +" style="fill: #fff2aa; stroke: #000000; stroke-linejoin: miter"/> - Facial Expression Recognition + 60.02 - +" style="fill: #fba35c; stroke: #000000; stroke-linejoin: miter"/> - 112x112 + 104.51 - +" style="fill: #b7e075; stroke: #000000; stroke-linejoin: miter"/> - 3.16 + 24.60 - +" style="fill: #fed884; stroke: #000000; stroke-linejoin: miter"/> - 32.53 + 76.97 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 604.36 + 2.66 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 15.99 + + FER + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 64.96 + Face Expression Recognition - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 811.32 + 112x112 - +" style="fill: #138c4a; stroke: #000000; stroke-linejoin: miter"/> - 31.07 + 3.32 - +" style="fill: #d3ec87; stroke: #000000; stroke-linejoin: miter"/> - 29.80 + 32.21 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 2.19 + 604.36 - +" style="fill: #8ecf67; stroke: #000000; stroke-linejoin: miter"/> - --- + 15.99 - +" style="fill: #feeb9d; stroke: #000000; stroke-linejoin: miter"/> - LPD-YuNet + 64.96 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - License Plate Detection + 811.32 - +" style="fill: #c5e67e; stroke: #000000; stroke-linejoin: miter"/> - 320x240 + 28.19 - +" style="fill: #c9e881; stroke: #000000; stroke-linejoin: miter"/> - 8.63 + 29.36 - +" style="fill: #f7fcb4; stroke: #000000; stroke-linejoin: miter"/> - 167.70 + 47.71 - +" style="fill: #a9da6c; stroke: #000000; stroke-linejoin: miter"/> - 3222.92 + 20.64 - +" style="fill: #e8f59f; stroke: #000000; stroke-linejoin: miter"/> - 57.57 + 40.38 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 283.75 + 2.19 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 4300.13 + + LPD_YuNet + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 56.12 + License Plate Detection - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 29.53 + 320x240 - +" style="fill: #097940; stroke: #000000; stroke-linejoin: miter"/> - 7.63 + 8.53 - +" style="fill: #fffab6; stroke: #000000; stroke-linejoin: miter"/> - --- + 192.61 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - YOLOX + 3222.92 - +" style="fill: #91d068; stroke: #000000; stroke-linejoin: miter"/> - Object Detection + 57.57 - +" style="fill: #fed07e; stroke: #000000; stroke-linejoin: miter"/> - 640x640 + 283.75 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 141.20 + 4300.13 - +" style="fill: #e3f399; stroke: #000000; stroke-linejoin: miter"/> - 1805.87 + 133.15 - +" style="fill: #dcf08f; stroke: #000000; stroke-linejoin: miter"/> - 38359.93 + 122.17 - +" style="fill: #feec9f; stroke: #000000; stroke-linejoin: miter"/> - 577.93 + 226.08 - +" style="fill: #89cc67; stroke: #000000; stroke-linejoin: miter"/> - 2749.22 + 53.50 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 49994.75 + --- - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 388.95 + 7.63 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 420.98 + + YOLOX + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 28.59 + Object Detection - - --- + 640x640 - +" style="fill: #63bc62; stroke: #000000; stroke-linejoin: miter"/> - NanoDet + 137.53 - +" style="fill: #eb5a3a; stroke: #000000; stroke-linejoin: miter"/> - Object Detection + 1932.97 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 416x416 + 38359.93 - +" style="fill: #e9f6a1; stroke: #000000; stroke-linejoin: miter"/> - 66.03 + 577.93 - +" style="fill: #a90426; stroke: #000000; stroke-linejoin: miter"/> - 225.10 + 2749.22 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 2303.55 + 49994.75 - +" style="fill: #fdc372; stroke: #000000; stroke-linejoin: miter"/> - 118.38 + 1228.13 - +" style="fill: #fec877; stroke: #000000; stroke-linejoin: miter"/> - 408.16 + 1189.83 - +" style="fill: #e75337; stroke: #000000; stroke-linejoin: miter"/> - 3360.20 + 1995.97 - +" style="fill: #fdc171; stroke: #000000; stroke-linejoin: miter"/> - 64.94 + 1238.91 - +" style="fill: #fed481; stroke: #000000; stroke-linejoin: miter"/> - 116.64 + 1103.24 - - 20.62 + 28.59 - + + - --- + + NanoDet + - - DB-IC15 (EN) + Object Detection - - Text Detection + 416x416 - +" style="fill: #3faa59; stroke: #000000; stroke-linejoin: miter"/> - 640x480 + 65.15 - +" style="fill: #b9e176; stroke: #000000; stroke-linejoin: miter"/> - 71.03 + 248.03 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 1862.75 + 2303.55 - +" style="fill: #73c264; stroke: #000000; stroke-linejoin: miter"/> - 49065.03 + 118.38 - +" style="fill: #e9f6a1; stroke: #000000; stroke-linejoin: miter"/> - 394.77 + 408.16 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 1908.87 + 3360.20 - +" style="fill: #addc6f; stroke: #000000; stroke-linejoin: miter"/> - 65681.91 + 215.57 - +" style="fill: #a9da6c; stroke: #000000; stroke-linejoin: miter"/> - 208.41 + 208.25 - +" style="fill: #daf08d; stroke: #000000; stroke-linejoin: miter"/> - --- + 343.08 - +" style="fill: #addc6f; stroke: #000000; stroke-linejoin: miter"/> - 17.15 + 214.99 - +" style="fill: #a7d96b; stroke: #000000; stroke-linejoin: miter"/> - --- + 200.50 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - DB-TD500 (EN&CN) + 20.62 - + + - Text Detection + + DB-IC15 (EN) + - - 640x480 + Text Detection - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 72.31 + 640x480 - +" style="fill: #51b35e; stroke: #000000; stroke-linejoin: miter"/> - 1878.45 + 75.82 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 49052.24 + 2025.09 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 392.52 + 49065.03 - +" style="fill: #e6f59d; stroke: #000000; stroke-linejoin: miter"/> - 1922.34 + 394.77 - +" style="fill: #af0926; stroke: #000000; stroke-linejoin: miter"/> - 65630.56 + 1908.87 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 210.51 + 65681.91 - +" style="fill: #fa9656; stroke: #000000; stroke-linejoin: miter"/> - --- + 1089.89 - +" style="fill: #fdb163; stroke: #000000; stroke-linejoin: miter"/> - 17.95 + 973.75 - +" style="fill: #e54e35; stroke: #000000; stroke-linejoin: miter"/> - --- + 1452.55 - +" style="fill: #cfeb85; stroke: #000000; stroke-linejoin: miter"/> - CRNN-EN + 303.12 - - Text Recognition + --- - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 100x32 + 17.15 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 20.16 + + DB-TD500 (EN&CN) + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 278.11 + Text Detection - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 2230.12 + 640x480 - +" style="fill: #4eb15d; stroke: #000000; stroke-linejoin: miter"/> - 77.51 + 74.80 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 464.58 + 2041.85 - - 3277.07 + 49052.24 - +" style="fill: #e6f59d; stroke: #000000; stroke-linejoin: miter"/> - 196.15 + 392.52 - +" style="fill: #af0926; stroke: #000000; stroke-linejoin: miter"/> - 125.30 + 1922.34 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 65630.56 - +" style="fill: #fa9656; stroke: #000000; stroke-linejoin: miter"/> - --- + 1089.94 - +" style="fill: #fdb365; stroke: #000000; stroke-linejoin: miter"/> - CRNN-CN + 961.44 - +" style="fill: #e65036; stroke: #000000; stroke-linejoin: miter"/> - Text Recognition + 1433.26 - +" style="fill: #cfeb85; stroke: #000000; stroke-linejoin: miter"/> - 100x32 + 302.58 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 23.07 + --- - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 297.48 + 17.95 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 2244.03 + + CRNN-EN + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 82.93 + Text Recognition - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 495.94 + 100*32 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 3330.69 + 20.43 - +" style="fill: #c3e67d; stroke: #000000; stroke-linejoin: miter"/> - 239.76 + 271.57 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 166.79 + 2230.12 - +" style="fill: #4eb15d; stroke: #000000; stroke-linejoin: miter"/> - --- + 77.51 - +" style="fill: #f5fbb2; stroke: #000000; stroke-linejoin: miter"/> - --- + 464.58 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - PP-ResNet + 3277.07 - +" style="fill: #c1e57b; stroke: #000000; stroke-linejoin: miter"/> - Image Classification + 269.52 - +" style="fill: #a7d96b; stroke: #000000; stroke-linejoin: miter"/> - 224x224 + 199.81 - +" style="fill: #cdea83; stroke: #000000; stroke-linejoin: miter"/> - 34.71 + 303.65 - +" style="fill: #219c52; stroke: #000000; stroke-linejoin: miter"/> - 463.93 + 45.60 - +" style="fill: #96d268; stroke: #000000; stroke-linejoin: miter"/> - 11793.09 + 172.06 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 178.87 + --- - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 759.81 + + CRNN-CN + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 15753.56 + Text Recognition - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 98.64 + 100*32 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 75.45 + 23.08 - +" style="fill: #c9e881; stroke: #000000; stroke-linejoin: miter"/> - 6.99 + 293.83 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 2244.03 - +" style="fill: #51b35e; stroke: #000000; stroke-linejoin: miter"/> - MobileNet-V1 + 82.93 - +" style="fill: #fbfdba; stroke: #000000; stroke-linejoin: miter"/> - Image Classification + 495.94 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 224x224 + 3330.69 - +" style="fill: #c9e881; stroke: #000000; stroke-linejoin: miter"/> - 5.90 + 290.82 - +" style="fill: #addc6f; stroke: #000000; stroke-linejoin: miter"/> - 72.33 + 217.07 - +" style="fill: #d5ed88; stroke: #000000; stroke-linejoin: miter"/> - 1546.16 + 329.84 - +" style="fill: #33a456; stroke: #000000; stroke-linejoin: miter"/> - 32.78 + 59.39 - +" style="fill: #9bd469; stroke: #000000; stroke-linejoin: miter"/> - 140.60 + 183.51 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 2091.13 + --- - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 33.18 + + PP-ResNet + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 145.66\* + Image Classification - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 5.15 + 224x224 - +" style="fill: #4eb15d; stroke: #000000; stroke-linejoin: miter"/> - --- + 35.40 - +" style="fill: #fa9656; stroke: #000000; stroke-linejoin: miter"/> - MobileNet-V2 + 547.70 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - Image Classification + 11793.09 - +" style="fill: #dff293; stroke: #000000; stroke-linejoin: miter"/> - 224x224 + 178.87 - +" style="fill: #e0422f; stroke: #000000; stroke-linejoin: miter"/> - 5.97 + 759.81 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 66.56 + 15753.56 - +" style="fill: #feda86; stroke: #000000; stroke-linejoin: miter"/> - 1166.56 + 376.88 - +" style="fill: #fed683; stroke: #000000; stroke-linejoin: miter"/> - 28.38 + 385.29 - +" style="fill: #f98e52; stroke: #000000; stroke-linejoin: miter"/> - 122.53 + 564.69 - +" style="fill: #cdea83; stroke: #000000; stroke-linejoin: miter"/> - 1583.25 + 148.58 - +" style="fill: #fee695; stroke: #000000; stroke-linejoin: miter"/> - 31.92 + 340.20 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 146.31\* + 6.99 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 5.41 + + MobileNet-V1 + - - --- + Image Classification - - PP-HumanSeg + 224x224 - +" style="fill: #0c7f43; stroke: #000000; stroke-linejoin: miter"/> - Human Segmentation + 6.25 - +" style="fill: #cbe982; stroke: #000000; stroke-linejoin: miter"/> - 192x192 + 74.51 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 8.81 + 1546.16 - +" style="fill: #7dc765; stroke: #000000; stroke-linejoin: miter"/> - 73.13 + 32.78 - +" style="fill: #fff8b4; stroke: #000000; stroke-linejoin: miter"/> - 1610.78 + 140.60 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 34.58 + 2091.13 - +" style="fill: #c3e67d; stroke: #000000; stroke-linejoin: miter"/> - 144.23 + 68.94 - +" style="fill: #bde379; stroke: #000000; stroke-linejoin: miter"/> - 2157.86 + 64.73 - +" style="fill: #ebf7a3; stroke: #000000; stroke-linejoin: miter"/> - 67.97 + 103.96 - +" style="fill: #c5e67e; stroke: #000000; stroke-linejoin: miter"/> - 74.77 + 69.24 - +" style="fill: #b7e075; stroke: #000000; stroke-linejoin: miter"/> - 6.94 + 60.75\* - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - --- + 5.15 - + + - WeChatQRCode + + MobileNet-V2 + - - QR Code Detection and Parsing + Image Classification - - 100x100 + 224x224 - +" style="fill: #08773f; stroke: #000000; stroke-linejoin: miter"/> - 1.29 + 6.00 - +" style="fill: #c1e57b; stroke: #000000; stroke-linejoin: miter"/> - 5.71 + 67.29 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 1166.56 - +" style="fill: #70c164; stroke: #000000; stroke-linejoin: miter"/> - --- + 28.38 - +" style="fill: #fafdb8; stroke: #000000; stroke-linejoin: miter"/> - --- + 122.53 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 1583.25 - +" style="fill: #b9e176; stroke: #000000; stroke-linejoin: miter"/> - --- + 62.12 - +" style="fill: #b3df72; stroke: #000000; stroke-linejoin: miter"/> - --- + 58.18 - +" style="fill: #dff293; stroke: #000000; stroke-linejoin: miter"/> - --- + 90.46 - +" style="fill: #b9e176; stroke: #000000; stroke-linejoin: miter"/> - --- + 62.12 - +" style="fill: #b1de71; stroke: #000000; stroke-linejoin: miter"/> - DaSiamRPN + 57.40\* - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - Object Tracking + 5.41 - + + - 1280x720 + + PP-HumanSeg + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 29.05 + Human Segmentation - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 712.94 + 192x192 - +" style="fill: #0a7b41; stroke: #000000; stroke-linejoin: miter"/> - 14738.64 + 8.20 - +" style="fill: #abdb6d; stroke: #000000; stroke-linejoin: miter"/> - 152.78 + 73.29 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 929.63 + 1610.78 - +" style="fill: #66bd63; stroke: #000000; stroke-linejoin: miter"/> - 19800.14 + 34.58 - +" style="fill: #e9f6a1; stroke: #000000; stroke-linejoin: miter"/> - 76.82 + 144.23 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - --- + 2157.86 - +" style="fill: #a0d669; stroke: #000000; stroke-linejoin: miter"/> - --- + 65.91 - +" style="fill: #a7d96b; stroke: #000000; stroke-linejoin: miter"/> - --- + 70.82 - +" style="fill: #cfeb85; stroke: #000000; stroke-linejoin: miter"/> - YoutuReID + 107.98 - +" style="fill: #bfe47a; stroke: #000000; stroke-linejoin: miter"/> - Person Re-Identification + 90.86 - - 128x256 + --- - - 30.39 + 6.94 - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 625.56 + + WeChatQRCode + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 11117.07 + QR Code Detection and Parsing - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 195.67 + 100x100 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 898.23 + 1.35 - +" style="fill: #8ecf67; stroke: #000000; stroke-linejoin: miter"/> - 14886.02 + 8.18 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 90.07 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 44.61 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 5.58 + --- - - --- + --- - +" style="fill: #6bbf64; stroke: #000000; stroke-linejoin: miter"/> - MP-PalmDet + 5.62 - +" style="fill: #66bd63; stroke: #000000; stroke-linejoin: miter"/> - Palm Detection + 5.29 - - 192x192 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 6.29 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 86.83 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 872.09 + --- - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 38.03 + + DaSiamRPN + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 142.23 + Object Tracking - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 1191.81 + 1280x720 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 83.20 + 29.46 - +" style="fill: #feffbe; stroke: #000000; stroke-linejoin: miter"/> - 33.81 + 762.56 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 5.17 + 14738.64 - +" style="fill: #69be63; stroke: #000000; stroke-linejoin: miter"/> - --- + 152.78 - +" style="fill: #fff0a6; stroke: #000000; stroke-linejoin: miter"/> - MP-HandPose + 929.63 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - Hand Pose Estimation + 19800.14 - +" style="fill: #d1ec86; stroke: #000000; stroke-linejoin: miter"/> - 224x224 + 466.19 - +" style="fill: #c7e77f; stroke: #000000; stroke-linejoin: miter"/> - 4.68 + 428.66 - +" style="fill: #f5fbb2; stroke: #000000; stroke-linejoin: miter"/> - 43.57 + 701.08 - +" style="fill: #2da155; stroke: #000000; stroke-linejoin: miter"/> - 460.56 + 76.54 - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 20.27 + --- - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 80.67 + --- - + +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> + - 636.22 + + YoutuReID + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 40.10 + Person Re-Identification - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 19.47 + 128x256 - +" style="fill: #54b45f; stroke: #000000; stroke-linejoin: miter"/> - 6.27 + 30.87 - +" style="fill: #cc2627; stroke: #000000; stroke-linejoin: miter"/> - --- + 676.15 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - MP-PersonDet + 11117.07 - +" style="fill: #fbfdba; stroke: #000000; stroke-linejoin: miter"/> - Person Detection + 195.67 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 224x224 + 898.23 - +" style="fill: #a50026; stroke: #000000; stroke-linejoin: miter"/> - 13.88 + 14886.02 - +" style="fill: #fca55d; stroke: #000000; stroke-linejoin: miter"/> - 98.52 + 411.49 - +" style="fill: #f99355; stroke: #000000; stroke-linejoin: miter"/> - 1326.56 + 439.53 - +" style="fill: #da362a; stroke: #000000; stroke-linejoin: miter"/> - 46.07 + 631.70 - +" style="fill: #d3ec87; stroke: #000000; stroke-linejoin: miter"/> - 191.41 + 126.65 - +" style="fill: #fed884; stroke: #000000; stroke-linejoin: miter"/> - 1835.97 + 307.90 - +" style="fill: #006837; stroke: #000000; stroke-linejoin: miter"/> - 56.69 + 5.58 - + + - --- + + MP-PalmDet + - +" style="fill: #ffffff; stroke: #000000; stroke-linejoin: miter"/> - 16.45 + Palm Detection - - --- + 192x192 + + + + + + 6.14 + + + + + + 91.48 + + + + + + 872.09 + + + + + + 38.03 + + + + + + 142.23 + + + + + + 1191.81 + + + + + + 69.60 + + + + + + 66.91 + + + + + + 105.23 + + + + + + 67.34 + + + + + + 95.92 + + + + + + 5.17 + + + + + + + + + MP-HandPose + + + + + + + Hand Pose Estimation + + + + + + 224x224 + + + + + + 4.68 + + + + + + 43.85 + + + + + + 460.56 + + + + + + 20.27 + + + + + + 80.67 + + + + + + 636.22 + + + + + + 41.02 + + + + + + 38.60 + + + + + + 57.95 + + + + + + 41.15 + + + + + + 44.36 + + + + + + 6.27 + + + + + + + + + MP-PersonDet + + + + + + + Person Detection + + + + + + 224x224 + + + + + + 13.88 + + + + + + 98.52 + + + + + + 1326.56 + + + + + + 46.07 + + + + + + 191.41 + + + + + + 1835.97 + + + + + + 98.38 + + + + + + 84.42 + + + + + + 134.10 + + + + + + 56.69 + + + + + + --- + + + + + + 16.45 - - \*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU. + + Units: All data in milliseconds (ms). + + + \*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU. - - + + diff --git a/benchmark/generate_table.py b/benchmark/generate_table.py index 8bad5e1..018c6d4 100644 --- a/benchmark/generate_table.py +++ b/benchmark/generate_table.py @@ -2,94 +2,193 @@ import re import matplotlib.pyplot as plt import matplotlib as mpl import numpy as np +import yaml -mpl.use("svg") # parse a '.md' file and find a table. return table information -def parse_table(filepath): +def parse_table(filepath, cfg): + # parse benchmark data + def _parse_benchmark_data(lines): + raw_data = [] + for l in lines: + l = l.strip() + # parse each line + m = re.match(r"(\d+\.?\d*)\s+(\d+\.?\d*)\s+(\d+\.?\d*)\s+\[([^]]*)]\s+(.*)", l) + if m: + raw_data.append(m.groups()) + return raw_data + + # find each cpu, gpu, npu block + def _find_all_platform_block(lines): + cur_start = None + cur_platform = None + platform_block = dict() + for i in range(len(lines)): + l = lines[i].strip() + # found start and end of a platform + if l.startswith("CPU") or l.startswith("GPU") or l.startswith("NPU"): + if cur_platform is not None: + platform_block[cur_platform] = (cur_start, i) + cur_platform = l[:-1] + cur_start = i + 1 + continue + if cur_platform is not None and i == len(lines) - 1: + platform_block[cur_platform] = (cur_start, i) + for key in platform_block: + r = platform_block[key] + platform_block[key] = _parse_benchmark_data(lines[r[0]:r[1]]) + + return platform_block + + # find device block + def _find_all_device_block(lines, level): + cur_start = None + cur_device_name = None + device_block = dict() + for i in range(len(lines)): + l = lines[i].strip() + m = re.match(r"^(#+)\s+(.*)", l) + # found start and end of a device + if m and len(m.group(1)) == level: + if cur_device_name is not None: + device_block[cur_device_name] = (cur_start, i) + cur_device_name = m.group(2) + cur_start = i + 1 + continue + if cur_device_name is not None and i == len(lines) - 1: + device_block[cur_device_name] = (cur_start, i) + + for key in device_block: + r = device_block[key] + device_block[key] = _find_all_platform_block(lines[r[0]:r[1]]) + + return device_block + + # find detail block + def _find_detail_block(lines, title, level): + start = None + end = len(lines) + for i in range(len(lines)): + l = lines[i].strip() + m = re.match(r"^(#+)\s+(.*)", l) + # found start of detailed results block + if m and len(m.group(1)) == level and m.group(2) == title: + start = i + 1 + continue + # found end of detailed results block + if start is not None and m and len(m.group(1)) <= level: + end = i + break + + return _find_all_device_block(lines[start:end], level + 1) + with open(filepath, "r", encoding="utf-8") as f: content = f.read() lines = content.split("\n") - header = [] - body = [] - - found_start = False # if found table start line - parse_done = False # if parse table done - for l in lines: - if found_start and parse_done: - break - l = l.strip() - if not l: - continue - if l.startswith("|") and l.endswith("|"): - if not found_start: - found_start = True - row = [c.strip() for c in l.split("|") if c.strip()] - if not header: - header = row - else: - body.append(row) - elif found_start: - parse_done = True + devices = cfg["Devices"] + models = cfg["Models"] + # display information of all devices + devices_display = [x['display_info'] for x in cfg["Devices"]] + header = ["Model", "Task", "Input Size"] + devices_display + body = [[x["name"], x["task"], x["input_size"]] + ["---"] * len(devices) for x in models] + table_raw_data = _find_detail_block(lines, title="Detailed Results", level=2) + + device_name_header = [f"{x['name']}-{x['platform']}" for x in devices] + device_name_header = [""] * (len(header) - len(device_name_header)) + device_name_header + # device name map to model col idx + device_name_to_col_idx = {k: v for v, k in enumerate(device_name_header)} + # model name map to model row idx + model_name_to_row_idx = {k[0]: v for v, k in enumerate(body)} + # convert raw data to usage data + for device in devices: + raw_data = table_raw_data[device["name"]][device["platform"]] + col_idx = device_name_to_col_idx[f"{device['name']}-{device['platform']}"] + for model in models: + # find which row idx of this model + row_idx = model_name_to_row_idx[model["name"]] + model_idxs = [i for i in range(len(raw_data)) if model["keyword"] in raw_data[i][-1]] + if len(model_idxs) > 0: + # only choose the first one + model_idx = model_idxs[0] + # choose mean as value + body[row_idx][col_idx] = raw_data[model_idx][0] + # remove used data + for idx in sorted(model_idxs, reverse=True): + raw_data.pop(idx) + + # handle suffix + for suffix in cfg["Suffixes"]: + row_idx = model_name_to_row_idx[suffix["model"]] + col_idx = device_name_to_col_idx[f"{suffix['device']}-{suffix['platform']}"] + body[row_idx][col_idx] += suffix["str"] + return header, body -# parse models information -def parse_data(models_info): - min_list = [] - max_list = [] - colors = [] - for model in models_info: - # remove \* - data = [x.replace("\\*", "") for x in model] - # get max data - max_data = -1 - max_idx = -1 - min_data = 9999999 - min_idx = -1 - - for i in range(len(data)): - try: - d = float(data[i]) - if d > max_data: - max_data = d - max_idx = i - if d < min_data: - min_data = d - min_idx = i - except: - pass - - min_list.append(min_idx) - max_list.append(max_idx) - - # calculate colors - color = [] - for t in data: - try: - t = (float(t) - min_data) / (max_data - min_data) - color.append(cmap(t)) - except: - color.append('white') - colors.append(color) - return colors, min_list, max_list +# render table and save +def render_table(header, body, save_path, cfg, cmap_type): + # parse models information and return some data + def _parse_data(models_info, cmap, cfg): + min_list = [] + max_list = [] + colors = [] + # model name map to idx + model_name_to_idx = {k["name"]: v for v, k in enumerate(cfg["Models"])} + for model in models_info: + # remove \* + data = [x.replace("\\*", "") for x in model] + # get max data + max_idx = -1 + min_data = 9999999 + min_idx = -1 + for i in range(len(data)): + try: + d = float(data[i]) + if d < min_data: + min_data = d + min_idx = i + except: + pass + # set all bigger than acceptable time to red color + idx = model_name_to_idx[model[0]] + acc_time = cfg["Models"][idx]["acceptable_time"] -if __name__ == '__main__': - hardware_info, models_info = parse_table("./README.md") - cmap = mpl.colormaps.get_cmap("RdYlGn_r") - # remove empty line - models_info.pop(0) - # remove reference - hardware_info = [re.sub(r'\[(.+?)]\(.+?\)', r'\1', r) for r in hardware_info] - models_info = [[re.sub(r'\[(.+?)]\(.+?\)', r'\1', c) for c in r] for r in models_info] - - table_colors, min_list, max_list = parse_data(models_info) - table_texts = [hardware_info] + models_info - table_colors = [['white'] * len(hardware_info)] + table_colors - # create a color bar. base width set to 1000, color map height set to 80 + min_list.append(min_idx) + max_list.append(max_idx) + + # calculate colors + color = [] + for t in data: + try: + t = float(t) + if t > acc_time: + # all bigger time will be set to red + color.append(cmap(1.)) + else: + # sqrt to make the result non-linear + t = np.sqrt((t - min_data) / (acc_time - min_data)) + color.append(cmap(t)) + except: + color.append('white') + colors.append(color) + return colors, min_list, max_list + + cmap = mpl.colormaps.get_cmap(cmap_type) + table_colors, min_list, max_list = _parse_data(body, cmap, cfg) + table_texts = [header] + body + table_colors = [['white'] * len(header)] + table_colors + + # create a figure, base width set to 1000, height set to 80 fig, axs = plt.subplots(nrows=3, figsize=(10, 0.8)) + # turn off labels and axis + for ax in axs: + ax.set_axis_off() + ax.set_xticks([]) + ax.set_yticks([]) + + # create and add a color map gradient = np.linspace(0, 1, 256) gradient = np.vstack((gradient, gradient)) axs[0].imshow(gradient, aspect='auto', cmap=cmap) @@ -101,6 +200,24 @@ if __name__ == '__main__': cellColours=table_colors, cellLoc="left", loc="upper left") + # set style of header, each url of hardware + ori_height = table[0, 0].get_height() + url_base = 'https://github.com/opencv/opencv_zoo/tree/master/benchmark#' + hw_urls = [f"{url_base}{x['name'].lower().replace(' ', '-')}" for x in cfg["Devices"]] + hw_urls = [""] * 3 + hw_urls + for col in range(len(header)): + cell = table[0, col] + cell.set_text_props(ha='center', weight='bold', linespacing=1.5, url=hw_urls[col]) + cell.set_url(hw_urls[col]) + cell.set_height(ori_height * 2.2) + + url_base = 'https://github.com/opencv/opencv_zoo/tree/master/models/' + model_urls = [f"{url_base}{x['folder']}" for x in cfg["Models"]] + model_urls = [""] + model_urls + for row in range(len(body) + 1): + cell = table[row, 0] + cell.set_text_props(url=model_urls[row]) + cell.set_url(model_urls[row]) # adjust table position table_pos = axs[1].get_position() @@ -122,24 +239,21 @@ if __name__ == '__main__': cell = table.get_celld()[(i + 1, min_list[i])] cell.set_text_props(weight='bold', color='white') + # draw table and trigger changing the column width value + fig.canvas.draw() + # calculate table height and width table_height = 0 table_width = 0 - # calculate table height and width for i in range(len(table_texts)): cell = table.get_celld()[(i, 0)] table_height += cell.get_height() for i in range(len(table_texts[0])): cell = table.get_celld()[(0, i)] - table_width += cell.get_width() + 0.1 + table_width += cell.get_width() # add notes for table - axs[2].text(0, -table_height - 0.8, "\*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU.", va='bottom', ha='left', fontsize=11, transform=axs[1].transAxes) - - # turn off labels - for ax in axs: - ax.set_axis_off() - ax.set_xticks([]) - ax.set_yticks([]) + axs[2].text(0, -table_height - 1, "Units: All data in milliseconds (ms).", va='bottom', ha='left', fontsize=11, transform=axs[1].transAxes) + axs[2].text(0, -table_height - 2, "\\*: Models are quantized in per-channel mode, which run slower than per-tensor quantized models on NPU.", va='bottom', ha='left', fontsize=11, transform=axs[1].transAxes) # adjust color map position to center cm_pos = axs[0].get_position() @@ -151,4 +265,13 @@ if __name__ == '__main__': ]) plt.rcParams['svg.fonttype'] = 'none' - plt.savefig("./color_table.svg", format='svg', bbox_inches="tight", pad_inches=0, metadata={'Date': None, 'Creator': None}) + plt.rcParams['svg.hashsalt'] = '11' # fix hash salt for avoiding id change + plt.savefig(save_path, format='svg', bbox_inches="tight", pad_inches=0, metadata={'Date': None, 'Creator': None}) + + +if __name__ == '__main__': + with open("table_config.yaml", 'r') as f: + cfg = yaml.safe_load(f) + + hw_info, model_info = parse_table("README.md", cfg) + render_table(hw_info, model_info, "color_table.svg", cfg, "RdYlGn_r") diff --git a/benchmark/table_config.yaml b/benchmark/table_config.yaml new file mode 100644 index 0000000..68c7644 --- /dev/null +++ b/benchmark/table_config.yaml @@ -0,0 +1,219 @@ +# model information +# - name: model name, used for display +# task: model task, used for display +# input_size: input size, used for display +# folder: which folder the model located in, used for jumping to model detail +# acceptable_time: maximum acceptable inference time, large ones will be marked red +# keyword: used to specify this model from all benchmark results +# +# device information +# - name: full device name used to identify the device block, and jump to device detail +# display_info: device information for display +# platform: used to identify benchmark result of specific platform +# +# suffix information +# - model: which model +# device: which device +# suffix: this suffix will be appended to end of this text + +Models: + - name: "YuNet" + task: "Face Detection" + input_size: "160x120" + folder: "face_detection_yunet" + acceptable_time: 50 + keyword: "face_detection_yunet" + + - name: "SFace" + task: "Face Recognition" + input_size: "112x112" + folder: "face_recognition_sface" + acceptable_time: 200 + keyword: "face_recognition_sface" + + - name: "FER" + task: "Face Expression Recognition" + input_size: "112x112" + folder: "facial_expression_recognition" + acceptable_time: 200 + keyword: "facial_expression_recognition_mobilefacenet" + + - name: "LPD_YuNet" + task: "License Plate Detection" + input_size: "320x240" + folder: "license_plate_detection_yunet" + acceptable_time: 700 + keyword: "license_plate_detection_lpd_yunet" + + - name: "YOLOX" + task: "Object Detection" + input_size: "640x640" + folder: "object_detection_yolox" + acceptable_time: 2800 + keyword: "object_detection_yolox" + + - name: "NanoDet" + task: "Object Detection" + input_size: "416x416" + folder: "object_detection_nanodet" + acceptable_time: 2000 + keyword: "object_detection_nanodet" + + - name: "DB-IC15 (EN)" + task: "Text Detection" + input_size: "640x480" + folder: "text_detection_db" + acceptable_time: 2000 + keyword: "text_detection_DB_IC15_resnet18" + + - name: "DB-TD500 (EN&CN)" + task: "Text Detection" + input_size: "640x480" + folder: "text_detection_db" + acceptable_time: 2000 + keyword: "text_detection_DB_TD500_resnet18" + + - name: "CRNN-EN" + task: "Text Recognition" + input_size: "100*32" + folder: "text_recognition_crnn" + acceptable_time: 2000 + keyword: "text_recognition_CRNN_EN" + + - name: "CRNN-CN" + task: "Text Recognition" + input_size: "100*32" + folder: "text_recognition_crnn" + acceptable_time: 2000 + keyword: "text_recognition_CRNN_CN" + + - name: "PP-ResNet" + task: "Image Classification" + input_size: "224x224" + folder: "image_classification_ppresnet" + acceptable_time: 1000 + keyword: "image_classification_ppresnet50" + + - name: "MobileNet-V1" + task: "Image Classification" + input_size: "224x224" + folder: "image_classification_mobilenet" + acceptable_time: 500 + keyword: "image_classification_mobilenetv1" + + - name: "MobileNet-V2" + task: "Image Classification" + input_size: "224x224" + folder: "image_classification_mobilenet" + acceptable_time: 500 + keyword: "image_classification_mobilenetv2" + + - name: "PP-HumanSeg" + task: "Human Segmentation" + input_size: "192x192" + folder: "human_segmentation_pphumanseg" + acceptable_time: 700 + keyword: "human_segmentation_pphumanseg" + + - name: "WeChatQRCode" + task: "QR Code Detection and Parsing" + input_size: "100x100" + folder: "qrcode_wechatqrcode" + acceptable_time: 100 + keyword: "WeChatQRCode" + + - name: "DaSiamRPN" + task: "Object Tracking" + input_size: "1280x720" + folder: "object_tracking_dasiamrpn" + acceptable_time: 3000 + keyword: "object_tracking_dasiamrpn" + + - name: "YoutuReID" + task: "Person Re-Identification" + input_size: "128x256" + folder: "person_reid_youtureid" + acceptable_time: 800 + keyword: "person_reid_youtu" + + - name: "MP-PalmDet" + task: "Palm Detection" + input_size: "192x192" + folder: "palm_detection_mediapipe" + acceptable_time: 500 + keyword: "palm_detection_mediapipe" + + - name: "MP-HandPose" + task: "Hand Pose Estimation" + input_size: "224x224" + folder: "handpose_estimation_mediapipe" + acceptable_time: 500 + keyword: "handpose_estimation_mediapipe" + + - name: "MP-PersonDet" + task: "Person Detection" + input_size: "224x224" + folder: "person_detection_mediapipe" + acceptable_time: 1300 + keyword: "person_detection_mediapipe" + + +Devices: + - name: "Intel 12700K" + display_info: "Intel\n12700K\nCPU" + platform: "CPU" + + - name: "Rasberry Pi 4B" + display_info: "Rasberry Pi 4B\nBCM2711\nCPU" + platform: "CPU" + + - name: "Toybrick RV1126" + display_info: "Toybrick\nRV1126\nCPU" + platform: "CPU" + + - name: "Khadas Edge2 (with RK3588)" + display_info: "Khadas Edge2\nRK3588S\nCPU" + platform: "CPU" + + - name: "Horizon Sunrise X3 PI" + display_info: "Horizon Sunrise Pi\nX3\nCPU" + platform: "CPU" + + - name: "MAIX-III AX-PI" + display_info: "MAIX-III AX-Pi\nAX620A\nCPU" + platform: "CPU" + + - name: "Jetson Nano B01" + display_info: "Jetson Nano\nB01\nCPU" + platform: "CPU" + + - name: "Khadas VIM3" + display_info: "Khadas VIM3\nA311D\nCPU" + platform: "CPU" + + - name: "Atlas 200 DK" + display_info: "Atlas 200 DK\nAscend 310\nCPU" + platform: "CPU" + + - name: "Jetson Nano B01" + display_info: "Jetson Nano\nB01\nGPU" + platform: "GPU (CUDA-FP32)" + + - name: "Khadas VIM3" + display_info: "Khadas VIM3\nA311D\nNPU" + platform: "NPU (TIMVX)" + + - name: "Atlas 200 DK" + display_info: "Atlas 200 DK\nAscend 310\nNPU" + platform: "NPU" + +Suffixes: + - model: "MobileNet-V1" + device: "Khadas VIM3" + platform: "NPU (TIMVX)" + str: "\\*" + + - model: "MobileNet-V2" + device: "Khadas VIM3" + platform: "NPU (TIMVX)" + str: "\\*" \ No newline at end of file -- GitLab