From ddee6c6d56287af0631583c06752a9e126c14a7b Mon Sep 17 00:00:00 2001 From: WenmuZhou Date: Sat, 26 Feb 2022 23:03:27 +0800 Subject: [PATCH] move preprocess of det to cpp and add opencl --- .../app/src/main/cpp/ppredictor.cpp | 34 +++- .../app/src/main/cpp/ppredictor.h | 3 +- .../baidu/paddle/lite/demo/ocr/Predictor.java | 147 ++++-------------- .../lite/demo/ocr/SettingsActivity.java | 51 ++---- .../app/src/main/res/values/strings.xml | 16 +- .../app/src/main/res/xml/settings.xml | 22 +-- 6 files changed, 87 insertions(+), 186 deletions(-) diff --git a/deploy/android_demo/app/src/main/cpp/ppredictor.cpp b/deploy/android_demo/app/src/main/cpp/ppredictor.cpp index dcbc7691..acea0a3f 100644 --- a/deploy/android_demo/app/src/main/cpp/ppredictor.cpp +++ b/deploy/android_demo/app/src/main/cpp/ppredictor.cpp @@ -2,9 +2,9 @@ #include "common.h" namespace ppredictor { -PPredictor::PPredictor(int thread_num, int net_flag, +PPredictor::PPredictor(int use_opencl, int thread_num, int net_flag, paddle::lite_api::PowerMode mode) - : _thread_num(thread_num), _net_flag(net_flag), _mode(mode) {} + : _use_opencl(use_opencl), _thread_num(thread_num), _net_flag(net_flag), _mode(mode) {} int PPredictor::init_nb(const std::string &model_content) { paddle::lite_api::MobileConfig config; @@ -19,6 +19,36 @@ int PPredictor::init_from_file(const std::string &model_content) { } template int PPredictor::_init(ConfigT &config) { + bool is_opencl_backend_valid = paddle::lite_api::IsOpenCLBackendValid(/*check_fp16_valid = false*/); + if (is_opencl_backend_valid) { + if (_use_opencl != 0) { + // Make sure you have write permission of the binary path. + // We strongly recommend each model has a unique binary name. + const std::string bin_path = "/data/local/tmp/"; + const std::string bin_name = "lite_opencl_kernel.bin"; + config.set_opencl_binary_path_name(bin_path, bin_name); + + // opencl tune option + // CL_TUNE_NONE: 0 + // CL_TUNE_RAPID: 1 + // CL_TUNE_NORMAL: 2 + // CL_TUNE_EXHAUSTIVE: 3 + const std::string tuned_path = "/data/local/tmp/"; + const std::string tuned_name = "lite_opencl_tuned.bin"; + config.set_opencl_tune(paddle::lite_api::CL_TUNE_NORMAL, tuned_path, tuned_name); + + // opencl precision option + // CL_PRECISION_AUTO: 0, first fp16 if valid, default + // CL_PRECISION_FP32: 1, force fp32 + // CL_PRECISION_FP16: 2, force fp16 + config.set_opencl_precision(paddle::lite_api::CL_PRECISION_FP32); + LOGI("device: running on gpu."); + } + } else { + LOGI("device: running on cpu."); + // you can give backup cpu nb model instead + // config.set_model_from_file(cpu_nb_model_dir); + } config.set_threads(_thread_num); config.set_power_mode(_mode); _predictor = paddle::lite_api::CreatePaddlePredictor(config); diff --git a/deploy/android_demo/app/src/main/cpp/ppredictor.h b/deploy/android_demo/app/src/main/cpp/ppredictor.h index 836861aa..40250764 100644 --- a/deploy/android_demo/app/src/main/cpp/ppredictor.h +++ b/deploy/android_demo/app/src/main/cpp/ppredictor.h @@ -22,7 +22,7 @@ public: class PPredictor : public PPredictor_Interface { public: PPredictor( - int thread_num, int net_flag = 0, + int use_opencl, int thread_num, int net_flag = 0, paddle::lite_api::PowerMode mode = paddle::lite_api::LITE_POWER_HIGH); virtual ~PPredictor() {} @@ -54,6 +54,7 @@ protected: template int _init(ConfigT &config); private: + int _use_opencl; int _thread_num; paddle::lite_api::PowerMode _mode; std::shared_ptr _predictor; diff --git a/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/Predictor.java b/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/Predictor.java index 32802f2f..b977bfb5 100644 --- a/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/Predictor.java +++ b/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/Predictor.java @@ -31,23 +31,19 @@ public class Predictor { protected float inferenceTime = 0; // Only for object detection protected Vector wordLabels = new Vector(); - protected String inputColorFormat = "BGR"; - protected long[] inputShape = new long[]{1, 3, 960}; - protected float[] inputMean = new float[]{0.485f, 0.456f, 0.406f}; - protected float[] inputStd = new float[]{1.0f / 0.229f, 1.0f / 0.224f, 1.0f / 0.225f}; + protected int detLongSize = 960; protected float scoreThreshold = 0.1f; protected Bitmap inputImage = null; protected Bitmap outputImage = null; protected volatile String outputResult = ""; - protected float preprocessTime = 0; protected float postprocessTime = 0; public Predictor() { } - public boolean init(Context appCtx, String modelPath, String labelPath, int cpuThreadNum, String cpuPowerMode) { - isLoaded = loadModel(appCtx, modelPath, cpuThreadNum, cpuPowerMode); + public boolean init(Context appCtx, String modelPath, String labelPath, int useOpencl, int cpuThreadNum, String cpuPowerMode) { + isLoaded = loadModel(appCtx, modelPath, useOpencl, cpuThreadNum, cpuPowerMode); if (!isLoaded) { return false; } @@ -56,49 +52,18 @@ public class Predictor { } - public boolean init(Context appCtx, String modelPath, String labelPath, int cpuThreadNum, String cpuPowerMode, - String inputColorFormat, - long[] inputShape, float[] inputMean, - float[] inputStd, float scoreThreshold) { - if (inputShape.length != 3) { - Log.e(TAG, "Size of input shape should be: 3"); - return false; - } - if (inputMean.length != inputShape[1]) { - Log.e(TAG, "Size of input mean should be: " + Long.toString(inputShape[1])); - return false; - } - if (inputStd.length != inputShape[1]) { - Log.e(TAG, "Size of input std should be: " + Long.toString(inputShape[1])); - return false; - } - if (inputShape[0] != 1) { - Log.e(TAG, "Only one batch is supported in the image classification demo, you can use any batch size in " + - "your Apps!"); - return false; - } - if (inputShape[1] != 1 && inputShape[1] != 3) { - Log.e(TAG, "Only one/three channels are supported in the image classification demo, you can use any " + - "channel size in your Apps!"); - return false; - } - if (!inputColorFormat.equalsIgnoreCase("BGR")) { - Log.e(TAG, "Only BGR color format is supported."); - return false; - } - boolean isLoaded = init(appCtx, modelPath, labelPath, cpuThreadNum, cpuPowerMode); + public boolean init(Context appCtx, String modelPath, String labelPath, int useOpencl, int cpuThreadNum, String cpuPowerMode, + int detLongSize, float scoreThreshold) { + boolean isLoaded = init(appCtx, modelPath, labelPath, useOpencl, cpuThreadNum, cpuPowerMode); if (!isLoaded) { return false; } - this.inputColorFormat = inputColorFormat; - this.inputShape = inputShape; - this.inputMean = inputMean; - this.inputStd = inputStd; + this.detLongSize = detLongSize; this.scoreThreshold = scoreThreshold; return true; } - protected boolean loadModel(Context appCtx, String modelPath, int cpuThreadNum, String cpuPowerMode) { + protected boolean loadModel(Context appCtx, String modelPath, int useOpencl, int cpuThreadNum, String cpuPowerMode) { // Release model if exists releaseModel(); @@ -118,12 +83,13 @@ public class Predictor { } OCRPredictorNative.Config config = new OCRPredictorNative.Config(); + config.useOpencl = useOpencl; config.cpuThreadNum = cpuThreadNum; config.cpuPower = cpuPowerMode; config.detModelFilename = realPath + File.separator + "det_db.nb"; config.recModelFilename = realPath + File.separator + "rec_crnn.nb"; config.clsModelFilename = realPath + File.separator + "cls.nb"; - Log.e("Predictor", "model path" + config.detModelFilename + " ; " + config.recModelFilename + ";" + config.clsModelFilename); + Log.i("Predictor", "model path" + config.detModelFilename + " ; " + config.recModelFilename + ";" + config.clsModelFilename); paddlePredictor = new OCRPredictorNative(config); this.cpuThreadNum = cpuThreadNum; @@ -170,82 +136,29 @@ public class Predictor { } - public boolean runModel() { + public boolean runModel(int run_det, int run_cls, int run_rec) { if (inputImage == null || !isLoaded()) { return false; } - // Pre-process image, and feed input tensor with pre-processed data - - Bitmap scaleImage = Utils.resizeWithStep(inputImage, Long.valueOf(inputShape[2]).intValue(), 32); - - Date start = new Date(); - int channels = (int) inputShape[1]; - int width = scaleImage.getWidth(); - int height = scaleImage.getHeight(); - float[] inputData = new float[channels * width * height]; - if (channels == 3) { - int[] channelIdx = null; - if (inputColorFormat.equalsIgnoreCase("RGB")) { - channelIdx = new int[]{0, 1, 2}; - } else if (inputColorFormat.equalsIgnoreCase("BGR")) { - channelIdx = new int[]{2, 1, 0}; - } else { - Log.i(TAG, "Unknown color format " + inputColorFormat + ", only RGB and BGR color format is " + - "supported!"); - return false; - } - - int[] channelStride = new int[]{width * height, width * height * 2}; - int[] pixels = new int[width * height]; - scaleImage.getPixels(pixels, 0, scaleImage.getWidth(), 0, 0, scaleImage.getWidth(), scaleImage.getHeight()); - for (int i = 0; i < pixels.length; i++) { - int color = pixels[i]; - float[] rgb = new float[]{(float) red(color) / 255.0f, (float) green(color) / 255.0f, - (float) blue(color) / 255.0f}; - inputData[i] = (rgb[channelIdx[0]] - inputMean[0]) / inputStd[0]; - inputData[i + channelStride[0]] = (rgb[channelIdx[1]] - inputMean[1]) / inputStd[1]; - inputData[i + channelStride[1]] = (rgb[channelIdx[2]] - inputMean[2]) / inputStd[2]; - } - } else if (channels == 1) { - int[] pixels = new int[width * height]; - scaleImage.getPixels(pixels, 0, scaleImage.getWidth(), 0, 0, scaleImage.getWidth(), scaleImage.getHeight()); - for (int i = 0; i < pixels.length; i++) { - int color = pixels[i]; - float gray = (float) (red(color) + green(color) + blue(color)) / 3.0f / 255.0f; - inputData[i] = (gray - inputMean[0]) / inputStd[0]; - } - } else { - Log.i(TAG, "Unsupported channel size " + Integer.toString(channels) + ", only channel 1 and 3 is " + - "supported!"); - return false; - } - float[] pixels = inputData; - Log.i(TAG, "pixels " + pixels[0] + " " + pixels[1] + " " + pixels[2] + " " + pixels[3] - + " " + pixels[pixels.length / 2] + " " + pixels[pixels.length / 2 + 1] + " " + pixels[pixels.length - 2] + " " + pixels[pixels.length - 1]); - Date end = new Date(); - preprocessTime = (float) (end.getTime() - start.getTime()); - // Warm up for (int i = 0; i < warmupIterNum; i++) { - paddlePredictor.runImage(inputData, width, height, channels, inputImage); + paddlePredictor.runImage(inputImage, detLongSize, run_det, run_cls, run_rec); } warmupIterNum = 0; // do not need warm // Run inference - start = new Date(); - ArrayList results = paddlePredictor.runImage(inputData, width, height, channels, inputImage); - end = new Date(); + Date start = new Date(); + ArrayList results = paddlePredictor.runImage(inputImage, detLongSize, run_det, run_cls, run_rec); + Date end = new Date(); inferenceTime = (end.getTime() - start.getTime()) / (float) inferIterNum; results = postprocess(results); - Log.i(TAG, "[stat] Preprocess Time: " + preprocessTime - + " ; Inference Time: " + inferenceTime + " ;Box Size " + results.size()); + Log.i(TAG, "[stat] Inference Time: " + inferenceTime + " ;Box Size " + results.size()); drawResults(results); return true; } - public boolean isLoaded() { return paddlePredictor != null && isLoaded; } @@ -282,10 +195,6 @@ public class Predictor { return outputResult; } - public float preprocessTime() { - return preprocessTime; - } - public float postprocessTime() { return postprocessTime; } @@ -310,6 +219,7 @@ public class Predictor { } } r.setLabel(word.toString()); + r.setClsLabel(r.getClsIdx() == 1 ? "180" : "0"); } return results; } @@ -319,14 +229,22 @@ public class Predictor { for (int i = 0; i < results.size(); i++) { OcrResultModel result = results.get(i); StringBuilder sb = new StringBuilder(""); - sb.append(result.getLabel()); - sb.append(" ").append(result.getConfidence()); - sb.append("; Points: "); - for (Point p : result.getPoints()) { - sb.append("(").append(p.x).append(",").append(p.y).append(") "); + if(result.getPoints().size()>0){ + sb.append("; Det: "); + for (Point p : result.getPoints()) { + sb.append("(").append(p.x).append(",").append(p.y).append(") "); + } + } + if(result.getLabel().length() > 0){ + sb.append("\n Rec: ").append(result.getLabel()); + sb.append(",").append(result.getConfidence()); + } + if(result.getClsIdx()!=-1){ + sb.append(" Cls: ").append(result.getClsLabel()); + sb.append(",").append(result.getClsConfidence()); } Log.i(TAG, sb.toString()); // show LOG in Logcat panel - outputResultSb.append(i + 1).append(": ").append(result.getLabel()).append("\n"); + outputResultSb.append(i + 1).append(": ").append(sb.toString()).append("\n"); } outputResult = outputResultSb.toString(); outputImage = inputImage; @@ -344,6 +262,9 @@ public class Predictor { for (OcrResultModel result : results) { Path path = new Path(); List points = result.getPoints(); + if(points.size()==0){ + continue; + } path.moveTo(points.get(0).x, points.get(0).y); for (int i = points.size() - 1; i >= 0; i--) { Point p = points.get(i); diff --git a/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/SettingsActivity.java b/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/SettingsActivity.java index c26fe6ea..477cd5d8 100644 --- a/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/SettingsActivity.java +++ b/deploy/android_demo/app/src/main/java/com/baidu/paddle/lite/demo/ocr/SettingsActivity.java @@ -20,16 +20,13 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha ListPreference etImagePath = null; ListPreference lpCPUThreadNum = null; ListPreference lpCPUPowerMode = null; - ListPreference lpInputColorFormat = null; - EditTextPreference etInputShape = null; - EditTextPreference etInputMean = null; - EditTextPreference etInputStd = null; + EditTextPreference etDetLongSize = null; EditTextPreference etScoreThreshold = null; List preInstalledModelPaths = null; List preInstalledLabelPaths = null; List preInstalledImagePaths = null; - List preInstalledInputShapes = null; + List preInstalledDetLongSizes = null; List preInstalledCPUThreadNums = null; List preInstalledCPUPowerModes = null; List preInstalledInputColorFormats = null; @@ -50,7 +47,7 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha preInstalledModelPaths = new ArrayList(); preInstalledLabelPaths = new ArrayList(); preInstalledImagePaths = new ArrayList(); - preInstalledInputShapes = new ArrayList(); + preInstalledDetLongSizes = new ArrayList(); preInstalledCPUThreadNums = new ArrayList(); preInstalledCPUPowerModes = new ArrayList(); preInstalledInputColorFormats = new ArrayList(); @@ -63,10 +60,7 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha preInstalledImagePaths.add(getString(R.string.IMAGE_PATH_DEFAULT)); preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); - preInstalledInputColorFormats.add(getString(R.string.INPUT_COLOR_FORMAT_DEFAULT)); - preInstalledInputShapes.add(getString(R.string.INPUT_SHAPE_DEFAULT)); - preInstalledInputMeans.add(getString(R.string.INPUT_MEAN_DEFAULT)); - preInstalledInputStds.add(getString(R.string.INPUT_STD_DEFAULT)); + preInstalledDetLongSizes.add(getString(R.string.DET_LONG_SIZE_DEFAULT)); preInstalledScoreThresholds.add(getString(R.string.SCORE_THRESHOLD_DEFAULT)); // Setup UI components @@ -89,11 +83,7 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); - lpInputColorFormat = - (ListPreference) findPreference(getString(R.string.INPUT_COLOR_FORMAT_KEY)); - etInputShape = (EditTextPreference) findPreference(getString(R.string.INPUT_SHAPE_KEY)); - etInputMean = (EditTextPreference) findPreference(getString(R.string.INPUT_MEAN_KEY)); - etInputStd = (EditTextPreference) findPreference(getString(R.string.INPUT_STD_KEY)); + etDetLongSize = (EditTextPreference) findPreference(getString(R.string.DET_LONG_SIZE_KEY)); etScoreThreshold = (EditTextPreference) findPreference(getString(R.string.SCORE_THRESHOLD_KEY)); } @@ -112,11 +102,7 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha editor.putString(getString(R.string.IMAGE_PATH_KEY), preInstalledImagePaths.get(modelIdx)); editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(modelIdx)); editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(modelIdx)); - editor.putString(getString(R.string.INPUT_COLOR_FORMAT_KEY), - preInstalledInputColorFormats.get(modelIdx)); - editor.putString(getString(R.string.INPUT_SHAPE_KEY), preInstalledInputShapes.get(modelIdx)); - editor.putString(getString(R.string.INPUT_MEAN_KEY), preInstalledInputMeans.get(modelIdx)); - editor.putString(getString(R.string.INPUT_STD_KEY), preInstalledInputStds.get(modelIdx)); + editor.putString(getString(R.string.DET_LONG_SIZE_KEY), preInstalledDetLongSizes.get(modelIdx)); editor.putString(getString(R.string.SCORE_THRESHOLD_KEY), preInstalledScoreThresholds.get(modelIdx)); editor.apply(); @@ -129,10 +115,7 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha etImagePath.setEnabled(enableCustomSettings); lpCPUThreadNum.setEnabled(enableCustomSettings); lpCPUPowerMode.setEnabled(enableCustomSettings); - lpInputColorFormat.setEnabled(enableCustomSettings); - etInputShape.setEnabled(enableCustomSettings); - etInputMean.setEnabled(enableCustomSettings); - etInputStd.setEnabled(enableCustomSettings); + etDetLongSize.setEnabled(enableCustomSettings); etScoreThreshold.setEnabled(enableCustomSettings); modelPath = sharedPreferences.getString(getString(R.string.MODEL_PATH_KEY), getString(R.string.MODEL_PATH_DEFAULT)); @@ -144,14 +127,8 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha getString(R.string.CPU_THREAD_NUM_DEFAULT)); String cpuPowerMode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), getString(R.string.CPU_POWER_MODE_DEFAULT)); - String inputColorFormat = sharedPreferences.getString(getString(R.string.INPUT_COLOR_FORMAT_KEY), - getString(R.string.INPUT_COLOR_FORMAT_DEFAULT)); - String inputShape = sharedPreferences.getString(getString(R.string.INPUT_SHAPE_KEY), - getString(R.string.INPUT_SHAPE_DEFAULT)); - String inputMean = sharedPreferences.getString(getString(R.string.INPUT_MEAN_KEY), - getString(R.string.INPUT_MEAN_DEFAULT)); - String inputStd = sharedPreferences.getString(getString(R.string.INPUT_STD_KEY), - getString(R.string.INPUT_STD_DEFAULT)); + String detLongSize = sharedPreferences.getString(getString(R.string.DET_LONG_SIZE_KEY), + getString(R.string.DET_LONG_SIZE_DEFAULT)); String scoreThreshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), getString(R.string.SCORE_THRESHOLD_DEFAULT)); etModelPath.setSummary(modelPath); @@ -164,14 +141,8 @@ public class SettingsActivity extends AppCompatPreferenceActivity implements Sha lpCPUThreadNum.setSummary(cpuThreadNum); lpCPUPowerMode.setValue(cpuPowerMode); lpCPUPowerMode.setSummary(cpuPowerMode); - lpInputColorFormat.setValue(inputColorFormat); - lpInputColorFormat.setSummary(inputColorFormat); - etInputShape.setSummary(inputShape); - etInputShape.setText(inputShape); - etInputMean.setSummary(inputMean); - etInputMean.setText(inputMean); - etInputStd.setSummary(inputStd); - etInputStd.setText(inputStd); + etDetLongSize.setSummary(detLongSize); + etDetLongSize.setText(detLongSize); etScoreThreshold.setText(scoreThreshold); etScoreThreshold.setSummary(scoreThreshold); } diff --git a/deploy/android_demo/app/src/main/res/values/strings.xml b/deploy/android_demo/app/src/main/res/values/strings.xml index 0228af09..6ee1f30f 100644 --- a/deploy/android_demo/app/src/main/res/values/strings.xml +++ b/deploy/android_demo/app/src/main/res/values/strings.xml @@ -1,5 +1,5 @@ - OCR Chinese + PaddleOCR CHOOSE_PRE_INSTALLED_MODEL_KEY ENABLE_CUSTOM_SETTINGS_KEY MODEL_PATH_KEY @@ -7,20 +7,14 @@ IMAGE_PATH_KEY CPU_THREAD_NUM_KEY CPU_POWER_MODE_KEY - INPUT_COLOR_FORMAT_KEY - INPUT_SHAPE_KEY - INPUT_MEAN_KEY - INPUT_STD_KEY + DET_LONG_SIZE_KEY SCORE_THRESHOLD_KEY - models/ocr_v2_for_cpu + models/ch_PP-OCRv2 labels/ppocr_keys_v1.txt - images/0.jpg + images/det_0.jpg 4 LITE_POWER_HIGH - BGR - 1,3,960 - 0.485, 0.456, 0.406 - 0.229,0.224,0.225 + 960 0.1 diff --git a/deploy/android_demo/app/src/main/res/xml/settings.xml b/deploy/android_demo/app/src/main/res/xml/settings.xml index 049727e8..8c2ea62a 100644 --- a/deploy/android_demo/app/src/main/res/xml/settings.xml +++ b/deploy/android_demo/app/src/main/res/xml/settings.xml @@ -47,26 +47,10 @@ android:entryValues="@array/cpu_power_mode_values"/> - - - + android:key="@string/DET_LONG_SIZE_KEY" + android:defaultValue="@string/DET_LONG_SIZE_DEFAULT" + android:title="det long size" />