diff --git a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java index 6a6665dd334d1c7a47fea04ef708b84498f0e357..7ad712f5e8af4237964be7ecc3b4d16a909bdc40 100755 --- a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java +++ b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java @@ -58,6 +58,7 @@ public class MainActivity extends Activity { private Context mContext = null; private int inputSize = 224; + private int[] ddims = {1, 3, 224, 224}; enum TYPE { googlenet @@ -121,14 +122,14 @@ public class MainActivity extends Activity { String assetPath = "pml_demo"; String sdcardPath = Environment.getExternalStorageDirectory() + File.separator + assetPath + File.separator + type; - //PML.load(sdcardPath); + PML.load(sdcardPath); String modelPath = Environment.getExternalStorageDirectory() + File.separator + assetPath + File.separator + "googlenet_combine" + File.separator + "model"; String paramPath = Environment.getExternalStorageDirectory() + File.separator + assetPath + File.separator + "googlenet_combine" + File.separator + "params"; - PML.loadCombined(modelPath, paramPath); +// PML.loadCombined(modelPath, paramPath); } }); @@ -351,8 +352,8 @@ public class MainActivity extends Activity { @Override public void onBackPressed() { super.onBackPressed(); - Log.d("mdl", "mdl clear"); - // clear mdl + Log.d("pml", "pml clear"); + // clear pml PML.clear(); } @@ -402,7 +403,7 @@ public class MainActivity extends Activity { float[] result = null; try { long start = System.currentTimeMillis(); - result = PML.predict(inputData); + result = PML.predictImage(inputData, ddims); long end = System.currentTimeMillis(); time = end - start; diff --git a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java index e67f04e47a77b28bfd8ce98866b1539797c217cd..7ff9af97cf1eac6c6b70a22f5f4478022656b860 100644 --- a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java +++ b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java @@ -2,14 +2,14 @@ package com.baidu.paddle; public class PML { /** - * Load - * @param modelPath + * Load seperated parameters + * @param modelDir * @return */ - public static native boolean load(String modelPath); + public static native boolean load(String modelDir); /** - * Load + * Load combined parameters * @param modelPath * @param paramPath * @return @@ -23,7 +23,20 @@ public class PML { * @param buf * @return */ - public static native float[] predict(float[] buf); + public static native float[] predictImage(float[] buf, int[]ddims); + + /** + * + * @param buf yuv420格式的字节数组 + * @param imgWidth yuv数据的宽 + * @param imgHeight yuv数据的高 + * @param ddims 输入数据的形状 + * @param meanValues 模型训练时各通道的均值 + * @return + */ + + public static native float[] predictYuv(byte[] buf, int imgWidth, int imgHeight, int[] ddims, float[]meanValues); + public static native void clear(); diff --git a/doc/development_doc.md b/doc/development_doc.md index 18a1974576d8e488fe7f4e70cad18ae088b794a9..da8eddd09f3d46a09b4593e820dd86bc24a4ab81 100644 --- a/doc/development_doc.md +++ b/doc/development_doc.md @@ -186,27 +186,51 @@ which to test : ##部署 Android应用可通过JNI接口调用底层C/C++，paddle-mobile对外提供的JNI接口如下： -##### 1 load接口 加载模型参数 - +##### 1 load接口 加载模型参数 +- 用于加载参数文件分散的模型 ``` -/* -*@param modelPath 模型文件路径 -*@return jboolean -*/ -JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_load(JNIEnv *env, - jclass thiz, - jstring modelPath); +/** + * Load seperated parameters + * @param modelDir + * @return + */ + public static native boolean load(String modelDir); +``` +- 用于加载参数文件合并的模型文件 ``` +/** + * Load combined parameters + * @param modelPath + * @param paramPath + * @return + */ + public static native boolean loadCombined(String modelPath,String paramPath); +``` ##### 2 predict接口 执行预测 - +- 接受预处理过的RGB数组的predict接口 ``` /** *@param buf 输入数据 *@return 输出数据 -JNIEXPORT jfloatArray JNICALL Java_com_baidu_paddle_PML_predict( +JNIEXPORT jfloatArray JNICALL Java_com_baidu_paddle_PML_predictImage( JNIEnv *env, jclass thiz, jfloatArray buf); ``` +- 接受原始yuv数据的predict接口 +``` + /** + * + * @param buf yuv420格式的字节数组 + * @param imgWidth yuv数据的宽 + * @param imgHeight yuv数据的高 + * @param ddims 输入数据的形状 + * @param meanValues 模型训练时各通道的均值 + * @return + */ + + public static native float[] predictYuv(byte[] buf, int imgWidth, int imgHeight, int[] ddims, float[]meanValues); + +``` ##### 3 clear接口 销毁实例、清理内存操作 ``` diff --git a/src/jni/paddle_mobile_jni.cpp b/src/jni/paddle_mobile_jni.cpp index b14f095c1d82f167c1e3f15897b907e730a4a5a8..47dbc08392977750151d0e6180814cb672668403 100644 --- a/src/jni/paddle_mobile_jni.cpp +++ b/src/jni/paddle_mobile_jni.cpp @@ -62,7 +62,7 @@ JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_load(JNIEnv *env, JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_loadCombined( JNIEnv *env, jclass thiz, jstring modelPath, jstring paramPath) { - ANDROIDLOGI("load invoked"); + ANDROIDLOGI("loadCombined invoked"); bool optimize = true; return getPaddleMobileInstance()->Load(jstring2cppstring(env, modelPath), jstring2cppstring(env, paramPath), @@ -70,7 +70,15 @@ JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_loadCombined( } JNIEXPORT jfloatArray JNICALL -Java_com_baidu_paddle_PML_predict(JNIEnv *env, jclass thiz, jfloatArray buf) { +Java_com_baidu_paddle_PML_predictImage(JNIEnv *env, jclass thiz, jfloatArray buf, jintArray ddims) { + ANDROIDLOGI("predictImage invoked"); + jsize ddim_size = env->GetArrayLength(ddims); + if(ddim_size != 4) { + ANDROIDLOGE("ddims size not equal to 4"); + } + jint *ddim_ptr = env->GetIntArrayElements(ddims, NULL); + framework::DDim ddim = framework::make_ddim({ddim_ptr[0], ddim_ptr[1], ddim_ptr[2], ddim_ptr[3]}); + int length = framework::product(ddim); jfloatArray result = NULL; int count = 0; float *dataPointer = nullptr; @@ -78,20 +86,104 @@ Java_com_baidu_paddle_PML_predict(JNIEnv *env, jclass thiz, jfloatArray buf) { dataPointer = env->GetFloatArrayElements(buf, NULL); } framework::Tensor input; - framework::DDim ddim = framework::make_ddim({1, 3, 224, 224}); input.Resize(ddim); auto input_ptr = input.mutable_data(); - for (int i = 0; i < framework::product(ddim); i++) { + for (int i = 0; i < length; i++) { input_ptr[i] = dataPointer[i]; } auto output = shared_paddle_mobile_instance->Predict(input); count = output->numel(); result = env->NewFloatArray(count); env->SetFloatArrayRegion(result, 0, count, output->data()); - ANDROIDLOGI("predict finished"); + env->ReleaseIntArrayElements(ddims,ddim_ptr, 0); + ANDROIDLOGI("predictImage finished"); return result; } +inline int yuv_to_rgb(int y, int u, int v, float *r, float *g, float *b) + { + int r1 = (int)(y + 1.370705 * (v - 128)); + int g1 = (int)(y - 0.698001 * (u - 128) - 0.703125 * (v - 128)); + int b1 = (int)(y + 1.732446 * (u - 128)); + + r1 = (int) fminf(255, fmaxf(0, r1)); + g1 = (int) fminf(255, fmaxf(0, g1)); + b1 = (int) fminf(255, fmaxf(0, b1)); + *r = r1; + *g = g1; + *b = b1; + + return 0; + } +void convert_nv21_to_matrix(uint8_t *nv21, float *matrix, int width, int height, int targetWidth, int targetHeight, float *means) { + const uint8_t *yData = nv21; + const uint8_t *vuData = nv21 + width * height; + + const int yRowStride = width; + const int vuRowStride = width; + + float scale_x = width * 1.0 / targetWidth; + float scale_y = height * 1.0 / targetHeight; + + for (int j = 0; j < targetHeight; ++j) { + int y = j * scale_y; + const uint8_t *pY = yData + y * yRowStride; + const uint8_t *pVU = vuData + (y >> 1) * vuRowStride; + for (int i = 0; i < targetWidth; ++i) { + int x = i * scale_x; + const int offset = ((x >> 1) << 1); + float r = 0; + float g = 0; + float b = 0; + yuv_to_rgb(pY[x], pVU[offset + 1], pVU[offset], &r, &g, &b); + int r_index = j * targetWidth + i; + int g_index = r_index + targetWidth * targetHeight; + int b_index = g_index + targetWidth * targetHeight; + matrix[r_index] = r - means[0]; + matrix[g_index] = g - means[1]; + matrix[b_index] = b - means[2]; + } + } + } + +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_PML_predictYuv(JNIEnv *env, jclass thiz, jbyteArray yuv_, jint imgwidth, jint imgHeight, jintArray ddims, +jfloatArray meanValues) { + ANDROIDLOGI("predictYuv invoked"); + jsize ddim_size = env->GetArrayLength(ddims); + if(ddim_size != 4) { + ANDROIDLOGE("ddims size not equal to 4"); + } + jint *ddim_ptr = env->GetIntArrayElements(ddims, NULL); + framework::DDim ddim = framework::make_ddim({ddim_ptr[0], ddim_ptr[1], ddim_ptr[2], ddim_ptr[3]}); + int length = framework::product(ddim); + float matrix[length]; + jbyte *yuv = env->GetByteArrayElements(yuv_, NULL); + float *meansPointer = nullptr; + if (nullptr != meanValues) { + meansPointer = env->GetFloatArrayElements(meanValues, NULL); + } + convert_nv21_to_matrix((uint8_t *)yuv,matrix, imgwidth, imgHeight, ddim[3], ddim[2], meansPointer ); + jfloatArray result = NULL; + int count = 0; + framework::Tensor input; + input.Resize(ddim); + auto input_ptr = input.mutable_data(); + for (int i = 0; i < length; i++) { + input_ptr[i] = matrix[i]; + } + auto output = shared_paddle_mobile_instance->Predict(input); + count = output->numel(); + result = env->NewFloatArray(count); + env->SetFloatArrayRegion(result, 0, count, output->data()); + env->ReleaseByteArrayElements(yuv_, yuv, 0); + env->ReleaseIntArrayElements(ddims, ddim_ptr, 0); + env->ReleaseFloatArrayElements(meanValues, meansPointer, 0); + ANDROIDLOGI("predictYuv finished"); + return result; + +} + JNIEXPORT void JNICALL Java_com_baidu_paddle_PML_clear(JNIEnv *env, jclass thiz) { getPaddleMobileInstance()->Clear(); diff --git a/src/jni/paddle_mobile_jni.h b/src/jni/paddle_mobile_jni.h index ab88816dcb7ec6ba88f12cb270812c4af0923b32..92a0e405902e71cb657af068358ae10a0e692410 100644 --- a/src/jni/paddle_mobile_jni.h +++ b/src/jni/paddle_mobile_jni.h @@ -33,6 +33,18 @@ JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_load(JNIEnv *env, JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_loadCombined( JNIEnv *env, jclass thiz, jstring modelPath, jstring paramPath); +/** + * object detection for anroid + */ +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_PML_predictImage(JNIEnv *env, jclass thiz, jfloatArray buf, jintArray ddims); + +/** + * object detection for anroid + */ +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_PML_predictYuv(JNIEnv *env, jclass thiz, jbyteArray yuv, jint imgwidth, jint imgHeight, jintArray ddims, jfloatArray meanValues); + /** * object detection for anroid */ diff --git a/src/operators/math/pooling.cpp b/src/operators/math/pooling.cpp index fc4d96645c1dd24e72ee65ea9b7a8dda63a26514..24db2e272e3124a223c22c6f687d868d42126f6b 100644 --- a/src/operators/math/pooling.cpp +++ b/src/operators/math/pooling.cpp @@ -60,8 +60,8 @@ class PoolFunctor { T *output_data = output->mutable_data(); for (int i = 0; i < batch_size; i++) { -#pragma omp parallel for for (int c = 0; c < output_channels; ++c) { +#pragma omp parallel for for (int ph = 0; ph < output_height; ++ph) { int hstart = ph * stride_height - padding_height; int hend = std::min(hstart + ksize_height, input_height);