From 62eebd63cab8db7e3c2f912ced309450fb08aef9 Mon Sep 17 00:00:00 2001 From: Gines Date: Mon, 25 Jun 2018 19:36:01 -0600 Subject: [PATCH] Accumulated improvements and bug fixes (#675) * Adam added * Cleaned adam.cpp * Added BVH file writer * Finished adam.cpp * Bvh and adam mesh avi with flags * reverse X * fix left/right, write right first * adding additional left_hand/right_hand * fix left_hand/right_hand * Forward Kinematics with Derivative * Pulled from official hand_model * BVH file duplicated * Removed warnings * Updated Adam * x10 IK speed up * Speed up for not computing Jac + smarter Jac copy * Merged official with OP_adam * Speed up of rendering and multi-thread with IK * Updated from hand_model branch * Cleaner examples * Fixed new scale * transmit json * Cleaner adam.cpp * Change * Better doc * Updated Adam code * Working with latest code * add foot fitting, head orientation fitting, set verbose for fitting * Face also fitted * t-pose improved * small regularization for fingers * Added extra wrist-fitting * mouth/eyes openness * Adam keeps last known 5 frames * setting flags for init output, rendering options * set a flag for zoom in * Cleaner code * Cleaner adam.cpp * Fixed typo * dist from root to foot * AdamFast multi-threaded * Added new element * Cleaner code * Cleaned warnings + less params shared * Almost multi-threaded * JointAngleEstimation class * Adam mostly integrated inside OpenPose * Recovered forgotten changes * Write video as Adam added * BVH writer as class * So Adam.cpp directly runs * Cleaner code * Updated code * Face and hand can be disabled * Switched shorcuts for channel selection * Notify user when pose value changes * BODY_25 as main one * BODY_25 as main one * Flag write_keypoint_json removed * Foot also fit * keep preivous param when joint target missing * Freeze last known pose by default * Added final BODY_25 model * adam facial expression * show_joint turn off everything * Mini-bug fixed in wrapper * Deprecated issue solved * Adam_fast_fit_init speed up * Donglai's advice * optional face fitting/joint angle keeping for arm only/multistage fitting flag * Fast solver option * Latest adam.cpp * Asio removed * Better doc * Improved calib * Cleaned SPinnaker * add a hand_enabled flag in Adam_FastFit_Initialized * remove extra transpose & 180d rotation in BVHWriter * UDP as own class * Fixed bug * Renamed some flags * Faster 3d triangulation with outlier warning * 3-D only if accurated enough * More strict outlier detection + incorporated handEnabled flag from IK * Non-linear 3d keypoint not done if outlier * Added BODY_65 * BODY_65 & visualization working * save computation when rigid body * joint angle keeping for knees & joint angle keeping trick for leg again * BODY_25 auto-downloaded * Cleanring eigen * Git pulled * cut face fitting time by half * BODY_65 support for Adam * Updated doc * Ransac added but commented out * Ransac working * RANSaC improved * fix face fitting out of bound error * No weird artifacts if no people * final speed up && twist detection * New Queue workers: Splitter/Assembler * Latency highly reduced for multi-view * Highly reduced latency without out-of-memory * Fixed 3-d bug * IdExtractor handles splitted frame * Foot not rendered standalone * Rm adam * Disabled in CMake --- README.md | 6 +- doc/installation.md | 1 + doc/library_add_new_module.md | 2 +- doc/library_overview.md | 8 +- doc/release_notes.md | 1 + doc/released_features.md | 6 +- examples/openpose/openpose.cpp | 4 +- .../1_custom_post_processing.cpp | 4 +- .../1_user_synchronous_postprocessing.cpp | 4 +- .../2_user_synchronous_input.cpp | 4 +- .../3_user_synchronous_output.cpp | 4 +- .../4_user_synchronous_all.cpp | 4 +- .../tutorial_wrapper/5_user_asynchronous.cpp | 4 +- .../6_user_asynchronous_output.cpp | 4 +- include/openpose/filestream/wBvhSaver.hpp | 3 +- include/openpose/filestream/wUdpSender.hpp | 33 +- include/openpose/gui/wGuiAdam.hpp | 11 +- include/openpose/pose/poseParameters.hpp | 7 +- .../openpose/pose/poseParametersRender.hpp | 8 +- include/openpose/producer/wDatumProducer.hpp | 48 ++- include/openpose/thread/headers.hpp | 1 + include/openpose/thread/threadManager.hpp | 80 ++-- include/openpose/thread/wIdGenerator.hpp | 8 +- include/openpose/thread/wQueueAssembler.hpp | 107 +++++ include/openpose/thread/wQueueOrderer.hpp | 42 +- include/openpose/wrapper/wrapper.hpp | 377 +++++++++--------- src/openpose/3d/jointAngleEstimation.cpp | 176 ++++---- src/openpose/3d/poseTriangulation.cpp | 92 ++++- src/openpose/core/datum.cpp | 5 + src/openpose/filestream/udpSender.cpp | 61 +-- src/openpose/pose/bodyPartConnectorBase.cpp | 14 +- src/openpose/pose/bodyPartConnectorBase.cu | 14 +- src/openpose/pose/poseParameters.cpp | 2 - src/openpose/thread/defineTemplates.cpp | 1 + 34 files changed, 729 insertions(+), 417 deletions(-) create mode 100644 include/openpose/thread/wQueueAssembler.hpp diff --git a/README.md b/README.md index d7fd783c..c7defd6b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - 15 or 18 or **25-keypoint body/foot keypoint estimation**. **Running time invariant to number of detected people**. - **2x21-keypoint hand keypoint estimation**. Currently, **running time depends** on **number of detected people**. - **70-keypoint face keypoint estimation**. Currently, **running time depends** on **number of detected people**. - - **3D real-time multi-person keypoint detection**: + - **3D real-time single-person keypoint detection**: - 3-D triangulation from multiple single views. - Synchronization of Flir cameras handled. - Compatible with Flir/Point Grey cameras, but provided C++ demos to add your custom input. @@ -37,8 +37,8 @@ ## Latest Features - Jun 2018: [**Combined body-foot model released! Faster and more accurate**](doc/installation.md)! -- Jun 2018: [**Python version**](doc/modules/python_module.md) released! -- Jun 2018: [**AMD graphic card version**](doc/installation.md) released! +- Jun 2018: [**Python API**](doc/modules/python_module.md) released! +- Jun 2018: [**OpenCL/AMD graphic card version**](doc/installation.md) released! - Jun 2018: [**Calibration toolbox**](doc/modules/calibration_module.md) released! - Jun 2018: [**Mac OSX version (CPU)**](doc/installation.md) released! - Mar 2018: [**CPU version**](doc/installation.md#cpu-version)! diff --git a/doc/installation.md b/doc/installation.md index 94fcac87..759c202b 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -102,6 +102,7 @@ The instructions in this section describe the steps to build OpenPose using CMak 1. Download and install CMake GUI: - Ubuntu: run the command `sudo apt-get install cmake-qt-gui`. Note: If you prefer to use CMake through the command line, see [Cmake Command Line Build](#cmake-command-line-build-ubuntu-only). - Windows: download and install the latest CMake win64-x64 msi installer from the [CMake website](https://cmake.org/download/), called `cmake-X.X.X-win64-x64.msi`. + - Mac: `brew cask install cmake`. 2. Nvidia GPU version prerequisites: 1. [**CUDA 8**](https://developer.nvidia.com/cuda-80-ga2-download-archive): - Ubuntu: Run `sudo ubuntu/install_cuda.sh` or alternatively download and install it from their website. diff --git a/doc/library_add_new_module.md b/doc/library_add_new_module.md index 75ccc503..a02228e3 100644 --- a/doc/library_add_new_module.md +++ b/doc/library_add_new_module.md @@ -8,7 +8,7 @@ In order to add a new module, these are the recommended steps in order to develo 2. Implement all the functionality in one `Worker` (i.e. inherit from `Worker` and implement all the functionality on that class). 1. The first letter of the class name should be `W` (e.g. `WHairExtractor`). 2. To initially simplify development: - 1. Initialize the Worker class with the specific std::shared_ptr> instead of directly using a template class. + 1. Initialize the Worker class with the specific std::shared_ptr> instead of directly using a template class (following the `examples/tutorial_wrapper` synchronous examples). 2. Use the whole op::Datum as unique argument of your auxiliary functions. 3. Use the OpenPose Wrapper in ThreadManagerMode::SingleThread mode (e.g. it allows you to directly use cv::imshow). 4. If you are using your own custom Caffe -> initially change the Caffe for your version. It should directly work. diff --git a/doc/library_overview.md b/doc/library_overview.md index 52139f48..735a17c3 100644 --- a/doc/library_overview.md +++ b/doc/library_overview.md @@ -70,8 +70,8 @@ The `Datum` class has all the variables that our Workers need to share to each o UserDatum : public op::Datum {/* op::Datum + extra variables */} // Worker and ThreadManager example initialization -op::WGui> userGUI(/* constructor arguments */); -op::ThreadManager> userThreadManager; +op::WGui> userGUI(/* constructor arguments */); +op::ThreadManager> userThreadManager; ``` Since `UserDatum` inherits from `op::Datum`, all the original OpenPose code will compile and run with your inherited version of `op::Datum`. @@ -147,7 +147,7 @@ Classes starting by the letter `W` + upper case letter (e.g. `WGui`) directly or The easiest way to create your own Worker is to inherit Worker, and implement the work() function such us it just calls a wrapper to your desired functionality (check the source code of some of our basic Workers). Since the Worker classes are templates, they are always compiled. Therefore, including your desired functionality in a different file will let you compile it only once. Otherwise, it would be compiled any time that any code which uses your worker is compiled. -All OpenPose Workers are templates, i.e. they are not only limited to work with the default std::vector. However, if you intend to use some of our Workers, your custom `TDatums` class (the one substituting std::vector) should implement the same variables and functions that those Workers use. The easiest solution is to inherit from `op::Datum` and extend its functionality. +All OpenPose Workers are templates, i.e. they are not only limited to work with the default op::Datum. However, if you intend to use some of our Workers, your custom `TDatums` class (the one substituting op::Datum) should implement the same variables and functions that those Workers use. The easiest solution is to inherit from `op::Datum` and extend its functionality. ### Creating New Workers @@ -171,7 +171,7 @@ All Workers wrap and call a non-Worker non-template equivalent which actually pe By separating functionality and their `Worker` wrappers, we get the good of both points, eliminating the cons. In this way, the user is able to: -1. Change `std::vector` for a custom class, implementing his own `Worker` templates, but using the already implemented functionality to create new custom `Worker` templates. +1. Change `std::shared_ptr>` for a custom class, implementing his own `Worker` templates, but using the already implemented functionality to create new custom `Worker` templates. 2. Create a `Worker` which wraps several non-`Worker`s classes. diff --git a/doc/release_notes.md b/doc/release_notes.md index 006e1364..dbd5c7af 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -226,6 +226,7 @@ OpenPose Library - Release Notes 2. It is only run if reprojction error is more than a minimum threshold (improve speed with already good quality results) and also less than another outlier threshold. 3. Outliers are removed from final result if >= 3 camera views. 4. Applied RANSAC if >=4 camera views. + 5. Latency highly reduced in multi-GPU setting. Each GPU process a different camera view, instead of a different time-instant sequence. 5. CMake: All libraries as single variable (simpler to add/remove libraries). 6. Datum includes extrinsic and intrinsic camera parameters. 7. Function `scaleKeypoints(Array& keypoints, const float scale)` also accepts 3D keypoints. diff --git a/doc/released_features.md b/doc/released_features.md index 53b8dbd5..69dc7dd2 100644 --- a/doc/released_features.md +++ b/doc/released_features.md @@ -1,9 +1,9 @@ OpenPose Library - Latest Released Features ==================================== -- Jun 2018: [**Combined body-foot model released! Faster and more accurate.**](./installation.md)! -- Jun 2018: [**Python version**](./modules/python_module.md) released! -- Jun 2018: [**AMD graphic card version**](./modules/calibration_module.md) released! +- Jun 2018: [**Combined body-foot model released! Faster and more accurate**](./installation.md)! +- Jun 2018: [**Python API**](./modules/python_module.md) released! +- Jun 2018: [**OpenCL/AMD graphic card version**](./modules/calibration_module.md) released! - Jun 2018: [**Calibration toolbox**](./modules/calibration_module.md) released! - Jun 2018: [**Mac OSX version (CPU)**](./installation.md) released! - Mar 2018: [**CPU version**](./installation.md#cpu-version)! diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 02c737ab..3170c0a2 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -235,8 +235,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); int openPoseDemo() { diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index e05534ec..79ae80f3 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -244,8 +244,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); int openPoseTutorialWrapper4() { diff --git a/examples/tutorial_wrapper/1_user_synchronous_postprocessing.cpp b/examples/tutorial_wrapper/1_user_synchronous_postprocessing.cpp index cfcbc421..93fa71d7 100644 --- a/examples/tutorial_wrapper/1_user_synchronous_postprocessing.cpp +++ b/examples/tutorial_wrapper/1_user_synchronous_postprocessing.cpp @@ -235,8 +235,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/examples/tutorial_wrapper/2_user_synchronous_input.cpp b/examples/tutorial_wrapper/2_user_synchronous_input.cpp index 799304ba..60c64e81 100644 --- a/examples/tutorial_wrapper/2_user_synchronous_input.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous_input.cpp @@ -235,8 +235,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/examples/tutorial_wrapper/3_user_synchronous_output.cpp b/examples/tutorial_wrapper/3_user_synchronous_output.cpp index 111450f7..ded2f992 100644 --- a/examples/tutorial_wrapper/3_user_synchronous_output.cpp +++ b/examples/tutorial_wrapper/3_user_synchronous_output.cpp @@ -235,8 +235,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/examples/tutorial_wrapper/4_user_synchronous_all.cpp b/examples/tutorial_wrapper/4_user_synchronous_all.cpp index a93bce40..e1cb5e2a 100644 --- a/examples/tutorial_wrapper/4_user_synchronous_all.cpp +++ b/examples/tutorial_wrapper/4_user_synchronous_all.cpp @@ -202,8 +202,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/examples/tutorial_wrapper/5_user_asynchronous.cpp b/examples/tutorial_wrapper/5_user_asynchronous.cpp index a992aa65..30010bca 100644 --- a/examples/tutorial_wrapper/5_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/5_user_asynchronous.cpp @@ -202,8 +202,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/examples/tutorial_wrapper/6_user_asynchronous_output.cpp b/examples/tutorial_wrapper/6_user_asynchronous_output.cpp index 8be60515..c7386192 100644 --- a/examples/tutorial_wrapper/6_user_asynchronous_output.cpp +++ b/examples/tutorial_wrapper/6_user_asynchronous_output.cpp @@ -227,8 +227,8 @@ DEFINE_string(write_video_adam, "", "Experimental, not avail " controls FPS."); DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g.: `~/Desktop/mocapResult.bvh`."); // UDP communication -DEFINE_string(udp_host, "127.0.0.1", "IP for UDP communication."); -DEFINE_string(udp_port, "8051", "Port number for UDP communication."); +DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); +DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); // If the user needs his own variables, he can inherit the op::Datum struct and add them diff --git a/include/openpose/filestream/wBvhSaver.hpp b/include/openpose/filestream/wBvhSaver.hpp index dfd160ac..1c01767c 100644 --- a/include/openpose/filestream/wBvhSaver.hpp +++ b/include/openpose/filestream/wBvhSaver.hpp @@ -57,7 +57,8 @@ namespace op const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Record BVH file const auto& tDatum = (*tDatums)[0]; - spBvhSaver->updateBvh(tDatum.adamPose, tDatum.adamTranslation, tDatum.j0Vec); + if (!tDatum.poseKeypoints3D.empty()) + spBvhSaver->updateBvh(tDatum.adamPose, tDatum.adamTranslation, tDatum.j0Vec); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); diff --git a/include/openpose/filestream/wUdpSender.hpp b/include/openpose/filestream/wUdpSender.hpp index af14bd14..b44c2604 100644 --- a/include/openpose/filestream/wUdpSender.hpp +++ b/include/openpose/filestream/wUdpSender.hpp @@ -57,21 +57,24 @@ namespace op // Send though UDP communication #ifdef USE_3D_ADAM_MODEL const auto& tDatum = (*tDatums)[0]; - const auto& adamPose = tDatum.adamPose; // Eigen::Matrix - const auto& adamTranslation = tDatum.adamTranslation; // Eigen::Vector3d(3, 1) - const auto adamFaceCoeffsExp = tDatum.adamFaceCoeffsExp; // Eigen::VectorXd resized to (200, 1) - //const float mouth_open = tDatum.mouthOpening; // tDatum.mouth_open; - //const float leye_open = tDatum.rightEyeOpening; // tDatum.leye_open; - //const float reye_open = tDatum.leftEyeOpening; // tDatum.reye_open; - //const float dist_root_foot = Datum.distanceRootFoot; // tDatum.dist_root_foot; - // m_adam_t: - // 1. Total translation (centimeters) of the root in camera/global coordinate representation. - // m_adam_pose: - // 1. First row is global rotation, in AngleAxis representation. Radians (not degrees!) - // 2. Rest are joint-angles in Euler-Angle representation. Degrees. - spUdpSender->sendJointAngles(adamPose.data(), adamPose.rows(), - adamTranslation.data(), - adamFaceCoeffsExp.data(), adamFaceCoeffsExp.rows()); + if (!tDatum.poseKeypoints3D.empty()) + { + const auto& adamPose = tDatum.adamPose; // Eigen::Matrix + const auto& adamTranslation = tDatum.adamTranslation; // Eigen::Vector3d(3, 1) + const auto adamFaceCoeffsExp = tDatum.adamFaceCoeffsExp; // Eigen::VectorXd resized to (200, 1) + //const float mouth_open = tDatum.mouthOpening; // tDatum.mouth_open; + //const float leye_open = tDatum.rightEyeOpening; // tDatum.leye_open; + //const float reye_open = tDatum.leftEyeOpening; // tDatum.reye_open; + //const float dist_root_foot = Datum.distanceRootFoot; // tDatum.dist_root_foot; + // m_adam_t: + // 1. Total translation (centimeters) of the root in camera/global coordinate representation. + // m_adam_pose: + // 1. First row is global rotation, in AngleAxis representation. Radians (not degrees!) + // 2. Rest are joint-angles in Euler-Angle representation. Degrees. + spUdpSender->sendJointAngles(adamPose.data(), adamPose.rows(), + adamTranslation.data(), + adamFaceCoeffsExp.data(), adamFaceCoeffsExp.rows()); + } #endif // Profiling speed Profiler::timerEnd(profilerKey); diff --git a/include/openpose/gui/wGuiAdam.hpp b/include/openpose/gui/wGuiAdam.hpp index bfe2c99e..fac8f32b 100644 --- a/include/openpose/gui/wGuiAdam.hpp +++ b/include/openpose/gui/wGuiAdam.hpp @@ -74,11 +74,12 @@ namespace op spGuiAdam->setImage(cvOutputDatas); // Update keypoints const auto& tDatum = (*tDatums)[0]; - spGuiAdam->generateMesh(tDatum.poseKeypoints3D, tDatum.faceKeypoints3D, tDatum.handKeypoints3D, - tDatum.adamPose.data(), tDatum.adamTranslation.data(), - tDatum.vtVec.data(), tDatum.vtVec.rows(), - tDatum.j0Vec.data(), tDatum.j0Vec.rows(), - tDatum.adamFaceCoeffsExp.data()); + if (!tDatum.poseKeypoints3D.empty()) + spGuiAdam->generateMesh(tDatum.poseKeypoints3D, tDatum.faceKeypoints3D, tDatum.handKeypoints3D, + tDatum.adamPose.data(), tDatum.adamTranslation.data(), + tDatum.vtVec.data(), tDatum.vtVec.rows(), + tDatum.j0Vec.data(), tDatum.j0Vec.rows(), + tDatum.adamFaceCoeffsExp.data()); } // Refresh/update GUI spGuiAdam->update(); diff --git a/include/openpose/pose/poseParameters.hpp b/include/openpose/pose/poseParameters.hpp index 9dc1e69e..37dd9f1d 100644 --- a/include/openpose/pose/poseParameters.hpp +++ b/include/openpose/pose/poseParameters.hpp @@ -8,7 +8,9 @@ namespace op { // Constant Global Parameters - // For OpenCL-NMS, (POSE_MAX_PEOPLE+1)*3(x,y,score) must be divisible by 32. Easy fix: POSE_MAX_PEOPLE = 32n - 1 + // For OpenCL-NMS in Ubuntu, (POSE_MAX_PEOPLE+1)*3(x,y,score) must be divisible by 32. Easy fix: + // POSE_MAX_PEOPLE = 32n - 1 + // For OpenCL-NMS in Windows, it must be by 64, so 64n - 1 const auto POSE_MAX_PEOPLE = 127u; // Model functions @@ -29,6 +31,9 @@ namespace op OP_API float getPoseDefaultConnectInterThreshold(const PoseModel poseModel); OP_API unsigned int getPoseDefaultMinSubsetCnt(const PoseModel poseModel); OP_API float getPoseDefaultConnectMinSubsetScore(const PoseModel poseModel); + + // const bool COCO_CHALLENGE = true; + const bool COCO_CHALLENGE = false; } #endif // OPENPOSE_POSE_POSE_PARAMETERS_HPP diff --git a/include/openpose/pose/poseParametersRender.hpp b/include/openpose/pose/poseParametersRender.hpp index d3a46651..87e0ef9d 100644 --- a/include/openpose/pose/poseParametersRender.hpp +++ b/include/openpose/pose/poseParametersRender.hpp @@ -170,8 +170,8 @@ namespace op 4,39, 39,40, 40,41, 41,42, 4,43, 43,44, 44,45, 45,46, 4,47, 47,48, 48,49, 49,50, 4,51, 51,52, 52,53, 53,54, 4,55, 55,56, 56,57, 57,58 #define POSE_BODY_59_SCALES_RENDER_GPU \ 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f, \ - 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, \ - 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f + 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, \ + 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f #define POSE_BODY_59_COLORS_RENDER_GPU \ 255.f, 0.f, 85.f, \ 255.f, 0.f, 0.f, \ @@ -242,8 +242,8 @@ namespace op 4,45, 45,46, 46,47, 47,48, 4,49, 49,50, 50,51, 51,52, 4,53, 53,54, 54,55, 55,56, 4,57, 57,58, 58,59, 59,60, 4,61, 61,62, 62,63, 63,64 #define POSE_BODY_65_SCALES_RENDER_GPU \ 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f,1.f, 1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f, \ - 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, \ - 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f,0.5f + 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, \ + 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f, 0.75f,0.75f,0.75f,0.75f,0.75f #define POSE_BODY_65_COLORS_RENDER_GPU \ 255.f, 0.f, 85.f, \ 255.f, 0.f, 0.f, \ diff --git a/include/openpose/producer/wDatumProducer.hpp b/include/openpose/producer/wDatumProducer.hpp index 4798470f..a6a69598 100644 --- a/include/openpose/producer/wDatumProducer.hpp +++ b/include/openpose/producer/wDatumProducer.hpp @@ -2,6 +2,7 @@ #define OPENPOSE_PRODUCER_W_DATUM_PRODUCER_HPP #include // std::numeric_limits +#include // std::queue #include #include #include @@ -20,6 +21,7 @@ namespace op private: std::shared_ptr> spDatumProducer; + std::queue mQueuedElements; DELETE_COPY(WDatumProducer); }; @@ -54,17 +56,43 @@ namespace op // Profiling speed const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Create and fill TDatums - const auto isRunningAndTDatums = spDatumProducer->checkIfRunningAndGetDatum(); - // Stop Worker if producer finished - if (!isRunningAndTDatums.first) - this->stop(); - // Profiling speed - Profiler::timerEnd(profilerKey); - Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); - // Debugging log - dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + std::shared_ptr tDatums; + // Producer + if (mQueuedElements.empty()) + { + bool isRunning; + std::tie(isRunning, tDatums) = spDatumProducer->checkIfRunningAndGetDatum(); + // Stop Worker if producer finished + if (!isRunning) + this->stop(); + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + // Equivalent to WQueueSplitter + // Queued elements - Multiple views --> Split views into different TDatums + if (tDatums != nullptr && tDatums->size() > 1) + { + // Add tDatums to mQueuedElements + for (auto i = 0u ; i < tDatums->size() ; i++) + { + auto& tDatum = (*tDatums)[i]; + tDatum.subId = i; + tDatum.subIdMax = tDatums->size()-1; + mQueuedElements.emplace( + std::make_shared(TDatumsNoPtr{tDatum})); + } + } + // Queued elements - Multiple views --> Return oldest view + if (!mQueuedElements.empty()) + { + tDatums = mQueuedElements.front(); + mQueuedElements.pop(); + } // Return TDatums - return isRunningAndTDatums.second; + return tDatums; } catch (const std::exception& e) { diff --git a/include/openpose/thread/headers.hpp b/include/openpose/thread/headers.hpp index 1c701a8b..15bab8b8 100644 --- a/include/openpose/thread/headers.hpp +++ b/include/openpose/thread/headers.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #endif // OPENPOSE_THREAD_HEADERS_HPP diff --git a/include/openpose/thread/threadManager.hpp b/include/openpose/thread/threadManager.hpp index 6029d0c6..1551b680 100644 --- a/include/openpose/thread/threadManager.hpp +++ b/include/openpose/thread/threadManager.hpp @@ -21,9 +21,11 @@ namespace op void setDefaultMaxSizeQueues(const long long defaultMaxSizeQueues = -1); - void add(const unsigned long long threadId, const std::vector& tWorkers, const unsigned long long queueInId, const unsigned long long queueOutId); + void add(const unsigned long long threadId, const std::vector& tWorkers, const unsigned long long queueInId, + const unsigned long long queueOutId); - void add(const unsigned long long threadId, const TWorker& tWorker, const unsigned long long queueInId, const unsigned long long queueOutId); + void add(const unsigned long long threadId, const TWorker& tWorker, const unsigned long long queueInId, + const unsigned long long queueOutId); void reset(); @@ -113,7 +115,9 @@ namespace op } template - void ThreadManager::add(const unsigned long long threadId, const std::vector& tWorkers, const unsigned long long queueInId, + void ThreadManager::add(const unsigned long long threadId, + const std::vector& tWorkers, + const unsigned long long queueInId, const unsigned long long queueOutId) { try @@ -127,7 +131,9 @@ namespace op } template - void ThreadManager::add(const unsigned long long threadId, const TWorker& tWorker, const unsigned long long queueInId, + void ThreadManager::add(const unsigned long long threadId, + const TWorker& tWorker, + const unsigned long long queueInId, const unsigned long long queueOutId) { try @@ -225,7 +231,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -243,7 +250,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -261,7 +269,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -279,7 +288,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -297,7 +307,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousOut) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousOut) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -315,7 +326,8 @@ namespace op { try { - if (mThreadManagerMode != ThreadManagerMode::Asynchronous && mThreadManagerMode != ThreadManagerMode::AsynchronousOut) + if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousOut) error("Not available for this ThreadManagerMode.", __LINE__, __FUNCTION__, __FILE__); if (mTQueues.empty()) error("ThreadManager already stopped or not started yet.", __LINE__, __FUNCTION__, __FILE__); @@ -350,8 +362,10 @@ namespace op try { for (const auto& threadWorkerQueue : threadWorkerQueues) - add({std::make_tuple(std::get<0>(threadWorkerQueue), std::vector{std::get<1>(threadWorkerQueue)}, - std::get<2>(threadWorkerQueue), std::get<3>(threadWorkerQueue))}); + add({std::make_tuple(std::get<0>(threadWorkerQueue), + std::vector{std::get<1>(threadWorkerQueue)}, + std::get<2>(threadWorkerQueue), + std::get<3>(threadWorkerQueue))}); } catch (const std::exception& e) { @@ -384,26 +398,34 @@ namespace op const auto queueOut = std::get<3>(threadWorkerQueue); std::shared_ptr> subThread; // If AsynchronousIn -> queue indexes are OK - if (mThreadManagerMode == ThreadManagerMode::Asynchronous || mThreadManagerMode == ThreadManagerMode::AsynchronousIn) + if (mThreadManagerMode == ThreadManagerMode::Asynchronous + || mThreadManagerMode == ThreadManagerMode::AsynchronousIn) { - if (mThreadManagerMode == ThreadManagerMode::AsynchronousIn && queueOut == mTQueues.size()) - subThread = {std::make_shared>(tWorkers, mTQueues.at(queueIn))}; + if (mThreadManagerMode == ThreadManagerMode::AsynchronousIn + && queueOut == mTQueues.size()) + subThread = {std::make_shared>( + tWorkers, mTQueues.at(queueIn))}; else - subThread = {std::make_shared>(tWorkers, mTQueues.at(queueIn), mTQueues.at(queueOut))}; + subThread = {std::make_shared>( + tWorkers, mTQueues.at(queueIn), mTQueues.at(queueOut))}; } // If !AsynchronousIn -> queue indexes - 1 - else if (queueOut != maxQueueIdSynchronous || mThreadManagerMode == ThreadManagerMode::AsynchronousOut) + else if (queueOut != maxQueueIdSynchronous + || mThreadManagerMode == ThreadManagerMode::AsynchronousOut) { // Queue in + out if (queueIn != 0) - subThread = {std::make_shared>(tWorkers, mTQueues.at(queueIn-1), mTQueues.at(queueOut-1))}; + subThread = {std::make_shared>( + tWorkers, mTQueues.at(queueIn-1), mTQueues.at(queueOut-1))}; // Case queue out (first TWorker(s)) else - subThread = {std::make_shared>(tWorkers, mTQueues.at(queueOut-1))}; + subThread = {std::make_shared>( + tWorkers, mTQueues.at(queueOut-1))}; } // Case queue in (last TWorker(s)) else if (queueIn != 0) // && queueOut == maxQueueIdSynchronous - subThread = {std::make_shared>(tWorkers, mTQueues.at(queueIn-1))}; + subThread = {std::make_shared>( + tWorkers, mTQueues.at(queueIn-1))}; // Case no queue else // if (queueIn == 0 && queueOut == maxQueueIdSynchronous) subThread = {std::make_shared>(tWorkers)}; @@ -431,7 +453,8 @@ namespace op { const auto currentThreadId = std::get<0>(threadWorkerQueue); if (currentThreadId - previousThreadId > 1) - error("Missing thread id " + std::to_string(currentThreadId) + " of " + std::to_string(maxThreadId) + ".", __LINE__, __FUNCTION__, __FILE__); + error("Missing thread id " + std::to_string(currentThreadId) + " of " + + std::to_string(maxThreadId) + ".", __LINE__, __FUNCTION__, __FILE__); previousThreadId = currentThreadId; } @@ -458,9 +481,11 @@ namespace op // Get max queue id to get queue size auto maxQueueId = std::get<3>(*mThreadWorkerQueues.cbegin()); for (const auto& threadWorkerQueue : mThreadWorkerQueues) - maxQueueId = fastMax(maxQueueId, fastMax(std::get<2>(threadWorkerQueue), std::get<3>(threadWorkerQueue))); + maxQueueId = fastMax( + maxQueueId, fastMax(std::get<2>(threadWorkerQueue), std::get<3>(threadWorkerQueue))); - // Check each queue id has at least a worker that uses it as input and another one as output. Special cases: + // Check each queue id has at least a worker that uses it as input and another one as output. + // Special cases: std::vector> usedQueueIds(maxQueueId+1, {false, false}); for (const auto& threadWorkerQueue : mThreadWorkerQueues) { @@ -475,9 +500,11 @@ namespace op for (auto i = 0ull ; i < usedQueueIds.size() ; i++) { if (!usedQueueIds[i].first) - error("Missing queue id " + std::to_string(i) + " (of " + std::to_string(maxQueueId) + ") as input.", __LINE__, __FUNCTION__, __FILE__); + error("Missing queue id " + std::to_string(i) + " (of " + + std::to_string(maxQueueId) + ") as input.", __LINE__, __FUNCTION__, __FILE__); if (!usedQueueIds[i].second) - error("Missing queue id " + std::to_string(i) + " (of " + std::to_string(maxQueueId) + ") as output.", __LINE__, __FUNCTION__, __FILE__); + error("Missing queue id " + std::to_string(i) + " (of " + + std::to_string(maxQueueId) + ") as output.", __LINE__, __FUNCTION__, __FILE__); } // Create Queues @@ -485,7 +512,8 @@ namespace op mTQueues.resize(maxQueueId+1); // First and last one are queues else if (mThreadManagerMode == ThreadManagerMode::Synchronous) mTQueues.resize(maxQueueId-1); // First and last one are not actually queues - else if (mThreadManagerMode == ThreadManagerMode::AsynchronousIn || mThreadManagerMode == ThreadManagerMode::AsynchronousOut) + else if (mThreadManagerMode == ThreadManagerMode::AsynchronousIn + || mThreadManagerMode == ThreadManagerMode::AsynchronousOut) mTQueues.resize(maxQueueId); // First or last one is queue else error("Unknown ThreadManagerMode", __LINE__, __FUNCTION__, __FILE__); diff --git a/include/openpose/thread/wIdGenerator.hpp b/include/openpose/thread/wIdGenerator.hpp index f4bdaba1..30b716e9 100644 --- a/include/openpose/thread/wIdGenerator.hpp +++ b/include/openpose/thread/wIdGenerator.hpp @@ -52,9 +52,13 @@ namespace op { // Add ID for (auto& tDatum : *tDatums) - tDatum.id = mGlobalCounter; + // To avoid overwritting ID if e.g., custom input has already filled it + if (tDatum.id == std::numeric_limits::max()) + tDatum.id = mGlobalCounter; // Increase ID - mGlobalCounter++; + const auto& tDatum = (*tDatums)[0]; + if (tDatum.subId == tDatum.subIdMax) + mGlobalCounter++; } } catch (const std::exception& e) diff --git a/include/openpose/thread/wQueueAssembler.hpp b/include/openpose/thread/wQueueAssembler.hpp new file mode 100644 index 00000000..2813d27d --- /dev/null +++ b/include/openpose/thread/wQueueAssembler.hpp @@ -0,0 +1,107 @@ +#ifndef OPENPOSE_THREAD_W_QUEUE_ASSEMBLER_HPP +#define OPENPOSE_THREAD_W_QUEUE_ASSEMBLER_HPP + +#include // std::queue +#include +#include +#include + +namespace op +{ + // Note: The goal of WQueueAssembler and WQueueSplitter (integrated in wDatumProducer) is to reduce the latency + // of OpenPose. E.g., if 4 cameras in stereo mode, without this, OpenPose would have to process all 4 cameras + // with the same GPU. In this way, this work is parallelized over GPUs (1 view for each). + // Pros: Latency highly recuded, same speed + // Cons: Requires these extra 2 classes and proper threads for them + template + class WQueueAssembler : public Worker + { + public: + explicit WQueueAssembler(); + + void initializationOnThread(); + + void work(TDatums& tDatums); + + private: + TDatums mNextTDatums; + + DELETE_COPY(WQueueAssembler); + }; +} + + + + + +// Implementation +#include +#include +namespace op +{ + template + WQueueAssembler::WQueueAssembler() + { + } + + template + void WQueueAssembler::initializationOnThread() + { + } + + template + void WQueueAssembler::work(TDatums& tDatums) + { + try + { + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + // Input TDatums -> enqueue it + if (checkNoNullNorEmpty(tDatums)) + { + // Security check + if (tDatums->size() > 1) + error("This function assumes that WQueueSplitter (inside WDatumProducer)" + " was applied in the first place, i.e., that there is only 1 element" + " per TDatums (size = " + std::to_string(tDatums->size()) + ").", + __LINE__, __FUNCTION__, __FILE__); + auto tDatum = (*tDatums)[0]; + // Single view --> Return + if (tDatum.subIdMax == 0) + return; + // Multiple view --> Merge views into different TDatums (1st frame) + if (mNextTDatums == nullptr) + mNextTDatums = std::make_shared(); + // Multiple view --> Merge views into different TDatums + mNextTDatums->emplace_back(tDatum); + // Last view - Return frame + if (mNextTDatums->back().subId == mNextTDatums->back().subIdMax) + { + tDatums = mNextTDatums; + mNextTDatums = nullptr; + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + // Non-last view - Return nothing + else + tDatums = nullptr; + } + // Sleep if no new tDatums to either pop or push + else + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + catch (const std::exception& e) + { + this->stop(); + tDatums = nullptr; + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + extern template class WQueueAssembler; +} + +#endif // OPENPOSE_THREAD_W_QUEUE_ASSEMBLER_HPP diff --git a/include/openpose/thread/wQueueOrderer.hpp b/include/openpose/thread/wQueueOrderer.hpp index e6bfed1f..cab87fe9 100644 --- a/include/openpose/thread/wQueueOrderer.hpp +++ b/include/openpose/thread/wQueueOrderer.hpp @@ -24,6 +24,7 @@ namespace op const unsigned int mMaxBufferSize; bool mStopWhenEmpty; unsigned long long mNextExpectedId; + unsigned long long mNextExpectedSubId; std::priority_queue, PointerContainerGreater> mPriorityQueueBuffer; DELETE_COPY(WQueueOrderer); @@ -43,7 +44,8 @@ namespace op WQueueOrderer::WQueueOrderer(const unsigned int maxBufferSize) : mMaxBufferSize{maxBufferSize}, mStopWhenEmpty{false}, - mNextExpectedId{0} + mNextExpectedId{0}, + mNextExpectedSubId{0} { } @@ -66,8 +68,22 @@ namespace op // T* to T auto& tDatumsNoPtr = *tDatums; // tDatums is the next expected, update counter - if (tDatumsNoPtr[0].id == mNextExpectedId) - mNextExpectedId++; + if (tDatumsNoPtr[0].id == mNextExpectedId && tDatumsNoPtr[0].subId == mNextExpectedSubId) + { + // If single-view + if (tDatumsNoPtr[0].subIdMax == 0) + mNextExpectedId++; + // If muilti-view system + else + { + mNextExpectedSubId++; + if (mNextExpectedSubId > tDatumsNoPtr[0].subIdMax) + { + mNextExpectedSubId = 0; + mNextExpectedId++; + } + } + } // Else push it to our buffered queue else { @@ -87,7 +103,9 @@ namespace op { // Retrieve frame if next is desired frame or if we want to stop this worker if (!mPriorityQueueBuffer.empty() - && ((*mPriorityQueueBuffer.top())[0].id == mNextExpectedId || mStopWhenEmpty)) + && (mStopWhenEmpty || + ((*mPriorityQueueBuffer.top())[0].id == mNextExpectedId + && (*mPriorityQueueBuffer.top())[0].subId == mNextExpectedSubId))) { tDatums = { mPriorityQueueBuffer.top() }; mPriorityQueueBuffer.pop(); @@ -97,9 +115,21 @@ namespace op if (checkNoNullNorEmpty(tDatums)) { const auto& tDatumsNoPtr = *tDatums; - mNextExpectedId = tDatumsNoPtr[0].id + 1; + // If single-view + if (tDatumsNoPtr[0].subIdMax == 0) + mNextExpectedId = tDatumsNoPtr[0].id + 1; + // If muilti-view system + else + { + mNextExpectedSubId = tDatumsNoPtr[0].subId + 1; + if (mNextExpectedSubId > tDatumsNoPtr[0].subIdMax) + { + mNextExpectedSubId = 0; + mNextExpectedId = tDatumsNoPtr[0].id + 1; + } + } } - // Sleep if no new tDatums to either pop + // Sleep if no new tDatums to either pop or push if (!checkNoNullNorEmpty(tDatums) && mPriorityQueueBuffer.size() < mMaxBufferSize / 2u) std::this_thread::sleep_for(std::chrono::milliseconds{1}); // If TDatum popped and/or pushed diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index 8b2783a8..97be0316 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -28,8 +28,9 @@ namespace op * Wrapper(ThreadManagerMode::Synchronous, nullptr, workersOutput, irrelevantBoolean, true) */ template>>, - typename TQueue = Queue>> + typename TDatumsSP = std::shared_ptr, + typename TWorker = std::shared_ptr>, + typename TQueue = Queue> class Wrapper { public: @@ -154,61 +155,61 @@ namespace op * Emplace (move) an element on the first (input) queue. * Only valid if ThreadManagerMode::Asynchronous or ThreadManagerMode::AsynchronousIn. * If the input queue is full or the Wrapper was stopped, it will return false and not emplace it. - * @param tDatums std::shared_ptr element to be emplaced. + * @param tDatums TDatumsSP element to be emplaced. * @return Boolean specifying whether the tDatums could be emplaced. */ - bool tryEmplace(std::shared_ptr& tDatums); + bool tryEmplace(TDatumsSP& tDatums); /** * Emplace (move) an element on the first (input) queue. * Similar to tryEmplace. * However, if the input queue is full, it will wait until it can emplace it. * If the Wrapper class is stopped before adding the element, it will return false and not emplace it. - * @param tDatums std::shared_ptr element to be emplaced. + * @param tDatums TDatumsSP element to be emplaced. * @return Boolean specifying whether the tDatums could be emplaced. */ - bool waitAndEmplace(std::shared_ptr& tDatums); + bool waitAndEmplace(TDatumsSP& tDatums); /** * Push (copy) an element on the first (input) queue. * Same as tryEmplace, but it copies the data instead of moving it. - * @param tDatums std::shared_ptr element to be pushed. + * @param tDatums TDatumsSP element to be pushed. * @return Boolean specifying whether the tDatums could be pushed. */ - bool tryPush(const std::shared_ptr& tDatums); + bool tryPush(const TDatumsSP& tDatums); /** * Push (copy) an element on the first (input) queue. * Same as waitAndEmplace, but it copies the data instead of moving it. - * @param tDatums std::shared_ptr element to be pushed. + * @param tDatums TDatumsSP element to be pushed. * @return Boolean specifying whether the tDatums could be pushed. */ - bool waitAndPush(const std::shared_ptr& tDatums); + bool waitAndPush(const TDatumsSP& tDatums); /** * Pop (retrieve) an element from the last (output) queue. * Only valid if ThreadManagerMode::Asynchronous or ThreadManagerMode::AsynchronousOut. * If the output queue is empty or the Wrapper was stopped, it will return false and not retrieve it. - * @param tDatums std::shared_ptr element where the retrieved element will be placed. + * @param tDatums TDatumsSP element where the retrieved element will be placed. * @return Boolean specifying whether the tDatums could be retrieved. */ - bool tryPop(std::shared_ptr& tDatums); + bool tryPop(TDatumsSP& tDatums); /** * Pop (retrieve) an element from the last (output) queue. * Similar to tryPop. * However, if the output queue is empty, it will wait until it can pop an element. * If the Wrapper class is stopped before popping the element, it will return false and not retrieve it. - * @param tDatums std::shared_ptr element where the retrieved element will be placed. + * @param tDatums TDatumsSP element where the retrieved element will be placed. * @return Boolean specifying whether the tDatums could be retrieved. */ - bool waitAndPop(std::shared_ptr& tDatums); + bool waitAndPop(TDatumsSP& tDatums); private: const ThreadManagerMode mThreadManagerMode; const std::shared_ptr, std::atomic>> spVideoSeek; bool mConfigured; - ThreadManager> mThreadManager; + ThreadManager mThreadManager; bool mUserInputWsOnNewThread; bool mUserPostProcessingWsOnNewThread; bool mUserOutputWsOnNewThread; @@ -217,15 +218,12 @@ namespace op // Workers std::vector mUserInputWs; TWorker wDatumProducer; - TWorker spWIdGenerator; TWorker spWScaleAndSizeExtractor; TWorker spWCvMatToOpInput; TWorker spWCvMatToOpOutput; std::vector> spWPoseExtractors; - TWorker mWQueueOrdererTriangulation; std::vector> spWPoseTriangulations; std::vector mPostProcessingWs; - TWorker mWQueueOrdererJointAngle; std::vector> spWJointAngleEstimations; std::vector mUserPostProcessingWs; std::vector mOutputWs; @@ -281,8 +279,8 @@ namespace op #include namespace op { - template - Wrapper::Wrapper(const ThreadManagerMode threadManagerMode) : + template + Wrapper::Wrapper(const ThreadManagerMode threadManagerMode) : mThreadManagerMode{threadManagerMode}, spVideoSeek{std::make_shared, std::atomic>>()}, mConfigured{false}, @@ -301,8 +299,8 @@ namespace op } } - template - Wrapper::~Wrapper() + template + Wrapper::~Wrapper() { try { @@ -315,8 +313,8 @@ namespace op } } - template - void Wrapper::disableMultiThreading() + template + void Wrapper::disableMultiThreading() { try { @@ -328,8 +326,9 @@ namespace op } } - template - void Wrapper::setWorkerInput(const TWorker& worker, const bool workerOnNewThread) + template + void Wrapper::setWorkerInput(const TWorker& worker, + const bool workerOnNewThread) { try { @@ -345,9 +344,9 @@ namespace op } } - template - void Wrapper::setWorkerPostProcessing(const TWorker& worker, - const bool workerOnNewThread) + template + void Wrapper::setWorkerPostProcessing(const TWorker& worker, + const bool workerOnNewThread) { try { @@ -363,8 +362,9 @@ namespace op } } - template - void Wrapper::setWorkerOutput(const TWorker& worker, const bool workerOnNewThread) + template + void Wrapper::setWorkerOutput(const TWorker& worker, + const bool workerOnNewThread) { try { @@ -380,10 +380,10 @@ namespace op } } - template - void Wrapper::configure(const WrapperStructPose& wrapperStructPose, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput) + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput) { try { @@ -396,11 +396,11 @@ namespace op } } - template - void Wrapper::configure(const WrapperStructPose& wrapperStructPose, - const WrapperStructFace& wrapperStructFace, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput) + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, + const WrapperStructFace& wrapperStructFace, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput) { try { @@ -413,11 +413,11 @@ namespace op } } - template - void Wrapper::configure(const WrapperStructPose& wrapperStructPose, - const WrapperStructHand& wrapperStructHand, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput) + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, + const WrapperStructHand& wrapperStructHand, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput) { try { @@ -430,21 +430,18 @@ namespace op } } - template - void Wrapper::configure(const WrapperStructPose& wrapperStructPose, - const WrapperStructFace& wrapperStructFace, - const WrapperStructHand& wrapperStructHand, - const WrapperStructExtra& wrapperStructExtra, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput) + template + void Wrapper::configure(const WrapperStructPose& wrapperStructPose, + const WrapperStructFace& wrapperStructFace, + const WrapperStructHand& wrapperStructHand, + const WrapperStructExtra& wrapperStructExtra, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput) { try { log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); - // Shortcut - typedef std::shared_ptr TDatumsPtr; - // Required parameters const auto renderOutput = wrapperStructPose.renderMode != RenderMode::None || wrapperStructFace.renderMode != RenderMode::None @@ -539,7 +536,7 @@ namespace op wrapperStructInput.producerSharedPtr, wrapperStructInput.frameFirst, wrapperStructInput.frameLast, spVideoSeek ); - wDatumProducer = std::make_shared>(datumProducer); + wDatumProducer = std::make_shared>(datumProducer); } else wDatumProducer = nullptr; @@ -556,15 +553,15 @@ namespace op wrapperStructPose.netInputSize, finalOutputSize, wrapperStructPose.scalesNumber, wrapperStructPose.scaleGap ); - spWScaleAndSizeExtractor = std::make_shared>(scaleAndSizeExtractor); + spWScaleAndSizeExtractor = std::make_shared>(scaleAndSizeExtractor); // Input cvMat to OpenPose input & output format const auto cvMatToOpInput = std::make_shared(wrapperStructPose.poseModel); - spWCvMatToOpInput = std::make_shared>(cvMatToOpInput); + spWCvMatToOpInput = std::make_shared>(cvMatToOpInput); if (renderOutput) { const auto cvMatToOpOutput = std::make_shared(); - spWCvMatToOpOutput = std::make_shared>(cvMatToOpOutput); + spWCvMatToOpOutput = std::make_shared>(cvMatToOpOutput); } // Pose estimators & renderers @@ -609,7 +606,7 @@ namespace op wrapperStructPose.poseModel, wrapperStructPose.renderThreshold, wrapperStructPose.blendOriginalFrame, alphaKeypoint, alphaHeatMap, wrapperStructPose.defaultPartToRender); - cpuRenderers.emplace_back(std::make_shared>(poseCpuRenderer)); + cpuRenderers.emplace_back(std::make_shared>(poseCpuRenderer)); } } log("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); @@ -639,9 +636,9 @@ namespace op const auto poseExtractor = std::make_shared( poseExtractorNets.at(i), keepTopNPeople, personIdExtractor, personTrackers, wrapperStructPose.numberPeopleMax, wrapperStructExtra.tracking); - spWPoseExtractors.at(i) = {std::make_shared>(poseExtractor)}; + spWPoseExtractors.at(i) = {std::make_shared>(poseExtractor)}; // // Just OpenPose keypoint detector - // spWPoseExtractors.at(i) = {std::make_shared>( + // spWPoseExtractors.at(i) = {std::make_shared>( // poseExtractorNets.at(i))}; } @@ -655,7 +652,7 @@ namespace op // const auto keepTopNPeople = std::make_shared( // wrapperStructPose.numberPeopleMax); // for (auto& wPose : spWPoseExtractors) - // wPose.emplace_back(std::make_shared>(keepTopNPeople)); + // wPose.emplace_back(std::make_shared>(keepTopNPeople)); // } } @@ -669,7 +666,7 @@ namespace op { const auto faceDetector = std::make_shared(wrapperStructPose.poseModel); for (auto& wPose : spWPoseExtractors) - wPose.emplace_back(std::make_shared>(faceDetector)); + wPose.emplace_back(std::make_shared>(faceDetector)); } // OpenCV face detector else @@ -681,7 +678,7 @@ namespace op // 1 FaceDetectorOpenCV per thread, OpenCV face detector is not thread-safe const auto faceDetectorOpenCV = std::make_shared(modelFolder); wPose.emplace_back( - std::make_shared>(faceDetectorOpenCV) + std::make_shared>(faceDetectorOpenCV) ); } } @@ -697,7 +694,7 @@ namespace op ); faceExtractorNets.emplace_back(faceExtractorNet); spWPoseExtractors.at(gpu).emplace_back( - std::make_shared>(faceExtractorNet)); + std::make_shared>(faceExtractorNet)); } } @@ -711,12 +708,12 @@ namespace op // If tracking if (wrapperStructHand.tracking) spWPoseExtractors.at(gpu).emplace_back( - std::make_shared>(handDetector) + std::make_shared>(handDetector) ); // If detection else spWPoseExtractors.at(gpu).emplace_back( - std::make_shared>(handDetector)); + std::make_shared>(handDetector)); // Hand keypoint extractor const auto netOutputSize = wrapperStructHand.netInputSize; const auto handExtractorNet = std::make_shared( @@ -727,12 +724,12 @@ namespace op ); handExtractorNets.emplace_back(handExtractorNet); spWPoseExtractors.at(gpu).emplace_back( - std::make_shared>(handExtractorNet) + std::make_shared>(handExtractorNet) ); // If tracking if (wrapperStructHand.tracking) spWPoseExtractors.at(gpu).emplace_back( - std::make_shared>(handDetector) + std::make_shared>(handDetector) ); } } @@ -740,7 +737,7 @@ namespace op // Pose renderer(s) if (!poseGpuRenderers.empty()) for (auto i = 0u; i < spWPoseExtractors.size(); i++) - spWPoseExtractors.at(i).emplace_back(std::make_shared>( + spWPoseExtractors.at(i).emplace_back(std::make_shared>( poseGpuRenderers.at(i) )); @@ -755,7 +752,7 @@ namespace op wrapperStructFace.alphaKeypoint, wrapperStructFace.alphaHeatMap); // Add worker - cpuRenderers.emplace_back(std::make_shared>(faceRenderer)); + cpuRenderers.emplace_back(std::make_shared>(faceRenderer)); } // GPU rendering else if (wrapperStructFace.renderMode == RenderMode::Gpu) @@ -779,7 +776,7 @@ namespace op } // Add worker spWPoseExtractors.at(i).emplace_back( - std::make_shared>(faceRenderer)); + std::make_shared>(faceRenderer)); } } else @@ -797,7 +794,7 @@ namespace op wrapperStructHand.alphaKeypoint, wrapperStructHand.alphaHeatMap); // Add worker - cpuRenderers.emplace_back(std::make_shared>(handRenderer)); + cpuRenderers.emplace_back(std::make_shared>(handRenderer)); } // GPU rendering else if (wrapperStructHand.renderMode == RenderMode::Gpu) @@ -821,7 +818,7 @@ namespace op } // Add worker spWPoseExtractors.at(i).emplace_back( - std::make_shared>(handRenderer)); + std::make_shared>(handRenderer)); } } else @@ -834,27 +831,22 @@ namespace op { // For all (body/face/hands): PoseTriangulations ~30 msec, 8 GPUS ~30 msec for keypoint estimation spWPoseTriangulations.resize(fastMax(1, int(spWPoseExtractors.size() / 4))); - if (spWPoseTriangulations.size() > 1) - mWQueueOrdererTriangulation = std::make_shared>(); for (auto i = 0u ; i < spWPoseTriangulations.size() ; i++) { const auto poseTriangulation = std::make_shared( wrapperStructExtra.minViews3d); - spWPoseTriangulations.at(i) = {std::make_shared>( + spWPoseTriangulations.at(i) = {std::make_shared>( poseTriangulation)}; } } // Itermediate workers (e.g. OpenPose format to cv::Mat, json & frames recorder, ...) mPostProcessingWs.clear(); - // Frame buffer and ordering - if (spWPoseExtractors.size() > 1u) - mPostProcessingWs.emplace_back(std::make_shared>()); // // Person ID identification (when no multi-thread and no dependency on tracking) // if (wrapperStructExtra.identification) // { // const auto personIdExtractor = std::make_shared(); // mPostProcessingWs.emplace_back( - // std::make_shared>(personIdExtractor) + // std::make_shared>(personIdExtractor) // ); // } // Frames processor (OpenPose format -> cv::Mat format) @@ -862,7 +854,7 @@ namespace op { mPostProcessingWs = mergeVectors(mPostProcessingWs, cpuRenderers); const auto opOutputToCvMat = std::make_shared(); - mPostProcessingWs.emplace_back(std::make_shared>(opOutputToCvMat)); + mPostProcessingWs.emplace_back(std::make_shared>(opOutputToCvMat)); } // Re-scale pose if desired // If desired scale is not the current input @@ -876,7 +868,7 @@ namespace op { // Then we must rescale the keypoints auto keypointScaler = std::make_shared(wrapperStructPose.keypointScale); - mPostProcessingWs.emplace_back(std::make_shared>(keypointScaler)); + mPostProcessingWs.emplace_back(std::make_shared>(keypointScaler)); } } @@ -889,14 +881,11 @@ namespace op if (wrapperStructExtra.ikThreads > 0) { spWJointAngleEstimations.resize(wrapperStructExtra.ikThreads); - if (spWJointAngleEstimations.size() > 1) - mWQueueOrdererJointAngle = std::make_shared>(); - // Pose extractor(s) for (auto i = 0u; i < spWJointAngleEstimations.size(); i++) { const auto jointAngleEstimation = std::make_shared(displayAdam); - spWJointAngleEstimations.at(i) = {std::make_shared>( + spWJointAngleEstimations.at(i) = {std::make_shared>( jointAngleEstimation)}; } } @@ -904,15 +893,13 @@ namespace op // Output workers mOutputWs.clear(); - if (spWJointAngleEstimations.size() > 1u) - mOutputWs.emplace_back(std::make_shared>()); // Send information (e.g., to Unity) though UDP client-server communication #ifdef USE_3D_ADAM_MODEL if (!wrapperStructOutput.udpHost.empty() && !wrapperStructOutput.udpPort.empty()) { const auto udpSender = std::make_shared(wrapperStructOutput.udpHost, wrapperStructOutput.udpPort); - mOutputWs.emplace_back(std::make_shared>(udpSender)); + mOutputWs.emplace_back(std::make_shared>(udpSender)); } #endif // Write people pose data on disk (json for OpenCV >= 3, xml, yml...) @@ -920,18 +907,18 @@ namespace op { const auto keypointSaver = std::make_shared(writeKeypointCleaned, wrapperStructOutput.writeKeypointFormat); - mOutputWs.emplace_back(std::make_shared>(keypointSaver)); + mOutputWs.emplace_back(std::make_shared>(keypointSaver)); if (wrapperStructFace.enable) - mOutputWs.emplace_back(std::make_shared>(keypointSaver)); + mOutputWs.emplace_back(std::make_shared>(keypointSaver)); if (wrapperStructHand.enable) - mOutputWs.emplace_back(std::make_shared>(keypointSaver)); + mOutputWs.emplace_back(std::make_shared>(keypointSaver)); } // Write OpenPose output data on disk in json format (body/hand/face keypoints, body part locations if // enabled, etc.) if (!writeJsonCleaned.empty()) { const auto peopleJsonSaver = std::make_shared(writeJsonCleaned); - mOutputWs.emplace_back(std::make_shared>(peopleJsonSaver)); + mOutputWs.emplace_back(std::make_shared>(peopleJsonSaver)); } // Write people pose data on disk (COCO validation json format) if (!wrapperStructOutput.writeCocoJson.empty()) @@ -940,7 +927,7 @@ namespace op const auto humanFormat = true; const auto cocoJsonSaver = std::make_shared(wrapperStructOutput.writeCocoJson, humanFormat, CocoJsonFormat::Body); - mOutputWs.emplace_back(std::make_shared>(cocoJsonSaver)); + mOutputWs.emplace_back(std::make_shared>(cocoJsonSaver)); } // Write people foot pose data on disk (COCO validation json format for foot data) if (!wrapperStructOutput.writeCocoFootJson.empty()) @@ -949,14 +936,14 @@ namespace op const auto humanFormat = true; const auto cocoJsonSaver = std::make_shared(wrapperStructOutput.writeCocoFootJson, humanFormat, CocoJsonFormat::Foot); - mOutputWs.emplace_back(std::make_shared>(cocoJsonSaver)); + mOutputWs.emplace_back(std::make_shared>(cocoJsonSaver)); } // Write frames as desired image format on hard disk if (!writeImagesCleaned.empty()) { const auto imageSaver = std::make_shared(writeImagesCleaned, wrapperStructOutput.writeImagesFormat); - mOutputWs.emplace_back(std::make_shared>(imageSaver)); + mOutputWs.emplace_back(std::make_shared>(imageSaver)); } // Write frames as *.avi video on hard disk const auto producerFps = (wrapperStructInput.producerSharedPtr == nullptr ? @@ -976,7 +963,7 @@ namespace op const auto videoSaver = std::make_shared( wrapperStructOutput.writeVideo, CV_FOURCC('M','J','P','G'), originalVideoFps, finalOutputSize ); - mOutputWs.emplace_back(std::make_shared>(videoSaver)); + mOutputWs.emplace_back(std::make_shared>(videoSaver)); } // Write joint angles as *.bvh file on hard disk #ifdef USE_3D_ADAM_MODEL @@ -985,7 +972,7 @@ namespace op const auto bvhSaver = std::make_shared( wrapperStructOutput.writeBvh, JointAngleEstimation::getTotalModel(), originalVideoFps ); - mOutputWs.emplace_back(std::make_shared>(bvhSaver)); + mOutputWs.emplace_back(std::make_shared>(bvhSaver)); } #endif // Write heat maps as desired image format on hard disk @@ -993,7 +980,7 @@ namespace op { const auto heatMapSaver = std::make_shared(writeHeatMapsCleaned, wrapperStructOutput.writeHeatMapsFormat); - mOutputWs.emplace_back(std::make_shared>(heatMapSaver)); + mOutputWs.emplace_back(std::make_shared>(heatMapSaver)); } // Add frame information for GUI const bool guiEnabled = (wrapperStructOutput.displayMode != DisplayMode::NoDisplay); @@ -1004,7 +991,7 @@ namespace op || mThreadManagerMode == ThreadManagerMode::AsynchronousOut)) { const auto guiInfoAdder = std::make_shared(numberThreads, guiEnabled); - mOutputWs.emplace_back(std::make_shared>(guiInfoAdder)); + mOutputWs.emplace_back(std::make_shared>(guiInfoAdder)); } // Minimal graphical user interface (GUI) spWGui = nullptr; @@ -1030,7 +1017,7 @@ namespace op wrapperStructOutput.writeVideoAdam ); // WGui - spWGui = {std::make_shared>(gui)}; + spWGui = {std::make_shared>(gui)}; #endif } // 3-D (+2-D) display @@ -1044,7 +1031,7 @@ namespace op wrapperStructPose.poseModel, wrapperStructOutput.displayMode ); // WGui - spWGui = {std::make_shared>(gui)}; + spWGui = {std::make_shared>(gui)}; } // 2-D display else if (wrapperStructOutput.displayMode == DisplayMode::Display2D) @@ -1055,7 +1042,7 @@ namespace op spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers ); // WGui - spWGui = {std::make_shared>(gui)}; + spWGui = {std::make_shared>(gui)}; } else error("Unknown DisplayMode.", __LINE__, __FUNCTION__, __FILE__); @@ -1070,8 +1057,8 @@ namespace op } } - template - void Wrapper::exec() + template + void Wrapper::exec() { try { @@ -1085,8 +1072,8 @@ namespace op } } - template - void Wrapper::start() + template + void Wrapper::start() { try { @@ -1100,8 +1087,8 @@ namespace op } } - template - void Wrapper::stop() + template + void Wrapper::stop() { try { @@ -1113,8 +1100,8 @@ namespace op } } - template - bool Wrapper::isRunning() const + template + bool Wrapper::isRunning() const { try { @@ -1127,8 +1114,8 @@ namespace op } } - template - bool Wrapper::tryEmplace(std::shared_ptr& tDatums) + template + bool Wrapper::tryEmplace(TDatumsSP& tDatums) { try { @@ -1144,8 +1131,8 @@ namespace op } } - template - bool Wrapper::waitAndEmplace(std::shared_ptr& tDatums) + template + bool Wrapper::waitAndEmplace(TDatumsSP& tDatums) { try { @@ -1161,8 +1148,8 @@ namespace op } } - template - bool Wrapper::tryPush(const std::shared_ptr& tDatums) + template + bool Wrapper::tryPush(const TDatumsSP& tDatums) { try { @@ -1178,8 +1165,8 @@ namespace op } } - template - bool Wrapper::waitAndPush(const std::shared_ptr& tDatums) + template + bool Wrapper::waitAndPush(const TDatumsSP& tDatums) { try { @@ -1195,8 +1182,8 @@ namespace op } } - template - bool Wrapper::tryPop(std::shared_ptr& tDatums) + template + bool Wrapper::tryPop(TDatumsSP& tDatums) { try { @@ -1212,8 +1199,8 @@ namespace op } } - template - bool Wrapper::waitAndPop(std::shared_ptr& tDatums) + template + bool Wrapper::waitAndPop(TDatumsSP& tDatums) { try { @@ -1229,8 +1216,8 @@ namespace op } } - template - void Wrapper::reset() + template + void Wrapper::reset() { try { @@ -1244,10 +1231,8 @@ namespace op spWCvMatToOpInput = nullptr; spWCvMatToOpOutput = nullptr; spWPoseExtractors.clear(); - mWQueueOrdererTriangulation = nullptr; spWPoseTriangulations.clear(); mPostProcessingWs.clear(); - mWQueueOrdererJointAngle = nullptr; spWJointAngleEstimations.clear(); mUserPostProcessingWs.clear(); mOutputWs.clear(); @@ -1260,12 +1245,12 @@ namespace op } } - template - void Wrapper::configureThreadManager() + template + void Wrapper::configureThreadManager() { try { - // The less number of queues -> the less lag + // The less number of queues -> the less threads opened, and potentially the less lag // Security checks if (!mConfigured) @@ -1287,59 +1272,48 @@ namespace op error("No output selected.", __LINE__, __FUNCTION__, __FILE__); } - // Thread Manager: + // Thread Manager // Clean previous thread manager (avoid configure to crash the program if used more than once) mThreadManager.reset(); mThreadId = 0ull; auto queueIn = 0ull; auto queueOut = 1ull; + // After producer + // ID generator (before any multi-threading or any function that requires the ID) + const auto wIdGenerator = std::make_shared>(); + std::vector workersAux{wIdGenerator}; + // Scale & cv::Mat to OP format + if (spWScaleAndSizeExtractor != nullptr) + workersAux = mergeVectors(workersAux, {spWScaleAndSizeExtractor}); + if (spWCvMatToOpInput != nullptr) + workersAux = mergeVectors(workersAux, {spWCvMatToOpInput}); + // cv::Mat to output format + if (spWCvMatToOpOutput != nullptr) + workersAux = mergeVectors(workersAux, {spWCvMatToOpOutput}); + + // Producer // If custom user Worker and uses its own thread - spWIdGenerator = std::make_shared>>(); if (!mUserInputWs.empty() && mUserInputWsOnNewThread) { // Thread 0, queues 0 -> 1 mThreadManager.add(mThreadId, mUserInputWs, queueIn++, queueOut++); - if (spWScaleAndSizeExtractor != nullptr && spWCvMatToOpInput != nullptr) - { - threadIdPP(); - // Thread 1, queues 1 -> 2 - if (spWCvMatToOpOutput == nullptr) - mThreadManager.add(mThreadId, {spWIdGenerator, spWScaleAndSizeExtractor, spWCvMatToOpInput}, - queueIn++, queueOut++); - else - mThreadManager.add(mThreadId, {spWIdGenerator, spWScaleAndSizeExtractor, spWCvMatToOpInput, - spWCvMatToOpOutput}, queueIn++, queueOut++); - } - else - mThreadManager.add(mThreadId, spWIdGenerator, queueIn++, queueOut++); - } - // If custom user Worker in same thread or producer on same thread - else - { - std::vector workersAux; - // Custom user Worker - if (!mUserInputWs.empty()) - workersAux = mergeVectors(workersAux, mUserInputWs); - // OpenPose producer - else if (wDatumProducer != nullptr) - workersAux = mergeVectors(workersAux, {wDatumProducer}); - // Otherwise - else if (mThreadManagerMode != ThreadManagerMode::Asynchronous - && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) - error("No input selected.", __LINE__, __FUNCTION__, __FILE__); - // ID generator - workersAux = mergeVectors(workersAux, {spWIdGenerator}); - // Scale & cv::Mat to OP format - if (spWScaleAndSizeExtractor != nullptr && spWCvMatToOpInput != nullptr) - workersAux = mergeVectors(workersAux, {spWScaleAndSizeExtractor, - spWCvMatToOpInput}); - // cv::Mat to output format - if (spWCvMatToOpOutput != nullptr) - workersAux = mergeVectors(workersAux, {spWCvMatToOpOutput}); - // Thread 0 or 1, queues 0 -> 1 - mThreadManager.add(mThreadId, workersAux, queueIn++, queueOut++); + threadIdPP(); } + // If custom user Worker in same thread + else if (!mUserInputWs.empty()) + workersAux = mergeVectors(mUserInputWs, workersAux); + // If OpenPose producer (same thread) + else if (wDatumProducer != nullptr) + workersAux = mergeVectors({wDatumProducer}, workersAux); + // Otherwise + else if (mThreadManagerMode != ThreadManagerMode::Asynchronous + && mThreadManagerMode != ThreadManagerMode::AsynchronousIn) + error("No input selected.", __LINE__, __FUNCTION__, __FILE__); + // Thread 0 or 1, queues 0 -> 1 + mThreadManager.add(mThreadId, workersAux, queueIn++, queueOut++); + // Increase thread threadIdPP(); + // Pose estimation & rendering // Thread 1 or 2...X, queues 1 -> 2, X = 2 + #GPUs if (!spWPoseExtractors.empty()) @@ -1351,6 +1325,15 @@ namespace op mThreadManager.add(mThreadId, wPose, queueIn, queueOut); threadIdPP(); } + queueIn++; + queueOut++; + // Sort frames - Required own thread + if (spWPoseExtractors.size() > 1u) + { + const auto wQueueOrderer = std::make_shared>(); + mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++); + threadIdPP(); + } } else { @@ -1358,34 +1341,45 @@ namespace op log("Multi-threading disabled, only 1 thread running. All GPUs have been disabled but the" " first one, which is defined by gpuNumberStart (e.g. in the OpenPose demo, it is set" " with the `--num_gpu_start` flag).", Priority::High); - mThreadManager.add(mThreadId, spWPoseExtractors.at(0), queueIn, queueOut); + mThreadManager.add(mThreadId, spWPoseExtractors.at(0), queueIn++, queueOut++); } - queueIn++; - queueOut++; } - // Adam/IK step + // Assemble all frames from same time instant (3-D module) + const auto wQueueAssembler = std::make_shared>(); + // 3-D reconstruction if (!spWPoseTriangulations.empty()) { + // Assemble frames + mThreadManager.add(mThreadId, wQueueAssembler, queueIn++, queueOut++); + threadIdPP(); + // 3-D reconstruction if (mMultiThreadEnabled) { - if (mWQueueOrdererTriangulation != nullptr) - mThreadManager.add(mThreadId, mWQueueOrdererTriangulation, queueIn++, queueOut++); for (auto& wPoseTriangulations : spWPoseTriangulations) { mThreadManager.add(mThreadId, wPoseTriangulations, queueIn, queueOut); threadIdPP(); } + queueIn++; + queueOut++; + // Sort frames + if (spWPoseTriangulations.size() > 1u) + { + const auto wQueueOrderer = std::make_shared>(); + mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++); + threadIdPP(); + } } else { if (spWPoseTriangulations.size() > 1) log("Multi-threading disabled, only 1 thread running for 3-D triangulation.", Priority::High); - mThreadManager.add(mThreadId, spWPoseTriangulations.at(0), queueIn, queueOut); + mThreadManager.add(mThreadId, spWPoseTriangulations.at(0), queueIn++, queueOut++); } - queueIn++; - queueOut++; } + else + mPostProcessingWs = mergeVectors({wQueueAssembler}, mPostProcessingWs); // Post processing workers if (!mPostProcessingWs.empty()) { @@ -1398,23 +1392,28 @@ namespace op { if (mMultiThreadEnabled) { - if (mWQueueOrdererJointAngle != nullptr) - mThreadManager.add(mThreadId, mWQueueOrdererJointAngle, queueIn++, queueOut++); for (auto& wJointAngleEstimator : spWJointAngleEstimations) { mThreadManager.add(mThreadId, wJointAngleEstimator, queueIn, queueOut); threadIdPP(); } + queueIn++; + queueOut++; + // Sort frames + if (spWJointAngleEstimations.size() > 1) + { + const auto wQueueOrderer = std::make_shared>(); + mThreadManager.add(mThreadId, wQueueOrderer, queueIn++, queueOut++); + threadIdPP(); + } } else { if (spWJointAngleEstimations.size() > 1) log("Multi-threading disabled, only 1 thread running for joint angle estimation.", Priority::High); - mThreadManager.add(mThreadId, spWJointAngleEstimations.at(0), queueIn, queueOut); + mThreadManager.add(mThreadId, spWJointAngleEstimations.at(0), queueIn++, queueOut++); } - queueIn++; - queueOut++; } // If custom user Worker and uses its own thread if (!mUserPostProcessingWs.empty()) @@ -1464,8 +1463,8 @@ namespace op } } - template - unsigned long long Wrapper::threadIdPP() + template + unsigned long long Wrapper::threadIdPP() { try { diff --git a/src/openpose/3d/jointAngleEstimation.cpp b/src/openpose/3d/jointAngleEstimation.cpp index 5114d915..527617aa 100644 --- a/src/openpose/3d/jointAngleEstimation.cpp +++ b/src/openpose/3d/jointAngleEstimation.cpp @@ -193,7 +193,6 @@ namespace op // error("JointAngleEstimation (`ik_threads` flag) buggy and not working yet, but we are working on it!" // " No coming soon...", __LINE__, __FUNCTION__, __FILE__); #ifndef USE_3D_ADAM_MODEL - UNUSED(ceresDisplayReport); error("OpenPose must be compiled with the `USE_3D_ADAM_MODEL` macro definition in order to use this" " functionality.", __LINE__, __FUNCTION__, __FILE__); #endif @@ -220,36 +219,60 @@ namespace op try { // Security checks - if (!poseKeypoints3D.empty() && poseKeypoints3D.getSize(1) != 19 && poseKeypoints3D.getSize(1) != 25) - error("Only working for BODY_19 or BODY_25/BODY_25_19 (#parts = " + if (!poseKeypoints3D.empty() && poseKeypoints3D.getSize(1) != 19 && poseKeypoints3D.getSize(1) != 25 + && poseKeypoints3D.getSize(1) != 65) + error("Only working for BODY_19 or BODY_25/BODY_25_19 or BODY_65 (#parts = " + std::to_string(poseKeypoints3D.getSize(2)) + ").", __LINE__, __FUNCTION__, __FILE__); // Shorter naming auto& frameParams = spImpl->mFrameParams; - // Reset to 0 all keypoints - Otherwise undefined behavior when fitting - // It must be done on every iteration, otherwise errors, e.g., if face - // was detected in frame i-1 but not in i - spImpl->mBodyJoints.setZero(); - spImpl->mFaceJoints.setZero(); - spImpl->mLHandJoints.setZero(); - spImpl->mRHandJoints.setZero(); - spImpl->mLFootJoints.setZero(); - spImpl->mRFootJoints.setZero(); // If keypoints detected if (!poseKeypoints3D.empty()) { + // Reset to 0 all keypoints - Otherwise undefined behavior when fitting + // It must be done on every iteration, otherwise errors, e.g., if face + // was detected in frame i-1 but not in i + spImpl->mBodyJoints.setZero(); + spImpl->mFaceJoints.setZero(); + spImpl->mLHandJoints.setZero(); + spImpl->mRHandJoints.setZero(); + spImpl->mLFootJoints.setZero(); + spImpl->mRFootJoints.setZero(); // Update body for (auto part = 0 ; part < 19; part++) updateKeypoint(spImpl->mBodyJoints, &poseKeypoints3D[{0, part, 0}], mapOPToAdam(part)); // Update left/right hand - for (auto hand = 0u ; hand < handKeypoints3D.size(); hand++) - if (!handKeypoints3D.at(hand).empty()) - for (auto part = 0 ; part < handKeypoints3D[hand].getSize(1); part++) - updateKeypoint((hand == 0 ? spImpl->mLHandJoints : spImpl->mRHandJoints), - &handKeypoints3D[hand][{0, part, 0}], - part); + if (poseKeypoints3D.getSize(1) == 65) + { + // Wrists + updateKeypoint(spImpl->mLHandJoints, + &poseKeypoints3D[{0, 7, 0}], + 0); + updateKeypoint(spImpl->mRHandJoints, + &poseKeypoints3D[{0, 4, 0}], + 0); + // Left + for (auto part = 0 ; part < 20; part++) + updateKeypoint(spImpl->mLHandJoints, + &poseKeypoints3D[{0, part+25, 0}], + part+1); + // Right + for (auto part = 0 ; part < 20; part++) + updateKeypoint(spImpl->mRHandJoints, + &poseKeypoints3D[{0, part+25+20, 0}], + part+1); + } + else + { + for (auto hand = 0u ; hand < handKeypoints3D.size(); hand++) + if (!handKeypoints3D.at(hand).empty()) + for (auto part = 0 ; part < handKeypoints3D[hand].getSize(1); part++) + updateKeypoint((hand == 0 ? spImpl->mLHandJoints : spImpl->mRHandJoints), + &handKeypoints3D[hand][{0, part, 0}], + part); + } // Update Foot data if (poseKeypoints3D.getSize(1) == 25) { @@ -272,77 +295,80 @@ namespace op part); // Meters --> cm spImpl->mBodyJoints *= 1e2; - if (!handKeypoints3D.at(0).empty()) + if (!handKeypoints3D.at(0).empty() || poseKeypoints3D.getSize(1) == 65) spImpl->mLHandJoints *= 1e2; - if (!handKeypoints3D.at(1).empty()) + if (!handKeypoints3D.at(1).empty() || poseKeypoints3D.getSize(1) == 65) spImpl->mRHandJoints *= 1e2; if (!faceKeypoints3D.empty()) spImpl->mFaceJoints *= 1e2; spImpl->mLFootJoints *= 1e2; spImpl->mRFootJoints *= 1e2; - } - // Initialization (e.g., first frame) - const bool fastVersion = false; - const bool freezeMissing = false; - const bool ceresDisplayReport = false; - // Fill Datum - if (!spImpl->mInitialized || !fastVersion) - { - if (!spImpl->mInitialized) - { - frameParams.m_adam_t(0) = spImpl->mBodyJoints(0, 2); - frameParams.m_adam_t(1) = spImpl->mBodyJoints(1, 2); - frameParams.m_adam_t(2) = spImpl->mBodyJoints(2, 2); - frameParams.m_adam_pose(0, 0) = 3.14159265358979323846264338327950288419716939937510582097494459; - spImpl->mInitialized = true; - } - // We make T-pose start with: - // 1. Root translation similar to current 3-d location of the mid-hip - // 2. x-orientation = 180, i.e., person standing up & looking to the camera - // 3. Because otherwise, if we call Adam_FastFit_Initialize twice (e.g., if a new person appears), - // it would use the latest ones from the last Adam_FastFit - // Fit initialization - // Adam_FastFit_Initialize only changes frameParams - const auto handEnabled = !handKeypoints3D[0].empty() || !handKeypoints3D[1].empty(); - const auto fitFaceExponents = !faceKeypoints3D.empty(); - const auto multistageFitting = true; - const auto fastSolver = true; - Adam_FastFit_Initialize(*spImpl->spTotalModel, frameParams, spImpl->mBodyJoints, spImpl->mRFootJoints, - spImpl->mLFootJoints, spImpl->mRHandJoints, spImpl->mLHandJoints, - spImpl->mFaceJoints, freezeMissing, ceresDisplayReport, - multistageFitting, handEnabled, fitFaceExponents, fastSolver); - // The following 2 operations takes ~12 msec - if (spImpl->mReturnJacobian) + // Initialization (e.g., first frame) + const bool fastVersion = false; + const bool freezeMissing = true; + const bool ceresDisplayReport = false; + // Fill Datum + if (!spImpl->mInitialized || !fastVersion) { - vtVec = spImpl->spTotalModel->m_meanshape - + spImpl->spTotalModel->m_shapespace_u * frameParams.m_adam_coeffs; - j0Vec = spImpl->spTotalModel->J_mu_ + spImpl->spTotalModel->dJdc_ * frameParams.m_adam_coeffs; - if (fastVersion) + if (!spImpl->mInitialized) { - spImpl->mVtVec = vtVec; - spImpl->mJ0Vec = j0Vec; + frameParams.m_adam_t(0) = spImpl->mBodyJoints(0, 2); + frameParams.m_adam_t(1) = spImpl->mBodyJoints(1, 2); + frameParams.m_adam_t(2) = spImpl->mBodyJoints(2, 2); + frameParams.m_adam_pose(0, 0) = 3.14159265358979323846264338327950288419716939937510582097494459; + spImpl->mInitialized = true; + } + // We make T-pose start with: + // 1. Root translation similar to current 3-d location of the mid-hip + // 2. x-orientation = 180, i.e., person standing up & looking to the camera + // 3. Because otherwise, if we call Adam_FastFit_Initialize twice (e.g., if a new person appears), + // it would use the latest ones from the last Adam_FastFit + // Fit initialization + // Adam_FastFit_Initialize only changes frameParams + const auto multistageFitting = true; + const auto handEnabled = !handKeypoints3D[0].empty() || !handKeypoints3D[1].empty() + || poseKeypoints3D.getSize(1) == 65; + const auto fitFaceExponents = !faceKeypoints3D.empty(); + const auto fastSolver = true; + Adam_FastFit_Initialize(*spImpl->spTotalModel, frameParams, spImpl->mBodyJoints, spImpl->mRFootJoints, + spImpl->mLFootJoints, spImpl->mRHandJoints, spImpl->mLHandJoints, + spImpl->mFaceJoints, freezeMissing, ceresDisplayReport, + multistageFitting, handEnabled, fitFaceExponents, fastSolver); + // The following 2 operations takes ~12 msec + if (spImpl->mReturnJacobian) + { + vtVec = spImpl->spTotalModel->m_meanshape + + spImpl->spTotalModel->m_shapespace_u * frameParams.m_adam_coeffs; + j0Vec = spImpl->spTotalModel->J_mu_ + spImpl->spTotalModel->dJdc_ * frameParams.m_adam_coeffs; + if (fastVersion) + { + spImpl->mVtVec = vtVec; + spImpl->mJ0Vec = j0Vec; + } } } - } - // Other frames if fastVersion - else // if (spImpl->mInitialized && fastVersion) - { - // Adam_FastFit only changes frameParams - Adam_FastFit(*spImpl->spTotalModel, frameParams, spImpl->mBodyJoints, spImpl->mRFootJoints, - spImpl->mLFootJoints, spImpl->mRHandJoints, spImpl->mLHandJoints, - spImpl->mFaceJoints, ceresDisplayReport); - if (spImpl->mReturnJacobian) + // Other frames if fastVersion + else // if (spImpl->mInitialized && fastVersion) { - vtVec = spImpl->mVtVec; - j0Vec = spImpl->mJ0Vec; + // Adam_FastFit only changes frameParams + Adam_FastFit(*spImpl->spTotalModel, frameParams, spImpl->mBodyJoints, spImpl->mRFootJoints, + spImpl->mLFootJoints, spImpl->mRHandJoints, spImpl->mLHandJoints, + spImpl->mFaceJoints, ceresDisplayReport); + if (spImpl->mReturnJacobian) + { + vtVec = spImpl->mVtVec; + j0Vec = spImpl->mJ0Vec; + } } + adamPose = frameParams.m_adam_pose; + adamTranslation = frameParams.m_adam_t; + adamFacecoeffsExp = frameParams.m_adam_facecoeffs_exp; + // // Not used anymore + // frameParams.mouth_open, frameParams.reye_open, frameParams.leye_open, frameParams.dist_root_foot } - adamPose = frameParams.m_adam_pose; - adamTranslation = frameParams.m_adam_t; - adamFacecoeffsExp = frameParams.m_adam_facecoeffs_exp; - // // Not used anymore - // frameParams.mouth_open, frameParams.reye_open, frameParams.leye_open, frameParams.dist_root_foot + else + spImpl->mInitialized = false; } catch (const std::exception& e) { diff --git a/src/openpose/3d/poseTriangulation.cpp b/src/openpose/3d/poseTriangulation.cpp index 5e9fc0ba..597515c1 100644 --- a/src/openpose/3d/poseTriangulation.cpp +++ b/src/openpose/3d/poseTriangulation.cpp @@ -165,33 +165,80 @@ namespace op // - Accuracy: initial reprojection error ~14-21, reduced ~5% with non-linear optimization // Basic triangulation - triangulate(reconstructedPoint, cameraMatrices, pointsOnEachCamera); + auto projectionError = triangulate(reconstructedPoint, cameraMatrices, pointsOnEachCamera); + + // Basic RANSAC (for >= 4 cameras if the reprojection error is higher than usual) + // 1. Run with all cameras (already done) + // 2. Run with all but 1 camera for each camera. + // 3. Use the one with minimum average reprojection error. + // Note: Meant to be used for up to 7-8 views. With more than that, it might not improve much. + // Set initial values + auto cameraMatricesFinal = cameraMatrices; + auto pointsOnEachCameraFinal = pointsOnEachCamera; + if (cameraMatrices.size() >= 4 + && projectionError > 0.5 * reprojectionMaxAcceptable + /*&& projectionError < 1.5 * reprojectionMaxAcceptable*/) + { + auto bestReprojection = projectionError; + auto bestReprojectionIndex = -1; // -1 means with all camera views + for (auto i = 0u; i < cameraMatrices.size(); ++i) + { + // Set initial values + auto cameraMatricesSubset = cameraMatrices; + auto pointsOnEachCameraSubset = pointsOnEachCamera; + // Remove camera i + cameraMatricesSubset.erase(cameraMatricesSubset.begin() + i); + pointsOnEachCameraSubset.erase(pointsOnEachCameraSubset.begin() + i); + // Remove camera i + const auto projectionErrorSubset = triangulate(reconstructedPoint, cameraMatricesSubset, + pointsOnEachCameraSubset); + // If projection doesn't change much, it usually means all points are bad. + if (projectionErrorSubset > 0.9 * projectionError + && projectionErrorSubset < 1.1 * projectionError) + { + bestReprojectionIndex = -1; + break; + } + // Save maximum + if (bestReprojection > projectionErrorSubset) + { + bestReprojection = projectionErrorSubset; + bestReprojectionIndex = i; + } + } + + if (bestReprojectionIndex != -1 && bestReprojection < 0.5 * reprojectionMaxAcceptable) + { + // Remove camera i + cameraMatricesFinal.erase(cameraMatricesFinal.begin() + bestReprojectionIndex); + pointsOnEachCameraFinal.erase(pointsOnEachCameraFinal.begin() + bestReprojectionIndex); + } + } #ifdef USE_CERES - const auto projectionErrorLinear = calcReprojectionError(reconstructedPoint, cameraMatrices, pointsOnEachCamera); // Empirically detected that reprojection error (for 4 cameras) only minimizes the error if initial // project error > ~2.5, and that it improves more the higher that error actually is // Therefore, we disable it for already accurate samples in order to get both: // - Speed // - Accuracy for already accurate samples - if (projectionErrorLinear > 3.0 - && projectionErrorLinear < 1.5*reprojectionMaxAcceptable) + if (projectionError > 3.0 + && projectionError < 1.5*reprojectionMaxAcceptable) { // Slow equivalent: double paramX[3]; paramX[i] = reconstructedPoint.at(i); double* paramX = (double*)reconstructedPoint.data; ceres::Problem problem; - for (auto i = 0u; i < cameraMatrices.size(); ++i) + for (auto i = 0u; i < cameraMatricesFinal.size(); ++i) { // Slow copy equivalent: - // double camParam[12]; memcpy(camParam, cameraMatrices[i].data, sizeof(double)*12); - const double* const camParam = (double*)cameraMatrices[i].data; + // double camParam[12]; memcpy(camParam, cameraMatricesFinal[i].data, sizeof(double)*12); + const double* const camParam = (double*)cameraMatricesFinal[i].data; // Each Residual block takes a point and a camera as input and outputs a 2 // dimensional residual. Internally, the cost function stores the observed // image location and compares the reprojection against the observation. ceres::CostFunction* cost_function = new ceres::AutoDiffCostFunction( new ReprojectionErrorForTriangulation( - pointsOnEachCamera[i].x, pointsOnEachCamera[i].y, camParam)); + pointsOnEachCameraFinal[i].x, pointsOnEachCameraFinal[i].y, camParam)); // Add to problem problem.AddResidualBlock(cost_function, //NULL, //squared loss @@ -224,9 +271,10 @@ namespace op // if (summary.initial_cost > summary.final_cost) // std::cout << summary.FullReport() << "\n"; + projectionError = calcReprojectionError(reconstructedPoint, cameraMatricesFinal, + pointsOnEachCameraFinal); // const auto reprojectionErrorDecrease = std::sqrt((summary.initial_cost - summary.final_cost) - // / double(cameraMatrices.size())); - // return reprojectionErrorDecrease; + // / double(cameraMatricesFinal.size())); } #else UNUSED(reprojectionMaxAcceptable); @@ -235,19 +283,19 @@ namespace op // // Check that our implementation gives similar result than OpenCV // // But we apply bundle adjustment + >2 views, so it should be better (and slower) than OpenCV one - // if (cameraMatrices.size() == 4) + // if (cameraMatricesFinal.size() == 4) // { // cv::Mat triangCoords4D; - // cv::triangulatePoints(cameraMatrices.at(0), cameraMatrices.at(3), - // std::vector{pointsOnEachCamera.at(0)}, - // std::vector{pointsOnEachCamera.at(3)}, triangCoords4D); + // cv::triangulatePoints(cameraMatricesFinal.at(0), cameraMatricesFinal.at(3), + // std::vector{pointsOnEachCameraFinal.at(0)}, + // std::vector{pointsOnEachCameraFinal.at(3)}, triangCoords4D); // triangCoords4D /= triangCoords4D.at(3); // std::cout << reconstructedPoint << "\n" // << triangCoords4D << "\n" // << cv::norm(reconstructedPoint-triangCoords4D) << "\n" << std::endl; // } - return calcReprojectionError(reconstructedPoint, cameraMatrices, pointsOnEachCamera); + return projectionError; } catch (const std::exception& e) { @@ -360,12 +408,6 @@ namespace op } const auto reprojectionErrorTotal = std::accumulate( reprojectionErrors.begin(), reprojectionErrors.end(), 0.0) / xyPoints.size(); - if (reprojectionErrorTotal > 60) - log("Unusual high re-projection error (averaged over #keypoints) of value " - + std::to_string(reprojectionErrorTotal) + " pixels, while the average for a good OpenPose" - " detection from 4 cameras is about 2-3 pixels. It might be simply a wrong OpenPose" - " detection. If this message appears very frequently, your calibration parameters" - " might be wrong.", Priority::High); // 3D points to pose // OpenCV alternative: @@ -374,6 +416,7 @@ namespace op // cv::triangulatePoints(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, // reconstructedcv::Points); // 20 pixels for 1280x1024 image + bool atLeastOnePointProjected = false; const auto lastChannelLength = keypoints3D.getSize(2); for (auto index = 0u; index < indexesUsed.size(); index++) { @@ -388,8 +431,15 @@ namespace op keypoints3D[baseIndex + 1] = xyzPoints[index].y; keypoints3D[baseIndex + 2] = xyzPoints[index].z; keypoints3D[baseIndex + 3] = 1.f; + atLeastOnePointProjected = true; } } + if (!atLeastOnePointProjected || reprojectionErrorTotal > 60) + log("Unusual high re-projection error (averaged over #keypoints) of value " + + std::to_string(reprojectionErrorTotal) + " pixels, while the average for a good OpenPose" + " detection from 4 cameras is about 2-3 pixels. It might be simply a wrong OpenPose" + " detection. If this message appears very frequently, your calibration parameters" + " might be wrong.", Priority::High); } // log("Reprojection error: " + std::to_string(reprojectionErrorTotal)); // To debug reprojection error } diff --git a/src/openpose/core/datum.cpp b/src/openpose/core/datum.cpp index 07153ce0..58108e74 100644 --- a/src/openpose/core/datum.cpp +++ b/src/openpose/core/datum.cpp @@ -18,6 +18,7 @@ namespace op subId{datum.subId}, subIdMax{datum.subIdMax}, name{datum.name}, + frameNumber{datum.frameNumber}, // Input image and rendered version cvInputData{datum.cvInputData}, inputNetData{datum.inputNetData}, @@ -81,6 +82,7 @@ namespace op subId = datum.subId; subIdMax = datum.subIdMax; name = datum.name; + frameNumber = datum.frameNumber; // Input image and rendered version cvInputData = datum.cvInputData; inputNetData = datum.inputNetData; @@ -146,6 +148,7 @@ namespace op id{datum.id}, subId{datum.subId}, subIdMax{datum.subIdMax}, + frameNumber{datum.frameNumber}, // Other parameters scaleInputToOutput{datum.scaleInputToOutput}, scaleNetToOutput{datum.scaleNetToOutput} @@ -218,6 +221,7 @@ namespace op subId = datum.subId; subIdMax = datum.subIdMax; std::swap(name, datum.name); + frameNumber = datum.frameNumber; // Input image and rendered version std::swap(cvInputData, datum.cvInputData); std::swap(inputNetData, datum.inputNetData); @@ -290,6 +294,7 @@ namespace op datum.subId = subId; datum.subIdMax = subIdMax; datum.name = name; + datum.frameNumber = frameNumber; // Input image and rendered version datum.cvInputData = cvInputData.clone(); datum.inputNetData.resize(inputNetData.size()); diff --git a/src/openpose/filestream/udpSender.cpp b/src/openpose/filestream/udpSender.cpp index 19602618..244addb7 100644 --- a/src/openpose/filestream/udpSender.cpp +++ b/src/openpose/filestream/udpSender.cpp @@ -114,45 +114,48 @@ namespace op #if defined(USE_ASIO) && defined(USE_EIGEN) try { - const Eigen::Map adamTranslation(adamTranslationPtr); - const Eigen::Map> adamPose( - adamPosePtr, adamPoseRows, 3); - const Eigen::Map adamFaceCoeffsExp(adamFaceCoeffsExpPtr, faceCoeffRows); - - const std::string prefix = "AnimData:"; - const std::string totalPositionString = "\"totalPosition\":" - + vectorToJson(adamTranslation(0), adamTranslation(1), adamTranslation(2)); - std::string jointAnglesString = "\"jointAngles\":["; - for (int i = 0; i < adamPoseRows; i++) + if (adamPosePtr != nullptr && adamTranslationPtr != nullptr && adamFaceCoeffsExpPtr != nullptr) { - jointAnglesString += vectorToJson(adamPose(i, 0), adamPose(i, 1), adamPose(i, 2)); - if (i != adamPoseRows - 1) + const Eigen::Map adamTranslation(adamTranslationPtr); + const Eigen::Map> adamPose( + adamPosePtr, adamPoseRows, 3); + const Eigen::Map adamFaceCoeffsExp(adamFaceCoeffsExpPtr, faceCoeffRows); + + const std::string prefix = "AnimData:"; + const std::string totalPositionString = "\"totalPosition\":" + + vectorToJson(adamTranslation(0), adamTranslation(1), adamTranslation(2)); + std::string jointAnglesString = "\"jointAngles\":["; + for (int i = 0; i < adamPoseRows; i++) { - jointAnglesString += ","; + jointAnglesString += vectorToJson(adamPose(i, 0), adamPose(i, 1), adamPose(i, 2)); + if (i != adamPoseRows - 1) + { + jointAnglesString += ","; + } } - } - jointAnglesString += "]"; + jointAnglesString += "]"; - std::string facialParamsString = "\"facialParams\":["; - for (int i = 0; i < faceCoeffRows; i++) - { - facialParamsString += std::to_string(adamFaceCoeffsExp(i)); - if (i != faceCoeffRows - 1) + std::string facialParamsString = "\"facialParams\":["; + for (int i = 0; i < faceCoeffRows; i++) { - facialParamsString += ","; + facialParamsString += std::to_string(adamFaceCoeffsExp(i)); + if (i != faceCoeffRows - 1) + { + facialParamsString += ","; + } } - } - facialParamsString += "]"; + facialParamsString += "]"; - // facialParamsString + std::to_string(mouth_open) + "," + std::to_string(leye_open) + "," + std::to_string(reye_open) + "]"; + // facialParamsString + std::to_string(mouth_open) + "," + std::to_string(leye_open) + "," + std::to_string(reye_open) + "]"; - // std::string rootHeightString = "\"rootHeight\":" + std::to_string(dist_root_foot); + // std::string rootHeightString = "\"rootHeight\":" + std::to_string(dist_root_foot); - const std::string data = prefix + "{" + facialParamsString - + "," + totalPositionString - + "," + jointAnglesString + "}"; + const std::string data = prefix + "{" + facialParamsString + + "," + totalPositionString + + "," + jointAnglesString + "}"; - spImpl->mUdpClient.send(data); + spImpl->mUdpClient.send(data); + } } catch (const std::exception& e) { diff --git a/src/openpose/pose/bodyPartConnectorBase.cpp b/src/openpose/pose/bodyPartConnectorBase.cpp index 6e546450..b9ae67e8 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cpp +++ b/src/openpose/pose/bodyPartConnectorBase.cpp @@ -301,7 +301,17 @@ namespace op validSubsetIndexes.reserve(fastMin((size_t)POSE_MAX_PEOPLE, subset.size())); for (auto index = 0u ; index < subset.size() ; index++) { - const auto subsetCounter = subset[index].first[subsetCounterIndex]; + auto subsetCounter = subset[index].first[subsetCounterIndex]; + // Foot keypoints do not affect subsetCounter (too many false positives, + // same foot usually appears as both left and right keypoints) + // Pros: Removed tons of false positives + // Cons: Standalone leg will never be recorded + if (!COCO_CHALLENGE && numberBodyParts == 25) + { + // No consider foot keypoints for that + for (auto i = 19 ; i < 25 ; i++) + subsetCounter -= (subset[index].first.at(i) > 0); + } const auto subsetScore = subset[index].second; if (subsetCounter >= minSubsetCnt && (subsetScore/subsetCounter) >= minSubsetScore) { @@ -310,7 +320,7 @@ namespace op if (numberPeople == POSE_MAX_PEOPLE) break; } - else if (subsetCounter < 1) + else if ((subsetCounter < 1 && numberBodyParts != 25) || subsetCounter < 0) error("Bad subsetCounter. Bug in this function if this happens.", __LINE__, __FUNCTION__, __FILE__); } diff --git a/src/openpose/pose/bodyPartConnectorBase.cu b/src/openpose/pose/bodyPartConnectorBase.cu index 575d7895..3f376024 100644 --- a/src/openpose/pose/bodyPartConnectorBase.cu +++ b/src/openpose/pose/bodyPartConnectorBase.cu @@ -303,7 +303,17 @@ namespace op validSubsetIndexes.reserve(fastMin((size_t)POSE_MAX_PEOPLE, subset.size())); for (auto index = 0u ; index < subset.size() ; index++) { - const auto subsetCounter = subset[index].first[subsetCounterIndex]; + auto subsetCounter = subset[index].first[subsetCounterIndex]; + // Foot keypoints do not affect subsetCounter (too many false positives, + // same foot usually appears as both left and right keypoints) + // Pros: Removed tons of false positives + // Cons: Standalone leg will never be recorded + if (!COCO_CHALLENGE && numberBodyParts == 25) + { + // No consider foot keypoints for that + for (auto i = 19 ; i < 25 ; i++) + subsetCounter -= (subset[index].first.at(i) > 0); + } const auto subsetScore = subset[index].second; if (subsetCounter >= minSubsetCnt && (subsetScore/subsetCounter) >= minSubsetScore) { @@ -312,7 +322,7 @@ namespace op if (numberPeople == POSE_MAX_PEOPLE) break; } - else if (subsetCounter < 1) + else if ((subsetCounter < 1 && numberBodyParts != 25) || subsetCounter < 0) error("Bad subsetCounter. Bug in this function if this happens.", __LINE__, __FUNCTION__, __FILE__); } diff --git a/src/openpose/pose/poseParameters.cpp b/src/openpose/pose/poseParameters.cpp index 15472b3d..c3738752 100644 --- a/src/openpose/pose/poseParameters.cpp +++ b/src/openpose/pose/poseParameters.cpp @@ -388,8 +388,6 @@ namespace op // Default Model Parameters // They might be modified on running time - // const bool COCO_CHALLENGE = true; - const bool COCO_CHALLENGE = false; const auto nmsT = (COCO_CHALLENGE ? 0.04f : 0.05f); const std::array POSE_DEFAULT_NMS_THRESHOLD{ nmsT, nmsT, 0.6f, 0.3f, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT diff --git a/src/openpose/thread/defineTemplates.cpp b/src/openpose/thread/defineTemplates.cpp index a3282b13..6e38caa6 100644 --- a/src/openpose/thread/defineTemplates.cpp +++ b/src/openpose/thread/defineTemplates.cpp @@ -17,5 +17,6 @@ namespace op DEFINE_TEMPLATE_DATUM(WorkerConsumer); DEFINE_TEMPLATE_DATUM(WorkerProducer); DEFINE_TEMPLATE_DATUM(WIdGenerator); + template class OP_API WQueueAssembler; DEFINE_TEMPLATE_DATUM(WQueueOrderer); } -- GitLab