diff --git a/doc/demo_overview.md b/doc/demo_overview.md index 8334052aecb264cf43c6b0dff838f2f151c3b397..479bd4b118209144ef95d7616277bcdf458609ff 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -184,8 +184,9 @@ Each flag is divided into flag name, default value, and description. - DEFINE_bool(hand_tracking, false, "Adding hand tracking might improve hand keypoints detection for webcam (if the frame rate is high enough, i.e. >7 FPS per GPU) and video. This is not person ID tracking, it simply looks for hands in positions at which hands were located in previous frames, but it does not guarantee the same person ID among frames."); 8. OpenPose 3-D Reconstruction -DEFINE_bool(3d, false, "Running OpenPose 3-D reconstruction demo: 1) Reading from a stereo camera system. 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction results. Note that it will only display 1 person. If multiple people is present, it will fail."); -DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per iteration, allowing tasks such as stereo camera processing (`--3d`). Note that `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the parameter folder as this number indicates."); +- DEFINE_bool(3d, false, "Running OpenPose 3-D reconstruction demo: 1) Reading from a stereo camera system. 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction results. Note that it will only display 1 person. If multiple people is present, it will fail."); +- DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will require all the cameras to see the keypoint in order to reconstruct it."); +- DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per iteration, allowing tasks such as stereo camera processing (`--3d`). Note that `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the parameter folder as this number indicates."); 9. OpenPose Rendering - DEFINE_int32(part_to_show, 0, "Prediction channel to visualize (default: 0). 0 for all the body parts, 1-18 for each body part heat map, 19 for the background heat map, 20 for all the body part heat maps together, 21 for all the PAFs, 22-40 for each body part pair PAF."); diff --git a/doc/quick_start.md b/doc/quick_start.md index 1ddbcc9b0f4565a86a5819dd5f7c8b27ca8be747..be647ec3606b3665642c53b0f871a6ef302d1b6b 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -147,13 +147,20 @@ build\x64\Release\OpenPoseDemo.exe --flir_camera --3d --number_people_max 1 --fa # Ubuntu (same flags for Windows version) # Optionally add `--face` and/or `--hand` to include face and/or hands # Assuming 3 cameras -# Note: We highly recommend to reduce `--output_resolution`. E.g. for 3 cameras recording at 1920x1080, the resulting image is (3x1920)x1080, so we recommend e.g. 1920x360 (x3 reduction). +# Note: We highly recommend to reduce `--output_resolution`. E.g. for 3 cameras recording at 1920x1080, the resulting image is (3x1920)x1080, so we recommend e.g. 640x360 (x3 reduction). # Video ./build/examples/openpose/openpose.bin --video output_folder_path/video.avi --3d_views 3 --3d --number_people_max 1 --output_resolution {desired_output_resolution} # Images ./build/examples/openpose/openpose.bin --image_dir output_folder_path/ --3d_views 3 --3d --number_people_max 1 --output_resolution {desired_output_resolution} ``` +5. Reconstruction when at least n visible views +``` +# Ubuntu (same flags for Windows version) +# Assuming >=2 cameras and reconstruction when at least 2 visible views +./build/examples/openpose/openpose.bin --flir_camera --3d --number_people_max 1 --3d_min_views 2 --output_resolution {desired_output_resolution} +``` + ## Expected Visual Results diff --git a/doc/release_notes.md b/doc/release_notes.md index 720a1ae5bcea95536d64c27e50c2edc3a70a089b..cadc2261ea2cabc60635f74aeae089bfb992c576 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -193,7 +193,8 @@ OpenPose Library - Release Notes 21. VideoSaver (`--write_video`) compatible with multi-camera setting. It will save all the different views concatenated. 22. OpenPose small GUI rescale the verbose text to the displayed image, to avoid the text to be either too big or small. 23. OpenPose small GUI shows the frame number w.r.t. the original producer, rather than the frame id. E.g., if video is started at frame 30, OpenPose will display 30 rather than 0 in the first frame. - 23. OpenPose GUI: 'l' and 'k' functionality swapped. + 24. OpenPose GUI: 'l' and 'k' functionality swapped. + 25. 3-D reconstruction module: Added flag `--3d_min_views` to select minimum number of cameras required for 3-D reconstruction. 2. Functions or parameters renamed: 1. Flag `no_display` renamed as `display`, able to select between `NoDisplay`, `Display2D`, `Display3D`, and `DisplayAll`. 2. 3-D reconstruction demo is now inside the OpenPose demo binary. diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index f1516d71107d25e15d55593824466ddd88e0c8c5..fcafbcec9adc8b5009b76db9cf63a8b40c134c1b 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -147,6 +147,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -276,7 +278,8 @@ int openPoseDemo() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d, FLAGS_identification}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views, + FLAGS_identification}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index 632f0d9659a1f66a8cee3ccb616fcb46761a53ff..74df6c961477297aa378e08d2961e1d6b63cbbb5 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -156,6 +156,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -278,7 +280,7 @@ int openPoseTutorialWrapper4() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp index 555365de611a959106c0c9eddbc802b1b61baa3e..caa8bcb98b625dce29170f602b00f0c68fb95f9a 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp @@ -146,6 +146,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -353,7 +355,7 @@ int openPoseTutorialWrapper1() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index 8293e76bf4f00a4401364a78e7f52a29c59f95e6..a23e9c0088a836d05f41c8bbc0a28f26e94d12ff 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -125,6 +125,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -443,7 +445,7 @@ int openPoseTutorialWrapper2() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/examples/tutorial_wrapper/3_user_asynchronous.cpp b/examples/tutorial_wrapper/3_user_asynchronous.cpp index 53f8971f6a4d4a20b55ead13eb65153a3012ece9..7705bff3181f77d8a7704ea6269f9ac8afd04e66 100644 --- a/examples/tutorial_wrapper/3_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/3_user_asynchronous.cpp @@ -125,6 +125,8 @@ DEFINE_bool(3d, false, "Running OpenPose 3-D re " 2) Performing 3-D reconstruction from the multiple views. 3) Displaying 3-D reconstruction" " results. Note that it will only display 1 person. If multiple people is present, it will" " fail."); +DEFINE_int32(3d_min_views, -1, "Minimum number of views required to reconstruct each keypoint. By default (-1), it will" + " require all the cameras to see the keypoint in order to reconstruct it."); DEFINE_int32(3d_views, 1, "Complementary option to `--image_dir` or `--video`. OpenPose will read as many images per" " iteration, allowing tasks such as stereo camera processing (`--3d`). Note that" " `--camera_parameters_folder` must be set. OpenPose must find as many `xml` files in the" @@ -384,7 +386,7 @@ int openPoseTutorialWrapper3() (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, (float)FLAGS_render_threshold, FLAGS_number_people_max, - enableGoogleLogging, FLAGS_3d}; + enableGoogleLogging, FLAGS_3d, FLAGS_3d_min_views}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), diff --git a/include/openpose/3d/poseTriangulation.hpp b/include/openpose/3d/poseTriangulation.hpp index c3210f74fcb7adf9bb3aa8decab345f3597653fc..5a997e58515890c80a3325f446fd1b5d86625859 100644 --- a/include/openpose/3d/poseTriangulation.hpp +++ b/include/openpose/3d/poseTriangulation.hpp @@ -6,8 +6,17 @@ namespace op { - OP_API Array reconstructArray(const std::vector>& keypointsVector, - const std::vector& matrixEachCamera); + class OP_API PoseTriangulation + { + public: + PoseTriangulation(const int minViews3d); + + Array reconstructArray(const std::vector>& keypointsVector, + const std::vector& cameraMatrices) const; + + private: + const int mMinViews3d; + }; } #endif // OPENPOSE_3D_POSE_TRIANGULATION_HPP diff --git a/include/openpose/3d/wPoseTriangulation.hpp b/include/openpose/3d/wPoseTriangulation.hpp index 08a316ccbfb985b2f8e5651dd903aa8e366a53ab..db949787971fd7e61ba4e8d1a8973e9562adf459 100644 --- a/include/openpose/3d/wPoseTriangulation.hpp +++ b/include/openpose/3d/wPoseTriangulation.hpp @@ -11,13 +11,15 @@ namespace op class WPoseTriangulation : public Worker { public: - explicit WPoseTriangulation(); + explicit WPoseTriangulation(const std::shared_ptr& poseTriangulation); void initializationOnThread(); void work(TDatums& tDatums); private: + const std::shared_ptr spPoseTriangulation; + DELETE_COPY(WPoseTriangulation); }; } @@ -31,7 +33,8 @@ namespace op namespace op { template - WPoseTriangulation::WPoseTriangulation() + WPoseTriangulation::WPoseTriangulation(const std::shared_ptr& poseTriangulation) : + spPoseTriangulation{poseTriangulation} { } @@ -66,10 +69,12 @@ namespace op cameraMatrices.emplace_back(datumsElement.cameraMatrix); } // Pose 3-D reconstruction - auto poseKeypoints3D = reconstructArray(poseKeypointVector, cameraMatrices); - auto faceKeypoints3D = reconstructArray(faceKeypointVector, cameraMatrices); - auto leftHandKeypoints3D = reconstructArray(leftHandKeypointVector, cameraMatrices); - auto rightHandKeypoints3D = reconstructArray(rightHandKeypointVector, cameraMatrices); + auto poseKeypoints3D = spPoseTriangulation->reconstructArray(poseKeypointVector, cameraMatrices); + auto faceKeypoints3D = spPoseTriangulation->reconstructArray(faceKeypointVector, cameraMatrices); + auto leftHandKeypoints3D = spPoseTriangulation->reconstructArray(leftHandKeypointVector, + cameraMatrices); + auto rightHandKeypoints3D = spPoseTriangulation->reconstructArray(rightHandKeypointVector, + cameraMatrices); // Assign to all tDatums for (auto& datumsElement : *tDatums) { diff --git a/include/openpose/gui/frameDisplayer.hpp b/include/openpose/gui/frameDisplayer.hpp index 214202927e9cefc4bf455a76d7f6f09a42b9a926..5e4ddd1ae0742c4ee0cde8d0be2fe6379affa92b 100644 --- a/include/openpose/gui/frameDisplayer.hpp +++ b/include/openpose/gui/frameDisplayer.hpp @@ -49,6 +49,11 @@ namespace op */ void displayFrame(const cv::Mat& frame, const int waitKeyValue = -1); + /** + * Analogous to the previous displayFrame, but first it horizontally concatenates all the frames + */ + void displayFrame(const std::vector& frames, const int waitKeyValue = -1); + private: const std::string mWindowName; Point mWindowedSize; diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index 25794669c69eca82e0f028692b8b9a1a89beb21e..3e85a5684ea7a7a9c733e053f120345324f5ceeb 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -798,8 +798,9 @@ namespace op // 3-D reconstruction if (wrapperStructPose.reconstruct3d) { + const auto poseTriangulation = std::make_shared(wrapperStructPose.minViews3d); mPostProcessingWs.emplace_back( - std::make_shared>() + std::make_shared>(poseTriangulation) ); } // Frames processor (OpenPose format -> cv::Mat format) diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index 52cb768f84f69f44633ac0398a726e5f9f6bc7e9..43566b235a33e0d88329d0b149ccc0c47d584b96 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -21,13 +21,13 @@ namespace op * @param threadManagerMode */ OP_API void wrapperConfigureSecurityChecks(const WrapperStructPose& wrapperStructPose, - const WrapperStructFace& wrapperStructFace, - const WrapperStructHand& wrapperStructHand, - const WrapperStructInput& wrapperStructInput, - const WrapperStructOutput& wrapperStructOutput, - const bool renderOutput, - const bool userOutputWsEmpty, - const ThreadManagerMode threadManagerMode); + const WrapperStructFace& wrapperStructFace, + const WrapperStructHand& wrapperStructHand, + const WrapperStructInput& wrapperStructInput, + const WrapperStructOutput& wrapperStructOutput, + const bool renderOutput, + const bool userOutputWsEmpty, + const ThreadManagerMode threadManagerMode); } #endif // OPENPOSE_WRAPPER_WRAPPER_AUXILIARY_HPP diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index 5946d0f193979225f314991589cc734f1f3541d7..f2e9e7611626b7a33c0c77ed6ad51c21a697f5a1 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -175,6 +175,12 @@ namespace op */ bool reconstruct3d; + /** + * Minimum number of views required to reconstruct each keypoint. + * By default (-1), it will require all the cameras to see the keypoint in order to reconstruct it. + */ + int minViews3d; + /** * Whether to return a person ID for each body skeleton, providing temporal consistency. */ @@ -198,7 +204,7 @@ namespace op const ScaleMode heatMapScale = ScaleMode::ZeroToOne, const bool addPartCandidates = false, const float renderThreshold = 0.05f, const int numberPeopleMax = -1, const bool enableGoogleLogging = true, const bool reconstruct3d = false, - const bool identification = false); + const int minViews3d = -1, const bool identification = false); }; } diff --git a/src/openpose/3d/poseTriangulation.cpp b/src/openpose/3d/poseTriangulation.cpp index 3ea26f04086205f44434f8a9fcd16cc7d42a6684..41621198977f7010a0f4e14a6b4cc77f15a8a926 100644 --- a/src/openpose/3d/poseTriangulation.cpp +++ b/src/openpose/3d/poseTriangulation.cpp @@ -3,7 +3,7 @@ namespace op { - double calcReprojectionError(const cv::Mat& X, const std::vector& M, + double calcReprojectionError(const cv::Mat& reconstructedPoint, const std::vector& M, const std::vector& points2d) { try @@ -11,7 +11,7 @@ namespace op auto averageError = 0.; for (auto i = 0u ; i < M.size() ; i++) { - cv::Mat imageX = M[i] * X; + cv::Mat imageX = M[i] * reconstructedPoint; imageX /= imageX.at(2,0); const auto error = std::sqrt(std::pow(imageX.at(0,0) - points2d[i].x,2) + std::pow(imageX.at(1,0) - points2d[i].y,2)); @@ -27,30 +27,34 @@ namespace op } } - void triangulate(cv::Mat& X, const std::vector& matrixEachCamera, + void triangulate(cv::Mat& reconstructedPoint, const std::vector& cameraMatrices, const std::vector& pointsOnEachCamera) { try { // Security checks - if (matrixEachCamera.empty() || matrixEachCamera.size() != pointsOnEachCamera.size()) - error("numberCameras.empty() || numberCameras.size() != pointsOnEachCamera.size()", - __LINE__, __FUNCTION__, __FILE__); + if (cameraMatrices.size() != pointsOnEachCamera.size()) + error("numberCameras.size() != pointsOnEachCamera.size() (" + std::to_string(cameraMatrices.size()) + + " vs. " + std::to_string(pointsOnEachCamera.size()) + ").", + __LINE__, __FUNCTION__, __FILE__); + if (cameraMatrices.empty()) + error("numberCameras.empty()", + __LINE__, __FUNCTION__, __FILE__); // Create and fill A - const auto numberCameras = (int)matrixEachCamera.size(); + const auto numberCameras = (int)cameraMatrices.size(); cv::Mat A = cv::Mat::zeros(numberCameras*2, 4, CV_64F); for (auto i = 0 ; i < numberCameras ; i++) { - cv::Mat temp = pointsOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) - - matrixEachCamera[i].rowRange(0,1); + cv::Mat temp = pointsOnEachCamera[i].x*cameraMatrices[i].rowRange(2,3) + - cameraMatrices[i].rowRange(0,1); temp.copyTo(A.rowRange(i*2, i*2+1)); - temp = pointsOnEachCamera[i].y*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(1,2); + temp = pointsOnEachCamera[i].y*cameraMatrices[i].rowRange(2,3) - cameraMatrices[i].rowRange(1,2); temp.copyTo(A.rowRange(i*2+1, i*2+2)); } // SVD on A cv::SVD svd{A}; - svd.solveZ(A,X); - X /= X.at(3); + svd.solveZ(A,reconstructedPoint); + reconstructedPoint /= reconstructedPoint.at(3); } catch (const std::exception& e) { @@ -59,20 +63,20 @@ namespace op } // TODO: ask for the missing function: TriangulationOptimization - double triangulateWithOptimization(cv::Mat& X, const std::vector& matrixEachCamera, + double triangulateWithOptimization(cv::Mat& reconstructedPoint, const std::vector& cameraMatrices, const std::vector& pointsOnEachCamera) { try { - triangulate(X, matrixEachCamera, pointsOnEachCamera); + triangulate(reconstructedPoint, cameraMatrices, pointsOnEachCamera); return 0.; - // return calcReprojectionError(X, matrixEachCamera, pointsOnEachCamera); + // return calcReprojectionError(X, cameraMatrices, pointsOnEachCamera); - // //if (matrixEachCamera.size() >= 3) - // //double beforeError = calcReprojectionError(&matrixEachCamera, pointsOnEachCamera, X); - // double change = TriangulationOptimization(&matrixEachCamera, pointsOnEachCamera, X); - // //double afterError = calcReprojectionError(&matrixEachCamera,pointsOnEachCamera,X); + // //if (cameraMatrices.size() >= 3) + // //double beforeError = calcReprojectionError(&cameraMatrices, pointsOnEachCamera, X); + // double change = TriangulationOptimization(&cameraMatrices, pointsOnEachCamera, X); + // //double afterError = calcReprojectionError(&cameraMatrices,pointsOnEachCamera,X); // //printfLog("!!Mine %.8f , inFunc %.8f \n",beforeError-afterError,change); // return change; } @@ -83,14 +87,30 @@ namespace op } } - Array reconstructArray(const std::vector>& keypointsVector, - const std::vector& matrixEachCamera) + PoseTriangulation::PoseTriangulation(const int minViews3d) : + mMinViews3d{minViews3d} + { + try + { + // Security checks + if (0 <= mMinViews3d && mMinViews3d < 2) + error("Minimum number of views must be at least 2 (e.g., `--3d_min_views 2`) or negative.", + __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + Array PoseTriangulation::reconstructArray(const std::vector>& keypointsVector, + const std::vector& cameraMatrices) const { try { Array keypoints3D; // Security checks - if (matrixEachCamera.size() < 2) + if (cameraMatrices.size() < 2) error("Only 1 camera detected. The 3-D reconstruction module can only be used with > 1 cameras" " simultaneously. E.g., using FLIR stereo cameras (`--flir_camera`).", __LINE__, __FUNCTION__, __FILE__); @@ -113,27 +133,33 @@ namespace op const auto threshold = 0.2f; std::vector indexesUsed; std::vector> xyPoints; + std::vector> cameraMatricesPerPoint; for (auto part = 0; part < numberBodyParts; part++) { // Create vector of points - auto missedPoint = false; + // auto missedPoint = false; std::vector xyPointsElement; + std::vector cameraMatricesElement; const auto baseIndex = part * lastChannelLength; - for (auto& keypoints : keypointsVector) + // for (auto& keypoints : keypointsVector) + for (auto i = 0u ; i < keypointsVector.size() ; i++) { + const auto& keypoints = keypointsVector[i]; if (keypoints[baseIndex+2] > threshold) - xyPointsElement.emplace_back(cv::Point2d{ keypoints[baseIndex], - keypoints[baseIndex+1]}); - else { - missedPoint = true; - break; + xyPointsElement.emplace_back(cv::Point2d{keypoints[baseIndex], + keypoints[baseIndex+1]}); + cameraMatricesElement.emplace_back(cameraMatrices[i]); } } - if (!missedPoint) + // If visible from all views (mMinViews3d < 0) + // or if visible for at least mMinViews3d views + if ((mMinViews3d < 0 && cameraMatricesElement.size() == cameraMatrices.size()) + || (mMinViews3d > 1 && mMinViews3d <= (int)xyPointsElement.size())) { indexesUsed.emplace_back(part); xyPoints.emplace_back(xyPointsElement); + cameraMatricesPerPoint.emplace_back(cameraMatricesElement); } } // 3D reconstruction @@ -144,16 +170,17 @@ namespace op std::vector xyzPoints(xyPoints.size()); for (auto i = 0u; i < xyPoints.size(); i++) { - cv::Mat X; - triangulateWithOptimization(X, matrixEachCamera, xyPoints[i]); - xyzPoints[i] = cv::Point3d{ X.at(0), X.at(1), X.at(2) }; + cv::Mat reconstructedPoint; + triangulateWithOptimization(reconstructedPoint, cameraMatricesPerPoint[i], xyPoints[i]); + xyzPoints[i] = cv::Point3d{reconstructedPoint.at(0), reconstructedPoint.at(1), + reconstructedPoint.at(2)}; } // 3D points to pose // OpenCV alternative: // http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#triangulatepoints // cv::Mat reconstructedPoints{4, firstcv::Points.size(), CV_64F}; - // cv::triangulatecv::Points(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, + // cv::triangulatePoints(cv::Mat::eye(3,4, CV_64F), M_3_1, firstcv::Points, secondcv::Points, // reconstructedcv::Points); const auto lastChannelLength = keypoints3D.getSize(2); for (auto index = 0u; index < indexesUsed.size(); index++) diff --git a/src/openpose/gui/frameDisplayer.cpp b/src/openpose/gui/frameDisplayer.cpp index 9ca87f4788268859ffce745dade3e8f57b4041b1..11f3b2aea11af0b50076f471dbc3c541fb2bcd5d 100644 --- a/src/openpose/gui/frameDisplayer.cpp +++ b/src/openpose/gui/frameDisplayer.cpp @@ -87,6 +87,9 @@ namespace op { try { + // Security check + if (frame.empty()) + error("Empty frame introduced.", __LINE__, __FUNCTION__, __FILE__); // If frame > window size --> Resize window if (mWindowedSize.x < frame.cols || mWindowedSize.y < frame.rows) { @@ -106,4 +109,33 @@ namespace op error(e.what(), __LINE__, __FUNCTION__, __FILE__); } } + + void FrameDisplayer::displayFrame(const std::vector& frames, const int waitKeyValue) + { + try + { + // No frames + if (frames.empty()) + displayFrame(cv::Mat(), waitKeyValue); + // 1 frame + else if (frames.size() == 1u) + displayFrame(frames[0], waitKeyValue); + // >= 2 frames + else + { + // Prepare final cvMat + // Concat (0) + cv::Mat cvMat = frames[0].clone(); + // Concat (1,size()-1) + for (auto i = 1u; i < frames.size(); i++) + cv::hconcat(cvMat, frames[i], cvMat); + // Display it + displayFrame(cvMat, waitKeyValue); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } } diff --git a/src/openpose/gui/gui.cpp b/src/openpose/gui/gui.cpp index 40e4fc7b9099c774d3052edc7dfd25b5c2a27df4..7d43335ff4e77349d4469a4538ae92e6e6c26316 100644 --- a/src/openpose/gui/gui.cpp +++ b/src/openpose/gui/gui.cpp @@ -177,12 +177,7 @@ namespace op { try { - // Check tDatum integrity - const bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutput.empty()); - - // Display - if (returnedIsValidFrame) - mFrameDisplayer.displayFrame(cvMatOutput, -1); + setImage(std::vector{cvMatOutput}); } catch (const std::exception& e) { @@ -194,36 +189,20 @@ namespace op { try { - // 0 image - if (cvMatOutputs.empty()) - setImage(cvMatOutputs[0]); - // 1 image - else if (cvMatOutputs.size() == 1) - setImage(cvMatOutputs[0]); - // > 1 image - else + // Check tDatum integrity + bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutputs.empty()); + for (const auto& cvMatOutput : cvMatOutputs) { - // Check tDatum integrity - bool returnedIsValidFrame = ((spIsRunning == nullptr || *spIsRunning) && !cvMatOutputs.empty()); - if (returnedIsValidFrame) + if (cvMatOutput.empty()) { - // Security checks - for (const auto& cvMatOutput : cvMatOutputs) - if (cvMatOutput.empty()) - returnedIsValidFrame = false; - // Prepare final cvMat - if (returnedIsValidFrame) - { - // Concat (0) - cv::Mat cvMat = cvMatOutputs[0].clone(); - // Concat (1,size()-1) - for (auto i = 1u; i < cvMatOutputs.size(); i++) - cv::hconcat(cvMat, cvMatOutputs[i], cvMat); - // Display - mFrameDisplayer.displayFrame(cvMat, -1); - } + returnedIsValidFrame = false; + break; } } + + // Display + if (returnedIsValidFrame) + mFrameDisplayer.displayFrame(cvMatOutputs, -1); } catch (const std::exception& e) { diff --git a/src/openpose/utilities/keypoint.cpp b/src/openpose/utilities/keypoint.cpp index d43c92e7b1928bc21abf84fcda41fc4d831d20aa..b9217d4939446b9f43bd94290dac4a668687c8c7 100644 --- a/src/openpose/utilities/keypoint.cpp +++ b/src/openpose/utilities/keypoint.cpp @@ -175,8 +175,8 @@ namespace op const auto thicknessRatio = fastMax(intRound(std::sqrt(area) * thicknessCircleRatio * ratioAreas), 2); // Negative thickness in cv::circle means that a filled circle is to be drawn. - const auto thicknessCircle = (ratioAreas > 0.05f ? thicknessRatio : -1); - const auto thicknessLine = intRound(thicknessRatio * thicknessLineRatioWRTCircle); + const auto thicknessCircle = fastMax(1, (ratioAreas > 0.05f ? thicknessRatio : -1)); + const auto thicknessLine = fastMax(1, intRound(thicknessRatio * thicknessLineRatioWRTCircle)); const auto radius = thicknessRatio / 2; // Draw lines diff --git a/src/openpose/wrapper/wrapperAuxiliary.cpp b/src/openpose/wrapper/wrapperAuxiliary.cpp index c2d8e3db732c77911908d5957bf0a4f556a7c01b..dc403b3820b7faf7269d2bbb755c52965b2ce821 100644 --- a/src/openpose/wrapper/wrapperAuxiliary.cpp +++ b/src/openpose/wrapper/wrapperAuxiliary.cpp @@ -115,6 +115,14 @@ namespace op if (wrapperStructPose.gpuNumber > 0) error("GPU number must be negative or 0 if CPU_ONLY is enabled.", __LINE__, __FUNCTION__, __FILE__); + // If num_gpu 0 --> output_resolution has no effect + if (wrapperStructPose.gpuNumber == 0 && + (wrapperStructPose.outputSize.x > 0 || wrapperStructPose.outputSize.y > 0)) + error("If `--num_gpu 0`, then `--output_resolution` has no effect, so either disable it or use" + " `--output_resolution -1x-1`. Current output size: (" + + std::to_string(wrapperStructPose.outputSize.x) + "x" + + std::to_string(wrapperStructPose.outputSize.y) + ").", + __LINE__, __FUNCTION__, __FILE__); // Net input resolution cannot be reshaped for Caffe OpenCL and MKL versions, only for CUDA version #if defined USE_MKL || defined CPU_ONLY // If image_dir and netInputSize == -1 --> error diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index 83a017b056dc36d99f2bf6eb4e46c86f9c3be30d..af98a046adf9ca9035bde3b8053eb35baddd9ccb 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -14,7 +14,7 @@ namespace op const ScaleMode heatMapScale_, const bool addPartCandidates_, const float renderThreshold_, const int numberPeopleMax_, const bool enableGoogleLogging_, const bool reconstruct3d_, - const bool identification_) : + const int minViews3d_, const bool identification_) : enable{enable_}, netInputSize{netInputSize_}, outputSize{outputSize_}, @@ -37,6 +37,7 @@ namespace op numberPeopleMax{numberPeopleMax_}, enableGoogleLogging{enableGoogleLogging_}, reconstruct3d{reconstruct3d_}, + minViews3d{minViews3d_}, identification{identification_} { }