diff --git a/doc/demo_overview.md b/doc/demo_overview.md index f658a957c6248bab34894f8045bff8154487259f..c69413bd7bce3617e86d16a961c75c37b000721e 100644 --- a/doc/demo_overview.md +++ b/doc/demo_overview.md @@ -162,11 +162,12 @@ Each flag is divided into flag name, default value, and description. - DEFINE_int32(scale_number, 1, "Number of scales to average."); - DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1. If you want to change the initial scale, you actually want to multiply the `net_resolution` by your desired initial scale."); -5. OpenPose Body Pose Heatmaps -- DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array, and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps (program speed will decrease). Not required for our library, enable it only if you intend to process this information later. If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +5. OpenPose Body Pose Heatmaps and Part Candidates +- DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps. If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will considerably decrease. Not required for OpenPose, enable it only if you intend to explicitly use this information later."); - DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to background."); - DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); - DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer rounded [0,255]; and 3 for no scaling."); +- DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the op::Datum::poseCandidates array with the body part candidates. Candidates refer to all the detected body parts, before being assembled into people. Note that the number of candidates is equal or higher than the number of final body parts (i.e. after being assembled into people). The empty body parts are filled with 0s. Program speed will slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly use this information."); 6. OpenPose Face - DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g. `model_folder`. Note that this will considerable slow down the performance and increse the required GPU memory. In addition, the greater number of people on the image, the slower OpenPose will be."); @@ -209,9 +210,10 @@ Each flag is divided into flag name, default value, and description. - DEFINE_string(write_images, "", "Directory to write rendered frames in `write_images_format` image format."); - DEFINE_string(write_images_format, "png", "File extension and format for `write_images`, e.g. png, jpg or bmp. Check the OpenCV function cv::imwrite for all compatible extensions."); - DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -- DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -- DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); -- DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -- DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -- DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be enabled."); +- DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +- DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +- DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag must be enabled."); - DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`. For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for floating values."); +- DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format with `write_keypoint_format`."); +- DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +- DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format, compatible with any OpenCV version."); diff --git a/doc/release_notes.md b/doc/release_notes.md index 77636edc8a85dd3864e3b076dd787046fa235ca2..4277aa9ea8dfa9274b046b690a898fd1408d9d6f 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -33,9 +33,9 @@ OpenPose Library - Release Notes 7. It prints out the error description before throwing the exception (so that it is written on the Windows cmd). 8. Highly improved documentation. 2. Functions or parameters renamed: - 1. Flag `write_pose` renamed as `write_keypoint` and it also applies to face and/or hands. - 2. Flag `write_pose_json` renamed as `write_keypoint_json` and it also applies to face and/or hands. - 3. Flag `write_pose_format` renamed as `write_keypoint_format` and it also applies to face and/or hands. + 1. Flag `--write_pose` renamed as `--write_keypoint` and it also applies to face and/or hands. + 2. Flag `--write_pose_json` renamed as `--write_keypoint_json` and it also applies to face and/or hands. + 3. Flag `--write_pose_format` renamed as `--write_keypoint_format` and it also applies to face and/or hands. 4. PoseSaver and its JSON variant renamed as KeypointSaver. 5. PoseJsonCocoSaver renamed as CocoJsonSaver. 3. Main bugs fixed: @@ -55,22 +55,22 @@ OpenPose Library - Release Notes 7. Rendering threshold for pose, face and hands becomes user-configurable. 8. Check() functions give more feedback. 9. WCocoJsonSaver finished and removed its 3599-image limit. - 10. Added `camera_fps` so generated video will use that frame rate. + 10. Added `--camera_fps` so generated video will use that frame rate. 11. Reduced the number of printed information messages. Default logging priority threshold increased to Priority::Max. 12. Google flags to OpenPose configuration parameters reader moved from each demo to utilities/flagsToOpenPose. 13. Nms classes do not use `numberParts` for `Reshape`, they deduce the value. 14. Improved documentation. 2. Functions or parameters renamed: 1. Render flags renamed in the demo in order to incorporate the CPU/GPU rendering. - 2. Keypoints saved in JSON files (`write_keypoint_json`) are now saved as `pose_keypoints`, `face_keypoints`, `hand_left_keypoints`, and `hand_right_keypoints`. They all were previously saved as `body_parts`. - 3. Flag `num_scales` renamed as `scale_number`. - 4. All hand and pose flags renamed such as they start by `hand_` and `face_` respectively. + 2. Keypoints saved in JSON files (`--write_keypoint_json`) are now saved as `pose_keypoints`, `face_keypoints`, `hand_left_keypoints`, and `hand_right_keypoints`. They all were previously saved as `body_parts`. + 3. Flag `--num_scales` renamed as `--scale_number`. + 4. All hand and pose flags renamed such as they start by `--hand_` and `--face_` respectively. 3. Main bugs fixed: 1. Fixed bug in Array::getConstCvMat() if mVolume=0, now returning empty cv::Mat. 2. Fixed bug: `--process_real_time` threw error with webcam. 3. Fixed bug: Face not working when input and output resolutions are different. 4. Fixed some bugs that prevented debug version to run. - 5. Face saved in JSON files were called `body_parts`. Now they are called `face_keypoints`. + 5. Face saved in JSON files were called `--body_parts`. Now they are called `--face_keypoints`. @@ -129,14 +129,14 @@ OpenPose Library - Release Notes 9. OpenPose successfully compiles if the flags `USE_CAFFE` and/or `USE_CUDA` are not enabled, although it will give an error saying they are required. 10. COCO JSON file outputs 0 as score for non-detected keypoints. 11. Added example for OpenPose for user asynchronous output and cleaned all `tutorial_wrapper/` examples. - 12. Added `-1` option for `net_resolution` in order to auto-select the best possible aspect ratio given the user input. + 12. Added `-1` option for `--net_resolution` in order to auto-select the best possible aspect ratio given the user input. 13. Net resolution can be dynamically changed (e.g. for images with different size). 14. Added example to add functionality/modules to OpenPose. - 15. Added `disable_multi_thread` flag in order to allow debug and/or highly reduce the latency (e.g. when using webcam in real-time). + 15. Added `--disable_multi_thread` flag in order to allow debug and/or highly reduce the latency (e.g. when using webcam in real-time). 16. Allowed to output images without any rendering. 2. Functions or parameters renamed: 1. OpenPose able to change its size and initial size dynamically: - 1. Flag `resolution` renamed as `output_resolution`. + 1. Flag `--resolution` renamed as `--output_resolution`. 2. FrameDisplayer, GuiInfoAdder and Gui constructors arguments modified (gui module). 3. OpOutputToCvMat constructor removed (core module). 4. New Renders classes to split GpuRenderers from CpuRenderers. @@ -151,14 +151,17 @@ OpenPose Library - Release Notes -## Current version (future OpenPose 1.3.0) +## Current version (future OpenPose 1.2.1) 1. Main improvements: 1. Heatmaps can be saved in floating format. - 2. More efficient non-processing version (i.e. if all keypoint extractors are disabled, and only image extraction and display/saving operations are performed). - 3. Heat maps scaling: Added `heatmaps_scale` to OpenPoseDemo, added option not to scale the heatmaps, and added custom `float` format to save heatmaps in floating format. + 2. More efficient non-processing version (i.e., if all keypoint extractors are disabled, and only image extraction and display/saving operations are performed). + 3. Heat maps scaling: Added `--heatmaps_scale` to OpenPoseDemo, added option not to scale the heatmaps, and added custom `float` format to save heatmaps in floating format. 4. Detector of the number of GPU also considers the initial GPU index given by the user. + 5. Added `--write_json` as new version of `--write_keypoint_json`. It includes the body part candidates (if enabled), as well as any extra information added in the future (e.g. person ID). + 6. Body part candidates can be retrieved in op::Datum and saved with `--write_json`. 2. Functions or parameters renamed: 1. `PoseParameters` splitted into `PoseParameters` and `PoseParametersRender` and const parameters turned into functions for more clarity. 3. Main bugs fixed: 1. Render working on images > 4K (#324). 2. Cleaned redundant arguments on `getAverageScore` and `getKeypointsArea`. + 3. Slight speed up when heatmaps must be returned to the user (not doing a double copy anymore). diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 7032a64a4a14e7800d8b9f300d21738d59b435d9..f8cbd8f3e6b176de2b02b431bae20e27014e16b4 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -87,18 +87,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -166,16 +174,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); int openPoseDemo() { @@ -202,6 +214,11 @@ int openPoseDemo() FLAGS_camera_resolution, FLAGS_camera_fps); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -222,8 +239,9 @@ int openPoseDemo() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging, FLAGS_identification}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging, + FLAGS_identification}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -243,7 +261,7 @@ int openPoseDemo() const op::WrapperStructOutput wrapperStructOutput{!FLAGS_no_display, !FLAGS_no_gui_verbose, FLAGS_fullscreen, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), - FLAGS_write_keypoint_json, FLAGS_write_coco_json, + writeJson, FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; // Configure wrapper diff --git a/examples/tests/handFromJsonTest.cpp b/examples/tests/handFromJsonTest.cpp index b788ecdf92f6c9f27aa3faa0e784163fb823dac5..69c9206b121f49357c179631a9822f7b463049be 100644 --- a/examples/tests/handFromJsonTest.cpp +++ b/examples/tests/handFromJsonTest.cpp @@ -30,7 +30,7 @@ DEFINE_bool(hand_tracking, false, ""); // Display DEFINE_bool(no_display, false, ""); // Result Saving -DEFINE_string(write_keypoint_json, "", ""); +DEFINE_string(write_json, "", ""); int handFromJsonTest() { @@ -67,7 +67,7 @@ int handFromJsonTest() op::flagsToRenderMode(1)}; // Configure wrapper opWrapper.configure(wrapperStructPose, wrapperStructHand, producerSharedPtr, FLAGS_hand_ground_truth, - FLAGS_write_keypoint_json, !FLAGS_no_display); + FLAGS_write_json, !FLAGS_no_display); // Start processing op::log("Starting thread(s)", op::Priority::High); diff --git a/examples/tests/hand_accuracy_test.sh b/examples/tests/hand_accuracy_test.sh index 1ebafba0df93359773116b2d566bbb202d5123fd..562435e209cb9900a7d01b0ed05227a82712fb56 100755 --- a/examples/tests/hand_accuracy_test.sh +++ b/examples/tests/hand_accuracy_test.sh @@ -11,10 +11,10 @@ # Read that script for details about all the paths and change them for your own paths. # Careful: - # If you are using the NAS, please do not override my files, i.e. please change the output paths (corresponding to the ones indicated by `--write_keypoint_json`, which is ). + # If you are using the NAS, please do not override my files, i.e. please change the output paths (corresponding to the ones indicated by `--write_json`, which is ). # In order to generate the JSON output: - # Uncomment the commented lines starting by `--write_keypoint_json` and `--no_display` + # Uncomment the commented lines starting by `--write_json` and `--no_display` # Step 2 - Running JSON output to get accuracy # Once you have the JSON files, run them with the script Tomas prepared for it, which in my case I use: @@ -32,12 +32,12 @@ HAND_TESTING_FOLDER="/media/posefs3b/Users/gines/openpose_train/dataset/hand_tes IMAGES_FOLDER=${HAND_TESTING_FOLDER}"0_images/" IMAGES_BB_FOLDER=${HAND_TESTING_FOLDER}"3_images_bounding_box" HAND_GROUND_TRUTH_FOLDER=${HAND_TESTING_FOLDER}"4_hand_detections" -KEYPOINT_JSON_FOLDER=${HAND_TESTING_FOLDER}"5_keypointJson/" +PEOPLE_JSON_FOLDER=${HAND_TESTING_FOLDER}"5_keypointJson/" # Variable paths SCALES=6 SUFFIX="_${SCALES}" -HAND_RESULTS_FOLDER_BASE=${KEYPOINT_JSON_FOLDER}"hand_keypoints_estimated" +HAND_RESULTS_FOLDER_BASE=${PEOPLE_JSON_FOLDER}"hand_keypoints_estimated" HAND_RESULTS_FOLDER_NO_BB=${HAND_RESULTS_FOLDER_BASE}"_old"${SUFFIX} HAND_RESULTS_FOLDER_BB=${HAND_RESULTS_FOLDER_BASE}"_BBox"${SUFFIX} HAND_RESULTS_FOLDER_BODY_59=${HAND_RESULTS_FOLDER_BASE}"_BODY_59" @@ -52,7 +52,7 @@ rm -rf $HAND_RESULTS_FOLDER_BB --hand_scale_number ${SCALES} --hand_scale_range 0.4 \ --image_dir ${IMAGES_BB_FOLDER} \ --hand_ground_truth ${HAND_GROUND_TRUTH_FOLDER} \ - --write_keypoint_json $HAND_RESULTS_FOLDER_BB \ + --write_json $HAND_RESULTS_FOLDER_BB \ --no_display @@ -65,7 +65,7 @@ rm -rf $HAND_RESULTS_FOLDER_NO_BB --hand \ --hand_scale_number ${SCALES} --hand_scale_range 0.4 \ --image_dir ${IMAGES_FOLDER} \ - --write_keypoint_json $HAND_RESULTS_FOLDER_NO_BB \ + --write_json $HAND_RESULTS_FOLDER_NO_BB \ --no_display @@ -77,5 +77,5 @@ rm -rf $HAND_RESULTS_FOLDER_BODY_59 ./build/examples/openpose/openpose.bin \ --model_pose BODY_59 \ --image_dir ${IMAGES_FOLDER} \ - --write_keypoint_json $HAND_RESULTS_FOLDER_BODY_59 \ + --write_json $HAND_RESULTS_FOLDER_BODY_59 \ --no_display diff --git a/examples/tests/wrapperHandFromJsonTest.hpp b/examples/tests/wrapperHandFromJsonTest.hpp index 06d293364a99da18f284579a2a50d01fc6639e87..9e227b537684205016ae1efd0ad80b66e9dd0486 100644 --- a/examples/tests/wrapperHandFromJsonTest.hpp +++ b/examples/tests/wrapperHandFromJsonTest.hpp @@ -26,7 +26,7 @@ namespace op const WrapperStructHand& wrapperStructHand, const std::shared_ptr& producerSharedPtr, const std::string& handGroundTruth, - const std::string& writeKeypointJson, + const std::string& writeJson, const bool displayGui = false); /** @@ -119,7 +119,7 @@ namespace op const WrapperStructHand& wrapperStructHand, const std::shared_ptr& producerSharedPtr, const std::string& handGroundTruth, - const std::string& writeKeypointJson, + const std::string& writeJson, const bool displayGui) { try @@ -134,7 +134,7 @@ namespace op error("The scale gap must be greater than 0 (it has no effect if the number of scales is 1).", __LINE__, __FUNCTION__, __FILE__); const std::string additionalMessage = " You could also set mThreadManagerMode = mThreadManagerMode::Asynchronous(Out) and/or add your own" " output worker class before calling this function."; - const auto savingSomething = !writeKeypointJson.empty(); + const auto savingSomething = !writeJson.empty(); if (!displayGui && !savingSomething) { const auto message = "No output is selected (`no_display`) and no results are generated (no `write_X` flags enabled). Thus," @@ -157,7 +157,7 @@ namespace op } // Proper format - const auto writeKeypointJsonCleaned = formatAsDirectory(writeKeypointJson); + const auto writeJsonCleaned = formatAsDirectory(writeJson); // Common parameters const auto finalOutputSize = wrapperStructPose.outputSize; @@ -243,10 +243,10 @@ namespace op mOutputWs.clear(); // Write people pose data on disk (json format) - if (!writeKeypointJsonCleaned.empty()) + if (!writeJsonCleaned.empty()) { - const auto keypointJsonSaver = std::make_shared(writeKeypointJsonCleaned); - mOutputWs.emplace_back(std::make_shared>(keypointJsonSaver)); + const auto jsonSaver = std::make_shared(writeJsonCleaned); + mOutputWs.emplace_back(std::make_shared>(jsonSaver)); } // Minimal graphical user interface (GUI) spWGui = nullptr; diff --git a/examples/tutorial_add_module/1_custom_post_processing.cpp b/examples/tutorial_add_module/1_custom_post_processing.cpp index f97d5570932d4830744f329bbb62f7e6c150d7be..940afe96639815f9d29343bad5688999abb0ca0d 100644 --- a/examples/tutorial_add_module/1_custom_post_processing.cpp +++ b/examples/tutorial_add_module/1_custom_post_processing.cpp @@ -96,18 +96,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -175,16 +183,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); int openPoseTutorialWrapper4() { @@ -211,6 +223,11 @@ int openPoseTutorialWrapper4() FLAGS_camera_resolution, FLAGS_camera_fps); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -231,8 +248,8 @@ int openPoseTutorialWrapper4() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -252,7 +269,7 @@ int openPoseTutorialWrapper4() const op::WrapperStructOutput wrapperStructOutput{!FLAGS_no_display, !FLAGS_no_gui_verbose, FLAGS_fullscreen, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), - FLAGS_write_keypoint_json, FLAGS_write_coco_json, + writeJson, FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; diff --git a/examples/tutorial_pose/1_extract_from_image.cpp b/examples/tutorial_pose/1_extract_from_image.cpp index 461dd0bcce6b3efa3d06cd3add3586ee783cf49a..a4fd3d8832d890220859d274841bead8f5a61f63 100644 --- a/examples/tutorial_pose/1_extract_from_image.cpp +++ b/examples/tutorial_pose/1_extract_from_image.cpp @@ -83,16 +83,13 @@ int openPoseTutorialPose1() if (FLAGS_scale_gap <= 0. && FLAGS_scale_number > 1) op::error("Incompatible flag configuration: scale_gap must be greater than 0 or scale_number = 1.", __LINE__, __FUNCTION__, __FILE__); - // Enabling Google Logging - const bool enableGoogleLogging = true; // Logging op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Step 3 - Initialize all required classes op::ScaleAndSizeExtractor scaleAndSizeExtractor(netInputSize, outputSize, FLAGS_scale_number, FLAGS_scale_gap); op::CvMatToOpInput cvMatToOpInput; op::CvMatToOpOutput cvMatToOpOutput; - op::PoseExtractorCaffe poseExtractorCaffe{poseModel, FLAGS_model_folder, - FLAGS_num_gpu_start, {}, op::ScaleMode::ZeroToOne, enableGoogleLogging}; + op::PoseExtractorCaffe poseExtractorCaffe{poseModel, FLAGS_model_folder, FLAGS_num_gpu_start}; op::PoseCpuRenderer poseRenderer{poseModel, (float)FLAGS_render_threshold, !FLAGS_disable_blending, (float)FLAGS_alpha_pose}; op::OpOutputToCvMat opOutputToCvMat; diff --git a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp index 5ce1f8d69dedc42392ad2a94d0972cb7966a7251..68030afacf0c3a6a8aa4906a428df1fbdd382f17 100644 --- a/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp +++ b/examples/tutorial_pose/2_extract_pose_or_heatmat_from_image.cpp @@ -88,18 +88,14 @@ int openPoseTutorialPose2() if (FLAGS_scale_gap <= 0. && FLAGS_scale_number > 1) op::error("Incompatible flag configuration: scale_gap must be greater than 0 or scale_number = 1.", __LINE__, __FUNCTION__, __FILE__); - // Enabling Google Logging - const bool enableGoogleLogging = true; // Logging op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Step 3 - Initialize all required classes op::ScaleAndSizeExtractor scaleAndSizeExtractor(netInputSize, outputSize, FLAGS_scale_number, FLAGS_scale_gap); op::CvMatToOpInput cvMatToOpInput; op::CvMatToOpOutput cvMatToOpOutput; - auto poseExtractorPtr = std::make_shared( - poseModel, FLAGS_model_folder, FLAGS_num_gpu_start, std::vector{}, op::ScaleMode::ZeroToOne, - enableGoogleLogging - ); + auto poseExtractorPtr = std::make_shared(poseModel, FLAGS_model_folder, + FLAGS_num_gpu_start); op::PoseGpuRenderer poseGpuRenderer{poseModel, poseExtractorPtr, (float)FLAGS_render_threshold, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap}; poseGpuRenderer.setElementToRender(FLAGS_part_to_show); diff --git a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp index 0310763803d7b9ee3896bcaf3731dcc57c216afb..52b5124bdd19612377b4a8b78e3fee3e81db29ee 100644 --- a/examples/tutorial_wrapper/1_user_asynchronous_output.cpp +++ b/examples/tutorial_wrapper/1_user_asynchronous_output.cpp @@ -86,18 +86,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -159,16 +167,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); // If the user needs his own variables, he can inherit the op::Datum struct and add them @@ -287,6 +299,11 @@ int openPoseTutorialWrapper1() FLAGS_camera_resolution, FLAGS_camera_fps); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -307,8 +324,8 @@ int openPoseTutorialWrapper1() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -329,8 +346,9 @@ int openPoseTutorialWrapper1() const bool guiVerbose = false; const bool fullScreen = false; const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_keypoint, - op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_keypoint_json, - FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, + op::stringToDataFormat(FLAGS_write_keypoint_format), + writeJson, FLAGS_write_coco_json, + FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; // Configure wrapper opWrapper.configure(wrapperStructPose, wrapperStructFace, wrapperStructHand, wrapperStructInput, diff --git a/examples/tutorial_wrapper/2_user_synchronous.cpp b/examples/tutorial_wrapper/2_user_synchronous.cpp index 293999ffb1076e0ac42737ac80ef11d789dd890b..e58140b2859736fa27998b2aa02ace1fe17d6672 100644 --- a/examples/tutorial_wrapper/2_user_synchronous.cpp +++ b/examples/tutorial_wrapper/2_user_synchronous.cpp @@ -69,18 +69,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -142,16 +150,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); // If the user needs his own variables, he can inherit the op::Datum struct and add them @@ -367,6 +379,11 @@ int openPoseTutorialWrapper2() const auto handNetInputSize = op::flagsToPoint(FLAGS_hand_net_resolution, "368x368 (multiples of 16)"); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -403,8 +420,8 @@ int openPoseTutorialWrapper2() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -421,8 +438,9 @@ int openPoseTutorialWrapper2() const bool guiVerbose = false; const bool fullScreen = false; const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_keypoint, - op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_keypoint_json, - FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, + op::stringToDataFormat(FLAGS_write_keypoint_format), + writeJson, FLAGS_write_coco_json, + FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; // Configure wrapper opWrapper.configure(wrapperStructPose, wrapperStructFace, wrapperStructHand, op::WrapperStructInput{}, diff --git a/examples/tutorial_wrapper/3_user_asynchronous.cpp b/examples/tutorial_wrapper/3_user_asynchronous.cpp index 791f8c2a8dfc09d480b980b4c4612801f12c434b..6aea73839137ad823fc1273683139a18e3cc1292 100644 --- a/examples/tutorial_wrapper/3_user_asynchronous.cpp +++ b/examples/tutorial_wrapper/3_user_asynchronous.cpp @@ -69,18 +69,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, false, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -142,16 +150,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); // If the user needs his own variables, he can inherit the op::Datum struct and add them @@ -325,6 +337,11 @@ int openPoseTutorialWrapper3() const auto handNetInputSize = op::flagsToPoint(FLAGS_hand_net_resolution, "368x368 (multiples of 16)"); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -344,8 +361,8 @@ int openPoseTutorialWrapper3() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -362,8 +379,9 @@ int openPoseTutorialWrapper3() const bool guiVerbose = false; const bool fullScreen = false; const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_keypoint, - op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_keypoint_json, - FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, + op::stringToDataFormat(FLAGS_write_keypoint_format), + writeJson, FLAGS_write_coco_json, + FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; // Configure wrapper op::log("Configuring OpenPose wrapper.", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); diff --git a/examples_beta/openpose3d/openpose3d.cpp b/examples_beta/openpose3d/openpose3d.cpp index 64ed973f50dd4e7cb26e0a7ad47fee19eea10f5f..088544a00e7722229f16c70613beaeee8ba4a49c 100644 --- a/examples_beta/openpose3d/openpose3d.cpp +++ b/examples_beta/openpose3d/openpose3d.cpp @@ -69,18 +69,26 @@ DEFINE_int32(scale_number, 1, "Number of scales to ave DEFINE_double(scale_gap, 0.3, "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1." " If you want to change the initial scale, you actually want to multiply the" " `net_resolution` by your desired initial scale."); -// OpenPose Body Pose Heatmaps -DEFINE_bool(heatmaps_add_parts, false, "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array," - " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps" - " (program speed will decrease). Not required for our library, enable it only if you intend" - " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it" - " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the" - " order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`."); +// OpenPose Body Pose Heatmaps and Part Candidates +DEFINE_bool(heatmaps_add_parts, false, "If true, it will fill op::Datum::poseHeatMaps array with the body part heatmaps, and" + " analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps." + " If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential" + " memory order: body parts + bkg + PAFs. It will follow the order on" + " POSE_BODY_PART_MAPPING in `src/openpose/pose/poseParameters.cpp`. Program speed will" + " considerably decrease. Not required for OpenPose, enable it only if you intend to" + " explicitly use this information later."); DEFINE_bool(heatmaps_add_bkg, false, "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to" " background."); DEFINE_bool(heatmaps_add_PAFs, false, "Same functionality as `add_heatmaps_parts`, but adding the PAFs."); DEFINE_int32(heatmaps_scale, 2, "Set 0 to scale op::Datum::poseHeatMaps in the range [-1,1], 1 for [0,1]; 2 for integer" " rounded [0,255]; and 3 for no scaling."); +DEFINE_bool(part_candidates, false, "Also enable `write_json` in order to save this information. If true, it will fill the" + " op::Datum::poseCandidates array with the body part candidates. Candidates refer to all" + " the detected body parts, before being assembled into people. Note that the number of" + " candidates is equal or higher than the number of final body parts (i.e. after being" + " assembled into people). The empty body parts are filled with 0s. Program speed will" + " slightly decrease. Not required for OpenPose, enable it only if you intend to explicitly" + " use this information."); // OpenPose Face DEFINE_bool(face, true, "Enables face keypoint detection. It will share some parameters from the body pose, e.g." " `model_folder`. Note that this will considerable slow down the performance and increse" @@ -143,16 +151,20 @@ DEFINE_string(write_images_format, "png", "File extension and form " function cv::imwrite for all compatible extensions."); DEFINE_string(write_video, "", "Full file path to write rendered frames in motion JPEG video format. It might fail if the" " final path does not finish in `.avi`. It internally uses cv::VideoWriter."); -DEFINE_string(write_keypoint, "", "Directory to write the people body pose keypoint data. Set format with `write_keypoint_format`."); -DEFINE_string(write_keypoint_format, "yml", "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available" - " for OpenCV < 3.0, use `write_keypoint_json` instead."); -DEFINE_string(write_keypoint_json, "", "Directory to write people pose data in *.json format, compatible with any OpenCV version."); -DEFINE_string(write_coco_json, "", "Full file path to write people pose data with *.json COCO validation format."); -DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag" +DEFINE_string(write_json, "", "Directory to write OpenPose output in JSON format. It includes body, hand, and face pose" + " keypoints, as well as pose candidates (if `--part_candidates` enabled)."); +DEFINE_string(write_coco_json, "", "Full file path to write people pose data with JSON COCO validation format."); +DEFINE_string(write_heatmaps, "", "Directory to write body pose heatmaps in PNG format. At least 1 `add_heatmaps_X` flag" " must be enabled."); DEFINE_string(write_heatmaps_format, "png", "File extension and format for `write_heatmaps`, analogous to `write_images_format`." " For lossless compression, recommended `png` for integer `heatmaps_scale` and `float` for" " floating values."); +DEFINE_string(write_keypoint, "", "(Deprecated, use `write_json`) Directory to write the people pose keypoint data. Set format" + " with `write_keypoint_format`."); +DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write_json`) File extension and format for `write_keypoint`: json, xml," + " yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead."); +DEFINE_string(write_keypoint_json, "", "(Deprecated, use `write_json`) Directory to write people pose data in JSON format," + " compatible with any OpenCV version."); int openpose3d() { @@ -176,6 +188,11 @@ int openpose3d() const auto handNetInputSize = op::flagsToPoint(FLAGS_hand_net_resolution, "368x368 (multiples of 16)"); // poseModel const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + const auto writeJson = (!FLAGS_write_json.empty() ? FLAGS_write_json : FLAGS_write_keypoint_json); + if (!FLAGS_write_keypoint.empty() || !FLAGS_write_keypoint_json.empty()) + op::log("Flags `write_keypoint` and `write_keypoint_json` are deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); // keypointScale const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); // heatmaps to add @@ -211,8 +228,8 @@ int openpose3d() (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose), poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, FLAGS_part_to_show, FLAGS_model_folder, - heatMapTypes, heatMapScale, (float)FLAGS_render_threshold, - enableGoogleLogging}; + heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, enableGoogleLogging}; // Face configuration (use op::WrapperStructFace{} to disable it) const op::WrapperStructFace wrapperStructFace{FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, FLAGS_render_pose), @@ -229,8 +246,9 @@ int openpose3d() const bool guiVerbose = true; const bool fullScreen = false; const op::WrapperStructOutput wrapperStructOutput{displayGui, guiVerbose, fullScreen, FLAGS_write_keypoint, - op::stringToDataFormat(FLAGS_write_keypoint_format), FLAGS_write_keypoint_json, - FLAGS_write_coco_json, FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, + op::stringToDataFormat(FLAGS_write_keypoint_format), + writeJson, FLAGS_write_coco_json, + FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_heatmaps, FLAGS_write_heatmaps_format}; // Configure wrapper opWrapper.configure(wrapperStructPose, wrapperStructFace, wrapperStructHand, op::WrapperStructInput{}, diff --git a/include/openpose/core/datum.hpp b/include/openpose/core/datum.hpp index 92f7971d114d3e533d181847e938e50646beb4a5..b184c0e33201106d61166c26a7030781e48cd0ce 100644 --- a/include/openpose/core/datum.hpp +++ b/include/openpose/core/datum.hpp @@ -85,7 +85,7 @@ namespace op /** * Body pose heatmaps (body parts, background and/or PAFs) for the whole image. - * This parameters is by default empty and disabled for performance. Each group (body parts, background and + * This parameter is by default empty and disabled for performance. Each group (body parts, background and * PAFs) can be individually enabled. * #heatmaps = #body parts (if enabled) + 1 (if background enabled) + 2 x #PAFs (if enabled). Each PAF has 2 * consecutive channels, one for x- and one for y-coordinates. @@ -97,6 +97,17 @@ namespace op */ Array poseHeatMaps; + /** + * Body pose candidates for the whole image. + * This parameter is by default empty and disabled for performance. It can be enabled with `candidates_body`. + * Candidates refer to all the detected body parts, before being assembled into people. Note that the number + * of candidates is equal or higher than the number of body parts after being assembled into people. + * Size: #body parts x min(part candidates, POSE_MAX_PEOPLE) x 3 (x,y,score). + * Rather than vector, it should ideally be: + * std::array>, #BP> poseCandidates; + */ + std::vector>> poseCandidates; + /** * Face detection locations (x,y,width,height) for each person in the image. * It is resized to cvInputData.size(). @@ -140,7 +151,7 @@ namespace op */ std::array, 2> handHeatMaps; - // ---------------------------------------- Other parameters ---------------------------------------- // + // ---------------------------------------- Other (internal) parameters ---------------------------------------- // /** * Scale ratio between the input Datum::cvInputData and the net input size. */ diff --git a/include/openpose/filestream/fileStream.hpp b/include/openpose/filestream/fileStream.hpp index 7b058bab1f2d0617cb96c2ff113a205bd91b6ec6..d80ed4ea57e93fb2510e15f823377d7747be17ef 100644 --- a/include/openpose/filestream/fileStream.hpp +++ b/include/openpose/filestream/fileStream.hpp @@ -36,12 +36,15 @@ namespace op const DataFormat format); // Json - Saving as *.json not available in OpenCV verions < 3.0, this function is a quick fix - OP_API void saveKeypointsJson(const Array& keypoints, const std::string& keypointName, - const std::string& fileName, const bool humanReadable); + OP_API void savePeopleJson(const Array& keypoints, + const std::vector>>& candidates, + const std::string& keypointName, const std::string& fileName, + const bool humanReadable); // It will save a bunch of Array elements - OP_API void saveKeypointsJson(const std::vector, std::string>>& keypointVector, - const std::string& fileName, const bool humanReadable); + OP_API void savePeopleJson(const std::vector, std::string>>& keypointVector, + const std::vector>>& candidates, + const std::string& fileName, const bool humanReadable); // Save/load image OP_API void saveImage(const cv::Mat& cvMat, const std::string& fullFilePath, diff --git a/include/openpose/filestream/headers.hpp b/include/openpose/filestream/headers.hpp index ecb821190a4040371b59c84ffdaf36a033868447..0b12393a87fd079815f05b8c279fada1d00558f3 100644 --- a/include/openpose/filestream/headers.hpp +++ b/include/openpose/filestream/headers.hpp @@ -9,15 +9,15 @@ #include #include #include -#include #include +#include #include #include #include #include #include #include -#include +#include #include #include diff --git a/include/openpose/filestream/jsonOfstream.hpp b/include/openpose/filestream/jsonOfstream.hpp index c4530d4c5e1eec736f0097c1f05fdd99d8dde2d0..0b6dbde40323569f4b4b3316e57eccef43679bcc 100644 --- a/include/openpose/filestream/jsonOfstream.hpp +++ b/include/openpose/filestream/jsonOfstream.hpp @@ -21,6 +21,8 @@ namespace op void arrayClose(); + void version(const std::string& version); + void key(const std::string& string); template diff --git a/include/openpose/filestream/keypointJsonSaver.hpp b/include/openpose/filestream/keypointJsonSaver.hpp deleted file mode 100644 index 8f84a8729d3439885b62c65efee7a23b964f920e..0000000000000000000000000000000000000000 --- a/include/openpose/filestream/keypointJsonSaver.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef OPENPOSE_FILESTREAM_KEYPOINT_JSON_SAVER_HPP -#define OPENPOSE_FILESTREAM_KEYPOINT_JSON_SAVER_HPP - -#include -#include - -namespace op -{ - class OP_API KeypointJsonSaver : public FileSaver - { - public: - KeypointJsonSaver(const std::string& directoryPath); - - void save(const std::vector, std::string>>& keypointVector, - const std::string& fileName, const bool humanReadable = true) const; - }; -} - -#endif // OPENPOSE_FILESTREAM_KEYPOINT_JSON_SAVER_HPP diff --git a/include/openpose/filestream/peopleJsonSaver.hpp b/include/openpose/filestream/peopleJsonSaver.hpp new file mode 100644 index 0000000000000000000000000000000000000000..4f07d1b630da52a33e08b1e601683459d502aa14 --- /dev/null +++ b/include/openpose/filestream/peopleJsonSaver.hpp @@ -0,0 +1,20 @@ +#ifndef OPENPOSE_FILESTREAM_PEOPLE_JSON_SAVER_HPP +#define OPENPOSE_FILESTREAM_PEOPLE_JSON_SAVER_HPP + +#include +#include + +namespace op +{ + class OP_API PeopleJsonSaver : public FileSaver + { + public: + PeopleJsonSaver(const std::string& directoryPath); + + void save(const std::vector, std::string>>& keypointVector, + const std::vector>>& candidates, + const std::string& fileName, const bool humanReadable = true) const; + }; +} + +#endif // OPENPOSE_FILESTREAM_PEOPLE_JSON_SAVER_HPP diff --git a/include/openpose/filestream/wKeypointJsonSaver.hpp b/include/openpose/filestream/wPeopleJsonSaver.hpp similarity index 72% rename from include/openpose/filestream/wKeypointJsonSaver.hpp rename to include/openpose/filestream/wPeopleJsonSaver.hpp index b0394dd92d991035f3f1f055944f7afc562ae4c4..a3be982e608bb7ea4031f96535c29a3cc10b3b84 100644 --- a/include/openpose/filestream/wKeypointJsonSaver.hpp +++ b/include/openpose/filestream/wPeopleJsonSaver.hpp @@ -1,26 +1,26 @@ -#ifndef OPENPOSE_FILESTREAM_W_KEYPOINT_JSON_SAVER_HPP -#define OPENPOSE_FILESTREAM_W_KEYPOINT_JSON_SAVER_HPP +#ifndef OPENPOSE_FILESTREAM_W_PEOPLE_JSON_SAVER_HPP +#define OPENPOSE_FILESTREAM_W_PEOPLE_JSON_SAVER_HPP #include -#include +#include #include namespace op { template - class WKeypointJsonSaver : public WorkerConsumer + class WPeopleJsonSaver : public WorkerConsumer { public: - explicit WKeypointJsonSaver(const std::shared_ptr& keypointJsonSaver); + explicit WPeopleJsonSaver(const std::shared_ptr& peopleJsonSaver); void initializationOnThread(); void workConsumer(const TDatums& tDatums); private: - const std::shared_ptr spKeypointJsonSaver; + const std::shared_ptr spPeopleJsonSaver; - DELETE_COPY(WKeypointJsonSaver); + DELETE_COPY(WPeopleJsonSaver); }; } @@ -33,18 +33,18 @@ namespace op namespace op { template - WKeypointJsonSaver::WKeypointJsonSaver(const std::shared_ptr& keypointJsonSaver) : - spKeypointJsonSaver{keypointJsonSaver} + WPeopleJsonSaver::WPeopleJsonSaver(const std::shared_ptr& peopleJsonSaver) : + spPeopleJsonSaver{peopleJsonSaver} { } template - void WKeypointJsonSaver::initializationOnThread() + void WPeopleJsonSaver::initializationOnThread() { } template - void WKeypointJsonSaver::workConsumer(const TDatums& tDatums) + void WPeopleJsonSaver::workConsumer(const TDatums& tDatums) { try { @@ -72,7 +72,7 @@ namespace op std::make_pair(tDatum.handKeypoints[1], "hand_right_keypoints") }; // Save keypoints - spKeypointJsonSaver->save(keypointVector, fileName, humanReadable); + spPeopleJsonSaver->save(keypointVector, tDatum.poseCandidates, fileName, humanReadable); } // Profiling speed Profiler::timerEnd(profilerKey); @@ -88,7 +88,7 @@ namespace op } } - COMPILE_TEMPLATE_DATUM(WKeypointJsonSaver); + COMPILE_TEMPLATE_DATUM(WPeopleJsonSaver); } -#endif // OPENPOSE_FILESTREAM_W_KEYPOINT_JSON_SAVER_HPP +#endif // OPENPOSE_FILESTREAM_W_PEOPLE_JSON_SAVER_HPP diff --git a/include/openpose/pose/poseExtractor.hpp b/include/openpose/pose/poseExtractor.hpp index 8cc1c3d606ed156d4e2aca124220b415252973b3..ddc5483cea80704e2f820772140e93121e6a9ffd 100644 --- a/include/openpose/pose/poseExtractor.hpp +++ b/include/openpose/pose/poseExtractor.hpp @@ -14,7 +14,8 @@ namespace op public: PoseExtractor(const PoseModel poseModel, const std::vector& heatMapTypes = {}, - const ScaleMode heatMapScale = ScaleMode::ZeroToOne); + const ScaleMode heatMapScale = ScaleMode::ZeroToOne, + const bool addPartCandidates = false); virtual ~PoseExtractor(); @@ -23,13 +24,19 @@ namespace op virtual void forwardPass(const std::vector>& inputNetData, const Point& inputDataSize, const std::vector& scaleRatios = {1.f}) = 0; + virtual const float* getCandidatesCpuConstPtr() const = 0; + + virtual const float* getCandidatesGpuConstPtr() const = 0; + virtual const float* getHeatMapCpuConstPtr() const = 0; virtual const float* getHeatMapGpuConstPtr() const = 0; virtual std::vector getHeatMapSize() const = 0; - Array getHeatMaps() const; + Array getHeatMapsCopy() const; + + std::vector>> getCandidatesCopy() const; virtual const float* getPoseGpuConstPtr() const = 0; @@ -59,6 +66,7 @@ namespace op private: const std::vector mHeatMapTypes; const ScaleMode mHeatMapScaleMode; + const bool mAddPartCandidates; std::array, (int)PoseProperty::Size> mProperties; std::thread::id mThreadId; diff --git a/include/openpose/pose/poseExtractorCaffe.hpp b/include/openpose/pose/poseExtractorCaffe.hpp index f691ea68f1601dec4e1760a98d2d6c52034c6c09..fa5246880abbc826f43542c1bdf45037e390409c 100644 --- a/include/openpose/pose/poseExtractorCaffe.hpp +++ b/include/openpose/pose/poseExtractorCaffe.hpp @@ -13,6 +13,7 @@ namespace op PoseExtractorCaffe(const PoseModel poseModel, const std::string& modelFolder, const int gpuId, const std::vector& heatMapTypes = {}, const ScaleMode heatMapScale = ScaleMode::ZeroToOne, + const bool addPartCandidates = false, const bool enableGoogleLogging = true); virtual ~PoseExtractorCaffe(); @@ -22,6 +23,10 @@ namespace op void forwardPass(const std::vector>& inputNetData, const Point& inputDataSize, const std::vector& scaleInputToNetInputs = {1.f}); + const float* getCandidatesCpuConstPtr() const; + + const float* getCandidatesGpuConstPtr() const; + const float* getHeatMapCpuConstPtr() const; const float* getHeatMapGpuConstPtr() const; diff --git a/include/openpose/pose/wPoseExtractor.hpp b/include/openpose/pose/wPoseExtractor.hpp index 727b9984bece6fee47c2c5d5f1be2283112ad12f..88447e5cf81b5d52f5df4dd1ebda15f12028d56e 100644 --- a/include/openpose/pose/wPoseExtractor.hpp +++ b/include/openpose/pose/wPoseExtractor.hpp @@ -61,7 +61,8 @@ namespace op spPoseExtractor->forwardPass(tDatum.inputNetData, Point{tDatum.cvInputData.cols, tDatum.cvInputData.rows}, tDatum.scaleInputToNetInputs); - tDatum.poseHeatMaps = spPoseExtractor->getHeatMaps().clone(); + tDatum.poseCandidates = spPoseExtractor->getCandidatesCopy(); + tDatum.poseHeatMaps = spPoseExtractor->getHeatMapsCopy(); tDatum.poseKeypoints = spPoseExtractor->getPoseKeypoints().clone(); tDatum.poseScores = spPoseExtractor->getPoseScores().clone(); tDatum.scaleNetToOutput = spPoseExtractor->getScaleNetToOutput(); diff --git a/include/openpose/wrapper/wrapper.hpp b/include/openpose/wrapper/wrapper.hpp index 99e235545ff328978be66327f65b1117a9728cc1..4913037517c3bfda318b4335270c4254812c2668 100644 --- a/include/openpose/wrapper/wrapper.hpp +++ b/include/openpose/wrapper/wrapper.hpp @@ -456,14 +456,14 @@ namespace op __LINE__, __FUNCTION__, __FILE__); if (!renderOutput && (!wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty())) { - const auto message = "In order to save the rendered frames (`write_images` or `write_video`), you" - " cannot disable `render_pose`."; + const auto message = "In order to save the rendered frames (`--write_images` or `--write_video`), you" + " cannot disable `--render_pose`."; log(message, Priority::High); } if (!wrapperStructOutput.writeHeatMaps.empty() && wrapperStructPose.heatMapTypes.empty()) { - const auto message = "In order to save the heatmaps (`write_heatmaps`), you need to pick which heat" - " maps you want to save: `heatmaps_add_X` flags or fill the" + const auto message = "In order to save the heatmaps (`--write_heatmaps`), you need to pick which heat" + " maps you want to save: `--heatmaps_add_X` flags or fill the" " wrapperStructPose.heatMapTypes."; error(message, __LINE__, __FUNCTION__, __FILE__); } @@ -473,7 +473,7 @@ namespace op { const auto message = "In order to save the heatmaps, you must either set" " wrapperStructPose.heatMapScale to ScaleMode::UnsignedChar (i.e. range [0, 255])" - " or `write_heatmaps_format` to `float` to storage floating numbers in binary" + " or `--write_heatmaps_format` to `float` to storage floating numbers in binary" " mode."; error(message, __LINE__, __FUNCTION__, __FILE__); } @@ -486,30 +486,30 @@ namespace op }; const auto savingSomething = ( !wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty() - || !wrapperStructOutput.writeKeypoint.empty() || !wrapperStructOutput.writeKeypointJson.empty() + || !wrapperStructOutput.writeKeypoint.empty() || !wrapperStructOutput.writeJson.empty() || !wrapperStructOutput.writeCocoJson.empty() || !wrapperStructOutput.writeHeatMaps.empty() ); if (!wrapperStructOutput.displayGui && !savingSomething) { - const auto message = "No output is selected (`no_display`) and no results are generated (no" - " `write_X` flags enabled). Thus, no output would be generated." + const auto message = "No output is selected (`--no_display`) and no results are generated (no" + " `--write_X` flags enabled). Thus, no output would be generated." + additionalMessage; error(message, __LINE__, __FUNCTION__, __FILE__); } if (wrapperStructInput.framesRepeat && savingSomething) { - const auto message = "Frames repetition (`frames_repeat`) is enabled as well as some writing" - " function (`write_X`). This program would never stop recording the same" + const auto message = "Frames repetition (`--frames_repeat`) is enabled as well as some writing" + " function (`--write_X`). This program would never stop recording the same" " frames over and over. Please, disable repetition or remove writing."; error(message, __LINE__, __FUNCTION__, __FILE__); } // Warnings if ((wrapperStructOutput.displayGui && wrapperStructOutput.guiVerbose) && !renderOutput) { - const auto message = "No render is enabled (e.g. `render_pose 0`), so you might also want to" - " remove the display (set `no_display` or `no_gui_verbose`). If you simply" - " want to use OpenPose to record video/images without keypoints, you only" - " need to set `num_gpu 0`." + additionalMessage; + const auto message = "No render is enabled (e.g. `--render_pose 0`), so you might also want to" + " remove the display (set `--no_display` or `--no_gui_verbose`). If you" + " simply want to use OpenPose to record video/images without keypoints, you" + " only need to set `--num_gpu 0`." + additionalMessage; log(message, Priority::High); } if (wrapperStructInput.realTimeProcessing && savingSomething) @@ -541,8 +541,8 @@ namespace op // Get total number GPUs const auto totalGpuNumber = getGpuNumber(); if (totalGpuNumber <= gpuNumberStart) - error("Number of initial GPUs (`number_gpu_start`) must be lower than the total number of used" - " GPUs (`number_gpu`)", __LINE__, __FUNCTION__, __FILE__); + error("Number of initial GPUs (`--number_gpu_start`) must be lower than the total number of used" + " GPUs (`--number_gpu`)", __LINE__, __FUNCTION__, __FILE__); gpuNumber = totalGpuNumber - gpuNumberStart; // Reset initial GPU to 0 (we want them all) // Logging message @@ -554,7 +554,7 @@ namespace op // Proper format const auto writeImagesCleaned = formatAsDirectory(wrapperStructOutput.writeImages); const auto writeKeypointCleaned = formatAsDirectory(wrapperStructOutput.writeKeypoint); - const auto writeKeypointJsonCleaned = formatAsDirectory(wrapperStructOutput.writeKeypointJson); + const auto writeJsonCleaned = formatAsDirectory(wrapperStructOutput.writeJson); const auto writeHeatMapsCleaned = formatAsDirectory(wrapperStructOutput.writeHeatMaps); const auto modelFolder = formatAsDirectory(wrapperStructPose.modelFolder); @@ -623,7 +623,7 @@ namespace op poseExtractors.emplace_back(std::make_shared( wrapperStructPose.poseModel, modelFolder, gpuId + gpuNumberStart, wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale, - wrapperStructPose.enableGoogleLogging + wrapperStructPose.addPartCandidates, wrapperStructPose.enableGoogleLogging )); // Pose renderers @@ -875,11 +875,12 @@ namespace op if (wrapperStructHand.enable) mOutputWs.emplace_back(std::make_shared>(keypointSaver)); } - // Write people pose data on disk (json format) - if (!writeKeypointJsonCleaned.empty()) + // Write OpenPose output data on disk in json format (body/hand/face keypoints, body part locations if + // enabled, etc.) + if (!writeJsonCleaned.empty()) { - const auto keypointJsonSaver = std::make_shared(writeKeypointJsonCleaned); - mOutputWs.emplace_back(std::make_shared>(keypointJsonSaver)); + const auto peopleJsonSaver = std::make_shared(writeJsonCleaned); + mOutputWs.emplace_back(std::make_shared>(peopleJsonSaver)); } // Write people pose data on disk (COCO validation json format) if (!wrapperStructOutput.writeCocoJson.empty()) @@ -1239,7 +1240,7 @@ namespace op if (spWPoses.size() > 1) log("Multi-threading disabled, only 1 thread running. All GPUs have been disabled but the" " first one, which is defined by gpuNumberStart (e.g. in the OpenPose demo, it is set" - " with the `num_gpu_start` flag).", Priority::High); + " with the `--num_gpu_start` flag).", Priority::High); mThreadManager.add(mThreadId, spWPoses.at(0), queueIn, queueOut); } queueIn++; diff --git a/include/openpose/wrapper/wrapperStructOutput.hpp b/include/openpose/wrapper/wrapperStructOutput.hpp index 83c23fbfbaeda32827d054144a21559b8950dac9..854081e33aa70689a74a92e11c455837e2833fd4 100644 --- a/include/openpose/wrapper/wrapperStructOutput.hpp +++ b/include/openpose/wrapper/wrapperStructOutput.hpp @@ -18,13 +18,14 @@ namespace op bool displayGui; /** - * Whether to add some information to the frame (number of frame, number people detected, etc.) after it is saved on disk - * and before it is displayed and/or returned to the user. + * Whether to add some information to the frame (number of frame, number people detected, etc.) after it is + * saved on disk and before it is displayed and/or returned to the user. */ bool guiVerbose; /** - * Whether to display the OpenPose small integrated GUI on fullscreen mode. It can be changed by interacting with the GUI itself. + * Whether to display the OpenPose small integrated GUI on fullscreen mode. It can be changed by interacting + * with the GUI itself. */ bool fullScreen; @@ -43,10 +44,14 @@ namespace op DataFormat writeKeypointFormat; /** - * Pose (x, y, score) locations saving folder location in JSON format (e.g. useful when needed JSON but using OpenCV < 3.0). + * Directory to write OpenPose output in JSON format. * If it is empty (default), it is disabled. + * It includes: + * - `people` field with body, hand, and face pose keypoints in (x, y, score) format. + * - `part_candidates` field with body part candidates in (x, y, score) format (if enabled with + * `--part_candidates`). */ - std::string writeKeypointJson; + std::string writeJson; /** * Pose (x, y, score) locations saving folder location in JSON COCO validation format. @@ -93,10 +98,13 @@ namespace op * It has the recommended and default values we recommend for each element of the struct. * Since all the elements of the struct are public, they can also be manually filled. */ - WrapperStructOutput(const bool displayGui = false, const bool guiVerbose = false, const bool fullScreen = false, const std::string& writeKeypoint = "", - const DataFormat writeKeypointFormat = DataFormat::Xml, const std::string& writeKeypointJson = "", const std::string& writeCocoJson = "", - const std::string& writeImages = "", const std::string& writeImagesFormat = "", const std::string& writeVideo = "", - const std::string& writeHeatMaps = "", const std::string& writeHeatMapsFormat = ""); + WrapperStructOutput(const bool displayGui = false, const bool guiVerbose = false, + const bool fullScreen = false, const std::string& writeKeypoint = "", + const DataFormat writeKeypointFormat = DataFormat::Xml, + const std::string& writeJson = "", const std::string& writeCocoJson = "", + const std::string& writeImages = "", const std::string& writeImagesFormat = "", + const std::string& writeVideo = "", const std::string& writeHeatMaps = "", + const std::string& writeHeatMapsFormat = ""); }; } diff --git a/include/openpose/wrapper/wrapperStructPose.hpp b/include/openpose/wrapper/wrapperStructPose.hpp index e7646d583a1efd0b775e71440884c326047fee55..dbe7c4b0b1496cd5235d88dbbaa676d94cba7b8c 100644 --- a/include/openpose/wrapper/wrapperStructPose.hpp +++ b/include/openpose/wrapper/wrapperStructPose.hpp @@ -133,6 +133,12 @@ namespace op */ ScaleMode heatMapScale; + /** + * Whether to add the body part candidates. + * Candidates refer to all the detected body parts, before being assembled into people. + */ + bool addPartCandidates; + /** * Rendering threshold. Only estimated keypoints whose score confidences are higher than this value will be * rendered. Generally, a high threshold (> 0.5) will only render very clear body parts; while small thresholds @@ -170,7 +176,7 @@ namespace op const float alphaHeatMap = POSE_DEFAULT_ALPHA_HEAT_MAP, const int defaultPartToRender = 0, const std::string& modelFolder = "models/", const std::vector& heatMapTypes = {}, - const ScaleMode heatMapScale = ScaleMode::ZeroToOne, + const ScaleMode heatMapScale = ScaleMode::ZeroToOne, const bool addPartCandidates = false, const float renderThreshold = 0.05f, const bool enableGoogleLogging = true, const bool identification = false); }; diff --git a/src/openpose/core/datum.cpp b/src/openpose/core/datum.cpp index 52d5840fc453e33d994119b5630403745144f2ad..52d2776060943698bb76543c333f35875869735d 100644 --- a/src/openpose/core/datum.cpp +++ b/src/openpose/core/datum.cpp @@ -24,6 +24,7 @@ namespace op poseIds{datum.poseIds}, poseScores{datum.poseScores}, poseHeatMaps{datum.poseHeatMaps}, + poseCandidates{datum.poseCandidates}, faceRectangles{datum.faceRectangles}, faceKeypoints{datum.faceKeypoints}, handRectangles{datum.handRectangles}, @@ -55,6 +56,7 @@ namespace op poseIds = datum.poseIds, poseScores = datum.poseScores, poseHeatMaps = datum.poseHeatMaps, + poseCandidates = datum.poseCandidates, faceRectangles = datum.faceRectangles, faceKeypoints = datum.faceKeypoints, handRectangles = datum.handRectangles, @@ -97,6 +99,7 @@ namespace op std::swap(poseIds, datum.poseIds); std::swap(poseScores, datum.poseScores); std::swap(poseHeatMaps, datum.poseHeatMaps); + std::swap(poseCandidates, datum.poseCandidates); std::swap(faceRectangles, datum.faceRectangles); std::swap(faceKeypoints, datum.faceKeypoints); std::swap(handRectangles, datum.handRectangles); @@ -130,6 +133,7 @@ namespace op std::swap(poseIds, datum.poseIds); std::swap(poseScores, datum.poseScores); std::swap(poseHeatMaps, datum.poseHeatMaps); + std::swap(poseCandidates, datum.poseCandidates); std::swap(faceRectangles, datum.faceRectangles); std::swap(faceKeypoints, datum.faceKeypoints); std::swap(handRectangles, datum.handRectangles); @@ -173,6 +177,7 @@ namespace op datum.poseIds = poseIds.clone(); datum.poseScores = poseScores.clone(); datum.poseHeatMaps = poseHeatMaps.clone(); + datum.poseCandidates = poseCandidates; datum.faceRectangles = faceRectangles; datum.faceKeypoints = faceKeypoints.clone(); datum.handRectangles = datum.handRectangles; diff --git a/src/openpose/filestream/CMakeLists.txt b/src/openpose/filestream/CMakeLists.txt index 6bc5ae4e8e68c3736e9533fdc5667d6b183497af..92bdee8266b3be685d6cf55161bea153b6f913b7 100644 --- a/src/openpose/filestream/CMakeLists.txt +++ b/src/openpose/filestream/CMakeLists.txt @@ -5,8 +5,8 @@ set(SOURCES cocoJsonSaver.cpp heatMapSaver.cpp imageSaver.cpp jsonOfstream.cpp - keypointJsonSaver.cpp keypointSaver.cpp + peopleJsonSaver.cpp videoSaver.cpp) add_library(openpose_filestream ${SOURCES}) @@ -15,4 +15,4 @@ install(TARGETS openpose_filestream EXPORT OpenPose RUNTIME DESTINATION bin LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib/openpose) \ No newline at end of file + ARCHIVE DESTINATION lib/openpose) diff --git a/src/openpose/filestream/defineTemplates.cpp b/src/openpose/filestream/defineTemplates.cpp index 7be52cd5b8bf913f8fdd8e1fbbb6c8c4c73c8843..85769e3e0cc798795df88470de9715185621589c 100644 --- a/src/openpose/filestream/defineTemplates.cpp +++ b/src/openpose/filestream/defineTemplates.cpp @@ -7,7 +7,7 @@ namespace op DEFINE_TEMPLATE_DATUM(WHandSaver); DEFINE_TEMPLATE_DATUM(WHeatMapSaver); DEFINE_TEMPLATE_DATUM(WImageSaver); - DEFINE_TEMPLATE_DATUM(WKeypointJsonSaver); + DEFINE_TEMPLATE_DATUM(WPeopleJsonSaver); DEFINE_TEMPLATE_DATUM(WPoseSaver); DEFINE_TEMPLATE_DATUM(WVideoSaver); } diff --git a/src/openpose/filestream/fileStream.cpp b/src/openpose/filestream/fileStream.cpp index 60c5bc2b40f0ef05fc88c37ef3ddd0096e73a84d..64cac7ac20df964543c7c9436bba99af91fb0f7b 100644 --- a/src/openpose/filestream/fileStream.cpp +++ b/src/openpose/filestream/fileStream.cpp @@ -42,6 +42,109 @@ namespace op return fileNameNoExtension + "." + dataFormatToString(format); } + void addKeypointsToJson(JsonOfstream& jsonOfstream, + const std::vector, std::string>>& keypointVector) + { + try + { + // Security checks + for (const auto& keypointPair : keypointVector) + if (!keypointPair.first.empty() && keypointPair.first.getNumberDimensions() != 3 ) + error("keypointVector.getNumberDimensions() != 3.", __LINE__, __FUNCTION__, __FILE__); + // Add people keypoints + jsonOfstream.key("people"); + jsonOfstream.arrayOpen(); + // Ger max numberPeople + auto numberPeople = 0; + for (auto vectorIndex = 0u ; vectorIndex < keypointVector.size() ; vectorIndex++) + numberPeople = fastMax(numberPeople, keypointVector[vectorIndex].first.getSize(0)); + for (auto person = 0 ; person < numberPeople ; person++) + { + jsonOfstream.objectOpen(); + for (auto vectorIndex = 0u ; vectorIndex < keypointVector.size() ; vectorIndex++) + { + const auto& keypoints = keypointVector[vectorIndex].first; + const auto& keypointName = keypointVector[vectorIndex].second; + const auto numberBodyParts = keypoints.getSize(1); + jsonOfstream.key(keypointName); + jsonOfstream.arrayOpen(); + // Body parts + for (auto bodyPart = 0 ; bodyPart < numberBodyParts ; bodyPart++) + { + const auto finalIndex = 3*(person*numberBodyParts + bodyPart); + jsonOfstream.plainText(keypoints[finalIndex]); + jsonOfstream.comma(); + jsonOfstream.plainText(keypoints[finalIndex+1]); + jsonOfstream.comma(); + jsonOfstream.plainText(keypoints[finalIndex+2]); + if (bodyPart < numberBodyParts-1) + jsonOfstream.comma(); + } + jsonOfstream.arrayClose(); + if (vectorIndex < keypointVector.size()-1) + jsonOfstream.comma(); + } + jsonOfstream.objectClose(); + if (person < numberPeople-1) + { + jsonOfstream.comma(); + jsonOfstream.enter(); + } + } + // Close bodies array + jsonOfstream.arrayClose(); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void addCandidatesToJson(JsonOfstream& jsonOfstream, + const std::vector>>& candidates) + { + try + { + // Add body part candidates + jsonOfstream.key("part_candidates"); + jsonOfstream.arrayOpen(); + // Ger max numberParts + auto numberParts = candidates.size(); + jsonOfstream.objectOpen(); + for (auto part = 0u ; part < numberParts ; part++) + { + // Open array + jsonOfstream.key(std::to_string(part)); + jsonOfstream.arrayOpen(); + // Iterate over part candidates + const auto& partCandidates = candidates[part]; + const auto numberPartCandidates = partCandidates.size(); + // Body part candidates + for (auto bodyPart = 0u ; bodyPart < numberPartCandidates ; bodyPart++) + { + const auto& candidate = partCandidates[bodyPart]; + jsonOfstream.plainText(candidate[0]); + jsonOfstream.comma(); + jsonOfstream.plainText(candidate[1]); + jsonOfstream.comma(); + jsonOfstream.plainText(candidate[2]); + if (bodyPart < numberPartCandidates-1) + jsonOfstream.comma(); + } + jsonOfstream.arrayClose(); + if (part < numberParts-1) + jsonOfstream.comma(); + } + jsonOfstream.objectClose(); + // Close array + jsonOfstream.arrayClose(); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + @@ -171,13 +274,15 @@ namespace op } } - void saveKeypointsJson(const Array& keypoints, const std::string& keypointName, const std::string& fileName, - const bool humanReadable) + void savePeopleJson(const Array& keypoints, + const std::vector>>& candidates, + const std::string& keypointName, const std::string& fileName, + const bool humanReadable) { try { - saveKeypointsJson( - std::vector, std::string>>{std::make_pair(keypoints, keypointName)}, + savePeopleJson( + std::vector, std::string>>{std::make_pair(keypoints, keypointName)}, candidates, fileName, humanReadable ); } @@ -187,8 +292,9 @@ namespace op } } - void saveKeypointsJson(const std::vector, std::string>>& keypointVector, - const std::string& fileName, const bool humanReadable) + void savePeopleJson(const std::vector, std::string>>& keypointVector, + const std::vector>>& candidates, + const std::string& fileName, const bool humanReadable) { try { @@ -199,52 +305,17 @@ namespace op // Record frame on desired path JsonOfstream jsonOfstream{fileName, humanReadable}; jsonOfstream.objectOpen(); - // Version - jsonOfstream.key("version"); - jsonOfstream.plainText("1.0"); + // Add version + jsonOfstream.version("1.1"); jsonOfstream.comma(); - // Bodies - jsonOfstream.key("people"); - jsonOfstream.arrayOpen(); - // Ger max numberPeople - auto numberPeople = 0; - for (auto vectorIndex = 0u ; vectorIndex < keypointVector.size() ; vectorIndex++) - numberPeople = fastMax(numberPeople, keypointVector[vectorIndex].first.getSize(0)); - for (auto person = 0 ; person < numberPeople ; person++) + // Add people keypoints + addKeypointsToJson(jsonOfstream, keypointVector); + // Add body part candidates + if (!candidates.empty()) { - jsonOfstream.objectOpen(); - for (auto vectorIndex = 0u ; vectorIndex < keypointVector.size() ; vectorIndex++) - { - const auto& keypoints = keypointVector[vectorIndex].first; - const auto& keypointName = keypointVector[vectorIndex].second; - const auto numberBodyParts = keypoints.getSize(1); - jsonOfstream.key(keypointName); - jsonOfstream.arrayOpen(); - // Body parts - for (auto bodyPart = 0 ; bodyPart < numberBodyParts ; bodyPart++) - { - const auto finalIndex = 3*(person*numberBodyParts + bodyPart); - jsonOfstream.plainText(keypoints[finalIndex]); - jsonOfstream.comma(); - jsonOfstream.plainText(keypoints[finalIndex+1]); - jsonOfstream.comma(); - jsonOfstream.plainText(keypoints[finalIndex+2]); - if (bodyPart < numberBodyParts-1) - jsonOfstream.comma(); - } - jsonOfstream.arrayClose(); - if (vectorIndex < keypointVector.size()-1) - jsonOfstream.comma(); - } - jsonOfstream.objectClose(); - if (person < numberPeople-1) - { - jsonOfstream.comma(); - jsonOfstream.enter(); - } + jsonOfstream.comma(); + addCandidatesToJson(jsonOfstream, candidates); } - // Close array - jsonOfstream.arrayClose(); // Close object jsonOfstream.objectClose(); } diff --git a/src/openpose/filestream/jsonOfstream.cpp b/src/openpose/filestream/jsonOfstream.cpp index a3ee6044ad7fc3f3164b46d3508c57930c1e747e..009551b9765ed7c5ca256d3429821782ac3e01dc 100644 --- a/src/openpose/filestream/jsonOfstream.cpp +++ b/src/openpose/filestream/jsonOfstream.cpp @@ -116,6 +116,19 @@ namespace op } } + void JsonOfstream::version(const std::string& version) + { + try + { + key("version"); + plainText(version); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + void JsonOfstream::key(const std::string& string) { try diff --git a/src/openpose/filestream/keypointJsonSaver.cpp b/src/openpose/filestream/keypointJsonSaver.cpp deleted file mode 100644 index e98f7b93fca16801f1f73c8aa431ed6ee5f8ed16..0000000000000000000000000000000000000000 --- a/src/openpose/filestream/keypointJsonSaver.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include - -namespace op -{ - KeypointJsonSaver::KeypointJsonSaver(const std::string& directoryPath) : - FileSaver{directoryPath} - { - } - - void KeypointJsonSaver::save(const std::vector, std::string>>& keypointVector, - const std::string& fileName, const bool humanReadable) const - { - try - { - // Record json - const auto finalFileName = getNextFileName(fileName) + ".json"; - saveKeypointsJson(keypointVector, finalFileName, humanReadable); - } - catch (const std::exception& e) - { - error(e.what(), __LINE__, __FUNCTION__, __FILE__); - } - } -} diff --git a/src/openpose/filestream/peopleJsonSaver.cpp b/src/openpose/filestream/peopleJsonSaver.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fd2108a8f190f5313115f026946bc0a56ed1a706 --- /dev/null +++ b/src/openpose/filestream/peopleJsonSaver.cpp @@ -0,0 +1,26 @@ +#include +#include + +namespace op +{ + PeopleJsonSaver::PeopleJsonSaver(const std::string& directoryPath) : + FileSaver{directoryPath} + { + } + + void PeopleJsonSaver::save(const std::vector, std::string>>& keypointVector, + const std::vector>>& candidates, + const std::string& fileName, const bool humanReadable) const + { + try + { + // Record json + const auto finalFileName = getNextFileName(fileName) + ".json"; + savePeopleJson(keypointVector, candidates, finalFileName, humanReadable); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } +} diff --git a/src/openpose/pose/poseExtractor.cpp b/src/openpose/pose/poseExtractor.cpp index 4c34d3e2b1a9fea230dd114a97057f0fdae87002..0fc670cbaa36956111a2b57e6a21e693d8a44b90 100644 --- a/src/openpose/pose/poseExtractor.cpp +++ b/src/openpose/pose/poseExtractor.cpp @@ -44,11 +44,12 @@ namespace op } PoseExtractor::PoseExtractor(const PoseModel poseModel, const std::vector& heatMapTypes, - const ScaleMode heatMapScale) : + const ScaleMode heatMapScale, const bool addPartCandidates) : mPoseModel{poseModel}, mNetOutputSize{0,0}, mHeatMapTypes{heatMapTypes}, - mHeatMapScaleMode{heatMapScale} + mHeatMapScaleMode{heatMapScale}, + mAddPartCandidates{addPartCandidates} { try { @@ -93,7 +94,7 @@ namespace op } } - Array PoseExtractor::getHeatMaps() const + Array PoseExtractor::getHeatMapsCopy() const { try { @@ -215,6 +216,43 @@ namespace op } } + std::vector>> PoseExtractor::getCandidatesCopy() const + { + try + { + // Security check + checkThread(); + // Initialization + std::vector>> candidates; + // Fill candidates + if (mAddPartCandidates) + { + const auto numberBodyParts = getPoseNumberBodyParts(mPoseModel); + candidates.resize(numberBodyParts); + const auto peaksArea = (POSE_MAX_PEOPLE+1) * 3; + // Memory copy + const auto* candidatesCpuPtr = getCandidatesCpuConstPtr(); + for (auto part = 0u ; part < numberBodyParts ; part++) + { + const auto numberPartCandidates = candidatesCpuPtr[part*peaksArea]; + candidates[part].resize(numberPartCandidates); + const auto* partCandidatesPtr = &candidatesCpuPtr[part*peaksArea+3]; + for (auto candidate = 0 ; candidate < numberPartCandidates ; candidate++) + candidates[part][candidate] = {partCandidatesPtr[3*candidate], + partCandidatesPtr[3*candidate+1], + partCandidatesPtr[3*candidate+2]}; + } + } + // Return + return candidates; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return std::vector>>{}; + } + } + Array PoseExtractor::getPoseKeypoints() const { try diff --git a/src/openpose/pose/poseExtractorCaffe.cpp b/src/openpose/pose/poseExtractorCaffe.cpp index facdb179fe4011cdf76c8c21d0860ad340e01362..718bffa502b251069118bd4f56ee8a6530ec521d 100644 --- a/src/openpose/pose/poseExtractorCaffe.cpp +++ b/src/openpose/pose/poseExtractorCaffe.cpp @@ -131,8 +131,9 @@ namespace op PoseExtractorCaffe::PoseExtractorCaffe(const PoseModel poseModel, const std::string& modelFolder, const int gpuId, const std::vector& heatMapTypes, - const ScaleMode heatMapScale, const bool enableGoogleLogging) : - PoseExtractor{poseModel, heatMapTypes, heatMapScale} + const ScaleMode heatMapScale, const bool addPartCandidates, + const bool enableGoogleLogging) : + PoseExtractor{poseModel, heatMapTypes, heatMapScale, addPartCandidates} #ifdef USE_CAFFE , upImpl{new ImplPoseExtractorCaffe{poseModel, gpuId, modelFolder, enableGoogleLogging}} #endif @@ -148,6 +149,7 @@ namespace op UNUSED(gpuId); UNUSED(heatMapTypes); UNUSED(heatMapScale); + UNUSED(addPartCandidates); error("OpenPose must be compiled with the `USE_CAFFE` macro definition in order to use this" " functionality.", __LINE__, __FUNCTION__, __FILE__); #endif @@ -300,6 +302,42 @@ namespace op } } + const float* PoseExtractorCaffe::getCandidatesCpuConstPtr() const + { + try + { + #ifdef USE_CAFFE + checkThread(); + return upImpl->spPeaksBlob->cpu_data(); + #else + return nullptr; + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return nullptr; + } + } + + const float* PoseExtractorCaffe::getCandidatesGpuConstPtr() const + { + try + { + #ifdef USE_CAFFE + checkThread(); + return upImpl->spPeaksBlob->gpu_data(); + #else + return nullptr; + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + return nullptr; + } + } + const float* PoseExtractorCaffe::getHeatMapCpuConstPtr() const { try diff --git a/src/openpose/wrapper/wrapperStructOutput.cpp b/src/openpose/wrapper/wrapperStructOutput.cpp index 65636ae4b1021e6ada198a5805294b03f41a7a24..352552847ebc9ddd8a4dca623e232a6684843502 100644 --- a/src/openpose/wrapper/wrapperStructOutput.cpp +++ b/src/openpose/wrapper/wrapperStructOutput.cpp @@ -2,16 +2,18 @@ namespace op { - WrapperStructOutput::WrapperStructOutput(const bool displayGui_, const bool guiVerbose_, const bool fullScreen_, const std::string& writeKeypoint_, - const DataFormat writeKeypointFormat_, const std::string& writeKeypointJson_, const std::string& writeCocoJson_, - const std::string& writeImages_, const std::string& writeImagesFormat_, const std::string& writeVideo_, - const std::string& writeHeatMaps_, const std::string& writeHeatMapsFormat_) : + WrapperStructOutput::WrapperStructOutput(const bool displayGui_, const bool guiVerbose_, const bool fullScreen_, + const std::string& writeKeypoint_, const DataFormat writeKeypointFormat_, + const std::string& writeJson_, const std::string& writeCocoJson_, + const std::string& writeImages_, const std::string& writeImagesFormat_, + const std::string& writeVideo_, const std::string& writeHeatMaps_, + const std::string& writeHeatMapsFormat_) : displayGui{displayGui_}, guiVerbose{guiVerbose_}, fullScreen{fullScreen_}, writeKeypoint{writeKeypoint_}, writeKeypointFormat{writeKeypointFormat_}, - writeKeypointJson{writeKeypointJson_}, + writeJson{writeJson_}, writeCocoJson{writeCocoJson_}, writeImages{writeImages_}, writeImagesFormat{writeImagesFormat_}, diff --git a/src/openpose/wrapper/wrapperStructPose.cpp b/src/openpose/wrapper/wrapperStructPose.cpp index 7dec2ffc3220a3cd14ec23a25c4d08ce65ce135e..e78e434495e939901bebdde7e9f5a4290fa8b1c2 100644 --- a/src/openpose/wrapper/wrapperStructPose.cpp +++ b/src/openpose/wrapper/wrapperStructPose.cpp @@ -11,8 +11,9 @@ namespace op const float alphaHeatMap_, const int defaultPartToRender_, const std::string& modelFolder_, const std::vector& heatMapTypes_, - const ScaleMode heatMapScale_, const float renderThreshold_, - const bool enableGoogleLogging_, const bool identification_) : + const ScaleMode heatMapScale_, const bool addPartCandidates_, + const float renderThreshold_, const bool enableGoogleLogging_, + const bool identification_) : enable{enable_}, netInputSize{netInputSize_}, outputSize{outputSize_}, @@ -30,6 +31,7 @@ namespace op modelFolder{modelFolder_}, heatMapTypes{heatMapTypes_}, heatMapScale{heatMapScale_}, + addPartCandidates{addPartCandidates_}, renderThreshold{renderThreshold_}, enableGoogleLogging{enableGoogleLogging_}, identification{identification_} diff --git a/windows/OpenPose/OpenPose.vcxproj b/windows/OpenPose/OpenPose.vcxproj index 988fd2244bbd856dfc4c0c8943fffc1247671336..ec4d76cd1c507cf1f6eab0e5d90447e7efb19b0d 100644 --- a/windows/OpenPose/OpenPose.vcxproj +++ b/windows/OpenPose/OpenPose.vcxproj @@ -151,15 +151,15 @@ - + - + @@ -290,8 +290,8 @@ - + diff --git a/windows/OpenPose/OpenPose.vcxproj.filters b/windows/OpenPose/OpenPose.vcxproj.filters index 9fd2fe7abb2717fd37278e097b05f6bc6d7c1fe9..83ba5968dd74e3189e33f3fa727854b2529e6b5d 100644 --- a/windows/OpenPose/OpenPose.vcxproj.filters +++ b/windows/OpenPose/OpenPose.vcxproj.filters @@ -248,10 +248,10 @@ Header Files\filestream - + Header Files\filestream - + Header Files\filestream @@ -272,7 +272,7 @@ Header Files\filestream - + Header Files\filestream @@ -657,10 +657,10 @@ Source Files\filestream - + Source Files\filestream - + Source Files\filestream