Added hand and face heatmaps (#260)

* Exposing first hand heatmaps through handExtractor

* Hand extractor exposes heatmaps for both hands of all detected persons.

* Clean-up hand heatmap code

* Face heatmap extraction similar to the hand-heatmap code

* Update faceExtractor.hpp

* Update faceExtractor.hpp

* Update handExtractor.hpp

* Update faceExtractor.cpp

* Update handExtractor.cpp

* Face heatmaps added to wrapper

* Added hand heatmaps to wrapper

* Cleaned code

* Added description in doc

doc/demo_overview.md

@@ -147,7 +147,7 @@ Each flag is divided into flag name, default value, and description.
 - DEFINE_string(net_resolution,           "656x368",      "Multiples of 16. If it is increased, the accuracy potentially increases. If it is decreased, the speed increases. For maximum speed-accuracy balance, it should keep the closest aspect ratio possible to the images or videos to be processed. Using `-1` in any of the dimensions, OP will choose the optimal resolution depending on the other value introduced by the user. E.g. the default `-1x368` is equivalent to `656x368` in 16:9 videos, e.g. full HD (1980x1080) and HD (1280x720) resolutions.");
 - DEFINE_int32(scale_number,              1,              "Number of scales to average.");
 - DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1. If you want to change the initial scale, you actually want to multiply the `net_resolution` by your desired initial scale.");
-- DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array (program speed will decrease). Not required for our library, enable it only if you intend to process this information later. If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`.");
+- DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array, and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps (program speed will decrease). Not required for our library, enable it only if you intend to process this information later. If more than one `add_heatmaps_X` flag is enabled, it will place then in sequential memory order: body parts + bkg + PAFs. It will follow the order on POSE_BODY_PART_MAPPING in `include/openpose/pose/poseParameters.hpp`.");
 - DEFINE_bool(heatmaps_add_bkg,           false,          "Same functionality as `add_heatmaps_parts`, but adding the heatmap corresponding to background.");
 - DEFINE_bool(heatmaps_add_PAFs,          false,          "Same functionality as `add_heatmaps_parts`, but adding the PAFs.");
 
@@ -196,5 +196,5 @@ Each flag is divided into flag name, default value, and description.
 - DEFINE_string(write_keypoint_format,    "yml",          "File extension and format for `write_keypoint`: json, xml, yaml & yml. Json not available for OpenCV < 3.0, use `write_keypoint_json` instead.");
 - DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 - DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-- DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be enabled.");
+- DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be enabled.");
 - DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`. Recommended `png` or any compressed and lossless format.");

doc/release_notes.md

@@ -120,12 +120,15 @@ OpenPose Library - Release Notes
         1. FrameDisplayer accepts variable size images by rescaling every time a frame with bigger width or height is displayed (gui module).
         2. OpOutputToCvMat & GuiInfoAdder does not require to know the output size at construction time, deduced from each image.
         3. CvMatToOutput and Renderers allow to keep input resolution as output for images (core module).
-    3. COCO JSON file outputs 0 as score for non-detected keypoints.
-    4. Added example for OpenPose for user asynchronous output and cleaned all `tutorial_wrapper/` examples.
-    5. Added `-1` option for `net_resolution` in order to auto-select the best possible aspect ratio given the user input.
+    3. Face and hand keypoint detectors now can return each keypoint heatmap.
+    4. COCO JSON file outputs 0 as score for non-detected keypoints.
+    5. Added example for OpenPose for user asynchronous output and cleaned all `tutorial_wrapper/` examples.
+    6. Added `-1` option for `net_resolution` in order to auto-select the best possible aspect ratio given the user input.
 2. Functions or parameters renamed:
     1. OpenPose able to change its size and initial size:
         1. Flag `resolution` renamed as `output_resolution`.
         2. FrameDisplayer, GuiInfoAdder and Gui constructors arguments modified (gui module).
         3. OpOutputToCvMat constructor removed (core module).
         4. New Renders classes to split GpuRenderers from CpuRenderers.
+3. Main bugs fixed:
+    1. Ubuntu installer script now works even if Python pip was not installed previously.

examples/openpose/openpose.cpp

@@ -76,7 +76,8 @@ DEFINE_int32(scale_number,              1,              "Number of scales to ave
 DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1."
                                                         " If you want to change the initial scale, you actually want to multiply the"
                                                         " `net_resolution` by your desired initial scale.");
-DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array"
+DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array,"
+                                                        " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps"
                                                         " (program speed will decrease). Not required for our library, enable it only if you intend"
                                                         " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it"
                                                         " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the"
@@ -156,8 +157,8 @@ DEFINE_string(write_keypoint_format,    "yml",          "File extension and form
                                                         " for OpenCV < 3.0, use `write_keypoint_json` instead.");
 DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be"
-                                                        " enabled.");
+DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag"
+                                                        " must be enabled.");
 DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`."
                                                         " Recommended `png` or any compressed and lossless format.");
 

examples/tutorial_wrapper/1_user_asynchronous_output.cpp

@@ -76,7 +76,8 @@ DEFINE_int32(scale_number,              1,              "Number of scales to ave
 DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1."
                                                         " If you want to change the initial scale, you actually want to multiply the"
                                                         " `net_resolution` by your desired initial scale.");
-DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array"
+DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array,"
+                                                        " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps"
                                                         " (program speed will decrease). Not required for our library, enable it only if you intend"
                                                         " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it"
                                                         " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the"
@@ -153,8 +154,8 @@ DEFINE_string(write_keypoint_format,    "yml",          "File extension and form
                                                         " for OpenCV < 3.0, use `write_keypoint_json` instead.");
 DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be"
-                                                        " enabled.");
+DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag"
+                                                        " must be enabled.");
 DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`."
                                                         " Recommended `png` or any compressed and lossless format.");
 
@@ -222,6 +223,28 @@ public:
             op::log("Face keypoints: " + datumsPtr->at(0).faceKeypoints.toString());
             op::log("Left hand keypoints: " + datumsPtr->at(0).handKeypoints[0].toString());
             op::log("Right hand keypoints: " + datumsPtr->at(0).handKeypoints[1].toString());
+            // Heatmaps
+            const auto& poseHeatMaps = datumsPtr->at(0).poseHeatMaps;
+            if (!poseHeatMaps.empty())
+            {
+                op::log("Pose heatmaps size: [" + std::to_string(poseHeatMaps.getSize(0)) + ", "
+                        + std::to_string(poseHeatMaps.getSize(1)) + ", "
+                        + std::to_string(poseHeatMaps.getSize(2)) + "]");
+                const auto& faceHeatMaps = datumsPtr->at(0).faceHeatMaps;
+                op::log("Face heatmaps size: [" + std::to_string(faceHeatMaps.getSize(0)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(1)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(2)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(3)) + "]");
+                const auto& handHeatMaps = datumsPtr->at(0).handHeatMaps;
+                op::log("Left hand heatmaps size: [" + std::to_string(handHeatMaps[0].getSize(0)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(1)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(2)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(3)) + "]");
+                op::log("Right hand heatmaps size: [" + std::to_string(handHeatMaps[1].getSize(0)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(1)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(2)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(3)) + "]");
+            }
         }
         else
             op::log("Nullptr or empty datumsPtr found.", op::Priority::High, __LINE__, __FUNCTION__, __FILE__);

examples/tutorial_wrapper/2_user_synchronous.cpp

@@ -59,7 +59,8 @@ DEFINE_int32(scale_number,              1,              "Number of scales to ave
 DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1."
                                                         " If you want to change the initial scale, you actually want to multiply the"
                                                         " `net_resolution` by your desired initial scale.");
-DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array"
+DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array,"
+                                                        " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps"
                                                         " (program speed will decrease). Not required for our library, enable it only if you intend"
                                                         " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it"
                                                         " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the"
@@ -136,8 +137,8 @@ DEFINE_string(write_keypoint_format,    "yml",          "File extension and form
                                                         " for OpenCV < 3.0, use `write_keypoint_json` instead.");
 DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be"
-                                                        " enabled.");
+DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag"
+                                                        " must be enabled.");
 DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`."
                                                         " Recommended `png` or any compressed and lossless format.");
 
@@ -289,6 +290,28 @@ public:
                 op::log("Face keypoints: " + datumsPtr->at(0).faceKeypoints.toString());
                 op::log("Left hand keypoints: " + datumsPtr->at(0).handKeypoints[0].toString());
                 op::log("Right hand keypoints: " + datumsPtr->at(0).handKeypoints[1].toString());
+                // Heatmaps
+                const auto& poseHeatMaps = datumsPtr->at(0).poseHeatMaps;
+                if (!poseHeatMaps.empty())
+                {
+                    op::log("Pose heatmaps size: [" + std::to_string(poseHeatMaps.getSize(0)) + ", "
+                            + std::to_string(poseHeatMaps.getSize(1)) + ", "
+                            + std::to_string(poseHeatMaps.getSize(2)) + "]");
+                    const auto& faceHeatMaps = datumsPtr->at(0).faceHeatMaps;
+                    op::log("Face heatmaps size: [" + std::to_string(faceHeatMaps.getSize(0)) + ", "
+                            + std::to_string(faceHeatMaps.getSize(1)) + ", "
+                            + std::to_string(faceHeatMaps.getSize(2)) + ", "
+                            + std::to_string(faceHeatMaps.getSize(3)) + "]");
+                    const auto& handHeatMaps = datumsPtr->at(0).handHeatMaps;
+                    op::log("Left hand heatmaps size: [" + std::to_string(handHeatMaps[0].getSize(0)) + ", "
+                            + std::to_string(handHeatMaps[0].getSize(1)) + ", "
+                            + std::to_string(handHeatMaps[0].getSize(2)) + ", "
+                            + std::to_string(handHeatMaps[0].getSize(3)) + "]");
+                    op::log("Right hand heatmaps size: [" + std::to_string(handHeatMaps[1].getSize(0)) + ", "
+                            + std::to_string(handHeatMaps[1].getSize(1)) + ", "
+                            + std::to_string(handHeatMaps[1].getSize(2)) + ", "
+                            + std::to_string(handHeatMaps[1].getSize(3)) + "]");
+                }
 
                 // Display rendered output image
                 cv::imshow("User worker GUI", datumsPtr->at(0).cvOutputData);

examples/tutorial_wrapper/3_user_asynchronous.cpp

@@ -59,7 +59,8 @@ DEFINE_int32(scale_number,              1,              "Number of scales to ave
 DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1."
                                                         " If you want to change the initial scale, you actually want to multiply the"
                                                         " `net_resolution` by your desired initial scale.");
-DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array"
+DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array,"
+                                                        " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps"
                                                         " (program speed will decrease). Not required for our library, enable it only if you intend"
                                                         " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it"
                                                         " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the"
@@ -136,8 +137,8 @@ DEFINE_string(write_keypoint_format,    "yml",          "File extension and form
                                                         " for OpenCV < 3.0, use `write_keypoint_json` instead.");
 DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be"
-                                                        " enabled.");
+DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag"
+                                                        " must be enabled.");
 DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`."
                                                         " Recommended `png` or any compressed and lossless format.");
 
@@ -260,6 +261,28 @@ public:
             op::log("Face keypoints: " + datumsPtr->at(0).faceKeypoints.toString());
             op::log("Left hand keypoints: " + datumsPtr->at(0).handKeypoints[0].toString());
             op::log("Right hand keypoints: " + datumsPtr->at(0).handKeypoints[1].toString());
+            // Heatmaps
+            const auto& poseHeatMaps = datumsPtr->at(0).poseHeatMaps;
+            if (!poseHeatMaps.empty())
+            {
+                op::log("Pose heatmaps size: [" + std::to_string(poseHeatMaps.getSize(0)) + ", "
+                        + std::to_string(poseHeatMaps.getSize(1)) + ", "
+                        + std::to_string(poseHeatMaps.getSize(2)) + "]");
+                const auto& faceHeatMaps = datumsPtr->at(0).faceHeatMaps;
+                op::log("Face heatmaps size: [" + std::to_string(faceHeatMaps.getSize(0)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(1)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(2)) + ", "
+                        + std::to_string(faceHeatMaps.getSize(3)) + "]");
+                const auto& handHeatMaps = datumsPtr->at(0).handHeatMaps;
+                op::log("Left hand heatmaps size: [" + std::to_string(handHeatMaps[0].getSize(0)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(1)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(2)) + ", "
+                        + std::to_string(handHeatMaps[0].getSize(3)) + "]");
+                op::log("Right hand heatmaps size: [" + std::to_string(handHeatMaps[1].getSize(0)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(1)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(2)) + ", "
+                        + std::to_string(handHeatMaps[1].getSize(3)) + "]");
+            }
         }
         else
             op::log("Nullptr or empty datumsPtr found.", op::Priority::High, __LINE__, __FUNCTION__, __FILE__);

examples_beta/openpose3d/openpose3d.cpp

@@ -59,7 +59,8 @@ DEFINE_int32(scale_number,              1,              "Number of scales to ave
 DEFINE_double(scale_gap,                0.3,            "Scale gap between scales. No effect unless scale_number > 1. Initial scale is always 1."
                                                         " If you want to change the initial scale, you actually want to multiply the"
                                                         " `net_resolution` by your desired initial scale.");
-DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array"
+DEFINE_bool(heatmaps_add_parts,         false,          "If true, it will add the body part heatmaps to the final op::Datum::poseHeatMaps array,"
+                                                        " and analogously face & hand heatmaps to op::Datum::faceHeatMaps & op::Datum::handHeatMaps"
                                                         " (program speed will decrease). Not required for our library, enable it only if you intend"
                                                         " to process this information later. If more than one `add_heatmaps_X` flag is enabled, it"
                                                         " will place then in sequential memory order: body parts + bkg + PAFs. It will follow the"
@@ -136,8 +137,8 @@ DEFINE_string(write_keypoint_format,    "yml",          "File extension and form
                                                         " for OpenCV < 3.0, use `write_keypoint_json` instead.");
 DEFINE_string(write_keypoint_json,      "",             "Directory to write people pose data in *.json format, compatible with any OpenCV version.");
 DEFINE_string(write_coco_json,          "",             "Full file path to write people pose data with *.json COCO validation format.");
-DEFINE_string(write_heatmaps,           "",             "Directory to write heatmaps in *.png format. At least 1 `add_heatmaps_X` flag must be"
-                                                        " enabled.");
+DEFINE_string(write_heatmaps,           "",             "Directory to write body pose heatmaps in *.png format. At least 1 `add_heatmaps_X` flag"
+                                                        " must be enabled.");
 DEFINE_string(write_heatmaps_format,    "png",          "File extension and format for `write_heatmaps`, analogous to `write_images_format`."
                                                         " Recommended `png` or any compressed and lossless format.");
 

include/openpose/core/datum.hpp

@@ -82,6 +82,13 @@ namespace op
          */
         Array<float> faceKeypoints;
 
+        /**
+         * Face pose heatmaps (face parts and/or background) for the whole image.
+         * Analogous of bodyHeatMaps applied to face. However, there is no PAFs and the size is different.
+         * Size: #people x #face parts (70) x output_net_height x output_net_width
+         */
+        Array<float> faceHeatMaps;
+
         /**
          * Hand detection locations (x,y,width,height) for each person in the image.
          * It is resized to cvInputData.size().
@@ -93,10 +100,17 @@ namespace op
          * Hand keypoints (x,y,score) locations for each person in the image.
          * It has been resized to the same resolution as `poseKeypoints`.
          * handKeypoints[0] corresponds to left hands, and handKeypoints[1] to right ones.
-         * Size: #people x #hand parts (21) x 3 ((x,y) coordinates + score)
+         * Size each Array: #people x #hand parts (21) x 3 ((x,y) coordinates + score)
          */
         std::array<Array<float>, 2> handKeypoints;
 
+        /**
+         * Face pose heatmaps (hand parts and/or background) for the whole image.
+         * Analogous of faceHeatMaps applied to face.
+         * Size each Array: #people x #hand parts (21) x output_net_height x output_net_width
+         */
+        std::array<Array<float>, 2> handHeatMaps;
+
         // -------------------------------------------------- Other parameters -------------------------------------------------- //
         float scaleInputToOutput; /**< Scale ratio between the input Datum::cvInputData and the output Datum::cvOutputData. */
 

include/openpose/face/faceExtractor.hpp

@@ -8,28 +8,38 @@
 #include <openpose/core/maximumCaffe.hpp>
 #include <openpose/core/net.hpp>
 #include <openpose/core/resizeAndMergeCaffe.hpp>
+#include <openpose/core/enumClasses.hpp>
 
 namespace op
 {
     class OP_API FaceExtractor
     {
     public:
-        explicit FaceExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize, const std::string& modelFolder, const int gpuId);
+        explicit FaceExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize,
+                               const std::string& modelFolder, const int gpuId,
+                               const std::vector<HeatMapType>& heatMapTypes = {},
+                               const ScaleMode heatMapScale = ScaleMode::ZeroToOne);
 
         void initializationOnThread();
 
-        void forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData, const float scaleInputToOutput);
+        void forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData,
+                         const float scaleInputToOutput);
 
         Array<float> getFaceKeypoints() const;
 
+        Array<float> getHeatMaps() const;
+
     private:
         const Point<int> mNetOutputSize;
-        const Point<int> mOutputSize;
         std::shared_ptr<Net> spNet;
         std::shared_ptr<ResizeAndMergeCaffe<float>> spResizeAndMergeCaffe;
         std::shared_ptr<MaximumCaffe<float>> spMaximumCaffe;
         Array<float> mFaceImageCrop;
         Array<float> mFaceKeypoints;
+        // HeatMaps parameters
+        const ScaleMode mHeatMapScaleMode;
+        const std::vector<HeatMapType> mHeatMapTypes;
+        Array<float> mHeatMaps;
         // Init with thread
         boost::shared_ptr<caffe::Blob<float>> spCaffeNetOutputBlob;
         std::shared_ptr<caffe::Blob<float>> spHeatMapsBlob;

include/openpose/face/wFaceExtractor.hpp

@@ -59,7 +59,8 @@ namespace op
                 for (auto& tDatum : *tDatums)
                 {
                     spFaceExtractor->forwardPass(tDatum.faceRectangles, tDatum.cvInputData, tDatum.scaleInputToOutput);
-                    tDatum.faceKeypoints = spFaceExtractor->getFaceKeypoints();
+                    tDatum.faceHeatMaps = spFaceExtractor->getHeatMaps().clone();
+                    tDatum.faceKeypoints = spFaceExtractor->getFaceKeypoints().clone();
                 }
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);

include/openpose/filestream/wHeatMapSaver.hpp

@@ -56,15 +56,17 @@ namespace op
                 const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__);
                 // T* to T
                 auto& tDatumsNoPtr = *tDatums;
-                // Record image(s) on disk
+                // Record pose heatmap image(s) on disk
                 std::vector<Array<float>> poseHeatMaps(tDatumsNoPtr.size());
                 for (auto i = 0u; i < tDatumsNoPtr.size(); i++)
                     poseHeatMaps[i] = tDatumsNoPtr[i].poseHeatMaps;
-                const auto fileName = (!tDatumsNoPtr[0].name.empty() ? tDatumsNoPtr[0].name : std::to_string(tDatumsNoPtr[0].id));
+                const auto fileName = (!tDatumsNoPtr[0].name.empty()
+                                       ? tDatumsNoPtr[0].name : std::to_string(tDatumsNoPtr[0].id)) + "_pose_heatmaps";
                 spHeatMapSaver->saveHeatMaps(poseHeatMaps, fileName);
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);
-                Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, Profiler::DEFAULT_X);
+                Profiler::printAveragedTimeMsOnIterationX(profilerKey,
+                                                          __LINE__, __FUNCTION__, __FILE__, Profiler::DEFAULT_X);
                 // Debugging log
                 dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
             }

include/openpose/hand/handExtractor.hpp

@@ -4,6 +4,7 @@
 #include <atomic>
 #include <thread>
 #include <opencv2/core/core.hpp> // cv::Mat
+#include <openpose/core/enumClasses.hpp>
 #include <openpose/core/common.hpp>
 #include <openpose/core/maximumCaffe.hpp>
 #include <openpose/core/net.hpp>
@@ -23,41 +24,48 @@ namespace op
          * @param netOutputSize Size of the final results. At the moment, it must be equal than netOutputSize.
          * @param modelFolder Folder where the models are located.
          * @param gpuId The GPU index (0-based) which the deep net will use.
-         * @param numberScales Number of scales to run. The more scales, the slower it will be but possibly also more accurate.
+         * @param numberScales Number of scales to run. The more scales, the slower it will be but possibly also more
+         * accurate.
          * @param rangeScales The range between the smaller and bigger scale.
          */
-        explicit HandExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize, const std::string& modelFolder, const int gpuId,
-                               const unsigned short numberScales = 1, const float rangeScales = 0.4f);
+        explicit HandExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize,
+                               const std::string& modelFolder, const int gpuId,
+                               const unsigned short numberScales = 1, const float rangeScales = 0.4f,
+                               const std::vector<HeatMapType>& heatMapTypes = {},
+                               const ScaleMode heatMapScale = ScaleMode::ZeroToOne);
 
         /**
-         * This function must be call before using any other function. It must also be called inside the thread in which the functions are going
-         * to be used.
+         * This function must be call before using any other function. It must also be called inside the thread in
+         * which the functions are going to be used.
          */
         void initializationOnThread();
 
         /**
          * This function extracts the hand keypoints for each detected hand in the image.
-         * @param fpsMode handRectangles Location of the hands in the image. It is a length-variable std::vector, where each index corresponds to
-         * a different person in the image. Internally the std::vector, a std::array of 2 elements: index 0 and 1 for left and right hand
-         * respectively. Inside each array element, a op::Rectangle<float> (similar to cv::Rect for floating values) with the position of that hand
-         * (or 0,0,0,0 if some hand is missing, e.g. if a specific person has only half of the body inside the image).
+         * @param fpsMode handRectangles Location of the hands in the image. It is a length-variable std::vector, where
+         * each index corresponds to a different person in the image. Internally the std::vector, a std::array of 2
+         * elements: index 0 and 1 for left and right hand respectively. Inside each array element, a
+         * op::Rectangle<float> (similar to cv::Rect for floating values) with the position of that hand (or 0,0,0,0 if
+         * some hand is missing, e.g. if a specific person has only half of the body inside the image).
          * @param cvInputData Original image in cv::Mat format and BGR format.
-         * @param scaleInputToOutput Desired scale of the final keypoints. Set to 1 if the desired size is the cvInputData size.
+         * @param scaleInputToOutput Desired scale of the final keypoints. Set to 1 if the desired size is the
+         * cvInputData size.
          */
         void forwardPass(const std::vector<std::array<Rectangle<float>, 2>> handRectangles, const cv::Mat& cvInputData,
                          const float scaleInputToOutput);
 
         /**
-         * This function returns the hand keypoins. VERY IMPORTANT: use getHandKeypoints().clone() if the keypoints are going to be edited
-         * in a different thread.
-         * @return And std::array with all the left hand keypoints (index 0) and all the right ones (index 1). Each Array<float> follows the pose
-         * structure, i.e. the first dimension corresponds to all the people in the image, the second to each specific keypoint, and the third
-         * one to (x, y, score).
+         * This function returns the hand keypoins. VERY IMPORTANT: use getHandKeypoints().clone() if the keypoints are
+         * going to be edited in a different thread.
+         * @return And std::array with all the left hand keypoints (index 0) and all the right ones (index 1). Each
+         * Array<float> follows the pose structure, i.e. the first dimension corresponds to all the people in the
+         * image, the second to each specific keypoint, and the third one to (x, y, score).
          */
         std::array<Array<float>, 2> getHandKeypoints() const;
 
+        std::array<Array<float>, 2> getHeatMaps() const;
+
     private:
-        // const bool mMultiScaleDetection;
         const std::pair<unsigned short, float> mMultiScaleNumberAndRange;
         const Point<int> mNetOutputSize;
         std::shared_ptr<Net> spNet;
@@ -65,6 +73,10 @@ namespace op
         std::shared_ptr<MaximumCaffe<float>> spMaximumCaffe;
         Array<float> mHandImageCrop;
         std::array<Array<float>, 2> mHandKeypoints;
+        // HeatMaps parameters
+        const ScaleMode mHeatMapScaleMode;
+        const std::vector<HeatMapType> mHeatMapTypes;
+        std::array<Array<float>, 2> mHeatMaps;
         // Init with thread
         boost::shared_ptr<caffe::Blob<float>> spCaffeNetOutputBlob;
         std::shared_ptr<caffe::Blob<float>> spHeatMapsBlob;
@@ -73,7 +85,10 @@ namespace op
 
         void checkThread() const;
 
-        void detectHandKeypoints(Array<float>& handCurrent, const float scaleInputToOutput, const int person, const cv::Mat& affineMatrix);
+        void detectHandKeypoints(Array<float>& handCurrent, const float scaleInputToOutput, const int person,
+                                 const cv::Mat& affineMatrix);
+
+        Array<float> getHeatMapsFromLastPass() const;
 
         DELETE_COPY(HandExtractor);
     };

include/openpose/hand/wHandExtractor.hpp

@@ -59,7 +59,11 @@ namespace op
                 for (auto& tDatum : *tDatums)
                 {
                     spHandExtractor->forwardPass(tDatum.handRectangles, tDatum.cvInputData, tDatum.scaleInputToOutput);
-                    tDatum.handKeypoints = spHandExtractor->getHandKeypoints();
+                    for (auto hand = 0 ; hand < 2 ; hand++)
+                    {
+                        tDatum.handHeatMaps[hand] = spHandExtractor->getHeatMaps()[hand].clone();
+                        tDatum.handKeypoints[hand] = spHandExtractor->getHandKeypoints()[hand].clone();
+                    }
                 }
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);

include/openpose/pose/wPoseExtractor.hpp

@@ -59,8 +59,8 @@ namespace op
                 for (auto& tDatum : *tDatums)
                 {
                     spPoseExtractor->forwardPass(tDatum.inputNetData, Point<int>{tDatum.cvInputData.cols, tDatum.cvInputData.rows}, tDatum.scaleRatios);
-                    tDatum.poseHeatMaps = spPoseExtractor->getHeatMaps();
-                    tDatum.poseKeypoints = spPoseExtractor->getPoseKeypoints();
+                    tDatum.poseHeatMaps = spPoseExtractor->getHeatMaps().clone();
+                    tDatum.poseKeypoints = spPoseExtractor->getPoseKeypoints().clone();
                     tDatum.scaleNetToOutput = spPoseExtractor->getScaleNetToOutput();
                 }
                 // Profiling speed

include/openpose/wrapper/wrapper.hpp

@@ -666,7 +666,7 @@ namespace op
                     const auto netOutputSize = wrapperStructFace.netInputSize;
                     const auto faceExtractor = std::make_shared<FaceExtractor>(
                         wrapperStructFace.netInputSize, netOutputSize, wrapperStructPose.modelFolder,
-                        gpu + gpuNumberStart
+                        gpu + gpuNumberStart, wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale
                     );
                     spWPoses.at(gpu).emplace_back(std::make_shared<WFaceExtractor<TDatumsPtr>>(faceExtractor));
                 }
@@ -691,7 +691,8 @@ namespace op
                     const auto netOutputSize = wrapperStructHand.netInputSize;
                     const auto handExtractor = std::make_shared<HandExtractor>(
                         wrapperStructHand.netInputSize, netOutputSize, wrapperStructPose.modelFolder,
-                        gpu + gpuNumberStart, wrapperStructHand.scalesNumber, wrapperStructHand.scaleRange
+                        gpu + gpuNumberStart, wrapperStructHand.scalesNumber, wrapperStructHand.scaleRange,
+                        wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScale
                     );
                     spWPoses.at(gpu).emplace_back(std::make_shared<WHandExtractor<TDatumsPtr>>(handExtractor));
                     // If tracking

src/openpose/face/faceExtractor.cpp

@@ -6,23 +6,70 @@
 #include <openpose/utilities/fastMath.hpp>
 #include <openpose/utilities/openCv.hpp>
 #include <openpose/face/faceExtractor.hpp>
- 
+
 namespace op
 {
-    FaceExtractor::FaceExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize, const std::string& modelFolder,
-                                 const int gpuId) :
+    void updateFaceHeatMapsForPerson(Array<float>& heatMaps, const int person, const ScaleMode heatMapScaleMode,
+                                     const float* heatMapsGpuPtr)
+    {
+        try
+        {
+            // Copy memory
+            const auto channelOffset = heatMaps.getVolume(2, 3);
+            const auto volumeBodyParts = FACE_NUMBER_PARTS * channelOffset;
+            auto totalOffset = 0u;
+            auto* heatMapsPtr = &heatMaps.getPtr()[person*volumeBodyParts];
+            // Copy face parts
+            cudaMemcpy(heatMapsPtr, heatMapsGpuPtr, volumeBodyParts * sizeof(float), cudaMemcpyDeviceToHost);
+            // Change from [0,1] to [-1,1]
+            if (heatMapScaleMode == ScaleMode::PlusMinusOne)
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]) * 2.f - 1.f;
+            // [0, 255]
+            else if (heatMapScaleMode == ScaleMode::UnsignedChar)
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = (float)intRound(fastTruncate(heatMapsPtr[i]) * 255.f);
+            // Avoid values outside original range
+            else
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]);
+            totalOffset += (unsigned int)volumeBodyParts;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
+    }
+
+    FaceExtractor::FaceExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize,
+                                 const std::string& modelFolder, const int gpuId,
+                                 const std::vector<HeatMapType>& heatMapTypes, const ScaleMode heatMapScale) :
         mNetOutputSize{netOutputSize},
-        spNet{std::make_shared<NetCaffe>(std::array<int,4>{1, 3, mNetOutputSize.y, mNetOutputSize.x}, modelFolder + FACE_PROTOTXT,
-                                         modelFolder + FACE_TRAINED_MODEL, gpuId)},
+        spNet{std::make_shared<NetCaffe>(std::array<int,4>{1, 3, mNetOutputSize.y, mNetOutputSize.x},
+                                         modelFolder + FACE_PROTOTXT, modelFolder + FACE_TRAINED_MODEL, gpuId)},
         spResizeAndMergeCaffe{std::make_shared<ResizeAndMergeCaffe<float>>()},
         spMaximumCaffe{std::make_shared<MaximumCaffe<float>>()},
-        mFaceImageCrop{mNetOutputSize.area()*3}
+        mFaceImageCrop{mNetOutputSize.area()*3},
+        mHeatMapScaleMode{heatMapScale},
+        mHeatMapTypes{heatMapTypes}
     {
         try
         {
-            checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__);
-            checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__);
-            checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.", __LINE__, __FUNCTION__, __FILE__);
+            // Error check
+            if (mHeatMapScaleMode != ScaleMode::ZeroToOne && mHeatMapScaleMode != ScaleMode::PlusMinusOne
+                && mHeatMapScaleMode != ScaleMode::UnsignedChar)
+                error("The ScaleMode heatMapScale must be ZeroToOne, PlusMinusOne or UnsignedChar.",
+                      __LINE__, __FUNCTION__, __FILE__);
+            checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            // Warnings
+            if (!mHeatMapTypes.empty())
+                log("Note only keypoint heatmaps are available with face heatmaps (no background nor PAFs).",
+                    Priority::Low, __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
         {
@@ -34,27 +81,25 @@ namespace op
     {
         try
         {
+            // Logging
             log("Starting initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
- 
             // Get thread id
             mThreadId = {std::this_thread::get_id()};
- 
             // Caffe net
             spNet->initializationOnThread();
             spCaffeNetOutputBlob = ((NetCaffe*)spNet.get())->getOutputBlob();
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
- 
             // HeatMaps extractor blob and layer
             spHeatMapsBlob = {std::make_shared<caffe::Blob<float>>(1,1,1,1)};
             const bool mergeFirstDimension = true;
-            spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}, FACE_CCN_DECREASE_FACTOR, mergeFirstDimension);
+            spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()},
+                                            FACE_CCN_DECREASE_FACTOR, mergeFirstDimension);
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
- 
             // Pose extractor blob and layer
             spPeaksBlob = {std::make_shared<caffe::Blob<float>>(1,1,1,1)};
             spMaximumCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()});
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
- 
+            // Logging
             log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
@@ -63,7 +108,8 @@ namespace op
         }
     }
 
-    void FaceExtractor::forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData, const float scaleInputToOutput)
+    void FaceExtractor::forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData,
+                                    const float scaleInputToOutput)
     {
         try
         {
@@ -80,6 +126,10 @@ namespace op
                 const auto numberPeople = (int)faceRectangles.size();
                 mFaceKeypoints.reset({numberPeople, (int)FACE_NUMBER_PARTS, 3}, 0);
 
+                // HeatMaps: define size
+                if (!mHeatMapTypes.empty())
+                    mHeatMaps.reset({numberPeople, FACE_NUMBER_PARTS, mNetOutputSize.y, mNetOutputSize.x});
+
                 // // Debugging
                 // cv::Mat cvInputDataCopy = cvInputData.clone();
                 // Extract face keypoints for each person
@@ -101,7 +151,8 @@ namespace op
                         // log(std::to_string(cvInputData.cols) + " " + std::to_string(cvInputData.rows));
                         // cv::rectangle(cvInputDataCopy,
                         //               cv::Point{(int)faceRectangle.x, (int)faceRectangle.y},
-                        //               cv::Point{(int)faceRectangle.bottomRight().x, (int)faceRectangle.bottomRight().y},
+                        //               cv::Point{(int)faceRectangle.bottomRight().x,
+                        //                         (int)faceRectangle.bottomRight().y},
                         //               cv::Scalar{0,255,0}, 2);
                         // Resize and shift image to face rectangle positions
                         const auto faceSize = fastMax(faceRectangle.width, faceRectangle.height);
@@ -125,9 +176,10 @@ namespace op
 
                         // 1. Caffe deep network
                         auto* inputDataGpuPtr = spNet->getInputDataGpuPtr();
-                        cudaMemcpy(inputDataGpuPtr, mFaceImageCrop.getPtr(), mNetOutputSize.area() * 3 * sizeof(float), cudaMemcpyHostToDevice);
+                        cudaMemcpy(inputDataGpuPtr, mFaceImageCrop.getPtr(), mNetOutputSize.area() * 3 * sizeof(float),
+                                   cudaMemcpyHostToDevice);
                         spNet->forwardPass();
-     
+
                         // 2. Resize heat maps + merge different scales
                         #ifndef CPU_ONLY
                             spResizeAndMergeCaffe->Forward_gpu({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()});
@@ -135,7 +187,7 @@ namespace op
                         #else
                             spResizeAndMergeCaffe->Forward_cpu({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()});
                         #endif
-     
+
                         // 3. Get peaks by Non-Maximum Suppression
                         #ifndef CPU_ONLY
                             spMaximumCaffe->Forward_gpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()});
@@ -143,7 +195,7 @@ namespace op
                         #else
                             spMaximumCaffe->Forward_cpu({spHeatMapsBlob.get()}, {spPeaksBlob.get()});
                         #endif
-     
+
                         const auto* facePeaksPtr = spPeaksBlob->mutable_cpu_data();
                         for (auto part = 0 ; part < mFaceKeypoints.getSize(1) ; part++)
                         {
@@ -151,7 +203,8 @@ namespace op
                             const auto x = facePeaksPtr[xyIndex];
                             const auto y = facePeaksPtr[xyIndex + 1];
                             const auto score = facePeaksPtr[xyIndex + 2];
-                            const auto baseIndex = mFaceKeypoints.getSize(2) * (part + person * mFaceKeypoints.getSize(1));
+                            const auto baseIndex = mFaceKeypoints.getSize(2)
+                                                 * (part + person * mFaceKeypoints.getSize(1));
                             mFaceKeypoints[baseIndex] = (float)(scaleInputToOutput * (Mscaling.at<double>(0,0) * x
                                                                                       + Mscaling.at<double>(0,1) * y
                                                                                       + Mscaling.at<double>(0,2)));
@@ -160,6 +213,10 @@ namespace op
                                                                                       + Mscaling.at<double>(1,2)));
                             mFaceKeypoints[baseIndex+2] = score;
                         }
+                        // HeatMaps: storing
+                        if (!mHeatMapTypes.empty())
+                            updateFaceHeatMapsForPerson(mHeatMaps, person, mHeatMapScaleMode,
+                                                        spHeatMapsBlob->gpu_data());
                     }
                 }
                 // // Debugging
@@ -188,12 +245,27 @@ namespace op
         }
     }
 
+    Array<float> FaceExtractor::getHeatMaps() const
+    {
+        try
+        {
+            checkThread();
+            return mHeatMaps;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Array<float>{};
+        }
+    }
+
     void FaceExtractor::checkThread() const
     {
         try
         {
-            if(mThreadId != std::this_thread::get_id())
-                error("The CPU/GPU pointer data cannot be accessed from a different thread.", __LINE__, __FUNCTION__, __FILE__);
+            if (mThreadId != std::this_thread::get_id())
+                error("The CPU/GPU pointer data cannot be accessed from a different thread.",
+                      __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
         {

src/openpose/filestream/heatMapSaver.cpp

@@ -27,7 +27,7 @@ namespace op
             if (!heatMaps.empty())
             {
                 // File path (no extension)
-                const auto fileNameNoExtension = getNextFileName(fileName) + "_heatmaps";
+                const auto fileNameNoExtension = getNextFileName(fileName);
 
                 // Get names for each heatMap
                 std::vector<std::string> fileNames(heatMaps.size());

src/openpose/hand/handExtractor.cpp

@@ -8,11 +8,12 @@
 #include <openpose/utilities/keypoint.hpp>
 #include <openpose/utilities/openCv.hpp>
 #include <openpose/hand/handExtractor.hpp>
- 
+
 namespace op
 {
-    void cropFrame(Array<float>& handImageCrop, cv::Mat& affineMatrix, const cv::Mat& cvInputData, const Rectangle<float>& handRectangle,
-                   const int netInputSide, const Point<int>& netOutputSize, const bool mirrorImage)
+    void cropFrame(Array<float>& handImageCrop, cv::Mat& affineMatrix, const cv::Mat& cvInputData,
+                   const Rectangle<float>& handRectangle, const int netInputSide,
+                   const Point<int>& netOutputSize, const bool mirrorImage)
     {
         try
         {
@@ -42,8 +43,8 @@ namespace op
         }
     }
 
-    void connectKeypoints(Array<float>& handCurrent, const float scaleInputToOutput, const int person, const cv::Mat& affineMatrix,
-                          const float* handPeaks)
+    void connectKeypoints(Array<float>& handCurrent, const float scaleInputToOutput, const int person,
+                          const cv::Mat& affineMatrix, const float* handPeaks)
     {
         try
         {
@@ -55,9 +56,11 @@ namespace op
                 const auto y = handPeaks[xyIndex + 1];
                 const auto score = handPeaks[xyIndex + 2];
                 const auto baseIndex = handCurrent.getSize(2) * (part + person * handCurrent.getSize(1));
-                handCurrent[baseIndex] = (float)(scaleInputToOutput * (affineMatrix.at<double>(0,0)*x + affineMatrix.at<double>(0,1)*y
+                handCurrent[baseIndex] = (float)(scaleInputToOutput
+                                       * (affineMatrix.at<double>(0,0)*x + affineMatrix.at<double>(0,1)*y
                                                                        + affineMatrix.at<double>(0,2)));
-                handCurrent[baseIndex+1] = (float)(scaleInputToOutput * (affineMatrix.at<double>(1,0)*x + affineMatrix.at<double>(1,1)*y
+                handCurrent[baseIndex+1] = (float)(scaleInputToOutput
+                                         * (affineMatrix.at<double>(1,0)*x + affineMatrix.at<double>(1,1)*y
                                                                        + affineMatrix.at<double>(1,2)));
                 handCurrent[baseIndex+2] = score;
             }
@@ -92,22 +95,69 @@ namespace op
         }
     }
 
+    void updateHandHeatMapsForPerson(Array<float>& heatMaps, const int person, const ScaleMode heatMapScaleMode,
+                                     const float* heatMapsGpuPtr)
+    {
+        try
+        {
+            // Copy memory
+            const auto channelOffset = heatMaps.getVolume(2, 3);
+            const auto volumeBodyParts = HAND_NUMBER_PARTS * channelOffset;
+            auto totalOffset = 0u;
+            auto* heatMapsPtr = &heatMaps.getPtr()[person*volumeBodyParts];
+            // Copy hand parts
+            cudaMemcpy(heatMapsPtr, heatMapsGpuPtr, volumeBodyParts * sizeof(float), cudaMemcpyDeviceToHost);
+            // Change from [0,1] to [-1,1]
+            if (heatMapScaleMode == ScaleMode::PlusMinusOne)
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]) * 2.f - 1.f;
+            // [0, 255]
+            else if (heatMapScaleMode == ScaleMode::UnsignedChar)
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = (float)intRound(fastTruncate(heatMapsPtr[i]) * 255.f);
+            // Avoid values outside original range
+            else
+                for (auto i = 0u ; i < volumeBodyParts ; i++)
+                    heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]);
+            totalOffset += (unsigned int)volumeBodyParts;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
+    }
+
     HandExtractor::HandExtractor(const Point<int>& netInputSize, const Point<int>& netOutputSize,
                                  const std::string& modelFolder, const int gpuId, const unsigned short numberScales,
-                                 const float rangeScales) :
+                                 const float rangeScales, const std::vector<HeatMapType>& heatMapTypes,
+                                 const ScaleMode heatMapScale) :
         mMultiScaleNumberAndRange{std::make_pair(numberScales, rangeScales)},
         mNetOutputSize{netOutputSize},
-        spNet{std::make_shared<NetCaffe>(std::array<int,4>{1, 3, mNetOutputSize.y, mNetOutputSize.x}, modelFolder + HAND_PROTOTXT,
-                                         modelFolder + HAND_TRAINED_MODEL, gpuId)},
+        spNet{std::make_shared<NetCaffe>(std::array<int,4>{1, 3, mNetOutputSize.y, mNetOutputSize.x},
+                                         modelFolder + HAND_PROTOTXT, modelFolder + HAND_TRAINED_MODEL, gpuId)},
         spResizeAndMergeCaffe{std::make_shared<ResizeAndMergeCaffe<float>>()},
         spMaximumCaffe{std::make_shared<MaximumCaffe<float>>()},
-        mHandImageCrop{mNetOutputSize.area()*3}
+        mHandImageCrop{mNetOutputSize.area()*3},
+        mHeatMapScaleMode{heatMapScale},
+        mHeatMapTypes{heatMapTypes}
     {
         try
         {
-            checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__);
-            checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.", __LINE__, __FUNCTION__, __FILE__);
-            checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.", __LINE__, __FUNCTION__, __FILE__);
+            // Error check
+            if (mHeatMapScaleMode != ScaleMode::ZeroToOne && mHeatMapScaleMode != ScaleMode::PlusMinusOne
+                && mHeatMapScaleMode != ScaleMode::UnsignedChar)
+                error("The ScaleMode heatMapScale must be ZeroToOne, PlusMinusOne or UnsignedChar.",
+                      __LINE__, __FUNCTION__, __FILE__);
+            checkE(netOutputSize.x, netInputSize.x, "Net input and output size must be equal.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            checkE(netOutputSize.y, netInputSize.y, "Net input and output size must be equal.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            checkE(netInputSize.x, netInputSize.y, "Net input size must be squared.",
+                   __LINE__, __FUNCTION__, __FILE__);
+            // Warnings
+            if (!mHeatMapTypes.empty())
+                log("Note only keypoint heatmaps are available with face heatmaps (no background nor PAFs).",
+                    Priority::Low, __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
         {
@@ -119,27 +169,25 @@ namespace op
     {
         try
         {
+            // Logging
             log("Starting initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
-
             // Get thread id
             mThreadId = {std::this_thread::get_id()};
-
             // Caffe net
             spNet->initializationOnThread();
             spCaffeNetOutputBlob = ((NetCaffe*)spNet.get())->getOutputBlob();
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
-
             // HeatMaps extractor blob and layer
             spHeatMapsBlob = {std::make_shared<caffe::Blob<float>>(1,1,1,1)};
             const bool mergeFirstDimension = true;
-            spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()}, HAND_CCN_DECREASE_FACTOR, mergeFirstDimension);
+            spResizeAndMergeCaffe->Reshape({spCaffeNetOutputBlob.get()}, {spHeatMapsBlob.get()},
+                                            HAND_CCN_DECREASE_FACTOR, mergeFirstDimension);
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
-
             // Pose extractor blob and layer
             spPeaksBlob = {std::make_shared<caffe::Blob<float>>(1,1,1,1)};
             spMaximumCaffe->Reshape({spHeatMapsBlob.get()}, {spPeaksBlob.get()});
             cudaCheck(__LINE__, __FUNCTION__, __FILE__);
- 
+            // Logging
             log("Finished initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
@@ -148,7 +196,8 @@ namespace op
         }
     }
 
-    void HandExtractor::forwardPass(const std::vector<std::array<Rectangle<float>, 2>> handRectangles, const cv::Mat& cvInputData,
+    void HandExtractor::forwardPass(const std::vector<std::array<Rectangle<float>, 2>> handRectangles,
+                                    const cv::Mat& cvInputData,
                                     const float scaleInputToOutput)
     {
         try
@@ -167,6 +216,13 @@ namespace op
                 mHandKeypoints[0].reset({numberPeople, (int)HAND_NUMBER_PARTS, 3}, 0);
                 mHandKeypoints[1].reset(mHandKeypoints[0].getSize(), 0);
 
+                // HeatMaps: define size
+                if (!mHeatMapTypes.empty())
+                {
+                    mHeatMaps[0].reset({numberPeople, HAND_NUMBER_PARTS, mNetOutputSize.y, mNetOutputSize.x});
+                    mHeatMaps[1].reset({numberPeople, HAND_NUMBER_PARTS, mNetOutputSize.y, mNetOutputSize.x});
+                }
+
                 // // Debugging
                 // cv::Mat cvInputDataCopied = cvInputData.clone();
                 // Extract hand keypoints for each person
@@ -203,7 +259,8 @@ namespace op
                                 // Parameters
                                 cv::Mat affineMatrix;
                                 // Resize image to hands positions + cv::Mat -> float*
-                                cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangle, netInputSide, mNetOutputSize, mirrorImage);
+                                cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangle, netInputSide,
+                                          mNetOutputSize, mirrorImage);
                                 // Deep net + Estimate keypoint locations
                                 detectHandKeypoints(handCurrent, scaleInputToOutput, person, affineMatrix);
                             }
@@ -217,28 +274,41 @@ namespace op
                                 for (auto i = 0 ; i < numberScales ; i++)
                                 {
                                     // Get current scale
-                                    const auto scale = initScale + mMultiScaleNumberAndRange.second * i / (numberScales-1.f);
+                                    const auto scale = initScale
+                                                     + mMultiScaleNumberAndRange.second * i / (numberScales-1.f);
                                     // Process hand
                                     Array<float> handEstimated({1, handCurrent.getSize(1), handCurrent.getSize(2)}, 0);
-                                    const auto handRectangleScale = recenter(handRectangle,
-                                                                             (float)(intRound(handRectangle.width * scale) / 2 * 2),
-                                                                             (float)(intRound(handRectangle.height * scale) / 2 * 2));
+                                    const auto handRectangleScale = recenter(
+                                        handRectangle,
+                                        (float)(intRound(handRectangle.width * scale) / 2 * 2),
+                                        (float)(intRound(handRectangle.height * scale) / 2 * 2)
+                                    );
                                     // // Debugging -> blue rectangle
                                     // cv::rectangle(cvInputDataCopied,
-                                    //               cv::Point{intRound(handRectangleScale.x), intRound(handRectangleScale.y)},
-                                    //               cv::Point{intRound(handRectangleScale.x + handRectangleScale.width),
-                                    //                         intRound(handRectangleScale.y + handRectangleScale.height)},
+                                    //               cv::Point{intRound(handRectangleScale.x),
+                                    //                         intRound(handRectangleScale.y)},
+                                    //               cv::Point{intRound(handRectangleScale.x
+                                    //                                  + handRectangleScale.width),
+                                    //                         intRound(handRectangleScale.y
+                                    //                                  + handRectangleScale.height)},
                                     //               cv::Scalar{255,0,0}, 2);
                                     // Parameters
                                     cv::Mat affineMatrix;
                                     // Resize image to hands positions + cv::Mat -> float*
-                                    cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangleScale, netInputSide, mNetOutputSize, mirrorImage);
+                                    cropFrame(mHandImageCrop, affineMatrix, cvInputData, handRectangleScale,
+                                              netInputSide, mNetOutputSize, mirrorImage);
                                     // Deep net + Estimate keypoint locations
                                     detectHandKeypoints(handEstimated, scaleInputToOutput, 0, affineMatrix);
-                                    if (i == 0 || getAverageScore(handEstimated, 0) > getAverageScore(handCurrent, person))
-                                        std::copy(handEstimated.getConstPtr(), handEstimated.getConstPtr() + handPtrArea, handCurrentPtr);
+                                    if (i == 0
+                                        || getAverageScore(handEstimated, 0) > getAverageScore(handCurrent, person))
+                                        std::copy(handEstimated.getConstPtr(),
+                                                  handEstimated.getConstPtr() + handPtrArea, handCurrentPtr);
                                 }
                             }
+                            // HeatMaps: storing
+                            if (!mHeatMapTypes.empty())
+                                updateHandHeatMapsForPerson(mHeatMaps[hand], person, mHeatMapScaleMode,
+                                                        spHeatMapsBlob->gpu_data());
                         }
                     }
                 }
@@ -271,12 +341,27 @@ namespace op
         }
     }
 
+    std::array<Array<float>, 2> HandExtractor::getHeatMaps() const
+    {
+        try
+        {
+            checkThread();
+            return mHeatMaps;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return std::array<Array<float>, 2>(); // Parentheses instead of braces to avoid error in GCC 4.8
+        }
+    }
+
     void HandExtractor::checkThread() const
     {
         try
         {
-            if(mThreadId != std::this_thread::get_id())
-                error("The CPU/GPU pointer data cannot be accessed from a different thread.", __LINE__, __FUNCTION__, __FILE__);
+            if (mThreadId != std::this_thread::get_id())
+                error("The CPU/GPU pointer data cannot be accessed from a different thread.",
+                      __LINE__, __FUNCTION__, __FILE__);
         }
         catch (const std::exception& e)
         {
@@ -284,15 +369,16 @@ namespace op
         }
     }
 
-    void HandExtractor::detectHandKeypoints(Array<float>& handCurrent, const float scaleInputToOutput, const int person,
-                                            const cv::Mat& affineMatrix)
+    void HandExtractor::detectHandKeypoints(Array<float>& handCurrent, const float scaleInputToOutput,
+                                            const int person, const cv::Mat& affineMatrix)
     {
         try
         {
             // Deep net
             // 1. Caffe deep network
             auto* inputDataGpuPtr = spNet->getInputDataGpuPtr();
-            cudaMemcpy(inputDataGpuPtr, mHandImageCrop.getConstPtr(), mNetOutputSize.area() * 3 * sizeof(float), cudaMemcpyHostToDevice);
+            cudaMemcpy(inputDataGpuPtr, mHandImageCrop.getConstPtr(), mNetOutputSize.area() * 3 * sizeof(float),
+                       cudaMemcpyHostToDevice);
             spNet->forwardPass();
 
             // 2. Resize heat maps + merge different scales

src/openpose/pose/poseExtractor.cpp

@@ -76,11 +76,17 @@ namespace op
 
     void PoseExtractor::initializationOnThread()
     {
-        // Get thread id
-        mThreadId = {std::this_thread::get_id()};
-
-        // Deep net initialization
-        netInitializationOnThread();
+        try
+        {
+            // Get thread id
+            mThreadId = {std::this_thread::get_id()};
+            // Deep net initialization
+            netInitializationOnThread();
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+        }
     }
 
     Array<float> PoseExtractor::getHeatMaps() const
@@ -88,75 +94,75 @@ namespace op
         try
         {
             checkThread();
-            Array<float> poseHeatMaps;
+            Array<float> heatMaps;
             if (!mHeatMapTypes.empty())
             {
                 // Allocate memory
                 const auto numberHeatMapChannels = getNumberHeatMapChannels(mHeatMapTypes, mPoseModel);
-                poseHeatMaps.reset({numberHeatMapChannels, mNetOutputSize.y, mNetOutputSize.x});
+                heatMaps.reset({numberHeatMapChannels, mNetOutputSize.y, mNetOutputSize.x});
 
                 // Copy memory
-                const auto channelOffset = poseHeatMaps.getVolume(1, 2);
+                const auto channelOffset = heatMaps.getVolume(1, 2);
                 const auto volumeBodyParts = POSE_NUMBER_BODY_PARTS[(int)mPoseModel] * channelOffset;
                 const auto volumePAFs = POSE_BODY_PART_PAIRS[(int)mPoseModel].size() * channelOffset;
                 unsigned int totalOffset = 0u;
                 if (heatMapTypesHas(mHeatMapTypes, HeatMapType::Parts))
                 {
-                    cudaMemcpy(poseHeatMaps.getPtr(), getHeatMapGpuConstPtr(), volumeBodyParts * sizeof(float), cudaMemcpyDeviceToHost);
+                    cudaMemcpy(heatMaps.getPtr(), getHeatMapGpuConstPtr(), volumeBodyParts * sizeof(float), cudaMemcpyDeviceToHost);
                     // Change from [0,1] to [-1,1]
                     if (mHeatMapScaleMode == ScaleMode::PlusMinusOne)
                         for (auto i = 0u ; i < volumeBodyParts ; i++)
-                            poseHeatMaps[i] = fastTruncate(poseHeatMaps[i]) * 2.f - 1.f;
+                            heatMaps[i] = fastTruncate(heatMaps[i]) * 2.f - 1.f;
                     // [0, 255]
                     else if (mHeatMapScaleMode == ScaleMode::UnsignedChar)
                         for (auto i = 0u ; i < volumeBodyParts ; i++)
-                            poseHeatMaps[i] = (float)intRound(fastTruncate(poseHeatMaps[i]) * 255.f);
+                            heatMaps[i] = (float)intRound(fastTruncate(heatMaps[i]) * 255.f);
                     // Avoid values outside original range
                     else
                         for (auto i = 0u ; i < volumeBodyParts ; i++)
-                            poseHeatMaps[i] = fastTruncate(poseHeatMaps[i]);
+                            heatMaps[i] = fastTruncate(heatMaps[i]);
                     totalOffset += (unsigned int)volumeBodyParts;
                 }
                 if (heatMapTypesHas(mHeatMapTypes, HeatMapType::Background))
                 {
-                    cudaMemcpy(poseHeatMaps.getPtr() + totalOffset, getHeatMapGpuConstPtr() + volumeBodyParts, channelOffset * sizeof(float), cudaMemcpyDeviceToHost);
+                    cudaMemcpy(heatMaps.getPtr() + totalOffset, getHeatMapGpuConstPtr() + volumeBodyParts, channelOffset * sizeof(float), cudaMemcpyDeviceToHost);
                     // Change from [0,1] to [-1,1]
-                    auto* poseHeatMapsPtr = poseHeatMaps.getPtr() + totalOffset;
+                    auto* heatMapsPtr = heatMaps.getPtr() + totalOffset;
                     if (mHeatMapScaleMode == ScaleMode::PlusMinusOne)
                         for (auto i = 0u ; i < channelOffset ; i++)
-                            poseHeatMapsPtr[i] = fastTruncate(poseHeatMapsPtr[i]) * 2.f - 1.f;
+                            heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]) * 2.f - 1.f;
                     // [0, 255]
                     else if (mHeatMapScaleMode == ScaleMode::UnsignedChar)
                         for (auto i = 0u ; i < channelOffset ; i++)
-                            poseHeatMapsPtr[i] = (float)intRound(fastTruncate(poseHeatMapsPtr[i]) * 255.f);
+                            heatMapsPtr[i] = (float)intRound(fastTruncate(heatMapsPtr[i]) * 255.f);
                     // Avoid values outside original range
                     else
                         for (auto i = 0u ; i < channelOffset ; i++)
-                            poseHeatMapsPtr[i] = fastTruncate(poseHeatMapsPtr[i]);
+                            heatMapsPtr[i] = fastTruncate(heatMapsPtr[i]);
                     totalOffset += (unsigned int)channelOffset;
                 }
                 if (heatMapTypesHas(mHeatMapTypes, HeatMapType::PAFs))
                 {
-                    cudaMemcpy(poseHeatMaps.getPtr() + totalOffset, getHeatMapGpuConstPtr() + volumeBodyParts + channelOffset, volumePAFs * sizeof(float), cudaMemcpyDeviceToHost);
+                    cudaMemcpy(heatMaps.getPtr() + totalOffset, getHeatMapGpuConstPtr() + volumeBodyParts + channelOffset, volumePAFs * sizeof(float), cudaMemcpyDeviceToHost);
                     // Change from [-1,1] to [0,1]. Note that PAFs are in [-1,1]
-                    auto* poseHeatMapsPtr = poseHeatMaps.getPtr() + totalOffset;
+                    auto* heatMapsPtr = heatMaps.getPtr() + totalOffset;
                     if (mHeatMapScaleMode == ScaleMode::ZeroToOne)
                         for (auto i = 0u ; i < volumePAFs ; i++)
-                            poseHeatMapsPtr[i] = fastTruncate(poseHeatMapsPtr[i], -1.f) * 0.5f + 0.5f;
+                            heatMapsPtr[i] = fastTruncate(heatMapsPtr[i], -1.f) * 0.5f + 0.5f;
                     // [0, 255]
                     else if (mHeatMapScaleMode == ScaleMode::UnsignedChar)
                         for (auto i = 0u ; i < volumePAFs ; i++)
-                            poseHeatMapsPtr[i] = (float)intRound(fastTruncate(poseHeatMapsPtr[i], -1.f) * 128.5f + 128.5f);
+                            heatMapsPtr[i] = (float)intRound(fastTruncate(heatMapsPtr[i], -1.f) * 128.5f + 128.5f);
                     // Avoid values outside original range
                     else
                         for (auto i = 0u ; i < volumePAFs ; i++)
-                            poseHeatMapsPtr[i] = fastTruncate(poseHeatMapsPtr[i], -1.f);
+                            heatMapsPtr[i] = fastTruncate(heatMapsPtr[i], -1.f);
                     totalOffset += (unsigned int)volumePAFs;
                 }
                 // Copy all at once
-                // cudaMemcpy(poseHeatMaps.getPtr(), getHeatMapGpuConstPtr(), poseHeatMaps.getVolume() * sizeof(float), cudaMemcpyDeviceToHost);
+                // cudaMemcpy(heatMaps.getPtr(), getHeatMapGpuConstPtr(), heatMaps.getVolume() * sizeof(float), cudaMemcpyDeviceToHost);
             }
-            return poseHeatMaps;
+            return heatMaps;
         }
         catch (const std::exception& e)
         {