Bug fixed: face not working with output resolution different to input

include/openpose/core/datum.hpp

@@ -94,9 +94,9 @@ namespace op
         Array<float> handKeypoints;
 
         // -------------------------------------------------- Other parameters -------------------------------------------------- //
-        double scaleInputToOutput; /**< Scale ratio between the input Datum::cvInputData and the output Datum::cvOutputData. */
+        float scaleInputToOutput; /**< Scale ratio between the input Datum::cvInputData and the output Datum::cvOutputData. */
 
-        double scaleNetToOutput; /**< Scale ratio between the net output and the final output Datum::cvOutputData. */
+        float scaleNetToOutput; /**< Scale ratio between the net output and the final output Datum::cvOutputData. */
 
         std::pair<int, std::string> elementRendered; /**< Pair with the element key id POSE_BODY_PART_MAPPING on `pose/poseParameters.hpp` and its mapped value (e.g. 1 and "Neck"). */
 

include/openpose/core/point.hpp

@@ -9,7 +9,7 @@ namespace op
         T x;
         T y;
 
-        Point(const T x = -1, const T y = -1);
+        Point(const T x = 0, const T y = 0);
 
         /**
          * Copy constructor.
@@ -120,16 +120,28 @@ namespace op
         // -------------------------------------------------- Basic Operators -------------------------------------------------- //
         Point<T>& operator+=(const Point<T>& point);
 
+        Point<T> operator+(const Point<T>& point);
+
         Point<T>& operator+=(const T value);
 
+        Point<T> operator+(const T value);
+
         Point<T>& operator-=(const Point<T>& point);
 
+        Point<T> operator-(const Point<T>& point);
+
         Point<T>& operator-=(const T value);
 
+        Point<T> operator-(const T value);
+
         Point<T>& operator*=(const T value);
 
+        Point<T> operator*(const T value);
+
         Point<T>& operator/=(const T value);
 
+        Point<T> operator/(const T value);
+
     };
 }
 

include/openpose/core/rectangle.hpp

@@ -60,6 +60,15 @@ namespace op
         {
             return width * height;
         }
+
+        // -------------------------------------------------- Basic Operators -------------------------------------------------- //
+        Rectangle<T>& operator*=(const T value);
+
+        Rectangle<T> operator*(const T value);
+
+        Rectangle<T>& operator/=(const T value);
+
+        Rectangle<T> operator/(const T value);
     };
 }
 

include/openpose/face/faceDetector.hpp

@@ -15,7 +15,7 @@ namespace op
     public:
         explicit FaceDetector(const PoseModel poseModel);
 
-        std::vector<Rectangle<float>> detectFaces(const Array<float>& poseKeypoints);
+        std::vector<Rectangle<float>> detectFaces(const Array<float>& poseKeypoints, const float scaleInputToOutput);
 
     private:
         const unsigned int mNeck;

include/openpose/face/faceExtractor.hpp

@@ -22,7 +22,7 @@ namespace op
 
         void initializationOnThread();
 
-        void forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData);
+        void forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData, const float scaleInputToOutput);
 
         Array<float> getFaceKeypoints() const;
 

include/openpose/face/wFaceDetector.hpp

@@ -59,7 +59,7 @@ namespace op
                 const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__);
                 // Detect people face
                 for (auto& tDatum : *tDatums)
-                    tDatum.faceRectangles = spFaceDetector->detectFaces(tDatum.poseKeypoints);
+                    tDatum.faceRectangles = spFaceDetector->detectFaces(tDatum.poseKeypoints, tDatum.scaleInputToOutput);
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);
                 Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__, Profiler::DEFAULT_X);

include/openpose/face/wFaceExtractor.hpp

@@ -61,7 +61,7 @@ namespace op
                 // Extract people face
                 for (auto& tDatum : *tDatums)
                 {
-                    spFaceExtractor->forwardPass(tDatum.faceRectangles, tDatum.cvInputData);
+                    spFaceExtractor->forwardPass(tDatum.faceRectangles, tDatum.cvInputData, tDatum.scaleInputToOutput);
                     tDatum.faceKeypoints = spFaceExtractor->getFaceKeypoints();
                 }
                 // Profiling speed

src/openpose/core/point.cpp

@@ -90,6 +90,20 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator+(const Point<T>& point)
+    {
+        try
+        {
+            return Point<T>{T(x + point.x), T(y + point.y)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     template<typename T>
     Point<T>& Point<T>::operator+=(const T value)
     {
@@ -107,6 +121,20 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator+(const T value)
+    {
+        try
+        {
+            return Point<T>{T(x + value), T(y + value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     template<typename T>
     Point<T>& Point<T>::operator-=(const Point<T>& point)
     {
@@ -124,6 +152,20 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator-(const Point<T>& point)
+    {
+        try
+        {
+            return Point<T>{T(x - point.x), T(y - point.y)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     template<typename T>
     Point<T>& Point<T>::operator-=(const T value)
     {
@@ -141,6 +183,20 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator-(const T value)
+    {
+        try
+        {
+            return Point<T>{T(x - value), T(y - value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     template<typename T>
     Point<T>& Point<T>::operator*=(const T value)
     {
@@ -158,6 +214,20 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator*(const T value)
+    {
+        try
+        {
+            return Point<T>{T(x * value), T(y * value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     template<typename T>
     Point<T>& Point<T>::operator/=(const T value)
     {
@@ -175,5 +245,19 @@ namespace op
         }
     }
 
+    template<typename T>
+    Point<T> Point<T>::operator/(const T value)
+    {
+        try
+        {
+            return Point<T>{T(x / value), T(y / value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Point<T>{};
+        }
+    }
+
     COMPILE_TEMPLATE_BASIC_TYPES(Point);
 }

src/openpose/core/rectangle.cpp

@@ -111,5 +111,71 @@ namespace op
         }
     }
 
+    template<typename T>
+    Rectangle<T>& Rectangle<T>::operator*=(const T value)
+    {
+        try
+        {
+            x *= value;
+            y *= value;
+            width *= value;
+            height *= value;
+            // Return
+            return *this;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return *this;
+        }
+    }
+
+    template<typename T>
+    Rectangle<T> Rectangle<T>::operator*(const T value)
+    {
+        try
+        {
+            return Rectangle<T>{T(x * value), T(y * value), T(width * value), T(height * value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Rectangle<T>{};
+        }
+    }
+
+    template<typename T>
+    Rectangle<T>& Rectangle<T>::operator/=(const T value)
+    {
+        try
+        {
+            x /= value;
+            y /= value;
+            width /= value;
+            height /= value;
+            // Return
+            return *this;
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return *this;
+        }
+    }
+
+    template<typename T>
+    Rectangle<T> Rectangle<T>::operator/(const T value)
+    {
+        try
+        {
+            return Rectangle<T>{T(x / value), T(y / value), T(width / value), T(height / value)};
+        }
+        catch (const std::exception& e)
+        {
+            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
+            return Rectangle<T>{};
+        }
+    }
+
     COMPILE_TEMPLATE_BASIC_TYPES(Rectangle);
 }

src/openpose/face/faceDetector.cpp

@@ -124,7 +124,7 @@ namespace op
         }
     }
 
-    std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints)
+    std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints, const float scaleInputToOutput)
     {
         try
         {
@@ -135,7 +135,7 @@ namespace op
             // Otherwise, get face position(s)
             if (!poseKeypoints.empty())
                 for (auto person = 0 ; person < numberPeople ; person++)
-                    faceRectangles.at(person) = getFaceFromPoseKeypoints(poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold);
+                    faceRectangles.at(person) = getFaceFromPoseKeypoints(poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold) / scaleInputToOutput;
             return faceRectangles;
         }
         catch (const std::exception& e)

src/openpose/face/faceExtractor.cpp

@@ -64,7 +64,7 @@ namespace op
         }
     }
 
-    void FaceExtractor::forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData)
+    void FaceExtractor::forwardPass(const std::vector<Rectangle<float>>& faceRectangles, const cv::Mat& cvInputData, const float scaleInputToOutput)
     {
         try
         {
@@ -77,7 +77,7 @@ namespace op
                 // Set face size
                 const auto numberPeople = (int)faceRectangles.size();
                 mFaceKeypoints.reset({numberPeople, FACE_NUMBER_PARTS, 3}, 0);
-
+// // Commented lines are for debugging
 // log("\nAreas:");
 // cv::Mat cvInputDataCopy = cvInputData.clone();
                 for (auto person = 0 ; person < numberPeople ; person++)
@@ -85,13 +85,13 @@ namespace op
                     // Only consider faces with a minimum pixel area
                     const auto faceAreaSquared = std::sqrt(faceRectangles.at(person).area());
                     // Get parts
-                    if (faceAreaSquared > 60)
+                    if (faceAreaSquared > 50)
                     {
                         const auto& faceRectangle = faceRectangles.at(person);
 // log(faceAreaSquared);
+// log(std::to_string(cvInputData.cols) + " " + std::to_string(cvInputData.rows));
 // cv::rectangle(cvInputDataCopy,
 //               cv::Point{(int)faceRectangle.x, (int)faceRectangle.y},
-//               // cv::Point{(int)(faceRectangle.x + faceRectangle.width), (int)(faceRectangle.y + faceRectangle.height)},
 //               cv::Point{(int)faceRectangle.bottomRight().x, (int)faceRectangle.bottomRight().y},
 //               cv::Scalar{255,0,255}, 2);
                         // Get face position(s)
@@ -111,6 +111,8 @@ namespace op
 
                         // cv::Mat -> float*
                         uCharCvMatToFloatPtr(mFaceImageCrop.getPtr(), faceImage, true);
+// if (person < 5)
+// cv::imshow("faceImage" + std::to_string(person), faceImage);
                         // 1. Caffe deep network
                         auto* inputDataGpuPtr = spNet->getInputDataGpuPtr();
                         cudaMemcpy(inputDataGpuPtr, mFaceImageCrop.getPtr(), mNetOutputSize.area() * 3 * sizeof(float), cudaMemcpyHostToDevice);
@@ -160,8 +162,8 @@ namespace op
                                 const auto y = facePeaksPtr[xyIndex + 1];
                                 const auto score = facePeaksPtr[xyIndex + 2];
                                 const auto baseIndex = (person * FACE_NUMBER_PARTS + part) * 3;
-                                mFaceKeypoints[baseIndex] = Mscaling.at<double>(0,0) * x + Mscaling.at<double>(0,1) * y + Mscaling.at<double>(0,2);
-                                mFaceKeypoints[baseIndex+1] = Mscaling.at<double>(1,0) * x + Mscaling.at<double>(1,1) * y + Mscaling.at<double>(1,2);
+                                mFaceKeypoints[baseIndex] = scaleInputToOutput * (Mscaling.at<double>(0,0) * x + Mscaling.at<double>(0,1) * y + Mscaling.at<double>(0,2));
+                                mFaceKeypoints[baseIndex+1] = scaleInputToOutput * (Mscaling.at<double>(1,0) * x + Mscaling.at<double>(1,1) * y + Mscaling.at<double>(1,2));
                                 mFaceKeypoints[baseIndex+2] = score;
                             }
                         }

src/openpose/producer/producer.cpp

@@ -69,7 +69,12 @@ namespace op
             check(fpsMode == ProducerFpsMode::RetrievalFps || fpsMode == ProducerFpsMode::OriginalFps, "Unknown ProducerFpsMode.", __LINE__, __FUNCTION__, __FILE__);
             // For webcam, ProducerFpsMode::OriginalFps == ProducerFpsMode::RetrievalFps, since the internal webcam cache will overwrite frames after it gets full
             if (mType == ProducerType::Webcam)
+            {
                 mProducerFpsMode = {ProducerFpsMode::RetrievalFps};
+                if (fpsMode == ProducerFpsMode::OriginalFps)
+                    log("The producer fps mode set to `OriginalFps` (flag `process_real_time` on the demo) is not necessary, it is already assumed for webcam.",
+                        Priority::Max, __LINE__, __FUNCTION__, __FILE__);
+            }
             // If no webcam
             else
             {