bodyPartConnectorBase.cpp 76.8 KB
Newer Older
G
Gines Hidalgo 已提交
1
#include <set>
G
Gines Hidalgo 已提交
2 3
#include <openpose/utilities/check.hpp>
#include <openpose/utilities/fastMath.hpp>
4
#include <openpose/utilities/keypoint.hpp>
G
Gines Hidalgo 已提交
5
#include <openpose/pose/poseParameters.hpp>
G
gineshidalgo99 已提交
6
#include <openpose/net/bodyPartConnectorBase.hpp>
G
gineshidalgo99 已提交
7 8 9

namespace op
{
G
gineshidalgo99 已提交
10
    template <typename T>
11 12 13
    inline T getScoreAB(
        const int i, const int j, const T* const candidateAPtr, const T* const candidateBPtr, const T* const mapX,
        const T* const mapY, const Point<int>& heatMapSize, const T interThreshold, const T interMinAboveThreshold)
G
gineshidalgo99 已提交
14 15 16
    {
        try
        {
17 18
            const auto vectorAToBX = candidateBPtr[3*j] - candidateAPtr[3*i];
            const auto vectorAToBY = candidateBPtr[3*j+1] - candidateAPtr[3*i+1];
G
gineshidalgo99 已提交
19 20
            const auto vectorAToBMax = fastMax(std::abs(vectorAToBX), std::abs(vectorAToBY));
            const auto numberPointsInLine = fastMax(
G
gineshidalgo99 已提交
21
                5, fastMin(25, positiveIntRound(std::sqrt(5*vectorAToBMax))));
G
gineshidalgo99 已提交
22 23 24 25
            const auto vectorNorm = T(std::sqrt( vectorAToBX*vectorAToBX + vectorAToBY*vectorAToBY ));
            // If the peaksPtr are coincident. Don't connect them.
            if (vectorNorm > 1e-6)
            {
26 27
                const auto sX = candidateAPtr[3*i];
                const auto sY = candidateAPtr[3*i+1];
G
gineshidalgo99 已提交
28 29 30 31 32 33 34 35 36 37
                const auto vectorAToBNormX = vectorAToBX/vectorNorm;
                const auto vectorAToBNormY = vectorAToBY/vectorNorm;

                auto sum = T(0);
                auto count = 0u;
                const auto vectorAToBXInLine = vectorAToBX/numberPointsInLine;
                const auto vectorAToBYInLine = vectorAToBY/numberPointsInLine;
                for (auto lm = 0; lm < numberPointsInLine; lm++)
                {
                    const auto mX = fastMax(
G
gineshidalgo99 已提交
38
                        0, fastMin(heatMapSize.x-1, positiveIntRound(sX + lm*vectorAToBXInLine)));
G
gineshidalgo99 已提交
39
                    const auto mY = fastMax(
G
gineshidalgo99 已提交
40
                        0, fastMin(heatMapSize.y-1, positiveIntRound(sY + lm*vectorAToBYInLine)));
G
gineshidalgo99 已提交
41 42 43 44 45 46 47 48
                    const auto idx = mY * heatMapSize.x + mX;
                    const auto score = (vectorAToBNormX*mapX[idx] + vectorAToBNormY*mapY[idx]);
                    if (score > interThreshold)
                    {
                        sum += score;
                        count++;
                    }
                }
G
gineshidalgo99 已提交
49
                if (count/T(numberPointsInLine) > interMinAboveThreshold)
G
gineshidalgo99 已提交
50 51 52 53 54 55 56 57 58 59 60
                    return sum/count;
            }
            return T(0);
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
            return T(0);
        }
    }

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
    template <typename T>
    void getKeypointCounter(
        int& personCounter, const std::vector<std::pair<std::vector<int>, T>>& peopleVector,
        const unsigned int index, const int indexFirst, const int indexLast, const int minimum)
    {
        try
        {
            // Count keypoints
            auto keypointCounter = 0;
            for (auto i = indexFirst ; i < indexLast ; i++)
                keypointCounter += (peopleVector[index].first.at(i) > 0);
            // If enough keypoints --> subtract them and keep them at least as big as minimum
            if (keypointCounter > minimum)
                personCounter += minimum-keypointCounter; // personCounter = non-considered keypoints + minimum
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
        }
    }

G
gineshidalgo99 已提交
82
    template <typename T>
G
gineshidalgo99 已提交
83
    std::vector<std::pair<std::vector<int>, T>> createPeopleVector(
84 85 86
        const T* const heatMapPtr, const T* const peaksPtr, const PoseModel poseModel, const Point<int>& heatMapSize,
        const int maxPeaks, const T interThreshold, const T interMinAboveThreshold,
        const std::vector<unsigned int>& bodyPartPairs, const unsigned int numberBodyParts,
G
gineshidalgo99 已提交
87
        const unsigned int numberBodyPartPairs, const Array<T>& pairScores)
G
gineshidalgo99 已提交
88 89 90
    {
        try
        {
G
gineshidalgo99 已提交
91 92
            if (poseModel != PoseModel::BODY_25 && poseModel != PoseModel::COCO_18
                && poseModel != PoseModel::MPI_15 && poseModel != PoseModel::MPI_15_4)
G
gineshidalgo99 已提交
93
                error("Model not implemented for CPU body connector.", __LINE__, __FUNCTION__, __FILE__);
G
gineshidalgo99 已提交
94

95 96
            // std::vector<std::pair<std::vector<int>, double>> refers to:
            //     - std::vector<int>: [body parts locations, #body parts found]
G
gineshidalgo99 已提交
97 98
            //     - double: person subset score
            std::vector<std::pair<std::vector<int>, T>> peopleVector;
99
            const auto& mapIdx = getPoseMapIndex(poseModel);
100
            const auto numberBodyPartsAndBkg = numberBodyParts + (addBkgChannel(poseModel) ? 1 : 0);
G
gineshidalgo99 已提交
101
            const auto vectorSize = numberBodyParts+1;
G
gineshidalgo99 已提交
102 103
            const auto peaksOffset = 3*(maxPeaks+1);
            const auto heatMapOffset = heatMapSize.area();
G
gineshidalgo99 已提交
104
            // Iterate over it PAF connection, e.g., neck-nose, neck-Lshoulder, etc.
G
gineshidalgo99 已提交
105
            for (auto pairIndex = 0u; pairIndex < numberBodyPartPairs; pairIndex++)
G
gineshidalgo99 已提交
106 107 108
            {
                const auto bodyPartA = bodyPartPairs[2*pairIndex];
                const auto bodyPartB = bodyPartPairs[2*pairIndex+1];
G
gineshidalgo99 已提交
109 110
                const auto* candidateAPtr = peaksPtr + bodyPartA*peaksOffset;
                const auto* candidateBPtr = peaksPtr + bodyPartB*peaksOffset;
G
gineshidalgo99 已提交
111 112
                const auto numberPeaksA = positiveIntRound(candidateAPtr[0]);
                const auto numberPeaksB = positiveIntRound(candidateBPtr[0]);
G
gineshidalgo99 已提交
113

G
gineshidalgo99 已提交
114
                // E.g., neck-nose connection. If one of them is empty (e.g., no noses detected)
G
gineshidalgo99 已提交
115 116
                // Add the non-empty elements into the peopleVector
                if (numberPeaksA == 0 || numberPeaksB == 0)
G
gineshidalgo99 已提交
117
                {
G
gineshidalgo99 已提交
118
                    // E.g., neck-nose connection. If no necks, add all noses
G
gineshidalgo99 已提交
119
                    // Change w.r.t. other
G
gineshidalgo99 已提交
120
                    if (numberPeaksA == 0) // numberPeaksB == 0 or not
G
gineshidalgo99 已提交
121
                    {
G
gineshidalgo99 已提交
122
                        // Non-MPI
G
gineshidalgo99 已提交
123
                        if (numberBodyParts != 15)
G
gineshidalgo99 已提交
124
                        {
G
gineshidalgo99 已提交
125
                            for (auto i = 1; i <= numberPeaksB; i++)
G
gineshidalgo99 已提交
126
                            {
G
gineshidalgo99 已提交
127
                                bool found = false;
G
gineshidalgo99 已提交
128
                                for (const auto& personVector : peopleVector)
G
gineshidalgo99 已提交
129
                                {
G
Gines Hidalgo 已提交
130
                                    const auto off = (int)bodyPartB*peaksOffset + i*3 + 2;
G
gineshidalgo99 已提交
131
                                    if (personVector.first[bodyPartB] == off)
G
gineshidalgo99 已提交
132
                                    {
G
gineshidalgo99 已提交
133
                                        found = true;
G
gineshidalgo99 已提交
134 135 136
                                        break;
                                    }
                                }
G
gineshidalgo99 已提交
137
                                // Add new personVector with this element
G
gineshidalgo99 已提交
138
                                if (!found)
G
gineshidalgo99 已提交
139
                                {
G
gineshidalgo99 已提交
140 141 142 143 144 145 146 147
                                    std::vector<int> rowVector(vectorSize, 0);
                                    // Store the index
                                    rowVector[ bodyPartB ] = bodyPartB*peaksOffset + i*3 + 2;
                                    // Last number in each row is the parts number of that person
                                    rowVector.back() = 1;
                                    const auto personScore = candidateBPtr[i*3+2];
                                    // Second last number in each row is the total score
                                    peopleVector.emplace_back(std::make_pair(rowVector, personScore));
G
gineshidalgo99 已提交
148 149 150
                                }
                            }
                        }
G
gineshidalgo99 已提交
151
                        // MPI
152
                        else
G
gineshidalgo99 已提交
153
                        {
G
gineshidalgo99 已提交
154
                            for (auto i = 1; i <= numberPeaksB; i++)
G
gineshidalgo99 已提交
155
                            {
G
gineshidalgo99 已提交
156
                                std::vector<int> rowVector(vectorSize, 0);
157 158 159
                                // Store the index
                                rowVector[ bodyPartB ] = bodyPartB*peaksOffset + i*3 + 2;
                                // Last number in each row is the parts number of that person
G
gineshidalgo99 已提交
160
                                rowVector.back() = 1;
161
                                // Second last number in each row is the total score
G
gineshidalgo99 已提交
162 163
                                const auto personScore = candidateBPtr[i*3+2];
                                peopleVector.emplace_back(std::make_pair(rowVector, personScore));
G
gineshidalgo99 已提交
164 165 166
                            }
                        }
                    }
G
gineshidalgo99 已提交
167
                    // E.g., neck-nose connection. If no noses, add all necks
G
gineshidalgo99 已提交
168
                    else // if (numberPeaksA != 0 && numberPeaksB == 0)
G
gineshidalgo99 已提交
169
                    {
G
gineshidalgo99 已提交
170
                        // Non-MPI
G
gineshidalgo99 已提交
171
                        if (numberBodyParts != 15)
G
gineshidalgo99 已提交
172
                        {
G
gineshidalgo99 已提交
173
                            for (auto i = 1; i <= numberPeaksA; i++)
G
gineshidalgo99 已提交
174
                            {
G
gineshidalgo99 已提交
175
                                bool found = false;
G
gineshidalgo99 已提交
176
                                const auto indexA = bodyPartA;
G
gineshidalgo99 已提交
177
                                for (const auto& personVector : peopleVector)
G
gineshidalgo99 已提交
178
                                {
G
Gines Hidalgo 已提交
179
                                    const auto off = (int)bodyPartA*peaksOffset + i*3 + 2;
G
gineshidalgo99 已提交
180
                                    if (personVector.first[indexA] == off)
G
gineshidalgo99 已提交
181
                                    {
G
gineshidalgo99 已提交
182
                                        found = true;
G
gineshidalgo99 已提交
183 184 185
                                        break;
                                    }
                                }
G
gineshidalgo99 已提交
186
                                if (!found)
G
gineshidalgo99 已提交
187
                                {
G
gineshidalgo99 已提交
188
                                    std::vector<int> rowVector(vectorSize, 0);
189 190 191
                                    // Store the index
                                    rowVector[ bodyPartA ] = bodyPartA*peaksOffset + i*3 + 2;
                                    // Last number in each row is the parts number of that person
G
gineshidalgo99 已提交
192
                                    rowVector.back() = 1;
193
                                    // Second last number in each row is the total score
G
gineshidalgo99 已提交
194 195
                                    const auto personScore = candidateAPtr[i*3+2];
                                    peopleVector.emplace_back(std::make_pair(rowVector, personScore));
G
gineshidalgo99 已提交
196 197 198
                                }
                            }
                        }
G
gineshidalgo99 已提交
199
                        // MPI
200
                        else
G
gineshidalgo99 已提交
201
                        {
G
gineshidalgo99 已提交
202
                            for (auto i = 1; i <= numberPeaksA; i++)
G
gineshidalgo99 已提交
203
                            {
G
gineshidalgo99 已提交
204
                                std::vector<int> rowVector(vectorSize, 0);
205 206 207
                                // Store the index
                                rowVector[ bodyPartA ] = bodyPartA*peaksOffset + i*3 + 2;
                                // Last number in each row is the parts number of that person
G
gineshidalgo99 已提交
208
                                rowVector.back() = 1;
209
                                // Second last number in each row is the total score
G
gineshidalgo99 已提交
210 211
                                const auto personScore = candidateAPtr[i*3+2];
                                peopleVector.emplace_back(std::make_pair(rowVector, personScore));
G
gineshidalgo99 已提交
212 213 214 215
                            }
                        }
                    }
                }
G
gineshidalgo99 已提交
216
                // E.g., neck-nose connection. If necks and noses, look for maximums
G
gineshidalgo99 已提交
217
                else // if (numberPeaksA != 0 && numberPeaksB != 0)
G
gineshidalgo99 已提交
218
                {
G
gineshidalgo99 已提交
219
                    // (score, indexA, indexB). Inverted order for easy std::sort
G
gineshidalgo99 已提交
220
                    std::vector<std::tuple<double, int, int>> allABConnections;
221 222
                    // Note: Problem of this function, if no right PAF between A and B, both elements are
                    // discarded. However, they should be added indepently, not discarded
R
Raaj 已提交
223
                    if (heatMapPtr != nullptr)
G
gineshidalgo99 已提交
224
                    {
225 226 227 228
                        const auto* mapX = heatMapPtr
                                         + (numberBodyPartsAndBkg + mapIdx[2*pairIndex]) * heatMapOffset;
                        const auto* mapY = heatMapPtr
                                         + (numberBodyPartsAndBkg + mapIdx[2*pairIndex+1]) * heatMapOffset;
G
gineshidalgo99 已提交
229
                        // E.g., neck-nose connection. For each neck
G
gineshidalgo99 已提交
230
                        for (auto i = 1; i <= numberPeaksA; i++)
G
gineshidalgo99 已提交
231
                        {
G
gineshidalgo99 已提交
232
                            // E.g., neck-nose connection. For each nose
G
gineshidalgo99 已提交
233
                            for (auto j = 1; j <= numberPeaksB; j++)
G
gineshidalgo99 已提交
234
                            {
G
gineshidalgo99 已提交
235
                                // Initial PAF
236 237 238
                                auto scoreAB = getScoreAB(
                                    i, j, candidateAPtr, candidateBPtr, mapX, mapY, heatMapSize, interThreshold,
                                    interMinAboveThreshold);
G
gineshidalgo99 已提交
239

G
gineshidalgo99 已提交
240
                                // E.g., neck-nose connection. If possible PAF between neck i, nose j --> add
G
gineshidalgo99 已提交
241 242 243
                                // parts score + connection score
                                if (scoreAB > 1e-6)
                                    allABConnections.emplace_back(std::make_tuple(scoreAB, i, j));
G
gineshidalgo99 已提交
244 245 246
                            }
                        }
                    }
G
gineshidalgo99 已提交
247
                    else if (!pairScores.empty())
R
Raaj 已提交
248
                    {
G
gineshidalgo99 已提交
249
                        const auto firstIndex = (int)pairIndex*pairScores.getSize(1)*pairScores.getSize(2);
G
gineshidalgo99 已提交
250
                        // E.g., neck-nose connection. For each neck
G
gineshidalgo99 已提交
251
                        for (auto i = 0; i < numberPeaksA; i++)
R
Raaj 已提交
252
                        {
G
gineshidalgo99 已提交
253
                            const auto iIndex = firstIndex + i*pairScores.getSize(2);
G
gineshidalgo99 已提交
254
                            // E.g., neck-nose connection. For each nose
G
gineshidalgo99 已提交
255
                            for (auto j = 0; j < numberPeaksB; j++)
R
Raaj 已提交
256
                            {
G
gineshidalgo99 已提交
257
                                const auto scoreAB = pairScores[iIndex + j];
R
Raaj 已提交
258

G
gineshidalgo99 已提交
259
                                // E.g., neck-nose connection. If possible PAF between neck i, nose j --> add
R
Raaj 已提交
260 261
                                // parts score + connection score
                                if (scoreAB > 1e-6)
G
gineshidalgo99 已提交
262 263
                                    // +1 because peaksPtr starts with counter
                                    allABConnections.emplace_back(std::make_tuple(scoreAB, i+1, j+1));
R
Raaj 已提交
264 265 266 267 268
                            }
                        }
                    }
                    else
                        error("Error. Should not reach here.", __LINE__, __FUNCTION__, __FILE__);
G
gineshidalgo99 已提交
269 270 271

                    // select the top minAB connection, assuming that each part occur only once
                    // sort rows in descending order based on parts + connection score
272
                    if (!allABConnections.empty())
G
gineshidalgo99 已提交
273 274
                        std::sort(allABConnections.begin(), allABConnections.end(),
                                  std::greater<std::tuple<double, int, int>>());
G
gineshidalgo99 已提交
275

276
                    std::vector<std::tuple<int, int, double>> abConnections; // (x, y, score)
G
gineshidalgo99 已提交
277
                    {
G
gineshidalgo99 已提交
278 279 280
                        const auto minAB = fastMin(numberPeaksA, numberPeaksB);
                        std::vector<int> occurA(numberPeaksA, 0);
                        std::vector<int> occurB(numberPeaksB, 0);
281
                        auto counter = 0;
G
gineshidalgo99 已提交
282
                        for (const auto& aBConnection : allABConnections)
G
gineshidalgo99 已提交
283
                        {
G
gineshidalgo99 已提交
284 285 286 287
                            const auto score = std::get<0>(aBConnection);
                            const auto indexA = std::get<1>(aBConnection);
                            const auto indexB = std::get<2>(aBConnection);
                            if (!occurA[indexA-1] && !occurB[indexB-1])
288
                            {
289 290
                                abConnections.emplace_back(std::make_tuple(
                                    bodyPartA*peaksOffset+indexA*3+2, bodyPartB*peaksOffset+indexB*3+2, score));
291 292 293
                                counter++;
                                if (counter==minAB)
                                    break;
G
gineshidalgo99 已提交
294 295
                                occurA[indexA-1] = 1;
                                occurB[indexB-1] = 1;
296
                            }
G
gineshidalgo99 已提交
297 298 299
                        }
                    }

G
gineshidalgo99 已提交
300
                    // Cluster all the body part candidates into peopleVector based on the part connection
301
                    if (!abConnections.empty())
G
gineshidalgo99 已提交
302
                    {
303 304
                        // initialize first body part connection 15&16
                        if (pairIndex==0)
G
gineshidalgo99 已提交
305
                        {
306
                            for (const auto& abConnection : abConnections)
307 308
                            {
                                std::vector<int> rowVector(numberBodyParts+3, 0);
309 310 311
                                const auto indexA = std::get<0>(abConnection);
                                const auto indexB = std::get<1>(abConnection);
                                const auto score = std::get<2>(abConnection);
312 313
                                rowVector[bodyPartPairs[0]] = indexA;
                                rowVector[bodyPartPairs[1]] = indexB;
G
gineshidalgo99 已提交
314
                                rowVector.back() = 2;
315
                                // add the score of parts and the connection
G
gineshidalgo99 已提交
316 317
                                const auto personScore = peaksPtr[indexA] + peaksPtr[indexB] + score;
                                peopleVector.emplace_back(std::make_pair(rowVector, personScore));
318
                            }
G
gineshidalgo99 已提交
319
                        }
320
                        // Add ears connections (in case person is looking to opposite direction to camera)
G
gineshidalgo99 已提交
321 322
                        // Note: This has some issues:
                        //     - It does not prevent repeating the same keypoint in different people
323 324
                        //     - Assuming I have nose,eye,ear as 1 person subset, and whole arm as another one, it
                        //       will not merge them both
G
gineshidalgo99 已提交
325 326
                        else if (
                            (numberBodyParts == 18 && (pairIndex==17 || pairIndex==18))
G
gineshidalgo99 已提交
327
                            || ((numberBodyParts == 19 || (numberBodyParts == 25)
G
gineshidalgo99 已提交
328
                                 || numberBodyParts == 59 || numberBodyParts == 65)
G
gineshidalgo99 已提交
329 330
                                && (pairIndex==18 || pairIndex==19))
                            )
331
                        {
332
                            for (const auto& abConnection : abConnections)
333
                            {
334 335
                                const auto indexA = std::get<0>(abConnection);
                                const auto indexB = std::get<1>(abConnection);
G
gineshidalgo99 已提交
336
                                for (auto& personVector : peopleVector)
337
                                {
G
gineshidalgo99 已提交
338 339 340
                                    auto& personVectorA = personVector.first[bodyPartA];
                                    auto& personVectorB = personVector.first[bodyPartB];
                                    if (personVectorA == indexA && personVectorB == 0)
G
gineshidalgo99 已提交
341
                                    {
G
gineshidalgo99 已提交
342
                                        personVectorB = indexB;
G
gineshidalgo99 已提交
343
                                        // // This seems to harm acc 0.1% for BODY_25
G
gineshidalgo99 已提交
344
                                        // personVector.first.back()++;
G
gineshidalgo99 已提交
345
                                    }
G
gineshidalgo99 已提交
346
                                    else if (personVectorB == indexB && personVectorA == 0)
G
gineshidalgo99 已提交
347
                                    {
G
gineshidalgo99 已提交
348
                                        personVectorA = indexA;
G
gineshidalgo99 已提交
349
                                        // // This seems to harm acc 0.1% for BODY_25
G
gineshidalgo99 已提交
350
                                        // personVector.first.back()++;
G
gineshidalgo99 已提交
351
                                    }
352
                                }
353 354
                            }
                        }
355
                        else
G
gineshidalgo99 已提交
356
                        {
G
gineshidalgo99 已提交
357
                            // A is already in the peopleVector, find its connection B
358
                            for (const auto& abConnection : abConnections)
G
gineshidalgo99 已提交
359
                            {
360 361
                                const auto indexA = std::get<0>(abConnection);
                                const auto indexB = std::get<1>(abConnection);
G
Gines Hidalgo 已提交
362
                                const auto score = T(std::get<2>(abConnection));
G
gineshidalgo99 已提交
363
                                bool found = false;
G
gineshidalgo99 已提交
364
                                for (auto& personVector : peopleVector)
G
gineshidalgo99 已提交
365
                                {
G
gineshidalgo99 已提交
366 367
                                    // Found partA in a peopleVector, add partB to same one.
                                    if (personVector.first[bodyPartA] == indexA)
G
gineshidalgo99 已提交
368
                                    {
G
gineshidalgo99 已提交
369 370 371
                                        personVector.first[bodyPartB] = indexB;
                                        personVector.first.back()++;
                                        personVector.second += peaksPtr[indexB] + score;
G
gineshidalgo99 已提交
372 373
                                        found = true;
                                        break;
G
gineshidalgo99 已提交
374 375
                                    }
                                }
G
gineshidalgo99 已提交
376
                                // Not found partA in peopleVector, add new peopleVector element
G
gineshidalgo99 已提交
377
                                if (!found)
G
gineshidalgo99 已提交
378
                                {
G
gineshidalgo99 已提交
379
                                    std::vector<int> rowVector(vectorSize, 0);
G
gineshidalgo99 已提交
380 381
                                    rowVector[bodyPartA] = indexA;
                                    rowVector[bodyPartB] = indexB;
G
gineshidalgo99 已提交
382 383 384
                                    rowVector.back() = 2;
                                    const auto personScore = peaksPtr[indexA] + peaksPtr[indexB] + score;
                                    peopleVector.emplace_back(std::make_pair(rowVector, personScore));
G
gineshidalgo99 已提交
385 386 387 388 389 390
                                }
                            }
                        }
                    }
                }
            }
G
gineshidalgo99 已提交
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
            return peopleVector;
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
            return {};
        }
    }

    template <typename T>
    std::vector<std::tuple<T, T, int, int, int>> pafPtrIntoVector(
        const Array<T>& pairScores, const T* const peaksPtr, const int maxPeaks,
        const std::vector<unsigned int>& bodyPartPairs, const unsigned int numberBodyPartPairs)
    {
        try
        {
            // Result is a std::vector<std::tuple<double, double, int, int, int>> with:
            // (totalScore, PAFscore, pairIndex, indexA, indexB)
            // totalScore is first to simplify later sorting
            std::vector<std::tuple<T, T, int, int, int>> pairConnections;

            // Get all PAF pairs in a single std::vector
            const auto peaksOffset = 3*(maxPeaks+1);
            for (auto pairIndex = 0u; pairIndex < numberBodyPartPairs; pairIndex++)
            {
                const auto bodyPartA = bodyPartPairs[2*pairIndex];
                const auto bodyPartB = bodyPartPairs[2*pairIndex+1];
                const auto* candidateAPtr = peaksPtr + bodyPartA*peaksOffset;
                const auto* candidateBPtr = peaksPtr + bodyPartB*peaksOffset;
G
gineshidalgo99 已提交
420 421
                const auto numberPeaksA = positiveIntRound(candidateAPtr[0]);
                const auto numberPeaksB = positiveIntRound(candidateBPtr[0]);
G
gineshidalgo99 已提交
422
                const auto firstIndex = (int)pairIndex*pairScores.getSize(1)*pairScores.getSize(2);
G
gineshidalgo99 已提交
423
                // E.g., neck-nose connection. For each neck
G
gineshidalgo99 已提交
424 425 426
                for (auto indexA = 0; indexA < numberPeaksA; indexA++)
                {
                    const auto iIndex = firstIndex + indexA*pairScores.getSize(2);
G
gineshidalgo99 已提交
427
                    // E.g., neck-nose connection. For each nose
G
gineshidalgo99 已提交
428 429 430 431
                    for (auto indexB = 0; indexB < numberPeaksB; indexB++)
                    {
                        const auto scoreAB = pairScores[iIndex + indexB];

G
gineshidalgo99 已提交
432
                        // E.g., neck-nose connection. If possible PAF between neck indexA, nose indexB --> add
G
gineshidalgo99 已提交
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
                        // parts score + connection score
                        if (scoreAB > 1e-6)
                        {
                            // totalScore - Only used for sorting
                            // // Original totalScore
                            // const auto totalScore = scoreAB;
                            // Improved totalScore
                            // Improved to avoid too much weight in the PAF between 2 elements, adding some weight
                            // on their confidence (avoid connecting high PAFs on very low-confident keypoints)
                            const auto indexScoreA = bodyPartA*peaksOffset + (indexA+1)*3 + 2;
                            const auto indexScoreB = bodyPartB*peaksOffset + (indexB+1)*3 + 2;
                            const auto totalScore = scoreAB
                                                  + T(0.1)*peaksPtr[indexScoreA]
                                                  + T(0.1)*peaksPtr[indexScoreB];
                            // +1 because peaksPtr starts with counter
                            pairConnections.emplace_back(
                                std::make_tuple(totalScore, scoreAB, pairIndex, indexA+1, indexB+1));
                        }
                    }
                }
            }

            // Sort rows in descending order based on its first element (`totalScore`)
            if (!pairConnections.empty())
                std::sort(pairConnections.begin(), pairConnections.end(),
                          std::greater<std::tuple<double, double, int, int, int>>());

            // Return result
            return pairConnections;
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
            return {};
        }
    }

    template <typename T>
    std::vector<std::pair<std::vector<int>, T>> pafVectorIntoPeopleVector(
        const std::vector<std::tuple<T, T, int, int, int>>& pairConnections, const T* const peaksPtr,
        const int maxPeaks, const std::vector<unsigned int>& bodyPartPairs, const unsigned int numberBodyParts)
    {
        try
        {
            // std::vector<std::pair<std::vector<int>, double>> refers to:
            //     - std::vector<int>: [body parts locations, #body parts found]
            //     - double: person subset score
            std::vector<std::pair<std::vector<int>, T>> peopleVector;
            const auto vectorSize = numberBodyParts+1;
            const auto peaksOffset = (maxPeaks+1);
            // Save which body parts have been already assigned
            std::vector<int> personAssigned(numberBodyParts*maxPeaks, -1);
G
Gines Hidalgo 已提交
485
            std::set<int, std::greater<int>> indexesToRemoveSortedSet;
G
gineshidalgo99 已提交
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
            // Iterate over each PAF pair connection detected
            // E.g., neck1-nose2, neck5-Lshoulder0, etc.
            for (const auto& pairConnection : pairConnections)
            {
                // Read pairConnection
                // // Total score - only required for previous sort
                // const auto totalScore = std::get<0>(pairConnection);
                const auto pafScore = std::get<1>(pairConnection);
                const auto pairIndex = std::get<2>(pairConnection);
                const auto indexA = std::get<3>(pairConnection);
                const auto indexB = std::get<4>(pairConnection);
                // Derived data
                const auto bodyPartA = bodyPartPairs[2*pairIndex];
                const auto bodyPartB = bodyPartPairs[2*pairIndex+1];

                const auto indexScoreA = (bodyPartA*peaksOffset + indexA)*3 + 2;
                const auto indexScoreB = (bodyPartB*peaksOffset + indexB)*3 + 2;
                // -1 because indexA and indexB are 1-based
                auto& aAssigned = personAssigned[bodyPartA*maxPeaks+indexA-1];
                auto& bAssigned = personAssigned[bodyPartB*maxPeaks+indexB-1];
                // Debugging
                #ifdef DEBUG
                    if (indexA-1 > peaksOffset || indexA <= 0)
                        error("Something is wrong: " + std::to_string(indexA)
                              + " vs. " + std::to_string(peaksOffset) + ". Contact us.",
                              __LINE__, __FUNCTION__, __FILE__);
                    if (indexB-1 > peaksOffset || indexB <= 0)
                        error("Something is wrong: " + std::to_string(indexB)
                              + " vs. " + std::to_string(peaksOffset) + ". Contact us.",
                              __LINE__, __FUNCTION__, __FILE__);
                #endif

                // Different cases:
                //     1. A & B not assigned yet: Create new person
                //     2. A assigned but not B: Add B to person with A (if no another B there)
                //     3. B assigned but not A: Add A to person with B (if no another A there)
                //     4. A & B already assigned to same person (circular/redundant PAF): Update person score
                //     5. A & B already assigned to different people: Merge people if keypoint intersection is null
                // 1. A & B not assigned yet: Create new person
                if (aAssigned < 0 && bAssigned < 0)
                {
                    // Keypoint indexes
                    std::vector<int> rowVector(vectorSize, 0);
                    rowVector[bodyPartA] = indexScoreA;
                    rowVector[bodyPartB] = indexScoreB;
                    // Number keypoints
                    rowVector.back() = 2;
                    // Score
                    const auto personScore = peaksPtr[indexScoreA] + peaksPtr[indexScoreB] + pafScore;
                    // Set associated personAssigned as assigned
G
Gines Hidalgo 已提交
536
                    aAssigned = (int)peopleVector.size();
G
gineshidalgo99 已提交
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
                    bAssigned = aAssigned;
                    // Create new personVector
                    peopleVector.emplace_back(std::make_pair(rowVector, personScore));
                }
                // 2. A assigned but not B: Add B to person with A (if no another B there)
                // or
                // 3. B assigned but not A: Add A to person with B (if no another A there)
                else if ((aAssigned >= 0 && bAssigned < 0)
                    || (aAssigned < 0 && bAssigned >= 0))
                {
                    // Assign person1 to one where xAssigned >= 0
                    const auto assigned1 = (aAssigned >= 0 ? aAssigned : bAssigned);
                    auto& assigned2 = (aAssigned >= 0 ? bAssigned : aAssigned);
                    const auto bodyPart2 = (aAssigned >= 0 ? bodyPartB : bodyPartA);
                    const auto indexScore2 = (aAssigned >= 0 ? indexScoreB : indexScoreA);
                    // Person index
                    auto& personVector = peopleVector[assigned1];
                    // Debugging
                    #ifdef DEBUG
                        const auto bodyPart1 = (aAssigned >= 0 ? bodyPartA : bodyPartB);
                        const auto indexScore1 = (aAssigned >= 0 ? indexScoreA : indexScoreB);
                        const auto index1 = (aAssigned >= 0 ? indexA : indexB);
                        if ((unsigned int)personVector.first.at(bodyPart1) != indexScore1)
                            error("Something is wrong: "
                                  + std::to_string((personVector.first[bodyPart1]-2)/3-bodyPart1*peaksOffset)
                                  + " vs. " + std::to_string((indexScore1-2)/3-bodyPart1*peaksOffset) + " vs. "
                                  + std::to_string(index1) + ". Contact us.",
                                  __LINE__, __FUNCTION__, __FILE__);
                    #endif
                    // If person with 1 does not have a 2 yet
                    if (personVector.first[bodyPart2] == 0)
                    {
                        // Update keypoint indexes
                        personVector.first[bodyPart2] = indexScore2;
                        // Update number keypoints
                        personVector.first.back()++;
                        // Update score
                        personVector.second += peaksPtr[indexScore2] + pafScore;
                        // Set associated personAssigned as assigned
                        assigned2 = assigned1;
                    }
                    // Otherwise, ignore this B because the previous one came from a higher PAF-confident score
                }
                // 4. A & B already assigned to same person (circular/redundant PAF): Update person score
                else if (aAssigned >=0 && bAssigned >=0 && aAssigned == bAssigned)
                    peopleVector[aAssigned].second += pafScore;
                // 5. A & B already assigned to different people: Merge people if keypoint intersection is null
                // I.e., that the keypoints in person A and B do not overlap
                else if (aAssigned >=0 && bAssigned >=0 && aAssigned != bAssigned)
                {
                    // Assign person1 to the one with lowest index for 2 reasons:
                    //     1. Speed up: Removing an element from std::vector is cheaper for latest elements
                    //     2. Avoid harder index update: Updated elements in person1ssigned would depend on
                    //        whether person1 > person2 or not: element = aAssigned - (person2 > person1 ? 1 : 0)
                    const auto assigned1 = (aAssigned < bAssigned ? aAssigned : bAssigned);
                    const auto assigned2 = (aAssigned < bAssigned ? bAssigned : aAssigned);
                    auto& person1 = peopleVector[assigned1].first;
                    const auto& person2 = peopleVector[assigned2].first;
                    // Check if complementary
G
Gines Hidalgo 已提交
596 597
                    // Defining found keypoint indexes in personA as kA, and analogously kB
                    // Complementary if and only if kA intersection kB = empty. I.e., no common keypoints
G
gineshidalgo99 已提交
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
                    bool complementary = true;
                    for (auto part = 0u ; part < numberBodyParts ; part++)
                    {
                        if (person1[part] > 0 && person2[part] > 0)
                        {
                            complementary = false;
                            break;
                        }
                    }
                    // If complementary, merge both people into 1
                    if (complementary)
                    {
                        // Update keypoint indexes
                        for (auto part = 0u ; part < numberBodyParts ; part++)
                            if (person1[part] == 0)
                                person1[part] = person2[part];
                        // Update number keypoints
                        person1.back() += person2.back();
                        // Update score
                        peopleVector[assigned1].second += peopleVector[assigned2].second + pafScore;
                        // Erase the non-merged person
G
Gines Hidalgo 已提交
619 620
                        // peopleVector.erase(peopleVector.begin()+assigned2); // x2 slower when removing on-the-fly
                        indexesToRemoveSortedSet.emplace(assigned2); // Add into set so we can remove them all at once
G
gineshidalgo99 已提交
621 622 623 624 625
                        // Update associated personAssigned (person indexes have changed)
                        for (auto& element : personAssigned)
                        {
                            if (element == assigned2)
                                element = assigned1;
G
Gines Hidalgo 已提交
626 627 628
                            // No need because I will only remove them at the very end
                            // else if (element > assigned2)
                            //     element--;
G
gineshidalgo99 已提交
629 630 631 632
                        }
                    }
                }
            }
G
Gines Hidalgo 已提交
633 634 635
            // Remove unused people
            for (const auto& index : indexesToRemoveSortedSet)
                peopleVector.erase(peopleVector.begin()+index);
G
gineshidalgo99 已提交
636 637
            // Return result
            return peopleVector;
638 639 640 641 642 643 644
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
            return {};
        }
    }
G
gineshidalgo99 已提交
645

646
    template <typename T>
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
    void getRoiDiameterAndBounds(
        Rectangle<int>& roi, int& diameter, int& indexFirstNon0, int& indexLastNon0,
        const std::vector<int>& personVector, const T* const peaksPtr,
        const int indexInit, const int indexEnd)
    {
        try
        {
            roi = Rectangle<int>{0,0,0,0};
            for (auto index = 0u ; index < personVector.size()-1 ; index++)
            {
                const auto x = peaksPtr[personVector[index]-2];
                const auto y = peaksPtr[personVector[index]-1];
                const auto score = peaksPtr[personVector[index]];
                if (roi.x > x)
                    roi.x = x;
                if (roi.y > y)
                    roi.y = y;
            }
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
        }
    }

    template <typename T>
    void removePeopleBelowThresholdsAndFillFaces(
G
gineshidalgo99 已提交
674
        std::vector<int>& validSubsetIndexes, int& numberPeople,
675 676 677
        std::vector<std::pair<std::vector<int>, T>>& peopleVector, const unsigned int numberBodyParts,
        const int minSubsetCnt, const T minSubsetScore, const bool maximizePositives, const T* const peaksPtr)
        // const int minSubsetCnt, const T minSubsetScore, const int maxPeaks, const bool maximizePositives)
678 679 680
    {
        try
        {
G
gineshidalgo99 已提交
681 682 683
            // Delete people below the following thresholds:
                // a) minSubsetCnt: removed if less than minSubsetCnt body parts
                // b) minSubsetScore: removed if global score smaller than this
684
                // c) maxPeaks (POSE_MAX_PEOPLE): keep first maxPeaks people above thresholds -> Not required
685 686
            numberPeople = 0;
            validSubsetIndexes.clear();
687 688 689 690 691 692 693 694 695
            // validSubsetIndexes.reserve(fastMin((size_t)maxPeaks, peopleVector.size())); // maxPeaks is not required
            validSubsetIndexes.reserve(peopleVector.size());
            // Face valid sets
            std::vector<int> faceValidSubsetIndexes;
            faceValidSubsetIndexes.reserve(peopleVector.size());
            // Face invalid sets
            std::vector<int> faceInvalidSubsetIndexes;
            faceInvalidSubsetIndexes.reserve(peopleVector.size());
            // For each person candidate
G
gineshidalgo99 已提交
696
            for (auto index = 0u ; index < peopleVector.size() ; index++)
G
gineshidalgo99 已提交
697
            {
G
gineshidalgo99 已提交
698
                auto personCounter = peopleVector[index].first.back();
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
                // Analog for hand/face keypoints
                if (numberBodyParts >= 135)
                {
                    // No consider face keypoints for personCounter
                    const auto currentCounter = personCounter;
                    getKeypointCounter(personCounter, peopleVector, index, 65, 135, 1);
                    const auto newCounter = personCounter;
                    if (personCounter == 0)
                    {
                        faceInvalidSubsetIndexes.emplace_back(index);
                        continue;
                    }
                    // If body is still valid and facial points were removed, then add to valid faces
                    else if (currentCounter != newCounter)
                        faceValidSubsetIndexes.emplace_back(index);
                    // No consider right hand keypoints for personCounter
                    getKeypointCounter(personCounter, peopleVector, index, 45, 65, 1);
                    // No consider left hand keypoints for personCounter
                    getKeypointCounter(personCounter, peopleVector, index, 25, 45, 1);
                }
G
gineshidalgo99 已提交
719
                // Foot keypoints do not affect personCounter (too many false positives,
720 721 722
                // same foot usually appears as both left and right keypoints)
                // Pros: Removed tons of false positives
                // Cons: Standalone leg will never be recorded
723
                // Solution: No consider foot keypoints for that
G
gineshidalgo99 已提交
724
                if (!maximizePositives && (numberBodyParts == 25 || numberBodyParts > 70))
725
                {
726 727 728 729 730 731 732 733
                    const auto currentCounter = personCounter;
                    getKeypointCounter(personCounter, peopleVector, index, 19, 25, 0);
                    const auto newCounter = personCounter;
                    // Problem: Same leg/foot keypoints are considered for both left and right keypoints.
                    // Solution: Remove legs that are duplicated and that do not have upper torso
                    // Result: Slight increase in COCO mAP and decrease in mAR + reducing a lot false positives!
                    if (newCounter != currentCounter && newCounter <= 4)
                        continue;
734
                }
735
                // Add only valid people
G
gineshidalgo99 已提交
736 737
                const auto personScore = peopleVector[index].second;
                if (personCounter >= minSubsetCnt && (personScore/personCounter) >= minSubsetScore)
G
gineshidalgo99 已提交
738 739 740
                {
                    numberPeople++;
                    validSubsetIndexes.emplace_back(index);
741 742 743
                    // // This is not required, it is OK if there are more people. No more GPU memory used.
                    // if (numberPeople == maxPeaks)
                    //     break;
G
gineshidalgo99 已提交
744
                }
745
                // Sanity check
G
gineshidalgo99 已提交
746
                else if ((personCounter < 1 && numberBodyParts != 25 && numberBodyParts < 70) || personCounter < 0)
G
gineshidalgo99 已提交
747
                    error("Bad personCounter (" + std::to_string(personCounter) + "). Bug in this"
748
                          " function if this happens.", __LINE__, __FUNCTION__, __FILE__);
G
gineshidalgo99 已提交
749
            }
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
//             // Random standalone facial keypoints --> Merge into a more complete face
//             if (numberPeople > 0 && faceInvalidSubsetIndexes.size() > 0)
//             {
//                 for (auto faceId = 0u ; faceId < faceInvalidSubsetIndexes.size() ; faceId++)
//                 {
//                     // Get ROI
//                     Rectangle<int> roi;
//                     int diameter;
//                     int indexFirstNon0;
//                     int indexLastNon0;
//                     const auto index = faceValidSubsetIndexes[faceId];
//                     getRoiDiameterAndBounds(
//                         roi, diameter, indexFirstNon0, indexLastNon0, peopleVector[index].first, peaksPtr, 65, 135);
//                     // const auto personCounter = peopleVector[index].first.back();
//                     // const auto x = peaksPtr[peopleVector[index].first[part]-2];
//                     // const auto y = peaksPtr[peopleVector[index].first[part]-1];
//                     // const auto score = peaksPtr[peopleVector[index].first[part]];
//                 }
//             }
            // If no people found --> Repeat with maximizePositives = true
            // Result: Increased COCO mAP because we catch more foot-only images
            if (numberPeople == 0 && !maximizePositives)
            {
                removePeopleBelowThresholdsAndFillFaces(
                    validSubsetIndexes, numberPeople, peopleVector, numberBodyParts, minSubsetCnt, minSubsetScore,
                    true, peaksPtr);
                // // Debugging
                // if (numberPeople > 0)
                //     log("Found " + std::to_string(numberPeople) + " people in second iteration");
            }
780 781 782 783 784 785
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
        }
    }
G
gineshidalgo99 已提交
786

787
    template <typename T>
788 789 790 791 792
    void peopleVectorToPeopleArray(
        Array<T>& poseKeypoints, Array<T>& poseScores, const T scaleFactor,
        const std::vector<std::pair<std::vector<int>, T>>& peopleVector, const std::vector<int>& validSubsetIndexes,
        const T* const peaksPtr, const int numberPeople, const unsigned int numberBodyParts,
        const unsigned int numberBodyPartPairs)
793 794 795
    {
        try
        {
796
            // Allocate memory (initialized to 0)
G
gineshidalgo99 已提交
797
            if (numberPeople > 0)
798
            {
799
                // Initialized to 0 for non-found keypoints in people
G
Gines Hidalgo 已提交
800
                poseKeypoints.reset({numberPeople, (int)numberBodyParts, 3}, 0.f);
801 802
                poseScores.reset(numberPeople);
            }
803
            // No people --> Empty Arrays
G
gineshidalgo99 已提交
804
            else
805
            {
806
                poseKeypoints.reset();
807 808
                poseScores.reset();
            }
809
            // Fill people keypoints
G
Gines Hidalgo 已提交
810
            const auto oneOverNumberBodyPartsAndPAFs = 1/T(numberBodyParts + numberBodyPartPairs);
811
            // For each person
G
gineshidalgo99 已提交
812
            for (auto person = 0u ; person < validSubsetIndexes.size() ; person++)
G
gineshidalgo99 已提交
813
            {
G
gineshidalgo99 已提交
814 815
                const auto& personPair = peopleVector[validSubsetIndexes[person]];
                const auto& personVector = personPair.first;
816
                // For each body part
G
Gines Hidalgo 已提交
817
                for (auto bodyPart = 0u; bodyPart < numberBodyParts; bodyPart++)
G
gineshidalgo99 已提交
818 819
                {
                    const auto baseOffset = (person*numberBodyParts + bodyPart) * 3;
G
gineshidalgo99 已提交
820
                    const auto bodyPartIndex = personVector[bodyPart];
G
gineshidalgo99 已提交
821 822
                    if (bodyPartIndex > 0)
                    {
823 824
                        poseKeypoints[baseOffset] = peaksPtr[bodyPartIndex-2] * scaleFactor;
                        poseKeypoints[baseOffset + 1] = peaksPtr[bodyPartIndex-1] * scaleFactor;
825
                        poseKeypoints[baseOffset + 2] = peaksPtr[bodyPartIndex];
G
gineshidalgo99 已提交
826 827
                    }
                }
G
Gines Hidalgo 已提交
828
                poseScores[person] = personPair.second * oneOverNumberBodyPartsAndPAFs;
G
gineshidalgo99 已提交
829 830 831 832 833 834 835 836
            }
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
        }
    }

G
gineshidalgo99 已提交
837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
//     template <typename T>
//     void connectDistanceStar(Array<T>& poseKeypoints, Array<T>& poseScores, const T* const heatMapPtr,
//                              const T* const peaksPtr, const PoseModel poseModel, const Point<int>& heatMapSize,
//                              const int maxPeaks, const T scaleFactor, const unsigned int numberBodyParts,
//                              const unsigned int bodyPartPairsSize)
//     {
//         try
//         {
//             // poseKeypoints from neck-part distances
//             if (poseModel == PoseModel::BODY_25D)
//             {
//                 const auto scaleDownFactor = 8;
//                 Array<T> poseKeypoints2 = poseKeypoints.clone();
//                 const auto rootIndex = 1;
//                 const auto rootNumberIndex = rootIndex*(maxPeaks+1)*3;
G
gineshidalgo99 已提交
852
//                 const auto numberPeople = positiveIntRound(peaksPtr[rootNumberIndex]);
G
gineshidalgo99 已提交
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
//                 poseKeypoints.reset({numberPeople, (int)numberBodyParts, 3}, 0);
//                 poseScores.reset(numberPeople, 0);
//                 // // 48 channels
//                 // const std::vector<float> AVERAGE{
//                 //     0.f, -2.76364f, -1.3345f, 0.f,   -1.95322f, 3.95679f, -1.20664f, 4.76543f,
//                 //     1.3345f, 0.f, 1.92318f, 3.96891f,   1.17999f, 4.7901f, 0.f, 7.72201f,
//                 //     -0.795236f, 7.74017f, -0.723963f,   11.209f, -0.651316f, 15.6972f,
//                 //     0.764623f, 7.74869f, 0.70755f,   11.2307f, 0.612832f, 15.7281f,
//                 //     -0.123134f, -3.43515f,   0.111775f, -3.42761f,
//                 //     -0.387066f, -3.16603f,   0.384038f, -3.15951f,
//                 //     0.344764f, 12.9666f, 0.624157f,   12.9057f, 0.195454f, 12.565f,
//                 //     -1.06074f, 12.9951f, -1.2427f,   12.9309f, -0.800837f, 12.5845f};
//                 // const std::vector<float> SIGMA{
//                 //     3.39629f, 3.15605f, 3.16913f, 1.8234f,   5.82252f, 5.05674f, 7.09876f, 6.64574f,
//                 //     3.16913f, 1.8234f, 5.79415f, 5.01424f,   7.03866f, 6.62427f, 5.52593f, 6.75962f,
//                 //     5.91224f, 6.87241f, 8.66473f,   10.1792f, 11.5871f, 13.6565f,
//                 //     5.86653f, 6.89568f, 8.68067f,   10.2127f, 11.5954f, 13.6722f,
//                 //     3.3335f, 3.49128f,   3.34476f, 3.50079f,
//                 //     2.93982f, 3.11151f,   2.95006f, 3.11004f,
//                 //     9.69408f, 7.58921f, 9.71193f,   7.44185f, 9.19343f, 7.11157f,
//                 //     9.16848f, 7.86122f, 9.07613f,   7.83682f, 8.91951f, 7.33715f};
//                 // 50 channels
//                 const std::vector<float> AVERAGE{
//                     0.f, -6.55251f,
//                     0.f, -4.15062f, -1.48818f, -4.15506f,   -2.22408f, -0.312264f, -1.42204f, 0.588495f,
//                     1.51044f, -4.14629f, 2.2113f, -0.312283f,   1.41081f, 0.612377f, -0.f, 3.41112f,
//                     -0.932306f, 3.45504f, -0.899812f,   6.79837f, -0.794223f, 11.4972f,
//                     0.919047f, 3.46442, 0.902314f,   6.81245f, 0.79518f, 11.5132f,
//                     -0.243982f, -7.07925f,   0.28065f, -7.07398f,
//                     -0.792812f, -7.09374f,   0.810145f, -7.06958f,
//                     0.582387f, 7.46846f, 0.889349f,   7.40577f, 0.465088f, 7.03969f,
//                     -0.96686f, 7.46148f, -1.20773f,   7.38834f, -0.762135f, 6.99575f};
//                 const std::vector<float> SIGMA{
//                     7.26789f, 9.70751f,
//                     6.29588f, 8.93472f, 6.97401f, 9.13746f,   7.49632f, 9.44757f, 8.06695f, 9.97319f,
//                     6.99726f, 9.14608f, 7.50529f, 9.43568f,   8.05888f, 9.98207f, 6.38929f, 9.29314f,
//                     6.71801f, 9.39271f, 8.00608f,   10.6141f, 10.3416f, 12.7812f,
//                     6.69875f, 9.41407f, 8.01876f,   10.637f, 10.3475f, 12.7849f,
//                     7.30923f, 9.7324f,   7.27886f, 9.73406f,
//                     7.35978f, 9.7289f,   7.28914f, 9.67711f,
//                     7.93153f, 8.10845f, 7.95577f,   8.01729f, 7.56865f, 7.87314f,
//                     7.4655f, 8.25336f, 7.43958f,   8.26333f, 7.33667f, 7.97446f};
//                 // To get ideal distance
//                 const auto numberBodyPartsAndBkgAndPAFChannels = numberBodyParts + 1 + bodyPartPairsSize;
//                 const auto heatMapOffset = heatMapSize.area();
//                 // For each person
//                 for (auto p = 0 ; p < numberPeople ; p++)
//                 {
//                     // For root (neck) position
//                     // bpOrig == rootIndex
//                     const auto rootXYSIndex = rootNumberIndex+3*(1+p);
//                     // Set (x,y,score)
//                     const auto rootX = scaleFactor*peaksPtr[rootXYSIndex];
//                     const auto rootY = scaleFactor*peaksPtr[rootXYSIndex+1];
//                     poseKeypoints[{p,rootIndex,0}] = rootX;
//                     poseKeypoints[{p,rootIndex,1}] = rootY;
//                     poseKeypoints[{p,rootIndex,2}] = peaksPtr[rootXYSIndex+2];
//                     // For each body part
//                     for (auto bpOrig = 0 ; bpOrig < (int)numberBodyParts ; bpOrig++)
//                     {
//                         if (bpOrig != rootIndex)
//                         {
//                             // // 48 channels
//                             // const auto bpChannel = (bpOrig < rootIndex ? bpOrig : bpOrig-1);
//                             // 50 channels
//                             const auto bpChannel = bpOrig;
//                             // Get ideal distance
//                             const auto offsetIndex = numberBodyPartsAndBkgAndPAFChannels + 2*bpChannel;
//                             const auto* mapX = heatMapPtr + offsetIndex * heatMapOffset;
//                             const auto* mapY = heatMapPtr + (offsetIndex+1) * heatMapOffset;
//                             const auto increaseRatio = scaleFactor*scaleDownFactor;
//                             // Set (x,y) coordinates from the distance
//                             const auto indexChannel = 2*bpChannel;
//                             // // Not refined method
G
gineshidalgo99 已提交
927 928
//                             // const auto index = positiveIntRound(rootY/scaleFactor)*heatMapSize.x
//                                                 + positiveIntRound(rootX/scaleFactor);
G
gineshidalgo99 已提交
929 930 931 932 933 934 935 936 937 938
//                             // const Point<T> neckPartDist{
//                             //     increaseRatio*(mapX[index]*SIGMA[indexChannel]+AVERAGE[indexChannel]),
//                             //     increaseRatio*(mapY[index]*SIGMA[indexChannel+1]+AVERAGE[indexChannel+1])};
//                             // poseKeypoints[{p,bpOrig,0}] = rootX + neckPartDist.x;
//                             // poseKeypoints[{p,bpOrig,1}] = rootY + neckPartDist.y;
//                             // Refined method
//                             const auto constant = 5;
//                             Point<T> neckPartDistRefined{0, 0};
//                             auto counterRefinements = 0;
//                             // We must keep it inside the image size
G
gineshidalgo99 已提交
939 940
//                             for (auto y = fastMax(0, positiveIntRound(rootY/scaleFactor) - constant);
//                                  y < fastMin(heatMapSize.y, positiveIntRound(rootY/scaleFactor) + constant+1) ; y++)
G
gineshidalgo99 已提交
941
//                             {
G
gineshidalgo99 已提交
942 943
//                                 for (auto x = fastMax(0, positiveIntRound(rootX/scaleFactor) - constant);
//                                      x < fastMin(heatMapSize.x, positiveIntRound(rootX/scaleFactor) + constant+1) ; x++)
G
gineshidalgo99 已提交
944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
//                                 {
//                                     const auto index = y*heatMapSize.x + x;
//                                     neckPartDistRefined.x += mapX[index];
//                                     neckPartDistRefined.y += mapY[index];
//                                     counterRefinements++;
//                                 }
//                             }
//                             neckPartDistRefined = Point<T>{
//                                 neckPartDistRefined.x*SIGMA[indexChannel]+counterRefinements*AVERAGE[indexChannel],
//                                 neckPartDistRefined.y*SIGMA[indexChannel+1]+counterRefinements*AVERAGE[indexChannel+1],
//                             };
//                             neckPartDistRefined *= increaseRatio/counterRefinements;
//                             const auto partX = rootX + neckPartDistRefined.x;
//                             const auto partY = rootY + neckPartDistRefined.y;
//                             poseKeypoints[{p,bpOrig,0}] = partX;
//                             poseKeypoints[{p,bpOrig,1}] = partY;
//                             // Set (temporary) body part score
//                             poseKeypoints[{p,bpOrig,2}] = T(0.0501);
//                             // Associate estimated keypoint with closest one
G
gineshidalgo99 已提交
963 964 965 966
//                             const auto xCleaned = fastMax(
//                                 0, fastMin(heatMapSize.x-1, positiveIntRound(partX/scaleFactor)));
//                             const auto yCleaned = fastMax(
//                                 0, fastMin(heatMapSize.y-1, positiveIntRound(partY/scaleFactor)));
G
gineshidalgo99 已提交
967 968 969 970 971 972
//                             const auto partConfidence = heatMapPtr[
//                                 bpOrig * heatMapOffset + yCleaned*heatMapSize.x + xCleaned];
//                             // If partConfidence is big enough, it means we are close to a keypoint
//                             if (partConfidence > T(0.05))
//                             {
//                                 const auto candidateNumberIndex = bpOrig*(maxPeaks+1)*3;
G
gineshidalgo99 已提交
973
//                                 const auto numberCandidates = positiveIntRound(peaksPtr[candidateNumberIndex]);
G
gineshidalgo99 已提交
974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
//                                 int closestIndex = -1;
//                                 T closetValue = std::numeric_limits<T>::max();
//                                 for (auto i = 0 ; i < numberCandidates ; i++)
//                                 {
//                                     const auto candidateXYSIndex = candidateNumberIndex+3*(1+i);
//                                     const auto diffX = partX-scaleFactor*peaksPtr[candidateXYSIndex];
//                                     const auto diffY = partY-scaleFactor*peaksPtr[candidateXYSIndex+1];
//                                     const auto dist = (diffX*diffX + diffY*diffY);
//                                     if (closetValue > dist)
//                                     {
//                                         closetValue = dist;
//                                         closestIndex = candidateXYSIndex;
//                                     }
//                                 }
//                                 if (closestIndex != -1)
//                                 {
//                                     poseKeypoints[{p,bpOrig,0}] = scaleFactor*peaksPtr[closestIndex];
//                                     poseKeypoints[{p,bpOrig,1}] = scaleFactor*peaksPtr[closestIndex+1];
//                                     // Set body part score
//                                     poseKeypoints[{p,bpOrig,2}] = peaksPtr[closestIndex+2];
//                                 }
//                             }
//                             // Set poseScore
//                             poseScores[p] += poseKeypoints[{p,bpOrig,2}];
//                         }
//                     }
//                 }
//             }
//         }
//         catch (const std::exception& e)
//         {
//             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
//         }
//     }
1008

G
gineshidalgo99 已提交
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
//     const std::vector<float> AVERAGE{
//         0.f, -6.55251f,
//         0.f, -4.15062f, -1.48818, -4.15506f,   -2.22408f, -0.312264f, -1.42204f, 0.588495f,
//         1.51044f, -4.14629f, 2.2113f, -0.312283f,   1.41081f, 0.612377f, -0.f, 3.41112f,
//         -0.932306f, 3.45504f, -0.899812f,   6.79837f, -0.794223f, 11.4972f,
//         0.919047f, 3.46442f, 0.902314f,   6.81245f, 0.79518f, 11.5132f,
//         -0.243982f, -7.07925f,   0.28065f, -7.07398f,
//         -0.792812f, -7.09374f,   0.810145f, -7.06958f,
//         0.582387f, 7.46846f, 0.889349f,   7.40577f, 0.465088f, 7.03969f,
//         -0.96686f, 7.46148f, -1.20773f,   7.38834f, -0.762135f, 6.99575f};
//     const std::vector<float> SIGMA{
//         7.26789f, 9.70751f,
//         6.29588f, 8.93472f, 6.97401f, 9.13746f,   7.49632f, 9.44757f, 8.06695f, 9.97319f,
//         6.99726f, 9.14608f, 7.50529f, 9.43568f,   8.05888f, 9.98207f, 6.38929f, 9.29314f,
//         6.71801f, 9.39271f, 8.00608f,   10.6141f, 10.3416f, 12.7812f,
//         6.69875f, 9.41407f, 8.01876f,   10.637f, 10.3475f, 12.7849f,
//         7.30923f, 9.7324f,   7.27886f, 9.73406f,
//         7.35978f, 9.7289f,   7.28914f, 9.67711f,
//         7.93153f, 8.10845f, 7.95577f,   8.01729f, 7.56865f, 7.87314f,
//         7.4655f, 8.25336f, 7.43958f,   8.26333f, 7.33667f, 7.97446f};
//     template <typename T>
//     std::array<T,3> regressPart(const Array<T>& person, const int rootIndex, const int targetIndex,
//                                 const int scaleDownFactor, const T* const heatMapPtr, const T* const peaksPtr,
//                                 const Point<int>& heatMapSize, const int maxPeaks, const T scaleFactor,
//                                 const unsigned int numberBodyPartsAndBkgAndPAFChannels)
//     {
//         try
//         {
//             std::array<T,3> result{0,0,0};
//             // poseKeypoints from neck-part distances
//             if (targetIndex != rootIndex && person[{rootIndex,2}] > T(0.05))
//             {
//                 // Set (x,y)
//                 const auto rootX = person[{rootIndex,0}];
//                 const auto rootY = person[{rootIndex,1}];
//                 // Get ideal distance
//                 const auto indexChannel = 2*targetIndex;
//                 const auto offsetIndex = numberBodyPartsAndBkgAndPAFChannels + indexChannel;
//                 const auto heatMapOffset = heatMapSize.area();
//                 const auto* mapX = heatMapPtr + offsetIndex * heatMapOffset;
//                 const auto* mapY = heatMapPtr + (offsetIndex+1) * heatMapOffset;
//                 const auto increaseRatio = scaleFactor*scaleDownFactor;
//                 // // Not refined method
G
gineshidalgo99 已提交
1052 1053
//                 // const auto index = positiveIntRound(rootY/scaleFactor)*heatMapSize.x
//                                     + positiveIntRound(rootX/scaleFactor);
G
gineshidalgo99 已提交
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
//                 // const Point<T> neckPartDist{
//                 //     increaseRatio*(mapX[index]*SIGMA[indexChannel]+AVERAGE[indexChannel]),
//                 //     increaseRatio*(mapY[index]*SIGMA[indexChannel+1]+AVERAGE[indexChannel+1])};
//                 // poseKeypoints[{p,targetIndex,0}] = rootX + neckPartDist.x;
//                 // poseKeypoints[{p,targetIndex,1}] = rootY + neckPartDist.y;
//                 // Refined method
//                 const auto constant = 5;
//                 Point<T> neckPartDistRefined{0, 0};
//                 auto counterRefinements = 0;
//                 // We must keep it inside the image size
G
gineshidalgo99 已提交
1064 1065
//                 for (auto y = fastMax(0, positiveIntRound(rootY/scaleFactor) - constant);
//                      y < fastMin(heatMapSize.y, positiveIntRound(rootY/scaleFactor) + constant+1) ; y++)
G
gineshidalgo99 已提交
1066
//                 {
G
gineshidalgo99 已提交
1067 1068
//                     for (auto x = fastMax(0, positiveIntRound(rootX/scaleFactor) - constant);
//                          x < fastMin(heatMapSize.x, positiveIntRound(rootX/scaleFactor) + constant+1) ; x++)
G
gineshidalgo99 已提交
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
//                     {
//                         const auto index = y*heatMapSize.x + x;
//                         neckPartDistRefined.x += mapX[index];
//                         neckPartDistRefined.y += mapY[index];
//                         counterRefinements++;
//                     }
//                 }
//                 neckPartDistRefined = Point<T>{
//                     neckPartDistRefined.x*SIGMA[indexChannel]+counterRefinements*AVERAGE[indexChannel],
//                     neckPartDistRefined.y*SIGMA[indexChannel+1]+counterRefinements*AVERAGE[indexChannel+1],
//                 };
//                 neckPartDistRefined *= increaseRatio/counterRefinements;
//                 const auto partX = rootX + neckPartDistRefined.x;
//                 const auto partY = rootY + neckPartDistRefined.y;
//                 result[0] = partX;
//                 result[1] = partY;
//                 // Set (temporary) body part score
//                 result[2] = T(0.0501);
//                 // Associate estimated keypoint with closest one
G
gineshidalgo99 已提交
1088 1089
//                 const auto xCleaned = fastMax(0, fastMin(heatMapSize.x-1, positiveIntRound(partX/scaleFactor)));
//                 const auto yCleaned = fastMax(0, fastMin(heatMapSize.y-1, positiveIntRound(partY/scaleFactor)));
G
gineshidalgo99 已提交
1090 1091 1092 1093 1094 1095
//                 const auto partConfidence = heatMapPtr[
//                     targetIndex * heatMapOffset + yCleaned*heatMapSize.x + xCleaned];
//                 // If partConfidence is big enough, it means we are close to a keypoint
//                 if (partConfidence > T(0.05))
//                 {
//                     const auto candidateNumberIndex = targetIndex*(maxPeaks+1)*3;
G
gineshidalgo99 已提交
1096
//                     const auto numberCandidates = positiveIntRound(peaksPtr[candidateNumberIndex]);
G
gineshidalgo99 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
//                     int closestIndex = -1;
//                     T closetValue = std::numeric_limits<T>::max();
//                     for (auto i = 0 ; i < numberCandidates ; i++)
//                     {
//                         const auto candidateXYSIndex = candidateNumberIndex+3*(1+i);
//                         const auto diffX = partX-scaleFactor*peaksPtr[candidateXYSIndex];
//                         const auto diffY = partY-scaleFactor*peaksPtr[candidateXYSIndex+1];
//                         const auto dist = (diffX*diffX + diffY*diffY);
//                         if (closetValue > dist)
//                         {
//                             closetValue = dist;
//                             closestIndex = candidateXYSIndex;
//                         }
//                     }
//                     if (closestIndex != -1)
//                     {
//                         result[0] = scaleFactor*peaksPtr[closestIndex];
//                         result[1] = scaleFactor*peaksPtr[closestIndex+1];
//                         // Set body part score
//                         result[2] = peaksPtr[closestIndex+2];
//                     }
//                 }
//             }
//             return result;
//         }
//         catch (const std::exception& e)
//         {
//             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
//             return std::array<T,3>{};
//         }
//     }

//     template <typename T>
//     void connectDistanceMultiStar(Array<T>& poseKeypoints, Array<T>& poseScores, const T* const heatMapPtr,
//                                   const T* const peaksPtr, const PoseModel poseModel, const Point<int>& heatMapSize,
//                                   const int maxPeaks, const T scaleFactor, const unsigned int numberBodyParts,
//                                   const unsigned int bodyPartPairsSize)
//     {
//         try
//         {
//             // poseKeypoints from neck-part distances
//             if (poseModel == PoseModel::BODY_25D)
//             {
//                 // Add all the root elements (necks)
//                 const std::vector<int> keypointsSize = {(int)numberBodyParts, 3};
//                 // Initial #people = number root elements
//                 const auto rootIndex = 1;
//                 std::vector<Array<T>> poseKeypointsTemp;
//                 // Iterate for each body part
//                 const std::array<int, 25> MAPPING{
//                     1, 8, 0, 2,5,9,12, 3,6,10,13, 15,16, 4,7,11,14, 17,18, 19,22,20,23,21,24};
//                 const auto numberBodyPartsAndBkgAndPAFChannels = numberBodyParts + 1 + bodyPartPairsSize;
//                 const auto scaleDownFactor = 8;
//                 for (auto index = 0u ; index < numberBodyParts ; index++)
//                 {
//                     const auto targetIndex = MAPPING[index];
//                     // Get all candidate keypoints
//                     const auto partNumberIndex = targetIndex*(maxPeaks+1)*3;
G
gineshidalgo99 已提交
1155
//                     const auto numberPartParts = positiveIntRound(peaksPtr[partNumberIndex]);
G
gineshidalgo99 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
//                     std::vector<std::array<T, 3>> currentPartCandidates(numberPartParts);
//                     for (auto i = 0u ; i < currentPartCandidates.size() ; i++)
//                     {
//                         const auto baseIndex = partNumberIndex+3*(i+1);
//                         currentPartCandidates[i][0] = scaleFactor*peaksPtr[baseIndex];
//                         currentPartCandidates[i][1] = scaleFactor*peaksPtr[baseIndex+1];
//                         currentPartCandidates[i][2] = peaksPtr[baseIndex+2];
//                     }
//                     // Detect new body part for existing people
//                     // For each temporary person --> Add new targetIndex part
//                     for (auto& person : poseKeypointsTemp)
//                     {
//                         // Estimate new body part w.r.t. each already-detected body part
//                         for (auto rootMapIndex = 0u ; rootMapIndex < index ; rootMapIndex++)
//                         {
//                             const auto rootIndex = MAPPING[rootMapIndex];
//                             if (person[{rootIndex,2}] > T(0.0501))
//                             {
//                                 const auto result = regressPart(
//                                     person, rootIndex, targetIndex, scaleDownFactor, heatMapPtr, peaksPtr,
//                                     heatMapSize, maxPeaks, scaleFactor, numberBodyPartsAndBkgAndPAFChannels);
//                                 if (person[{targetIndex,2}] < result[2])
//                                 {
//                                     person[{targetIndex,0}] = result[0];
//                                     person[{targetIndex,1}] = result[1];
//                                     person[{targetIndex,2}] = result[2];
//                                 }
//                             }
//                         }
//                     }
//                     // Add leftovers body parts as new people
// if (targetIndex == rootIndex)
// {
//                     const auto currentSize = poseKeypointsTemp.size();
//                     poseKeypointsTemp.resize(currentSize+currentPartCandidates.size());
//                     for (auto p = 0u ; p < currentPartCandidates.size() ; p++)
//                     {
//                         poseKeypointsTemp[currentSize+p] = Array<T>(keypointsSize, 0.f);
//                         const auto baseIndex = 3*targetIndex;
//                         poseKeypointsTemp[currentSize+p][baseIndex  ] = currentPartCandidates[p][0];
//                         poseKeypointsTemp[currentSize+p][baseIndex+1] = currentPartCandidates[p][1];
//                         poseKeypointsTemp[currentSize+p][baseIndex+2] = currentPartCandidates[p][2];
//                     }

// }
//                 }
//                 // poseKeypoints: Reformat poseKeypointsTemp as poseKeypoints
//                 poseKeypoints.reset({(int)poseKeypointsTemp.size(), (int)numberBodyParts, 3}, 0);
//                 poseScores.reset(poseKeypoints.getSize(0), 0.f);
//                 const auto keypointArea = poseKeypoints.getSize(1)*poseKeypoints.getSize(2);
//                 for (auto p = 0 ; p < poseKeypoints.getSize(0) ; p++)
//                 {
//                     const auto pIndex = p*keypointArea;
//                     for (auto part = 0 ; part < poseKeypoints.getSize(1) ; part++)
//                     {
//                         const auto baseIndexTemp = 3*part;
//                         const auto baseIndex = pIndex+baseIndexTemp;
//                         poseKeypoints[baseIndex  ] = poseKeypointsTemp[p][baseIndexTemp];
//                         poseKeypoints[baseIndex+1] = poseKeypointsTemp[p][baseIndexTemp+1];
//                         poseKeypoints[baseIndex+2] = poseKeypointsTemp[p][baseIndexTemp+2];
//                         // Set poseScore
//                         poseScores[p] += poseKeypoints[baseIndex+2];
//                     }
//                 }
//             }
//         }
//         catch (const std::exception& e)
//         {
//             error(e.what(), __LINE__, __FUNCTION__, __FILE__);
//         }
//     }
1227 1228

    template <typename T>
1229 1230 1231 1232 1233
    void connectBodyPartsCpu(
        Array<T>& poseKeypoints, Array<T>& poseScores, const T* const heatMapPtr, const T* const peaksPtr,
        const PoseModel poseModel, const Point<int>& heatMapSize, const int maxPeaks, const T interMinAboveThreshold,
        const T interThreshold, const int minSubsetCnt, const T minSubsetScore, const T scaleFactor,
        const bool maximizePositives)
1234 1235 1236 1237 1238 1239
    {
        try
        {
            // Parts Connection
            const auto& bodyPartPairs = getPosePartPairs(poseModel);
            const auto numberBodyParts = getPoseNumberBodyParts(poseModel);
G
Gines Hidalgo 已提交
1240
            const auto numberBodyPartPairs = (unsigned int)(bodyPartPairs.size() / 2);
1241 1242 1243 1244 1245
            if (numberBodyParts == 0)
                error("Invalid value of numberBodyParts, it must be positive, not " + std::to_string(numberBodyParts),
                      __LINE__, __FUNCTION__, __FILE__);
            // std::vector<std::pair<std::vector<int>, double>> refers to:
            //     - std::vector<int>: [body parts locations, #body parts found]
G
gineshidalgo99 已提交
1246
            //     - double: person subset score
1247
            auto peopleVector = createPeopleVector(
1248
                heatMapPtr, peaksPtr, poseModel, heatMapSize, maxPeaks, interThreshold, interMinAboveThreshold,
G
gineshidalgo99 已提交
1249
                bodyPartPairs, numberBodyParts, numberBodyPartPairs);
1250 1251 1252 1253 1254 1255
            // Delete people below the following thresholds:
                // a) minSubsetCnt: removed if less than minSubsetCnt body parts
                // b) minSubsetScore: removed if global score smaller than this
                // c) maxPeaks (POSE_MAX_PEOPLE): keep first maxPeaks people above thresholds
            int numberPeople;
            std::vector<int> validSubsetIndexes;
1256 1257 1258
            // validSubsetIndexes.reserve(fastMin((size_t)maxPeaks, peopleVector.size()));
            validSubsetIndexes.reserve(peopleVector.size());
            removePeopleBelowThresholdsAndFillFaces(
G
gineshidalgo99 已提交
1259
                validSubsetIndexes, numberPeople, peopleVector, numberBodyParts, minSubsetCnt, minSubsetScore,
1260
                maximizePositives, peaksPtr);
1261
            // Fill and return poseKeypoints
1262 1263 1264
            peopleVectorToPeopleArray(
                poseKeypoints, poseScores, scaleFactor, peopleVector, validSubsetIndexes, peaksPtr, numberPeople,
                numberBodyParts, numberBodyPartPairs);
G
gineshidalgo99 已提交
1265
            // Experimental code
1266
            if (poseModel == PoseModel::BODY_25D)
G
gineshidalgo99 已提交
1267 1268 1269
                error("BODY_25D is an experimental branch which is not usable.", __LINE__, __FUNCTION__, __FILE__);
//                 connectDistanceMultiStar(poseKeypoints, poseScores, heatMapPtr, peaksPtr, poseModel, heatMapSize,
//                                          maxPeaks, scaleFactor, numberBodyParts, bodyPartPairs.size());
1270 1271 1272 1273 1274 1275 1276 1277 1278
//                 connectDistanceStar(poseKeypoints, poseScores, heatMapPtr, peaksPtr, poseModel, heatMapSize,
//                                     maxPeaks, scaleFactor, numberBodyParts, bodyPartPairs.size());
        }
        catch (const std::exception& e)
        {
            error(e.what(), __LINE__, __FUNCTION__, __FILE__);
        }
    }

G
Gines Hidalgo 已提交
1279 1280 1281 1282
    template OP_API void connectBodyPartsCpu(
        Array<float>& poseKeypoints, Array<float>& poseScores, const float* const heatMapPtr,
        const float* const peaksPtr, const PoseModel poseModel, const Point<int>& heatMapSize, const int maxPeaks,
        const float interMinAboveThreshold, const float interThreshold, const int minSubsetCnt,
G
gineshidalgo99 已提交
1283
        const float minSubsetScore, const float scaleFactor, const bool maximizePositives);
G
Gines Hidalgo 已提交
1284 1285 1286 1287
    template OP_API void connectBodyPartsCpu(
        Array<double>& poseKeypoints, Array<double>& poseScores, const double* const heatMapPtr,
        const double* const peaksPtr, const PoseModel poseModel, const Point<int>& heatMapSize, const int maxPeaks,
        const double interMinAboveThreshold, const double interThreshold, const int minSubsetCnt,
G
gineshidalgo99 已提交
1288
        const double minSubsetScore, const double scaleFactor, const bool maximizePositives);
G
gineshidalgo99 已提交
1289

G
Gines Hidalgo 已提交
1290
    template OP_API std::vector<std::pair<std::vector<int>, float>> createPeopleVector(
G
gineshidalgo99 已提交
1291 1292 1293 1294 1295
        const float* const heatMapPtr, const float* const peaksPtr, const PoseModel poseModel,
        const Point<int>& heatMapSize, const int maxPeaks, const float interThreshold,
        const float interMinAboveThreshold, const std::vector<unsigned int>& bodyPartPairs,
        const unsigned int numberBodyParts, const unsigned int numberBodyPartPairs,
        const Array<float>& precomputedPAFs);
G
Gines Hidalgo 已提交
1296
    template OP_API std::vector<std::pair<std::vector<int>, double>> createPeopleVector(
G
gineshidalgo99 已提交
1297 1298 1299 1300 1301 1302
        const double* const heatMapPtr, const double* const peaksPtr, const PoseModel poseModel,
        const Point<int>& heatMapSize, const int maxPeaks, const double interThreshold,
        const double interMinAboveThreshold, const std::vector<unsigned int>& bodyPartPairs,
        const unsigned int numberBodyParts, const unsigned int numberBodyPartPairs,
        const Array<double>& precomputedPAFs);

1303
    template OP_API void removePeopleBelowThresholdsAndFillFaces(
G
gineshidalgo99 已提交
1304
        std::vector<int>& validSubsetIndexes, int& numberPeople,
1305 1306 1307 1308
        std::vector<std::pair<std::vector<int>, float>>& peopleVector,
        const unsigned int numberBodyParts, const int minSubsetCnt, const float minSubsetScore,
        const bool maximizePositives, const float* const peaksPtr);
    template OP_API void removePeopleBelowThresholdsAndFillFaces(
G
gineshidalgo99 已提交
1309
        std::vector<int>& validSubsetIndexes, int& numberPeople,
1310 1311 1312
        std::vector<std::pair<std::vector<int>, double>>& peopleVector,
        const unsigned int numberBodyParts, const int minSubsetCnt, const double minSubsetScore,
        const bool maximizePositives, const double* const peaksPtr);
G
gineshidalgo99 已提交
1313

G
Gines Hidalgo 已提交
1314
    template OP_API void peopleVectorToPeopleArray(
G
gineshidalgo99 已提交
1315 1316 1317 1318 1319
        Array<float>& poseKeypoints, Array<float>& poseScores, const float scaleFactor,
        const std::vector<std::pair<std::vector<int>, float>>& peopleVector,
        const std::vector<int>& validSubsetIndexes, const float* const peaksPtr,
        const int numberPeople, const unsigned int numberBodyParts,
        const unsigned int numberBodyPartPairs);
G
Gines Hidalgo 已提交
1320
    template OP_API void peopleVectorToPeopleArray(
G
gineshidalgo99 已提交
1321 1322 1323 1324 1325 1326
        Array<double>& poseKeypoints, Array<double>& poseScores, const double scaleFactor,
        const std::vector<std::pair<std::vector<int>, double>>& peopleVector,
        const std::vector<int>& validSubsetIndexes, const double* const peaksPtr,
        const int numberPeople, const unsigned int numberBodyParts,
        const unsigned int numberBodyPartPairs);

G
Gines Hidalgo 已提交
1327
    template OP_API std::vector<std::tuple<float, float, int, int, int>> pafPtrIntoVector(
G
gineshidalgo99 已提交
1328 1329
        const Array<float>& pairScores, const float* const peaksPtr, const int maxPeaks,
        const std::vector<unsigned int>& bodyPartPairs, const unsigned int numberBodyPartPairs);
G
Gines Hidalgo 已提交
1330
    template OP_API std::vector<std::tuple<double, double, int, int, int>> pafPtrIntoVector(
G
gineshidalgo99 已提交
1331 1332 1333
        const Array<double>& pairScores, const double* const peaksPtr, const int maxPeaks,
        const std::vector<unsigned int>& bodyPartPairs, const unsigned int numberBodyPartPairs);

G
Gines Hidalgo 已提交
1334
    template OP_API std::vector<std::pair<std::vector<int>, float>> pafVectorIntoPeopleVector(
G
gineshidalgo99 已提交
1335 1336 1337
        const std::vector<std::tuple<float, float, int, int, int>>& pairConnections,
        const float* const peaksPtr, const int maxPeaks, const std::vector<unsigned int>& bodyPartPairs,
        const unsigned int numberBodyParts);
G
Gines Hidalgo 已提交
1338
    template OP_API std::vector<std::pair<std::vector<int>, double>> pafVectorIntoPeopleVector(
G
gineshidalgo99 已提交
1339 1340 1341
        const std::vector<std::tuple<double, double, int, int, int>>& pairConnections,
        const double* const peaksPtr, const int maxPeaks, const std::vector<unsigned int>& bodyPartPairs,
        const unsigned int numberBodyParts);
G
gineshidalgo99 已提交
1342
}