From 9e1d6e418a3b9003b3a42db13f49a9818bec3765 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Mon, 12 Feb 2018 11:09:41 -0500 Subject: [PATCH] 3D updating bug fixed, dlog in experimental module --- CMakeLists.txt | 54 ++++++++++++------- doc/installation.md | 4 +- doc/openpose_3d_reconstruction_demo.md | 8 ++- examples/CMakeLists.txt | 4 +- examples/experimental_3d/3d.cpp | 4 +- include/openpose/experimental/3d/datum3D.hpp | 1 + .../openpose/experimental/3d/flirReader.hpp | 42 +++++++++++++++ include/openpose/experimental/3d/headers.hpp | 4 +- .../openpose/experimental/3d/pointGrey.hpp | 39 -------------- .../experimental/3d/reconstruction3D.hpp | 8 +-- .../producer/imageDirectoryReader.hpp | 4 +- src/openpose/experimental/3d/CMakeLists.txt | 2 +- .../3d/{pointGrey.cpp => flirReader.cpp} | 49 +++++++++-------- .../experimental/3d/reconstruction3D.cpp | 27 ++++++---- src/openpose/experimental/3d/renderer.cpp | 30 ++++++----- .../producer/imageDirectoryReader.cpp | 10 ++-- ubuntu/Makefile.config.Ubuntu14_cuda7.example | 3 +- ubuntu/Makefile.config.Ubuntu14_cuda8.example | 3 +- ubuntu/Makefile.config.Ubuntu16_cuda7.example | 3 +- ubuntu/Makefile.config.Ubuntu16_cuda8.example | 3 +- .../Makefile.config.Ubuntu16_cuda8_JetsonTX2 | 3 +- ubuntu/Makefile.example | 20 ++++--- windows/OpenPose/OpenPose.vcxproj | 4 +- windows/OpenPose/OpenPose.vcxproj.filters | 6 +-- 24 files changed, 194 insertions(+), 141 deletions(-) create mode 100644 include/openpose/experimental/3d/flirReader.hpp delete mode 100644 include/openpose/experimental/3d/pointGrey.hpp rename src/openpose/experimental/3d/{pointGrey.cpp => flirReader.cpp} (95%) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9f97267..a80e5fd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,7 +144,8 @@ if (${GPU_MODE} MATCHES "CUDA") endif (${GPU_MODE} MATCHES "CUDA") # Suboptions for OpenPose 3D Reconstruction demo -option(BUILD_MODULE_3D "Build OpenPose 3D Reconstruction module." OFF) +option(WITH_3D_RENDERER "Add OpenPose 3D renderer module (it requires FreeGLUT library)." OFF) +option(WITH_FLIR_CAMERA "Add FLIR (formerly Point Grey) camera code (requires Spinnaker SDK already installed)." OFF) # Download the models option(DOWNLOAD_COCO_MODEL "Download COCO model." ON) @@ -171,10 +172,14 @@ if (${GPU_MODE} MATCHES "CUDA") endif () # Adding 3D -if (BUILD_MODULE_3D) +if (WITH_3D_RENDERER) # OpenPose flags - add_definitions(-DBUILD_MODULE_3D) -endif (BUILD_MODULE_3D) + add_definitions(-DWITH_3D_RENDERER) +endif (WITH_3D_RENDERER) +if (WITH_FLIR_CAMERA) + # OpenPose flags + add_definitions(-DWITH_FLIR_CAMERA) +endif (WITH_FLIR_CAMERA) if (UNIX AND NOT APPLE) if (${GPU_MODE} MATCHES "CUDA") @@ -193,19 +198,21 @@ if (UNIX AND NOT APPLE) endif (OpenCV_CONFIG_FILE) # 3D - if (BUILD_MODULE_3D) + if (WITH_3D_RENDERER) # GLUT find_package(GLUT REQUIRED) + endif (WITH_3D_RENDERER) + if (WITH_FLIR_CAMERA) # Spinnaker find_package(Spinnaker) if (NOT SPINNAKER_FOUND) - message(FATAL_ERROR "Spinnaker not found. Either turn off the `BUILD_MODULE_3D` option or specify the path of - Spinnaker includes and libs.") + message(FATAL_ERROR "Spinnaker not found. Either turn off the `WITH_FLIR_CAMERA` option or specify the path to + the Spinnaker includes and libs.") endif (NOT SPINNAKER_FOUND) # Otherwise error if 3D module enabled, and then disabled (with Spinnaker not installed) - else (BUILD_MODULE_3D) + else (WITH_FLIR_CAMERA) set(SPINNAKER_LIB "") - endif (BUILD_MODULE_3D) + endif (WITH_FLIR_CAMERA) # OpenMP if (${GPU_MODE} MATCHES "CPU_ONLY") @@ -271,9 +278,9 @@ if (WIN32) download_zip("opencv_310.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} 1e5240a64b814b3c0b822f136be78ad7) download_zip("caffe3rdparty_2017_07_14.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} ec0f800c8fb337e33304f3375bd06a80) download_zip("caffe_2018_01_18.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} 4b8e548cc7ea20abea472950dd5301bd) - if (BUILD_MODULE_3D) + if (WITH_3D_RENDERER) download_zip("freeglut_2018_01_14.zip" ${OP_WIN_URL} ${FIND_LIB_PREFIX} BB182187285E06880F0EDE3A39530091) - endif (BUILD_MODULE_3D) + endif (WITH_3D_RENDERER) message(STATUS "Windows dependencies downloaded.") find_library(OpenCV_LIBS opencv_world310 HINTS ${FIND_LIB_PREFIX}/opencv/x64/vc14/lib) @@ -284,20 +291,24 @@ if (WIN32) find_library(OpenCV_LIBS opencv_world310 HINTS ${FIND_LIB_PREFIX}/opencv/x64/vc14/lib) find_library(Caffe_LIB caffe HINTS ${FIND_LIB_PREFIX}/caffe/lib) find_library(Caffe_Proto_LIB caffeproto HINTS ${FIND_LIB_PREFIX}/caffe/lib) - if (BUILD_MODULE_3D) - find_library(SPINNAKER_LIB spinnaker_v140 HINTS ${FIND_LIB_PREFIX}/spinnaker/lib) + if (WITH_3D_RENDERER) find_library(GLUT_LIBRARY freeglut HINTS ${FIND_LIB_PREFIX}/freeglut/lib) message(STATUS "\${GLUT_LIBRARY} = ${GLUT_LIBRARY}") - endif (BUILD_MODULE_3D) + endif (WITH_3D_RENDERER) + if (WITH_FLIR_CAMERA) + find_library(SPINNAKER_LIB spinnaker_v140 HINTS ${FIND_LIB_PREFIX}/spinnaker/lib) + endif (WITH_FLIR_CAMERA) set(Caffe_LIBS ${Caffe_LIB};${Caffe_Proto_LIB}) set(OpenCV_INCLUDE_DIRS "3rdparty/windows/opencv/include") set(Caffe_INCLUDE_DIRS "3rdparty/windows/caffe/include;3rdparty/windows/caffe/include2") set(Boost_INCLUDE_DIRS "3rdparty/windows/caffe3rdparty/include/boost-1_61") set(WINDOWS_INCLUDE_DIRS "3rdparty/windows/caffe3rdparty/include") - if (BUILD_MODULE_3D) - set(SPINNAKER_INCLUDE_DIRS "3rdparty/windows/spinnaker/include") + if (WITH_3D_RENDERER) set(GLUT_INCLUDE_DIRS "3rdparty/windows/freeglut/include") - endif (BUILD_MODULE_3D) + endif (WITH_3D_RENDERER) + if (WITH_FLIR_CAMERA) + set(SPINNAKER_INCLUDE_DIRS "3rdparty/windows/spinnaker/include") + endif (WITH_FLIR_CAMERA) set(Caffe_FOUND 1) endif (WIN32) @@ -445,9 +456,12 @@ if (${GPU_MODE} MATCHES "CUDA") ${CUDA_INCLUDE_DIRS}) endif () # 3D -if (BUILD_MODULE_3D) - include_directories(${GLUT_INCLUDE_DIRS} ${SPINNAKER_INCLUDE_DIRS}) -endif (BUILD_MODULE_3D) +if (WITH_3D_RENDERER) + include_directories(${GLUT_INCLUDE_DIRS}) +endif (WITH_3D_RENDERER) +if (WITH_FLIR_CAMERA) + include_directories(${SPINNAKER_INCLUDE_DIRS}) +endif (WITH_FLIR_CAMERA) # Windows includes if (WIN32) include_directories( diff --git a/doc/installation.md b/doc/installation.md index 9e3b4d2a..71210a1f 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -198,7 +198,7 @@ By default, the body MPI model is not downloaded. You can download it by turning #### OpenPose 3D Reconstruction Module and Demo You can include the 3D reconstruction module by: -1. Install the FLIR camera driver and software, Spinnaker SDK. It is a propietary software, so we cannot provide direct download link. +1. Install the FLIR camera software, Spinnaker SDK. It is a propietary software, so we cannot provide direct download link. Note: You might skip this step if you intend to use the 3-D OpenPose module with a different camera brand. 1. Ubuntu: Get and install the latest Spinnaker SKD version in their default path. OpenPose will automatically find it. Otherwise, set the right path with CMake. 2. Windows: Donwload the latest Spinnaker SKD version from [https://www.ptgrey.com/support/downloads](https://www.ptgrey.com/support/downloads). - Copy `{PointGreyParentDirectory}\Point Grey Research\Spinnaker\bin64\vs2015\` as `{OpenPoseDirectory}\3rdparty\windows\spinnaker\bin\`. You can remove all the *.exe files. @@ -216,7 +216,7 @@ You can include the 3D reconstruction module by: - Copy `{freeglutParentDirectory}\freeglut\bin\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\bin\`. - Copy `{freeglutParentDirectory}\freeglut\include\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\include\`. - Copy `{freeglutParentDirectory}\freeglut\lib\x64\` as `{OpenPoseDirectory}\3rdparty\windows\freeglut\lib\`. -3. Follow the CMake installation steps, and set the `BUILD_MODULE_3D` option. +3. Follow the CMake installation steps. In addition, set the `WITH_FLIR_CAMERA` (only if Spinnaker was installed) and `WITH_3D_RENDERER` options. 4. In Windows, after openning the OpenPose visual studio solution: 1. Right-click on `Solution 'OpenPose'` of the `Solution Explorer` window, usually placed at the top-right part of the VS screen. 2. Click on `Properties`. Go to `Configuration Properties` -> `Configuration` and check `Build` for the `OpenPose3DReconstruction` project. diff --git a/doc/openpose_3d_reconstruction_demo.md b/doc/openpose_3d_reconstruction_demo.md index cd70cbfd..0e6af548 100644 --- a/doc/openpose_3d_reconstruction_demo.md +++ b/doc/openpose_3d_reconstruction_demo.md @@ -8,7 +8,8 @@ 4. [Camera Calibration](#camera-calibration) 5. [Camera Ordering](#camera-ordering) 6. [Installing the OpenPose 3-D Reconstruction Module](#installing-the-openpose-3-d-reconstruction-module) -7. [Known Bug](#known-bug) +7. [Using a Different Camera Brand](#using-a-different-camera-brand) +8. [Known Bug](#known-bug) @@ -87,6 +88,11 @@ It should be similar to the following image. +## Using a Different Camera Brand +You can copy and modify the OpenPose 3-D demo to use any camera brand, by modifying the frames producer. For that, you would need to provide your custom code to retrieve synchronized images from your cameras, as well as their intrinsic and extrinsic camera parameters. + + + ## Known Bug FreeGLUT is a quite light library. Due to that, there is a known bug in the 3D module: diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 1f19c3db..8076483c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,11 +1,9 @@ +add_subdirectory(experimental_3d) add_subdirectory(openpose) add_subdirectory(tutorial_add_module) add_subdirectory(tutorial_pose) add_subdirectory(tutorial_thread) add_subdirectory(tutorial_wrapper) -if (BUILD_MODULE_3D) - add_subdirectory(experimental_3d) -endif (BUILD_MODULE_3D) if (UNIX AND NOT APPLE) add_subdirectory(tests) endif (UNIX AND NOT APPLE) diff --git a/examples/experimental_3d/3d.cpp b/examples/experimental_3d/3d.cpp index 1259983b..947f409d 100644 --- a/examples/experimental_3d/3d.cpp +++ b/examples/experimental_3d/3d.cpp @@ -206,7 +206,7 @@ int openpose() // Initializing the user custom classes // Frames producer (e.g. video, webcam, ...) - auto wPointGrey = std::make_shared(); + auto wFlirReader = std::make_shared(); // Processing auto wReconstruction3D = std::make_shared(); // GUI (Display) @@ -215,7 +215,7 @@ int openpose() op::Wrapper> opWrapper; // Add custom input const auto workerInputOnNewThread = true; - opWrapper.setWorkerInput(wPointGrey, workerInputOnNewThread); + opWrapper.setWorkerInput(wFlirReader, workerInputOnNewThread); // Add custom processing const auto workerProcessingOnNewThread = true; opWrapper.setWorkerPostProcessing(wReconstruction3D, workerProcessingOnNewThread); diff --git a/include/openpose/experimental/3d/datum3D.hpp b/include/openpose/experimental/3d/datum3D.hpp index a5732644..9a518102 100644 --- a/include/openpose/experimental/3d/datum3D.hpp +++ b/include/openpose/experimental/3d/datum3D.hpp @@ -14,6 +14,7 @@ namespace op Array faceKeypoints3D; Array leftHandKeypoints3D; Array rightHandKeypoints3D; + cv::Mat cameraParameterMatrix; }; } diff --git a/include/openpose/experimental/3d/flirReader.hpp b/include/openpose/experimental/3d/flirReader.hpp new file mode 100644 index 00000000..a5524cac --- /dev/null +++ b/include/openpose/experimental/3d/flirReader.hpp @@ -0,0 +1,42 @@ +#ifndef OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP +#define OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP + +#include +#include +#include + +namespace op +{ + /** + * FLIR (Point-Grey) camera producer + * Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from WorkerProducer. + * This worker: + * 1. Set hardware trigger and the buffer to get the latest obtained frame. + * 2. Read images from FLIR cameras. + * 3. Turn them into std::vector. + * 4. Return the resulting images wrapped into a std::shared_ptr>. + * The HW trigger + reading FLIR camera code is highly based on the Spinnaker SDK examples + * `AcquisitionMultipleCamera` and specially `Trigger` + * (located in `src/`). See them for more details about the cameras. + * See `examples/tutorial_wrapper/` for more details about inhering the WorkerProducer class. + */ + class OP_API WFlirReader : public WorkerProducer>> + { + public: + WFlirReader(); + + ~WFlirReader(); + + void initializationOnThread(); + + std::shared_ptr> workProducer(); + + private: + // PIMPL idiom + // http://www.cppsamples.com/common-tasks/pimpl.html + struct ImplWFlirReader; + std::unique_ptr upImpl; + }; +} + +#endif // OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP diff --git a/include/openpose/experimental/3d/headers.hpp b/include/openpose/experimental/3d/headers.hpp index 59ee5b12..1658f06b 100644 --- a/include/openpose/experimental/3d/headers.hpp +++ b/include/openpose/experimental/3d/headers.hpp @@ -5,8 +5,8 @@ #include // Datum3D #include -// PointGrey cameras -#include +// Flir (Point-Grey) cameras +#include // 3D reconstruction #include // OpenGL Renderer diff --git a/include/openpose/experimental/3d/pointGrey.hpp b/include/openpose/experimental/3d/pointGrey.hpp deleted file mode 100644 index 01d94604..00000000 --- a/include/openpose/experimental/3d/pointGrey.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP -#define OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP - -#include -#include -#include - -namespace op -{ - // Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from WorkerProducer. - // This worker: - // 1. Set hardware trigger and the buffer to get the latest obtained frame. - // 2. Read images from FLIR cameras. - // 3. Turn them into std::vector. - // 4. Return the resulting images wrapped into a std::shared_ptr>. - // The HW trigger + reading FLIR camera code is highly based on the Spinnaker SDK examples `AcquisitionMultipleCamera` - // and specially `Trigger` - // (located in `src/`). See them for more details about the cameras. - // See `examples/tutorial_wrapper/` for more details about inhering the WorkerProducer class. - class OP_API WPointGrey : public WorkerProducer>> - { - public: - WPointGrey(); - - ~WPointGrey(); - - void initializationOnThread(); - - std::shared_ptr> workProducer(); - - private: - // PIMPL idiom - // http://www.cppsamples.com/common-tasks/pimpl.html - struct ImplWPointGrey; - std::unique_ptr upImpl; - }; -} - -#endif // OPENPOSE_EXPERIMENTAL_3D_POINT_GREY_HPP diff --git a/include/openpose/experimental/3d/reconstruction3D.hpp b/include/openpose/experimental/3d/reconstruction3D.hpp index 3bf35c4c..5101c694 100644 --- a/include/openpose/experimental/3d/reconstruction3D.hpp +++ b/include/openpose/experimental/3d/reconstruction3D.hpp @@ -9,11 +9,11 @@ namespace op { // Following OpenPose `tutorial_wrapper/` examples, we create our own class inherited from Worker. // This worker will do 3-D reconstruction - // We apply the simple Direct linear transformation (DLT) algorithm, asumming each keypoint (e.g. right hip) is seen - // by all the cameras. + // We apply the simple Direct linear transformation (DLT) algorithm, asumming each keypoint (e.g. right hip) is + // seen by all the cameras. // No non-linear minimization used, and if some camera misses the point, it is not reconstructed. - // See `examples/tutorial_wrapper/` for more details about inhering the Worker class and using it for post-processing - // purposes. + // See `examples/tutorial_wrapper/` for more details about inhering the Worker class and using it for + // post-processing purposes. class OP_API WReconstruction3D : public Worker>> { public: diff --git a/include/openpose/producer/imageDirectoryReader.hpp b/include/openpose/producer/imageDirectoryReader.hpp index dd003b92..b988dbf5 100644 --- a/include/openpose/producer/imageDirectoryReader.hpp +++ b/include/openpose/producer/imageDirectoryReader.hpp @@ -15,8 +15,8 @@ namespace op { public: /** - * Constructor of ImageDirectoryReader. It sets the image directory path from which the images will be loaded and - * generates a std::vector with the list of images on that directory. + * Constructor of ImageDirectoryReader. It sets the image directory path from which the images will be loaded + * and generates a std::vector with the list of images on that directory. * @param imageDirectoryPath const std::string parameter with the folder path containing the images. */ explicit ImageDirectoryReader(const std::string& imageDirectoryPath); diff --git a/src/openpose/experimental/3d/CMakeLists.txt b/src/openpose/experimental/3d/CMakeLists.txt index b510b50f..a048e567 100644 --- a/src/openpose/experimental/3d/CMakeLists.txt +++ b/src/openpose/experimental/3d/CMakeLists.txt @@ -1,6 +1,6 @@ set(SOURCES_OP_3D cameraParameters.cpp - pointGrey.cpp + flirReader.cpp reconstruction3D.cpp renderer.cpp) diff --git a/src/openpose/experimental/3d/pointGrey.cpp b/src/openpose/experimental/3d/flirReader.cpp similarity index 95% rename from src/openpose/experimental/3d/pointGrey.cpp rename to src/openpose/experimental/3d/flirReader.cpp index b49756e1..e04bdf21 100644 --- a/src/openpose/experimental/3d/pointGrey.cpp +++ b/src/openpose/experimental/3d/flirReader.cpp @@ -1,20 +1,20 @@ #include #include #include -#ifdef BUILD_MODULE_3D +#ifdef WITH_FLIR_CAMERA #include #endif #include #include -#include +#include namespace op { - #ifdef BUILD_MODULE_3D + #ifdef WITH_FLIR_CAMERA /* * This function converts between Spinnaker::ImagePtr container to cv::Mat container used in OpenCV. */ - cv::Mat pointGreyToCvMat(const Spinnaker::ImagePtr &imagePtr) + cv::Mat flirReaderToCvMat(const Spinnaker::ImagePtr &imagePtr) { try { @@ -266,10 +266,10 @@ namespace op for (auto i = 0u; i < imagePtrs.size(); i++) { // Baseline - // cvMats.emplace_back(pointGreyToCvMat(imagePtrs.at(i)).clone()); + // cvMats.emplace_back(flirReaderToCvMat(imagePtrs.at(i)).clone()); // Undistort // http://docs.opencv.org/2.4/modules/imgproc/doc/geometric_transformations.html#undistort - auto auxCvMat = pointGreyToCvMat(imagePtrs.at(i)); + auto auxCvMat = flirReaderToCvMat(imagePtrs.at(i)); cvMats.emplace_back(); cv::undistort(auxCvMat, cvMats[i], getIntrinsics(i), getDistorsion(i)); } @@ -321,29 +321,29 @@ namespace op } #endif - struct WPointGrey::ImplWPointGrey + struct WFlirReader::ImplWFlirReader { - #ifdef BUILD_MODULE_3D + #ifdef WITH_FLIR_CAMERA bool mInitialized; Spinnaker::CameraList mCameraList; Spinnaker::SystemPtr mSystemPtr; - ImplWPointGrey() : + ImplWFlirReader() : mInitialized{false} { } #endif }; - WPointGrey::WPointGrey() - #ifdef BUILD_MODULE_3D - : upImpl{new ImplWPointGrey{}} + WFlirReader::WFlirReader() + #ifdef WITH_FLIR_CAMERA + : upImpl{new ImplWFlirReader{}} #endif { try { - #ifndef BUILD_MODULE_3D - error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + #ifndef WITH_FLIR_CAMERA + error("OpenPose must be compiled with `WITH_FLIR_CAMERA` in order to use this class.", __LINE__, __FUNCTION__, __FILE__); #endif } @@ -353,9 +353,9 @@ namespace op } } - WPointGrey::~WPointGrey() + WFlirReader::~WFlirReader() { - #ifdef BUILD_MODULE_3D + #ifdef WITH_FLIR_CAMERA try { if (upImpl->mInitialized) @@ -417,9 +417,9 @@ namespace op #endif } - void WPointGrey::initializationOnThread() + void WFlirReader::initializationOnThread() { - #ifdef BUILD_MODULE_3D + #ifdef WITH_FLIR_CAMERA try { upImpl->mInitialized = true; @@ -600,13 +600,15 @@ namespace op #endif } - std::shared_ptr> WPointGrey::workProducer() + std::shared_ptr> WFlirReader::workProducer() { try { - #ifdef BUILD_MODULE_3D + #ifdef WITH_FLIR_CAMERA try { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Profiling speed const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // Get image from each camera @@ -616,7 +618,8 @@ namespace op for (auto i = 0u ; i < cvMats.size() ; i++) { datums3d->at(i).cvInputData = cvMats.at(i); - datums3d->at(i).cvOutputData = (*datums3d)[i].cvInputData; + (*datums3d)[i].cvOutputData = (*datums3d)[i].cvInputData; + (*datums3d)[i].cameraParameterMatrix = getM(i); } // Profiling speed if (!cvMats.empty()) @@ -624,6 +627,8 @@ namespace op Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); } + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Return Datum return datums3d; } @@ -634,7 +639,7 @@ namespace op return nullptr; } #else - error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + error("OpenPose must be compiled with `WITH_FLIR_CAMERA` in order to use this class.", __LINE__, __FUNCTION__, __FILE__); return nullptr; #endif diff --git a/src/openpose/experimental/3d/reconstruction3D.cpp b/src/openpose/experimental/3d/reconstruction3D.cpp index cb31bdf1..d5b8849a 100644 --- a/src/openpose/experimental/3d/reconstruction3D.cpp +++ b/src/openpose/experimental/3d/reconstruction3D.cpp @@ -1,5 +1,4 @@ #include -#include #include namespace op @@ -7,7 +6,7 @@ namespace op double calcReprojectionError(const cv::Mat& X, const std::vector& M, const std::vector& pt2D) { auto averageError = 0.; - for(unsigned int i = 0 ; i < M.size() ; i++) + for (auto i = 0u ; i < M.size() ; i++) { cv::Mat imageX = M[i] * X; imageX /= imageX.at(2,0); @@ -31,7 +30,8 @@ namespace op cv::Mat A = cv::Mat::zeros(numberCameras*2, 4, CV_64F); for (auto i = 0 ; i < numberCameras ; i++) { - cv::Mat temp = pointOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(0,1); + cv::Mat temp = pointOnEachCamera[i].x*matrixEachCamera[i].rowRange(2,3) + - matrixEachCamera[i].rowRange(0,1); temp.copyTo(A.rowRange(i*2,i*2+1)); temp = pointOnEachCamera[i].y*matrixEachCamera[i].rowRange(2,3) - matrixEachCamera[i].rowRange(1,2); temp.copyTo(A.rowRange(i*2+1,i*2+2)); @@ -42,7 +42,7 @@ namespace op X /= X.at(3); } - // TODO: ask Hanbyul for the missing function: TriangulationOptimization + // TODO: ask for the missing function: TriangulationOptimization double triangulateWithOptimization(cv::Mat& X, const std::vector& matrixEachCamera, const std::vector& pointOnEachCamera) { @@ -57,7 +57,8 @@ namespace op return 0.; } - void reconstructArray(Array& keypoints3D, const std::vector>& keypointsVector) + void reconstructArray(Array& keypoints3D, const std::vector>& keypointsVector, + const std::vector& matrixEachCamera) { // Get number body parts auto detectionMissed = false; @@ -106,7 +107,7 @@ namespace op for (auto i = 0u; i < xyPoints.size(); i++) { cv::Mat X; - triangulateWithOptimization(X, getMs(), xyPoints[i]); + triangulateWithOptimization(X, matrixEachCamera, xyPoints[i]); xyzPoints[i] = cv::Point3d{ X.at(0), X.at(1), X.at(2) }; } @@ -143,10 +144,13 @@ namespace op // datum.poseKeypoints: Array with the estimated pose try { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Profiling speed const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); if (datumsPtr != nullptr && !datumsPtr->empty()) { + std::vector matrixEachCamera; std::vector> poseKeypointVector; std::vector> faceKeypointVector; std::vector> leftHandKeypointVector; @@ -157,18 +161,21 @@ namespace op faceKeypointVector.emplace_back(datumsElement.faceKeypoints); leftHandKeypointVector.emplace_back(datumsElement.handKeypoints[0]); rightHandKeypointVector.emplace_back(datumsElement.handKeypoints[1]); + matrixEachCamera.emplace_back(datumsElement.cameraParameterMatrix); } // Pose 3-D reconstruction - reconstructArray(datumsPtr->at(0).poseKeypoints3D, poseKeypointVector); + reconstructArray(datumsPtr->at(0).poseKeypoints3D, poseKeypointVector, matrixEachCamera); // Face 3-D reconstruction - reconstructArray(datumsPtr->at(0).faceKeypoints3D, faceKeypointVector); + reconstructArray(datumsPtr->at(0).faceKeypoints3D, faceKeypointVector, matrixEachCamera); // Left hand 3-D reconstruction - reconstructArray(datumsPtr->at(0).leftHandKeypoints3D, leftHandKeypointVector); + reconstructArray(datumsPtr->at(0).leftHandKeypoints3D, leftHandKeypointVector, matrixEachCamera); // Right hand 3-D reconstruction - reconstructArray(datumsPtr->at(0).rightHandKeypoints3D, rightHandKeypointVector); + reconstructArray(datumsPtr->at(0).rightHandKeypoints3D, rightHandKeypointVector, matrixEachCamera); // Profiling speed Profiler::timerEnd(profilerKey); Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); } } catch (const std::exception& e) diff --git a/src/openpose/experimental/3d/renderer.cpp b/src/openpose/experimental/3d/renderer.cpp index 738947aa..a4466136 100644 --- a/src/openpose/experimental/3d/renderer.cpp +++ b/src/openpose/experimental/3d/renderer.cpp @@ -1,6 +1,6 @@ #include #include -#ifdef BUILD_MODULE_3D +#ifdef WITH_3D_RENDERER #include #include // glutLeaveMainLoop #include @@ -15,7 +15,7 @@ namespace op { const bool LOG_VERBOSE_3D_RENDERER = false; - #ifdef BUILD_MODULE_3D + #ifdef WITH_3D_RENDERER struct Keypoints3D { Array mPoseKeypoints; @@ -206,7 +206,7 @@ namespace op } // this is the actual idle function - void idleFunc() + void idleFunction() { glutPostRedisplay(); glutSwapBuffers(); @@ -381,8 +381,7 @@ namespace op char *my_argv[] = { NULL }; int my_argc = 0; glutInit(&my_argc, my_argv); - - // setup the size, position, and display mode for new windows + // Setup the size, position, and display mode for new windows glutInitWindowSize(1280, 720); glutInitWindowPosition(200, 0); // glutSetOption(GLUT_MULTISAMPLE,8); @@ -390,14 +389,14 @@ namespace op // button, but it does not work (tested in Ubuntu) // https://stackoverflow.com/questions/3799803/is-it-possible-to-make-a-window-withouth-a-top-in-glut glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH | GLUT_MULTISAMPLE); - - // create and set up a window + // Create and set up a window glutCreateWindow(GUI_NAME.c_str()); initGraphics(); glutDisplayFunc(renderMain); glutMouseFunc(mouseButton); glutMotionFunc(mouseMotion); - glutIdleFunc(idleFunc); + // Only required if glutMainLoop() called + // glutIdleFunc(idleFunction); // Full screen would fix the problem of disabling `x` button // glutFullScreen(); // Key presses @@ -414,12 +413,12 @@ namespace op { try { - #ifdef BUILD_MODULE_3D + #ifdef WITH_3D_RENDERER // Update sPoseModel sPoseModel = poseModel; #else UNUSED(poseModel); - error("OpenPose must be compiled with `BUILD_MODULE_3D` in order to use this class.", + error("OpenPose must be compiled with `WITH_3D_RENDERER` in order to use this class.", __LINE__, __FUNCTION__, __FILE__); #endif } @@ -433,7 +432,7 @@ namespace op { try { - #ifdef BUILD_MODULE_3D + #ifdef WITH_3D_RENDERER glutLeaveMainLoop(); #endif } @@ -445,7 +444,7 @@ namespace op void WRender3D::initializationOnThread() { - #ifdef BUILD_MODULE_3D + #ifdef WITH_3D_RENDERER try { // Init display @@ -462,9 +461,11 @@ namespace op void WRender3D::workConsumer(const std::shared_ptr>& datumsPtr) { - #ifdef BUILD_MODULE_3D + #ifdef WITH_3D_RENDERER try { + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Profiling speed const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); // User's displaying/saving/other processing here @@ -504,9 +505,12 @@ namespace op this->stop(); // OpenCL - Run main loop event // It is run outside loop, or it would get visually stuck if loop to slow + idleFunction(); glutMainLoopEvent(); // This alternative can only be called once, and it will block the thread until program exit // glutMainLoop(); + // Debugging log + dLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); } catch (const std::exception& e) { diff --git a/src/openpose/producer/imageDirectoryReader.cpp b/src/openpose/producer/imageDirectoryReader.cpp index f08a0c81..a6ed8dca 100644 --- a/src/openpose/producer/imageDirectoryReader.cpp +++ b/src/openpose/producer/imageDirectoryReader.cpp @@ -10,8 +10,11 @@ namespace op try { // Get files on directory with the desired extensions - const std::vector extensions{"bmp", "dib", "pbm", "pgm", "ppm", "sr", "ras", // Completely supported by OpenCV - "jpg", "jpeg", "png"}; // Most of them supported by OpenCV + const std::vector extensions{ + // Completely supported by OpenCV + "bmp", "dib", "pbm", "pgm", "ppm", "sr", "ras", + // Most of them supported by OpenCV + "jpg", "jpeg", "png"}; const auto imagePaths = getFilesOnDirectory(imageDirectoryPath, extensions); // Check #files > 0 @@ -45,7 +48,8 @@ namespace op try { auto frame = loadImage(mFilePaths.at(mFrameNameCounter++).c_str(), CV_LOAD_IMAGE_COLOR); - // Check frame integrity. This function also checks width/height changes. However, if it is performed after setWidth/setHeight this is performed over the new resolution (so they always match). + // Check frame integrity. This function also checks width/height changes. However, if it is performed + // after setWidth/setHeight this is performed over the new resolution (so they always match). checkFrameIntegrity(frame); // Update size, since images might have different size between each one of them mResolution = Point{frame.cols, frame.rows}; diff --git a/ubuntu/Makefile.config.Ubuntu14_cuda7.example b/ubuntu/Makefile.config.Ubuntu14_cuda7.example index 43128b2c..877cff7c 100644 --- a/ubuntu/Makefile.config.Ubuntu14_cuda7.example +++ b/ubuntu/Makefile.config.Ubuntu14_cuda7.example @@ -53,7 +53,8 @@ DEEP_NET := caffe CAFFE_DIR := 3rdparty/caffe/distribute # OpenPose 3-D Reconstruction -# BUILD_MODULE_3D := 1 +# WITH_3D_RENDERER := 1 +# WITH_FLIR_CAMERA := 1 # Spinnaker directory SPINNAKER_DIR := /usr/include/spinnaker diff --git a/ubuntu/Makefile.config.Ubuntu14_cuda8.example b/ubuntu/Makefile.config.Ubuntu14_cuda8.example index cb448414..21dc26ee 100644 --- a/ubuntu/Makefile.config.Ubuntu14_cuda8.example +++ b/ubuntu/Makefile.config.Ubuntu14_cuda8.example @@ -59,7 +59,8 @@ DEEP_NET := caffe CAFFE_DIR := 3rdparty/caffe/distribute # OpenPose 3-D Reconstruction -# BUILD_MODULE_3D := 1 +# WITH_3D_RENDERER := 1 +# WITH_FLIR_CAMERA := 1 # Spinnaker directory SPINNAKER_DIR := /usr/include/spinnaker diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda7.example b/ubuntu/Makefile.config.Ubuntu16_cuda7.example index 3fbd60fb..4a2159d8 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda7.example +++ b/ubuntu/Makefile.config.Ubuntu16_cuda7.example @@ -53,7 +53,8 @@ DEEP_NET := caffe CAFFE_DIR := 3rdparty/caffe/distribute # OpenPose 3-D Reconstruction -# BUILD_MODULE_3D := 1 +# WITH_3D_RENDERER := 1 +# WITH_FLIR_CAMERA := 1 # Spinnaker directory SPINNAKER_DIR := /usr/include/spinnaker diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda8.example b/ubuntu/Makefile.config.Ubuntu16_cuda8.example index 1830999f..92fcf077 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda8.example +++ b/ubuntu/Makefile.config.Ubuntu16_cuda8.example @@ -59,7 +59,8 @@ DEEP_NET := caffe CAFFE_DIR := 3rdparty/caffe/distribute # OpenPose 3-D Reconstruction -# BUILD_MODULE_3D := 1 +# WITH_3D_RENDERER := 1 +# WITH_FLIR_CAMERA := 1 # Spinnaker directory SPINNAKER_DIR := /usr/include/spinnaker diff --git a/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 b/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 index 75563a59..0d2e9419 100644 --- a/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 +++ b/ubuntu/Makefile.config.Ubuntu16_cuda8_JetsonTX2 @@ -59,7 +59,8 @@ DEEP_NET := caffe CAFFE_DIR := 3rdparty/caffe/distribute # OpenPose 3-D Reconstruction -# BUILD_MODULE_3D := 1 +# WITH_3D_RENDERER := 1 +# WITH_FLIR_CAMERA := 1 # Spinnaker directory SPINNAKER_DIR := /usr/include/spinnaker diff --git a/ubuntu/Makefile.example b/ubuntu/Makefile.example index c3f991f0..7fba9af3 100644 --- a/ubuntu/Makefile.example +++ b/ubuntu/Makefile.example @@ -71,17 +71,23 @@ else endif -# Spinnaker -BUILD_MODULE_3D ?= 0 -ifneq ($(BUILD_MODULE_3D), 0) - COMMON_FLAGS += -DBUILD_MODULE_3D - # Spinnaker SDK - LIBRARIES += Spinnaker - INCLUDE_DIRS += $(SPINNAKER_DIR) +# 3-D +# 3-D renderer +WITH_3D_RENDERER ?= 0 +ifneq ($(WITH_3D_RENDERER), 0) + COMMON_FLAGS += -DWITH_3D_RENDERER # FreeGLUT LIBRARIES += glut # LIBRARIES += GLU GL glut endif +# Spinnaker SDK +WITH_FLIR_CAMERA ?= 0 +ifneq ($(WITH_FLIR_CAMERA), 0) + COMMON_FLAGS += -DWITH_FLIR_CAMERA + # Spinnaker SDK + LIBRARIES += Spinnaker + INCLUDE_DIRS += $(SPINNAKER_DIR) +endif ############################## # Get all source files diff --git a/windows/OpenPose/OpenPose.vcxproj b/windows/OpenPose/OpenPose.vcxproj index 72445ecf..826a7bfd 100644 --- a/windows/OpenPose/OpenPose.vcxproj +++ b/windows/OpenPose/OpenPose.vcxproj @@ -125,8 +125,8 @@ + - @@ -284,7 +284,7 @@ - + diff --git a/windows/OpenPose/OpenPose.vcxproj.filters b/windows/OpenPose/OpenPose.vcxproj.filters index e69527f2..4bfb14db 100644 --- a/windows/OpenPose/OpenPose.vcxproj.filters +++ b/windows/OpenPose/OpenPose.vcxproj.filters @@ -560,10 +560,10 @@ Header Files\experimental\3d - + Header Files\experimental\3d - + Header Files\experimental\3d @@ -657,7 +657,7 @@ Source Files\experimental\3d - + Source Files\experimental\3d -- GitLab