From 50ce91e2af0c2c28c59e8f5bd9ae49cf57587689 Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Thu, 24 May 2018 22:22:53 -0400 Subject: [PATCH] Improved flir camera image quality --- doc/release_notes.md | 26 ++++---- src/openpose/producer/spinnakerWrapper.cpp | 69 +++++++++++++++------- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/doc/release_notes.md b/doc/release_notes.md index 657df72c..60854c0a 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -214,18 +214,20 @@ OpenPose Library - Release Notes ## Current version - future OpenPose 1.3.1 1. Main improvements: 1. New calibration module: Intrinsic and extrinsic camera calibration toolbox based on OpenCV. - 2. Flir cameras: Added software trigger and a dedicated thread to keep reading images so latency is removed and runtime is faster (analogously to webcamReader). - 3. Flir cameras: Undistortion of the images is x3.5 faster per camera, i.e., x3.5 Flir camera producer reading (it was multi-threaded). - 4. Flir cameras: Added flag `flir_camera_index` to allow running on all the cameras at once, or only on 1 camera at the time. - 5. Flir cameras: Added flag `frame_keep_distortion` not to undistort the images. E.g., useful when recording images for camera calibration. - 6. 3-D reconstruction: Added non-linear minimization to further improve 3-D triangulation accuracy by ~5% (Ubuntu only). - 7. CMake: All libraries as single variable (simpler to add/remove libraries). - 8. Datum includes extrinsic and intrinsic camera parameters. - 9. Function `scaleKeypoints(Array& keypoints, const float scale)` also accepts 3D keypoints. - 10. 3D keypoints and camera parameters in meters (instead of millimeters) in order to reduce numerical errors. - 11. New `PoseExtractor` class to contain future ID and tracking algorithms as well as the current OpenPose keypoint detection algorithm. - 12. Added initial alpha versions of the `tracking` and `identification` modules (for now disabled but available in the source code), including `PersonIdExtractor` and `PersonTracker`. `PersonIdExtractor` includes greedy matrix OP-LK matching. - 13. Added catchs to all demos for higher debug information. + 2. Improvements involving Flir cameras: + 1. Added software trigger and a dedicated thread to keep reading images so latency is removed and runtime is faster (analogously to webcamReader). + 2. Undistortion of the images is x3.5 faster per camera, i.e., x3.5 Flir camera producer reading w.r.t previous multi-threaded version, which was x number_cameras faster than the original version. + 3. Added flag `flir_camera_index` to allow running on all the cameras at once, or only on 1 camera at the time. + 4. Added flag `frame_keep_distortion` not to undistort the images. E.g., useful when recording images for camera calibration. + 5. Changed Spinnaker::DEFAULT image extraction mode by Spinnaker::IPP, which does not show a pixelated image while keeping very similar runtime. + 3. 3-D reconstruction: Added non-linear minimization to further improve 3-D triangulation accuracy by ~5% (Ubuntu only). + 4. CMake: All libraries as single variable (simpler to add/remove libraries). + 5. Datum includes extrinsic and intrinsic camera parameters. + 6. Function `scaleKeypoints(Array& keypoints, const float scale)` also accepts 3D keypoints. + 7. 3D keypoints and camera parameters in meters (instead of millimeters) in order to reduce numerical errors. + 8. New `PoseExtractor` class to contain future ID and tracking algorithms as well as the current OpenPose keypoint detection algorithm. + 9. Added initial alpha versions of the `tracking` and `identification` modules (for now disabled but available in the source code), including `PersonIdExtractor` and `PersonTracker`. `PersonIdExtractor` includes greedy matrix OP-LK matching. + 10. Added catchs to all demos for higher debug information. 2. Functions or parameters renamed: 1. Removed scale parameter from hand and face rectangle extractor (causing wrong results if custom `--output_resolution`). 2. Functions `scaleKeypoints`, other than `scaleKeypoints(Array& keypoints, const float scale)`, renamed as `scaleKeypoints2d`. diff --git a/src/openpose/producer/spinnakerWrapper.cpp b/src/openpose/producer/spinnakerWrapper.cpp index a0f18dfe..88dd5c7a 100644 --- a/src/openpose/producer/spinnakerWrapper.cpp +++ b/src/openpose/producer/spinnakerWrapper.cpp @@ -167,31 +167,50 @@ namespace op // Colors // http://softwareservices.ptgrey.com/Spinnaker/latest/group___camera_defs__h.html#ggabd5af55aaa20bcb0644c46241c2cbad1a33a1c8a1f6dbcb4a4eaaaf6d4d7ff1d1 // PixelFormat_BGR8 - // Time tests - // const auto reps = 1e3; - // const auto begin = std::chrono::high_resolution_clock::now(); - // for (auto asdf = 0 ; asdf < reps ; asdf++){ - // ~ 1.5 ms but pixeled - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); - return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); + + // // Time tests + // // DEFAULT + // const auto reps = 1e2; + // const auto begin1 = std::chrono::high_resolution_clock::now(); + // for (auto asdf = 0 ; asdf < reps ; asdf++) + // const auto imagePtrTemp = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); + // const auto durationMs1 = std::chrono::duration_cast( + // std::chrono::high_resolution_clock::now()-begin1 + // ).count() * 1e-6; + // // EDGE_SENSING + // const auto begin2 = std::chrono::high_resolution_clock::now(); + // for (auto asdf = 0 ; asdf < reps ; asdf++) + // const auto imagePtrTemp = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::EDGE_SENSING); + // const auto durationMs2 = std::chrono::duration_cast( + // std::chrono::high_resolution_clock::now()-begin2 + // ).count() * 1e-6; + // // IPP + // const auto begin3 = std::chrono::high_resolution_clock::now(); + // for (auto asdf = 0 ; asdf < reps ; asdf++) + // const auto imagePtrTemp = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::IPP); + // const auto durationMs3 = std::chrono::duration_cast( + // std::chrono::high_resolution_clock::now()-begin3 + // ).count() * 1e-6; + // // Print times + // log("Time (ms) 1: " + std::to_string(durationMs1 / reps), Priority::High); + // log("Time (ms) 2: " + std::to_string(durationMs2 / reps), Priority::High); + // log("Time (ms) 3: " + std::to_string(durationMs3 / reps), Priority::High); + + // Return right one + // ~ 1.3 ms but pixeled + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DEFAULT); // ~0.5 ms but BW - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::NO_COLOR_PROCESSING); + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::NO_COLOR_PROCESSING); // ~6 ms, looks as good as best - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::HQ_LINEAR); - // ~2 ms default << edge << best - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::EDGE_SENSING); + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::HQ_LINEAR); + // ~2.2 ms default << edge << best + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::EDGE_SENSING); // ~115, too slow - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::RIGOROUS); - // ~2 ms, slightly worse than HQ_LINEAR - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::IPP); + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::RIGOROUS); + // ~1.7 ms, slightly worse than HQ_LINEAR + return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::IPP); // ~30 ms, ideally best quality? - // imagePtr = imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DIRECTIONAL_FILTER); - // imagePtr = imagePtr; - // } - // durationMs = std::chrono::duration_cast( - // std::chrono::high_resolution_clock::now()-begin - // ).count() * 1e-6; - // log("Time conversion (ms): " + std::to_string(durationMs / reps), Priority::High); + // return imagePtr->Convert(Spinnaker::PixelFormat_BGR8, Spinnaker::DIRECTIONAL_FILTER); } /* @@ -404,6 +423,7 @@ namespace op // cameraIntrinsics instead of cv::getOptimalNewCameraMatrix to // avoid black borders cameraIntrinsics, + // #include for next line // cv::getOptimalNewCameraMatrix(cameraIntrinsics, // cameraDistorsions, // imageSize, 1, @@ -838,6 +858,9 @@ namespace op for (auto i = 0u; i < serialNumbers.size(); i++) log("Camera " + std::to_string(i) + " serial number set to " + serialNumbers[i] + "...", Priority::High); + if (upImpl->mCameraIndex >= 0) + log("Only using camera index " + std::to_string(upImpl->mCameraIndex) + ", i.e., serial number " + + serialNumbers[upImpl->mCameraIndex] + "...", Priority::High); // Read camera parameters from SN if (upImpl->mUndistortImage) @@ -867,7 +890,9 @@ namespace op else upImpl->mResolution = Point{cvMats[0].cols, cvMats[0].rows}; - log("\nRunning for all cameras...\n\n*** IMAGE ACQUISITION ***\n", Priority::High); + const std::string numberCameras = std::to_string(upImpl->mCameraIndex < 0 ? serialNumbers.size() : 1); + log("\nRunning for " + numberCameras + " out of " + std::to_string(serialNumbers.size()) + + " camera(s)...\n\n*** IMAGE ACQUISITION ***\n", Priority::High); } catch (const Spinnaker::Exception& e) { -- GitLab