Merge pull request #16094 from saskatchewancatch:issue-16053

* Add eps error checking for approxPolyDP to allow sensible values only
for epsilon value of Douglas-Peucker algorithm.

* Review changes for PR

modules/imgproc/src/approx.cpp

@@ -677,6 +677,13 @@ void cv::approxPolyDP( InputArray _curve, OutputArray _approxCurve,
 {
     CV_INSTRUMENT_REGION();
 
+    //Prevent unreasonable error values (Douglas-Peucker algorithm)
+    //from being used.
+    if (epsilon < 0.0 || !(epsilon < 1e30))
+    {
+        CV_Error(CV_StsOutOfRange, "Epsilon not valid.");
+    }
+
     Mat curve = _curve.getMat();
     int npoints = curve.checkVector(2), depth = curve.depth();
     CV_Assert( npoints >= 0 && (depth == CV_32S || depth == CV_32F));

modules/imgproc/test/test_approxpoly.cpp

@@ -355,4 +355,25 @@ _exit_:
 
 TEST(Imgproc_ApproxPoly, accuracy) { CV_ApproxPolyTest test; test.safe_run(); }
 
+//Tests to make sure that unreasonable epsilon (error)
+//values never get passed to the Douglas-Peucker algorithm.
+TEST(Imgproc_ApproxPoly, bad_epsilon)
+{
+    std::vector<Point2f> inputPoints;
+    inputPoints.push_back(Point2f(0.0f, 0.0f));
+    std::vector<Point2f> outputPoints;
+
+    double eps = std::numeric_limits<double>::infinity();
+    ASSERT_ANY_THROW(approxPolyDP(inputPoints, outputPoints, eps, false));
+
+    eps = 9e99;
+    ASSERT_ANY_THROW(approxPolyDP(inputPoints, outputPoints, eps, false));
+
+    eps = -1e-6;
+    ASSERT_ANY_THROW(approxPolyDP(inputPoints, outputPoints, eps, false));
+
+    eps = NAN;
+    ASSERT_ANY_THROW(approxPolyDP(inputPoints, outputPoints, eps, false));
+}
+
 }} // namespace