Fixed MSMF file capture error while opening the file containing unsupported video stream format

Applied in PR3260 to 2.4 branch

modules/videoio/src/cap_msmf.cpp

@@ -3848,18 +3848,25 @@ bool CvCaptureFile_MSMF::open(const char* filename)
         hr = enumerateCaptureFormats(videoFileSource);
     }
 
-    if (SUCCEEDED(hr))
+    if( captureFormats.empty() )
+    {
+        isOpened = false;
+    }
+    else
     {
-        hr = ImageGrabberThread::CreateInstance(&grabberThread, videoFileSource, (unsigned int)-2, true);
+        if (SUCCEEDED(hr))
+        {
+            hr = ImageGrabberThread::CreateInstance(&grabberThread, videoFileSource, (unsigned int)-2, true);
+        }
+
+        isOpened = SUCCEEDED(hr);
     }
 
-    if (SUCCEEDED(hr))
+    if (isOpened)
     {
         grabberThread->start();
     }
 
-    isOpened = SUCCEEDED(hr);
-
     return isOpened;
 }
 
@@ -3991,7 +3998,9 @@ HRESULT CvCaptureFile_MSMF::enumerateCaptureFormats(IMFMediaSource *pSource)
             goto done;
         }
         MediaType MT = FormatReader::Read(pType.Get());
-        captureFormats.push_back(MT);
+        // We can capture only RGB video.
+        if( MT.MF_MT_SUBTYPE == MFVideoFormat_RGB24 )
+            captureFormats.push_back(MT);
     }
 
 done: