Only filter InvalidOperation from the GetPackageSpecsAsync call

src/VisualStudio/Core/Def/Packaging/PackageInstallerServiceFactory.cs

@@ -502,12 +502,13 @@ private void ProcessProjectChange(Solution solution, ProjectId projectId)
                 // Nuget may throw an ArgumentException when there is something about the project 
                 // they do not like/support.
             }
-            catch (InvalidOperationException)
+            catch (InvalidOperationException e) when (e.StackTrace.Contains("NuGet.PackageManagement.VisualStudio.NetCorePackageReferenceProject.GetPackageSpecsAsync"))
             {
                 // NuGet throws an InvalidOperationException if details
                 // for the project fail to load. We don't need to report
                 // these, and can assume that this will work on a future
                 // project change
+                // This should be removed with https://github.com/dotnet/roslyn/issues/33187
             }
             catch (Exception e) when (FatalError.ReportWithoutCrash(e))
             {