[SPARK-2549] Functions defined inside of other functions trigger failures

Author: Prashant Sharma <prashant.s@imaginea.com>

Closes #1510 from ScrapCodes/SPARK-2549/fun-in-fun and squashes the following commits:

9458bc5 [Prashant Sharma] Tested by removing an inner function from excludes.
bc03b1c [Prashant Sharma] SPARK-2549 Functions defined inside of other functions trigger failures

project/MimaExcludes.scala

@@ -32,108 +32,83 @@ import com.typesafe.tools.mima.core._
  */
 object MimaExcludes {
 
-  def excludes(version: String) = version match {
-    case v if v.startsWith("1.1") =>
-      Seq(
-        MimaBuild.excludeSparkPackage("deploy"),
-        MimaBuild.excludeSparkPackage("graphx")
-      ) ++
-      closures.map(method => ProblemFilters.exclude[MissingMethodProblem](method)) ++
-      Seq(
-        // Adding new method to JavaRDLike trait - we should probably mark this as a developer API.
-        ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.api.java.JavaRDDLike.partitions"),
-        // We made a mistake earlier (ed06500d3) in the Java API to use default parameter values
-        // for countApproxDistinct* functions, which does not work in Java. We later removed
-        // them, and use the following to tell Mima to not care about them.
-        ProblemFilters.exclude[IncompatibleResultTypeProblem](
-          "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
-        ProblemFilters.exclude[IncompatibleResultTypeProblem](
-          "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.api.java.JavaPairRDD.countApproxDistinct$default$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey$default$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.api.java.JavaRDD.countApproxDistinct$default$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.api.java.JavaRDDLike.countApproxDistinct$default$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.api.java.JavaDoubleRDD.countApproxDistinct$default$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.storage.MemoryStore.Entry"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$debugChildren$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$firstDebugString$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$shuffleDebugString$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$debugString$1"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$"
-            + "createZero$1")
-      ) ++
-      Seq(
-        ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.flume.FlumeReceiver.this")
-      ) ++
-      Seq( // Ignore some private methods in ALS.
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures"),
-        ProblemFilters.exclude[MissingMethodProblem]( // The only public constructor is the one without arguments.
-          "org.apache.spark.mllib.recommendation.ALS.this"),
-        ProblemFilters.exclude[MissingMethodProblem](
-          "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$<init>$default$7"),
-        ProblemFilters.exclude[IncompatibleMethTypeProblem](
-          "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures")
-      ) ++
-      MimaBuild.excludeSparkClass("mllib.linalg.distributed.ColumnStatisticsAggregator") ++
-      MimaBuild.excludeSparkClass("rdd.ZippedRDD") ++
-      MimaBuild.excludeSparkClass("rdd.ZippedPartition") ++
-      MimaBuild.excludeSparkClass("util.SerializableHyperLogLog") ++
-      MimaBuild.excludeSparkClass("storage.Values") ++
-      MimaBuild.excludeSparkClass("storage.Entry") ++
-      MimaBuild.excludeSparkClass("storage.MemoryStore$Entry") ++
-      Seq(
-        ProblemFilters.exclude[IncompatibleMethTypeProblem](
-          "org.apache.spark.mllib.tree.impurity.Gini.calculate"),
-        ProblemFilters.exclude[IncompatibleMethTypeProblem](
-          "org.apache.spark.mllib.tree.impurity.Entropy.calculate"),
-        ProblemFilters.exclude[IncompatibleMethTypeProblem](
-          "org.apache.spark.mllib.tree.impurity.Variance.calculate")
-      )
-    case v if v.startsWith("1.0") =>
-      Seq(
-        MimaBuild.excludeSparkPackage("api.java"),
-        MimaBuild.excludeSparkPackage("mllib"),
-        MimaBuild.excludeSparkPackage("streaming")
-      ) ++
-      MimaBuild.excludeSparkClass("rdd.ClassTags") ++
-      MimaBuild.excludeSparkClass("util.XORShiftRandom") ++
-      MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++
-      MimaBuild.excludeSparkClass("graphx.VertexRDD") ++
-      MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++
-      MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++
-      MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++
-      MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++
-      MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
-      MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++
-      MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
-      MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++
-      MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
-    case _ => Seq()
-  }
-
-  private val closures = Seq(
-    "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$mergeMaps$1",
-    "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$countPartition$1",
-    "org.apache.spark.rdd.RDD.org$apache$spark$rdd$RDD$$distributePartition$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$mergeValue$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$writeToFile$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$reducePartition$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$writeShard$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$mergeCombiners$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$process$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$createCombiner$1",
-    "org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$mergeMaps$1"
-  )
+    def excludes(version: String) =
+      version match {
+        case v if v.startsWith("1.1") =>
+          Seq(
+            MimaBuild.excludeSparkPackage("deploy"),
+            MimaBuild.excludeSparkPackage("graphx")
+          ) ++
+          Seq(
+            // Adding new method to JavaRDLike trait - we should probably mark this as a developer API.
+            ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.api.java.JavaRDDLike.partitions"),
+            // We made a mistake earlier (ed06500d3) in the Java API to use default parameter values
+            // for countApproxDistinct* functions, which does not work in Java. We later removed
+            // them, and use the following to tell Mima to not care about them.
+            ProblemFilters.exclude[IncompatibleResultTypeProblem](
+              "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
+            ProblemFilters.exclude[IncompatibleResultTypeProblem](
+              "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.api.java.JavaPairRDD.countApproxDistinct$default$1"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey$default$1"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.api.java.JavaRDD.countApproxDistinct$default$1"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.api.java.JavaRDDLike.countApproxDistinct$default$1"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.api.java.JavaDoubleRDD.countApproxDistinct$default$1"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.storage.MemoryStore.Entry")
+          ) ++
+          Seq(
+            ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.flume.FlumeReceiver.this")
+          ) ++
+          Seq( // Ignore some private methods in ALS.
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures"),
+            ProblemFilters.exclude[MissingMethodProblem]( // The only public constructor is the one without arguments.
+              "org.apache.spark.mllib.recommendation.ALS.this"),
+            ProblemFilters.exclude[MissingMethodProblem](
+              "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$<init>$default$7"),
+            ProblemFilters.exclude[IncompatibleMethTypeProblem](
+              "org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures")
+          ) ++
+          MimaBuild.excludeSparkClass("mllib.linalg.distributed.ColumnStatisticsAggregator") ++
+          MimaBuild.excludeSparkClass("rdd.ZippedRDD") ++
+          MimaBuild.excludeSparkClass("rdd.ZippedPartition") ++
+          MimaBuild.excludeSparkClass("util.SerializableHyperLogLog") ++
+          MimaBuild.excludeSparkClass("storage.Values") ++
+          MimaBuild.excludeSparkClass("storage.Entry") ++
+          MimaBuild.excludeSparkClass("storage.MemoryStore$Entry") ++
+          Seq(
+            ProblemFilters.exclude[IncompatibleMethTypeProblem](
+              "org.apache.spark.mllib.tree.impurity.Gini.calculate"),
+            ProblemFilters.exclude[IncompatibleMethTypeProblem](
+              "org.apache.spark.mllib.tree.impurity.Entropy.calculate"),
+            ProblemFilters.exclude[IncompatibleMethTypeProblem](
+              "org.apache.spark.mllib.tree.impurity.Variance.calculate")
+          )
+        case v if v.startsWith("1.0") =>
+          Seq(
+            MimaBuild.excludeSparkPackage("api.java"),
+            MimaBuild.excludeSparkPackage("mllib"),
+            MimaBuild.excludeSparkPackage("streaming")
+          ) ++
+          MimaBuild.excludeSparkClass("rdd.ClassTags") ++
+          MimaBuild.excludeSparkClass("util.XORShiftRandom") ++
+          MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++
+          MimaBuild.excludeSparkClass("graphx.VertexRDD") ++
+          MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++
+          MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++
+          MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++
+          MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++
+          MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
+          MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++
+          MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
+          MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++
+          MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
+        case _ => Seq()
+      }
 }

tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala

@@ -99,9 +99,25 @@ object GenerateMIMAIgnore {
     (ignoredClasses.flatMap(c => Seq(c, c.replace("$", "#"))).toSet, ignoredMembers.toSet)
   }
 
+  /** Scala reflection does not let us see inner function even if they are upgraded
+    * to public for some reason. So had to resort to java reflection to get all inner
+    * functions with $$ in there name.
+    */
+  def getInnerFunctions(classSymbol: unv.ClassSymbol): Seq[String] = {
+    try {
+      Class.forName(classSymbol.fullName, false, classLoader).getMethods.map(_.getName)
+        .filter(_.contains("$$")).map(classSymbol.fullName + "." + _)
+    } catch {
+      case t: Throwable =>
+        println("[WARN] Unable to detect inner functions for class:" + classSymbol.fullName)
+        Seq.empty[String]
+    }
+  }
+
   private def getAnnotatedOrPackagePrivateMembers(classSymbol: unv.ClassSymbol) = {
     classSymbol.typeSignature.members
-      .filter(x => isPackagePrivate(x) || isDeveloperApi(x) || isExperimental(x)).map(_.fullName)
+      .filter(x => isPackagePrivate(x) || isDeveloperApi(x) || isExperimental(x)).map(_.fullName) ++
+      getInnerFunctions(classSymbol)
   }
 
   def main(args: Array[String]) {
@@ -121,7 +137,8 @@ object GenerateMIMAIgnore {
     name.endsWith("$class") ||
     name.contains("$sp") ||
     name.contains("hive") ||
-    name.contains("Hive")
+    name.contains("Hive") ||
+    name.contains("repl")
   }
 
   /**