From 38a35c4a5e2662d8b8770e47850a62d0b6cde67b Mon Sep 17 00:00:00 2001 From: StephanEwen Date: Sat, 29 Jun 2013 21:37:37 +0200 Subject: [PATCH] Cleaned up connected components example and tests. --- ...orksetConnectedComponentsCoGroupTest.java} | 6 ++--- ...va => WorksetConnectedComponentsTest.java} | 4 ++-- .../plandump/CompiledPlanDumpTest.java | 2 +- .../plandump/PreviewPlanDumpTest.java | 2 +- pact/pact-examples/pom.xml | 22 +++++++++++++++++++ .../DuplicateLongInputFormat.java | 2 +- .../LongLongInputFormat.java | 2 +- .../WorksetConnectedComponents.java | 2 +- ...orksetConnectedComponentsWithCoGroup.java} | 8 +++---- .../CoGroupConnectedComponentsITCase.java | 4 ++-- .../iterative/ConnectedComponentsITCase.java | 2 +- .../ConnectedComponentsNepheleITCase.java | 6 ++--- 12 files changed, 42 insertions(+), 20 deletions(-) rename pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/{CoGroupConnectedComponentsTest.java => WorksetConnectedComponentsCoGroupTest.java} (95%) rename pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/{IncrementalConnectedComponentsTest.java => WorksetConnectedComponentsTest.java} (97%) rename pact/pact-examples/src/main/java/eu/stratosphere/pact/example/{iterative => connectedcomponents}/DuplicateLongInputFormat.java (96%) rename pact/pact-examples/src/main/java/eu/stratosphere/pact/example/{iterative => connectedcomponents}/LongLongInputFormat.java (96%) rename pact/pact-examples/src/main/java/eu/stratosphere/pact/example/{iterative => connectedcomponents}/WorksetConnectedComponents.java (99%) rename pact/pact-examples/src/main/java/eu/stratosphere/pact/example/{iterative/CoGroupConnectedComponents.java => connectedcomponents/WorksetConnectedComponentsWithCoGroup.java} (94%) diff --git a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/CoGroupConnectedComponentsTest.java b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsCoGroupTest.java similarity index 95% rename from pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/CoGroupConnectedComponentsTest.java rename to pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsCoGroupTest.java index baf1a101470..79aa07ceae6 100644 --- a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/CoGroupConnectedComponentsTest.java +++ b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsCoGroupTest.java @@ -28,7 +28,7 @@ import eu.stratosphere.pact.compiler.plan.candidate.SourcePlanNode; import eu.stratosphere.pact.compiler.plan.candidate.WorksetIterationPlanNode; import eu.stratosphere.pact.compiler.plandump.PlanJSONDumpGenerator; import eu.stratosphere.pact.compiler.plantranslate.NepheleJobGraphGenerator; -import eu.stratosphere.pact.example.iterative.CoGroupConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponentsWithCoGroup; import eu.stratosphere.pact.runtime.shipping.ShipStrategyType; import eu.stratosphere.pact.runtime.task.DriverStrategy; import eu.stratosphere.pact.runtime.task.util.LocalStrategy; @@ -36,7 +36,7 @@ import eu.stratosphere.pact.runtime.task.util.LocalStrategy; /** * */ -public class CoGroupConnectedComponentsTest extends CompilerTestBase { +public class WorksetConnectedComponentsCoGroupTest extends CompilerTestBase { private static final String VERTEX_SOURCE = "Vertices"; @@ -55,7 +55,7 @@ public class CoGroupConnectedComponentsTest extends CompilerTestBase { @Test public void testWorksetConnectedComponents() { - CoGroupConnectedComponents cc = new CoGroupConnectedComponents(); + WorksetConnectedComponentsWithCoGroup cc = new WorksetConnectedComponentsWithCoGroup(); Plan plan = cc.getPlan(String.valueOf(DEFAULT_PARALLELISM), IN_FILE, IN_FILE, OUT_FILE, String.valueOf(100)); diff --git a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/IncrementalConnectedComponentsTest.java b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsTest.java similarity index 97% rename from pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/IncrementalConnectedComponentsTest.java rename to pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsTest.java index 03bb08a4910..5a01adc7635 100644 --- a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/IncrementalConnectedComponentsTest.java +++ b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/iterations/WorksetConnectedComponentsTest.java @@ -29,7 +29,7 @@ import eu.stratosphere.pact.compiler.plan.candidate.SourcePlanNode; import eu.stratosphere.pact.compiler.plan.candidate.WorksetIterationPlanNode; import eu.stratosphere.pact.compiler.plandump.PlanJSONDumpGenerator; import eu.stratosphere.pact.compiler.plantranslate.NepheleJobGraphGenerator; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents; import eu.stratosphere.pact.runtime.shipping.ShipStrategyType; import eu.stratosphere.pact.runtime.task.DriverStrategy; import eu.stratosphere.pact.runtime.task.util.LocalStrategy; @@ -37,7 +37,7 @@ import eu.stratosphere.pact.runtime.task.util.LocalStrategy; /** * */ -public class IncrementalConnectedComponentsTest extends CompilerTestBase { +public class WorksetConnectedComponentsTest extends CompilerTestBase { private static final String VERTEX_SOURCE = "Vertices"; diff --git a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/CompiledPlanDumpTest.java b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/CompiledPlanDumpTest.java index a2ba76669f2..247df2fb266 100644 --- a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/CompiledPlanDumpTest.java +++ b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/CompiledPlanDumpTest.java @@ -23,7 +23,7 @@ import org.junit.Test; import eu.stratosphere.pact.common.plan.Plan; import eu.stratosphere.pact.compiler.CompilerTestBase; import eu.stratosphere.pact.compiler.plan.candidate.OptimizedPlan; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents; import eu.stratosphere.pact.example.kmeans.KMeansIterative; import eu.stratosphere.pact.example.kmeans.KMeansSingleStep; import eu.stratosphere.pact.example.relational.TPCHQuery3; diff --git a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/PreviewPlanDumpTest.java b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/PreviewPlanDumpTest.java index 5723f38796d..24d6ed08665 100644 --- a/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/PreviewPlanDumpTest.java +++ b/pact/pact-compiler-tests/src/test/java/eu/stratosphere/pact/compiler/plandump/PreviewPlanDumpTest.java @@ -25,7 +25,7 @@ import org.junit.Test; import eu.stratosphere.pact.common.plan.Plan; import eu.stratosphere.pact.compiler.PactCompiler; import eu.stratosphere.pact.compiler.plan.DataSinkNode; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents; import eu.stratosphere.pact.example.kmeans.KMeansIterative; import eu.stratosphere.pact.example.kmeans.KMeansSingleStep; import eu.stratosphere.pact.example.relational.TPCHQuery3; diff --git a/pact/pact-examples/pom.xml b/pact/pact-examples/pom.xml index 76f882124b3..84aac8f632b 100644 --- a/pact/pact-examples/pom.xml +++ b/pact/pact-examples/pom.xml @@ -40,6 +40,28 @@ 2.4 + + + ConnectedComponents + package + + jar + + + ConnectedComponents + + + + eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents + + + + + **/connectedcomponents/*.class + + + + DanglingPageRank diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/DuplicateLongInputFormat.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/DuplicateLongInputFormat.java similarity index 96% rename from pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/DuplicateLongInputFormat.java rename to pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/DuplicateLongInputFormat.java index 144e5cc806f..5ea5dee7907 100644 --- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/DuplicateLongInputFormat.java +++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/DuplicateLongInputFormat.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.pact.example.iterative; +package eu.stratosphere.pact.example.connectedcomponents; import eu.stratosphere.pact.common.io.TextInputFormat; import eu.stratosphere.pact.common.type.PactRecord; diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/LongLongInputFormat.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/LongLongInputFormat.java similarity index 96% rename from pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/LongLongInputFormat.java rename to pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/LongLongInputFormat.java index e3919a50230..2c95f758f97 100644 --- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/LongLongInputFormat.java +++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/LongLongInputFormat.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.pact.example.iterative; +package eu.stratosphere.pact.example.connectedcomponents; import eu.stratosphere.pact.common.io.TextInputFormat; import eu.stratosphere.pact.common.type.PactRecord; diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/WorksetConnectedComponents.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponents.java similarity index 99% rename from pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/WorksetConnectedComponents.java rename to pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponents.java index d2f776f5ef1..0adc1129f68 100644 --- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/WorksetConnectedComponents.java +++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponents.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.pact.example.iterative; +package eu.stratosphere.pact.example.connectedcomponents; import java.util.Iterator; diff --git a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/CoGroupConnectedComponents.java b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponentsWithCoGroup.java similarity index 94% rename from pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/CoGroupConnectedComponents.java rename to pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponentsWithCoGroup.java index 2201e874dc7..e747435403e 100644 --- a/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/iterative/CoGroupConnectedComponents.java +++ b/pact/pact-examples/src/main/java/eu/stratosphere/pact/example/connectedcomponents/WorksetConnectedComponentsWithCoGroup.java @@ -13,7 +13,7 @@ * **********************************************************************************************************************/ -package eu.stratosphere.pact.example.iterative; +package eu.stratosphere.pact.example.connectedcomponents; import java.util.Iterator; @@ -32,14 +32,14 @@ import eu.stratosphere.pact.common.stubs.StubAnnotation.ConstantFieldsFirst; import eu.stratosphere.pact.common.stubs.StubAnnotation.ConstantFieldsSecond; import eu.stratosphere.pact.common.type.PactRecord; import eu.stratosphere.pact.common.type.base.PactLong; -import eu.stratosphere.pact.example.iterative.DuplicateLongInputFormat; -import eu.stratosphere.pact.example.iterative.LongLongInputFormat; +import eu.stratosphere.pact.example.connectedcomponents.DuplicateLongInputFormat; +import eu.stratosphere.pact.example.connectedcomponents.LongLongInputFormat; import eu.stratosphere.pact.generic.contract.WorksetIteration; /** * */ -public class CoGroupConnectedComponents implements PlanAssembler, PlanAssemblerDescription { +public class WorksetConnectedComponentsWithCoGroup implements PlanAssembler, PlanAssemblerDescription { public static final class NeighborWithComponentIDJoin extends MatchStub { diff --git a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/CoGroupConnectedComponentsITCase.java b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/CoGroupConnectedComponentsITCase.java index 84025c41453..71c1109d95e 100644 --- a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/CoGroupConnectedComponentsITCase.java +++ b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/CoGroupConnectedComponentsITCase.java @@ -24,7 +24,7 @@ import org.junit.runners.Parameterized.Parameters; import eu.stratosphere.nephele.configuration.Configuration; import eu.stratosphere.pact.common.plan.Plan; -import eu.stratosphere.pact.example.iterative.CoGroupConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponentsWithCoGroup; import eu.stratosphere.pact.test.iterative.nephele.ConnectedComponentsNepheleITCase; import eu.stratosphere.pact.test.util.TestBase2; @@ -60,7 +60,7 @@ public class CoGroupConnectedComponentsITCase extends TestBase2 { int maxIterations = config.getInteger("ConnectedComponents#NumIterations", 1); String[] params = { String.valueOf(dop) , verticesPath, edgesPath, resultPath, String.valueOf(maxIterations) }; - CoGroupConnectedComponents cc = new CoGroupConnectedComponents(); + WorksetConnectedComponentsWithCoGroup cc = new WorksetConnectedComponentsWithCoGroup(); return cc.getPlan(params); } diff --git a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/ConnectedComponentsITCase.java b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/ConnectedComponentsITCase.java index f7627ef5ad1..95770318441 100644 --- a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/ConnectedComponentsITCase.java +++ b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/ConnectedComponentsITCase.java @@ -24,7 +24,7 @@ import org.junit.runners.Parameterized.Parameters; import eu.stratosphere.nephele.configuration.Configuration; import eu.stratosphere.pact.common.plan.Plan; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents; import eu.stratosphere.pact.test.iterative.nephele.ConnectedComponentsNepheleITCase; import eu.stratosphere.pact.test.util.TestBase2; diff --git a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/nephele/ConnectedComponentsNepheleITCase.java b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/nephele/ConnectedComponentsNepheleITCase.java index e353972d6c9..a43f960a71a 100644 --- a/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/nephele/ConnectedComponentsNepheleITCase.java +++ b/pact/pact-tests/src/test/java/eu/stratosphere/pact/test/iterative/nephele/ConnectedComponentsNepheleITCase.java @@ -43,9 +43,9 @@ import eu.stratosphere.pact.common.stubs.aggregators.LongSumAggregator; import eu.stratosphere.pact.common.type.PactRecord; import eu.stratosphere.pact.common.type.base.PactLong; import eu.stratosphere.pact.common.type.base.parser.DecimalTextLongParser; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents.MinimumComponentIDReduce; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents.NeighborWithComponentIDJoin; -import eu.stratosphere.pact.example.iterative.WorksetConnectedComponents.UpdateComponentIdMatch; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents.MinimumComponentIDReduce; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents.NeighborWithComponentIDJoin; +import eu.stratosphere.pact.example.connectedcomponents.WorksetConnectedComponents.UpdateComponentIdMatch; import eu.stratosphere.pact.generic.types.TypeComparatorFactory; import eu.stratosphere.pact.generic.types.TypePairComparatorFactory; import eu.stratosphere.pact.generic.types.TypeSerializerFactory; -- GitLab