Skip to content

F
flink

doujutun3207 / flink
与 Fork 源项目一致

从无法访问的项目Fork

通知 24
Star 0
Fork 0
F
flink

体验新版 GitCode，发现更多精彩内容 >>

提交 f90d0575 编写于 作者: F Fabian Hueske
浏览文件
操作

- Move time and memory intensive tests to integration tests in pact modules 

- Removed log level fixing from pact sort tests
- Changed info logs to debug logs in pact-runtime tests
- Removed invalid character from WebLogGenerator.java
上级 59a383cd
隐藏空白更改
内联 并排
Showing with 271 addition and 411 deletion
pact/pact-examples/src/main/java/eu/stratosphere/pact/example/relational/generator/WebLogGenerator.java
浏览文件 @ f90d0575
...@@ -218,7 +218,7 @@ public class WebLogGenerator { ...@@ -218,7 +218,7 @@ public class WebLogGenerator {
for (int i = 0; i < noVisits; i++) { for (int i = 0; i < noVisits; i++) {
int year = yearFilter - 3 + rand.nextInt(6); // yearFilter 3 int year = yearFilter - 3 + rand.nextInt(6); // yearFilter 3
int month = rand.nextInt(12) + 1; // month between 1 and 12 int month = rand.nextInt(12) + 1; // month between 1 and 12
int day = rand.nextInt(27) + 1; // day between 1 and 28 int day = rand.nextInt(27) + 1; // day between 1 and 28
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/hash/InMemoryHashMergeIteratorTest.java
浏览文件 @ f90d0575
...@@ -29,7 +29,6 @@ import eu.stratosphere.nephele.io.Reader; ...@@ -29,7 +29,6 @@ import eu.stratosphere.nephele.io.Reader;
import eu.stratosphere.pact.common.type.Key; import eu.stratosphere.pact.common.type.Key;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.common.type.Value; import eu.stratosphere.pact.common.type.Value;
import eu.stratosphere.pact.runtime.hash.InMemoryHashMatchIterator;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator; import eu.stratosphere.pact.runtime.test.util.TestData.Generator;
import eu.stratosphere.pact.runtime.test.util.TestData.RecordReaderMock; import eu.stratosphere.pact.runtime.test.util.TestData.RecordReaderMock;
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/io/MemoryAccessSpeedTest.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/io/MemoryAccessSpeedITCase.java
浏览文件 @ f90d0575
...@@ -23,7 +23,7 @@ import org.junit.Before; ...@@ -23,7 +23,7 @@ import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
public class MemoryAccessSpeedTest { public class MemoryAccessSpeedITCase {
private static final int ARRAY_LENGTH = 1024 * 1024 * 164; private static final int ARRAY_LENGTH = 1024 * 1024 * 164;
private static final int SEGMENT_OFFSET = 1024 * 1024 * 16; private static final int SEGMENT_OFFSET = 1024 * 1024 * 16;
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestBufferSortable.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/BufferSortableTest.java
浏览文件 @ f90d0575
...@@ -22,8 +22,6 @@ import java.util.Iterator; ...@@ -22,8 +22,6 @@ import java.util.Iterator;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
...@@ -42,9 +40,6 @@ import eu.stratosphere.nephele.services.memorymanager.MemorySegment; ...@@ -42,9 +40,6 @@ import eu.stratosphere.nephele.services.memorymanager.MemorySegment;
import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager; import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory; import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory;
import eu.stratosphere.pact.runtime.sort.BufferSortable;
import eu.stratosphere.pact.runtime.sort.DeserializerComparator;
import eu.stratosphere.pact.runtime.sort.QuickSort;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator.KeyMode; import eu.stratosphere.pact.runtime.test.util.TestData.Generator.KeyMode;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode; import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode;
...@@ -52,8 +47,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode; ...@@ -52,8 +47,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode;
/** /**
* @author Erik Nijkamp * @author Erik Nijkamp
*/ */
public class TestBufferSortable { public class BufferSortableTest {
private static final Log LOG = LogFactory.getLog(TestBufferSortable.class); private static final Log LOG = LogFactory.getLog(BufferSortableTest.class);
private static final long SEED = 649180756312423613L; private static final long SEED = 649180756312423613L;
...@@ -67,26 +62,12 @@ public class TestBufferSortable { ...@@ -67,26 +62,12 @@ public class TestBufferSortable {
private MemoryManager memoryManager; private MemoryManager memoryManager;
private static Level rootLevel, pkqLevel;
@BeforeClass @BeforeClass
public static void beforeClass() { public static void beforeClass() {
Logger rootLogger = Logger.getRootLogger();
rootLevel = rootLogger.getLevel();
rootLogger.setLevel(Level.INFO);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(BufferSortable.class.getPackage().getName());
pkqLevel = pkgLogger.getLevel();
pkgLogger.setLevel(Level.DEBUG);
} }
@AfterClass @AfterClass
public static void afterClass() { public static void afterClass() {
Logger rootLogger = Logger.getRootLogger();
rootLogger.setLevel(rootLevel);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(BufferSortable.class.getPackage().getName());
pkgLogger.setLevel(pkqLevel);
} }
@Before @Before
...@@ -132,7 +113,7 @@ public class TestBufferSortable { ...@@ -132,7 +113,7 @@ public class TestBufferSortable {
writtenPairs++; writtenPairs++;
pair = generator.next(); pair = generator.next();
} }
LOG.info("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of " LOG.debug("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of "
+ MEMORY_SIZE + " bytes."); + MEMORY_SIZE + " bytes.");
memory = buffer.unbind(); memory = buffer.unbind();
} }
...@@ -146,7 +127,7 @@ public class TestBufferSortable { ...@@ -146,7 +127,7 @@ public class TestBufferSortable {
while (buffer.read(pair)) { while (buffer.read(pair)) {
readPairs++; readPairs++;
} }
LOG.info("Read " + readPairs + " pairs from buffer."); LOG.debug("Read " + readPairs + " pairs from buffer.");
memory = buffer.unbind(); memory = buffer.unbind();
} }
...@@ -177,7 +158,7 @@ public class TestBufferSortable { ...@@ -177,7 +158,7 @@ public class TestBufferSortable {
writtenPairs++; writtenPairs++;
pair = generator.next(); pair = generator.next();
} }
LOG.info("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of " + 1024 LOG.debug("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of " + 1024
+ " bytes."); + " bytes.");
limit = buffer.getPosition(); limit = buffer.getPosition();
memory = buffer.unbind(); memory = buffer.unbind();
...@@ -194,7 +175,7 @@ public class TestBufferSortable { ...@@ -194,7 +175,7 @@ public class TestBufferSortable {
LOG.debug("-> " + pair); LOG.debug("-> " + pair);
readPairs++; readPairs++;
} }
LOG.info("Read " + readPairs + " pairs from buffer."); LOG.debug("Read " + readPairs + " pairs from buffer.");
memory = buffer.unbind(); memory = buffer.unbind();
} }
...@@ -221,7 +202,7 @@ public class TestBufferSortable { ...@@ -221,7 +202,7 @@ public class TestBufferSortable {
writtenBytes += generator.sizeOf(pair) + Integer.SIZE / 8; writtenBytes += generator.sizeOf(pair) + Integer.SIZE / 8;
pair = generator.next(); pair = generator.next();
} }
LOG.info("Occupied " + writtenBytes + " of " + MEMORY_SIZE + " bytes."); LOG.debug("Occupied " + writtenBytes + " of " + MEMORY_SIZE + " bytes.");
memory = buffer.unbind(); memory = buffer.unbind();
} }
...@@ -245,7 +226,7 @@ public class TestBufferSortable { ...@@ -245,7 +226,7 @@ public class TestBufferSortable {
while (unsortedBuffer.write(generator.next())) { while (unsortedBuffer.write(generator.next())) {
writtenPairs++; writtenPairs++;
} }
LOG.info("Written " + writtenPairs + " pairs."); LOG.debug("Written " + writtenPairs + " pairs.");
} }
...@@ -335,7 +316,7 @@ public class TestBufferSortable { ...@@ -335,7 +316,7 @@ public class TestBufferSortable {
writtenPairs++; writtenPairs++;
pair = generator.next(); pair = generator.next();
} }
LOG.info("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of " LOG.debug("Written " + writtenPairs + " pairs to buffer which occupied " + writtenBytes + " of "
+ MEMORY_SIZE + " bytes."); + MEMORY_SIZE + " bytes.");
it = buffer.getIterator(); it = buffer.getIterator();
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestCombiningUnilateralSortMerger.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/CombiningUnilateralSortMergerITCase.java
浏览文件 @ f90d0575
...@@ -23,8 +23,6 @@ import junit.framework.Assert; ...@@ -23,8 +23,6 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
...@@ -40,8 +38,6 @@ import eu.stratosphere.pact.common.stub.ReduceStub; ...@@ -40,8 +38,6 @@ import eu.stratosphere.pact.common.stub.ReduceStub;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.common.type.base.PactInteger; import eu.stratosphere.pact.common.type.base.PactInteger;
import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory; import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory;
import eu.stratosphere.pact.runtime.sort.CombiningUnilateralSortMerger;
import eu.stratosphere.pact.runtime.sort.SortMerger;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
import eu.stratosphere.pact.runtime.test.util.TestData.Key; import eu.stratosphere.pact.runtime.test.util.TestData.Key;
import eu.stratosphere.pact.runtime.test.util.TestData.Value; import eu.stratosphere.pact.runtime.test.util.TestData.Value;
...@@ -51,8 +47,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode; ...@@ -51,8 +47,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode;
/** /**
* @author Fabian Hueske * @author Fabian Hueske
*/ */
public class TestCombiningUnilateralSortMerger { public class CombiningUnilateralSortMergerITCase {
private static final Log LOG = LogFactory.getLog(TestCombiningUnilateralSortMerger.class); private static final Log LOG = LogFactory.getLog(CombiningUnilateralSortMergerITCase.class);
private static final long SEED = 649180756312423613L; private static final long SEED = 649180756312423613L;
...@@ -72,30 +68,13 @@ public class TestCombiningUnilateralSortMerger { ...@@ -72,30 +68,13 @@ public class TestCombiningUnilateralSortMerger {
private MemoryManager memoryManager; private MemoryManager memoryManager;
private static Level rootLevel, pkqLevel;
@BeforeClass @BeforeClass
public static void beforeClass() { public static void beforeClass() {
Logger rootLogger = Logger.getRootLogger();
rootLevel = rootLogger.getLevel();
rootLogger.setLevel(Level.INFO);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(
CombiningUnilateralSortMerger.class.getPackage().getName());
pkqLevel = pkgLogger.getLevel();
pkgLogger.setLevel(Level.DEBUG);
ioManager = new IOManager(); ioManager = new IOManager();
} }
@AfterClass @AfterClass
public static void afterClass() { public static void afterClass() {
Logger rootLogger = Logger.getRootLogger();
rootLogger.setLevel(rootLevel);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(
CombiningUnilateralSortMerger.class.getPackage().getName());
pkgLogger.setLevel(pkqLevel);
} }
@Before @Before
...@@ -126,7 +105,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -126,7 +105,7 @@ public class TestCombiningUnilateralSortMerger {
final Comparator<TestData.Key> keyComparator = new TestData.KeyComparator(); final Comparator<TestData.Key> keyComparator = new TestData.KeyComparator();
MockRecordReader<KeyValuePair<TestData.Key, PactInteger>> reader = new MockRecordReader<KeyValuePair<TestData.Key, PactInteger>>(); MockRecordReader<KeyValuePair<TestData.Key, PactInteger>> reader = new MockRecordReader<KeyValuePair<TestData.Key, PactInteger>>();
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, PactInteger> merger = new CombiningUnilateralSortMerger<TestData.Key, PactInteger>( SortMerger<TestData.Key, PactInteger> merger = new CombiningUnilateralSortMerger<TestData.Key, PactInteger>(
new TestCountCombiner(), memoryManager, ioManager, 6, 1024 * 1024 * 8, 1024 * 1024 * 64, 128, new TestCountCombiner(), memoryManager, ioManager, 6, 1024 * 1024 * 8, 1024 * 1024 * 64, 128,
keySerialization, valSerialization, keyComparator, reader, OFFSETS_PERCENTAGE, null, true); keySerialization, valSerialization, keyComparator, reader, OFFSETS_PERCENTAGE, null, true);
...@@ -167,14 +146,14 @@ public class TestCombiningUnilateralSortMerger { ...@@ -167,14 +146,14 @@ public class TestCombiningUnilateralSortMerger {
MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>(); MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>();
// merge iterator // merge iterator
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, TestData.Value> merger = new CombiningUnilateralSortMerger<TestData.Key, TestData.Value>( SortMerger<TestData.Key, TestData.Value> merger = new CombiningUnilateralSortMerger<TestData.Key, TestData.Value>(
new TestCountCombiner2(), memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2, new TestCountCombiner2(), memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2,
keySerialization, valSerialization, keyComparator, reader, OFFSETS_PERCENTAGE, null, true); keySerialization, valSerialization, keyComparator, reader, OFFSETS_PERCENTAGE, null, true);
Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator(); Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator();
// emit data // emit data
LOG.info("emitting data"); LOG.debug("emitting data");
TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
ValueMode.FIX_LENGTH); ValueMode.FIX_LENGTH);
for (int i = 0; i < NUM_PAIRS; i++) { for (int i = 0; i < NUM_PAIRS; i++) {
...@@ -186,7 +165,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -186,7 +165,7 @@ public class TestCombiningUnilateralSortMerger {
reader.close(); reader.close();
// check order // check order
LOG.info("checking results"); LOG.debug("checking results");
KeyValuePair<TestData.Key, TestData.Value> pair1 = null; KeyValuePair<TestData.Key, TestData.Value> pair1 = null;
while (iterator.hasNext()) { while (iterator.hasNext()) {
KeyValuePair<TestData.Key, TestData.Value> pair2 = iterator.next(); KeyValuePair<TestData.Key, TestData.Value> pair2 = iterator.next();
...@@ -223,13 +202,13 @@ public class TestCombiningUnilateralSortMerger { ...@@ -223,13 +202,13 @@ public class TestCombiningUnilateralSortMerger {
* MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key, * MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key,
* N_Integer>>(); * N_Integer>>();
* // merge iterator * // merge iterator
* LOG.info("initializing sortmerger"); * LOG.debug("initializing sortmerger");
* SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new * SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new
* TestCountCombiner(), memoryManager, ioManager, 10, 1024*16, 1024*1024*12, 2, keySerialization, valSerialization, * TestCountCombiner(), memoryManager, ioManager, 10, 1024*16, 1024*1024*12, 2, keySerialization, valSerialization,
* keyComparator, reader, 0.5f); * keyComparator, reader, 0.5f);
* Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator(); * Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator();
* // emit data * // emit data
* LOG.info("emitting data"); * LOG.debug("emitting data");
* TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, * TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
* ValueMode.FIX_LENGTH); * ValueMode.FIX_LENGTH);
* for (int i = 0; i < NUM_PAIRS; i++) * for (int i = 0; i < NUM_PAIRS; i++)
...@@ -240,7 +219,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -240,7 +219,7 @@ public class TestCombiningUnilateralSortMerger {
* } * }
* reader.close(); * reader.close();
* // check order * // check order
* LOG.info("checking results"); * LOG.debug("checking results");
* int pairsEmitted = 0; * int pairsEmitted = 0;
* KeyValuePair<TestData.Key, N_Integer> pair1 = null; * KeyValuePair<TestData.Key, N_Integer> pair1 = null;
* while (iterator.hasNext()) * while (iterator.hasNext())
...@@ -279,14 +258,14 @@ public class TestCombiningUnilateralSortMerger { ...@@ -279,14 +258,14 @@ public class TestCombiningUnilateralSortMerger {
* MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key, * MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key,
* N_Integer>>(); * N_Integer>>();
* // merge iterator * // merge iterator
* LOG.info("initializing sortmerger"); * LOG.debug("initializing sortmerger");
* SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new * SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new
* TestCountCombiner(), memoryManager, ioManager, 3, 1024*1024*64, 1024*1024*64, 256, keySerialization, * TestCountCombiner(), memoryManager, ioManager, 3, 1024*1024*64, 1024*1024*64, 256, keySerialization,
* valSerialization, keyComparator, reader, 0.2f); * valSerialization, keyComparator, reader, 0.2f);
* Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator(); * Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator();
* // emit data * // emit data
* long start = System.currentTimeMillis(); * long start = System.currentTimeMillis();
* LOG.info("emitting data"); * LOG.debug("emitting data");
* TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, * TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
* ValueMode.RANDOM_LENGTH, CreationMode.MUTABLE); * ValueMode.RANDOM_LENGTH, CreationMode.MUTABLE);
* long bytesWritten = 0; * long bytesWritten = 0;
...@@ -295,7 +274,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -295,7 +274,7 @@ public class TestCombiningUnilateralSortMerger {
* if(i % (PAIRS / 20) == 0 || i == PAIRS) * if(i % (PAIRS / 20) == 0 || i == PAIRS)
* { * {
* long mb = bytesWritten / 1024 / 1024; * long mb = bytesWritten / 1024 / 1024;
* LOG.info("emitted "+(int)(100.0 * i / PAIRS)+"% (" + i + " pairs, "+mb+" mb)"); * LOG.debug("emitted "+(int)(100.0 * i / PAIRS)+"% (" + i + " pairs, "+mb+" mb)");
* } * }
* KeyValuePair<TestData.Key, N_Integer> pair = null; // generator.next(); * KeyValuePair<TestData.Key, N_Integer> pair = null; // generator.next();
* // bytesWritten += generator.sizeOf(pair); * // bytesWritten += generator.sizeOf(pair);
...@@ -303,7 +282,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -303,7 +282,7 @@ public class TestCombiningUnilateralSortMerger {
* } * }
* reader.close(); * reader.close();
* // check order * // check order
* LOG.info("checking results"); * LOG.debug("checking results");
* int pairsEmitted = 0; * int pairsEmitted = 0;
* KeyValuePair<TestData.Key, N_Integer> pair1 = null; * KeyValuePair<TestData.Key, N_Integer> pair1 = null;
* while (iterator.hasNext()) * while (iterator.hasNext())
...@@ -319,7 +298,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -319,7 +298,7 @@ public class TestCombiningUnilateralSortMerger {
* // log * // log
* if(pairsEmitted % (PAIRS / 20) == 0 || pairsEmitted == PAIRS-1) * if(pairsEmitted % (PAIRS / 20) == 0 || pairsEmitted == PAIRS-1)
* { * {
* LOG.info("checked "+(int)(100.0 * pairsEmitted / PAIRS)+"% (" + pairsEmitted + " pairs)"); * LOG.debug("checked "+(int)(100.0 * pairsEmitted / PAIRS)+"% (" + pairsEmitted + " pairs)");
* } * }
* } * }
* Assert.assertTrue(PAIRS == pairsEmitted); * Assert.assertTrue(PAIRS == pairsEmitted);
...@@ -328,7 +307,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -328,7 +307,7 @@ public class TestCombiningUnilateralSortMerger {
* long diff = end - start; * long diff = end - start;
* long secs = diff / 1000; * long secs = diff / 1000;
* long mb = bytesWritten / 1024 / 1024; * long mb = bytesWritten / 1024 / 1024;
* LOG.info("sorting a workload of " + PAIRS + " pairs ("+mb+"mb) took "+secs+" seconds -> " + (1.0*mb)/secs + * LOG.debug("sorting a workload of " + PAIRS + " pairs ("+mb+"mb) took "+secs+" seconds -> " + (1.0*mb)/secs +
* "mb/s"); * "mb/s");
* } * }
*/ */
...@@ -348,13 +327,13 @@ public class TestCombiningUnilateralSortMerger { ...@@ -348,13 +327,13 @@ public class TestCombiningUnilateralSortMerger {
* MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key, * MockRecordReader<KeyValuePair<TestData.Key, N_Integer>> reader = new MockRecordReader<KeyValuePair<TestData.Key,
* N_Integer>>(); * N_Integer>>();
* // merge iterator * // merge iterator
* LOG.info("initializing sortmerger"); * LOG.debug("initializing sortmerger");
* SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new * SortMerger<TestData.Key, N_Integer> merger = new CombiningUnilateralSortMerger<TestData.Key, N_Integer>(new
* TestCountCombiner(), memoryManager, ioManager, 1, 1024*1024*4, 1024*1024*12, 2, keySerialization, * TestCountCombiner(), memoryManager, ioManager, 1, 1024*1024*4, 1024*1024*12, 2, keySerialization,
* valSerialization, keyComparator, reader, LOW_OFFSETS_PERCENTAGE); * valSerialization, keyComparator, reader, LOW_OFFSETS_PERCENTAGE);
* Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator(); * Iterator<KeyValuePair<TestData.Key, N_Integer>> iterator = merger.getIterator();
* // emit data * // emit data
* LOG.info("emitting data"); * LOG.debug("emitting data");
* TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, * TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
* ValueMode.FIX_LENGTH); * ValueMode.FIX_LENGTH);
* for (int i = 0; i < NUM_PAIRS; i++) * for (int i = 0; i < NUM_PAIRS; i++)
...@@ -363,7 +342,7 @@ public class TestCombiningUnilateralSortMerger { ...@@ -363,7 +342,7 @@ public class TestCombiningUnilateralSortMerger {
* } * }
* reader.close(); * reader.close();
* // check order * // check order
* LOG.info("checking results"); * LOG.debug("checking results");
* int pairsEmitted = 0; * int pairsEmitted = 0;
* KeyValuePair<TestData.Key, N_Integer> pair1 = null; * KeyValuePair<TestData.Key, N_Integer> pair1 = null;
* while (iterator.hasNext()) * while (iterator.hasNext())
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestDeserializerComparator.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/DeserializerComparatorTest.java
浏览文件 @ f90d0575
...@@ -33,7 +33,7 @@ import eu.stratosphere.pact.runtime.test.util.TestData; ...@@ -33,7 +33,7 @@ import eu.stratosphere.pact.runtime.test.util.TestData;
/** /**
* @author Erik Nijkamp * @author Erik Nijkamp
*/ */
public class TestDeserializerComparator { public class DeserializerComparatorTest {
@Test @Test
public void testCompare() throws Exception { public void testCompare() throws Exception {
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestMergeIterator.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/MergeIteratorTest.java
浏览文件 @ f90d0575
...@@ -20,7 +20,6 @@ import java.util.Comparator; ...@@ -20,7 +20,6 @@ import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import org.apache.log4j.Level;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
...@@ -28,33 +27,19 @@ import org.junit.BeforeClass; ...@@ -28,33 +27,19 @@ import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.runtime.sort.MergeIterator;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
/** /**
* @author Erik Nijkamp * @author Erik Nijkamp
*/ */
public class TestMergeIterator { public class MergeIteratorTest {
private static Level rootLevel, pkqLevel;
@BeforeClass @BeforeClass
public static void beforeClass() { public static void beforeClass() {
Logger rootLogger = Logger.getRootLogger();
rootLevel = rootLogger.getLevel();
rootLogger.setLevel(Level.INFO);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(MergeIterator.class.getPackage().getName());
pkqLevel = pkgLogger.getLevel();
pkgLogger.setLevel(Level.DEBUG);
} }
@AfterClass @AfterClass
public static void afterClass() { public static void afterClass() {
Logger rootLogger = Logger.getRootLogger();
rootLogger.setLevel(rootLevel);
Logger pkgLogger = rootLogger.getLoggerRepository().getLogger(MergeIterator.class.getPackage().getName());
pkgLogger.setLevel(pkqLevel);
} }
private Iterator<KeyValuePair<TestData.Key, TestData.Value>> newIterator(final int[] keys, final String[] values) { private Iterator<KeyValuePair<TestData.Key, TestData.Value>> newIterator(final int[] keys, final String[] values) {
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestSortMergeMatchIterator.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/SortMergeMatchIteratorITCase.java
浏览文件 @ f90d0575
...@@ -23,8 +23,6 @@ import java.util.List; ...@@ -23,8 +23,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
...@@ -39,8 +37,6 @@ import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager; ...@@ -39,8 +37,6 @@ import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager;
import eu.stratosphere.pact.common.type.Key; import eu.stratosphere.pact.common.type.Key;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.common.type.Value; import eu.stratosphere.pact.common.type.Value;
import eu.stratosphere.pact.runtime.sort.SortMergeMatchIterator;
import eu.stratosphere.pact.runtime.sort.UnilateralSortMerger;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator; import eu.stratosphere.pact.runtime.test.util.TestData.Generator;
import eu.stratosphere.pact.runtime.test.util.TestData.RecordReaderMock; import eu.stratosphere.pact.runtime.test.util.TestData.RecordReaderMock;
...@@ -50,7 +46,7 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode; ...@@ -50,7 +46,7 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode;
/** /**
* @author Erik Nijkamp * @author Erik Nijkamp
*/ */
public class TestSortMergeMatchIterator { public class SortMergeMatchIteratorITCase {
// total memory // total memory
public static final int MEMORY_SIZE = 1024 * 1024 * 64; public static final int MEMORY_SIZE = 1024 * 1024 * 64;
...@@ -91,31 +87,14 @@ public class TestSortMergeMatchIterator { ...@@ -91,31 +87,14 @@ public class TestSortMergeMatchIterator {
private MemoryManager memoryManager; private MemoryManager memoryManager;
// logging
private static Level rootLevel, pkqLevel;
@BeforeClass @BeforeClass
public static void beforeClass() { public static void beforeClass() {
Logger rootLogger = Logger.getRootLogger();
rootLevel = rootLogger.getLevel();
rootLogger.setLevel(Level.INFO);
Logger pkgLogger = rootLogger.getLoggerRepository()
.getLogger(UnilateralSortMerger.class.getPackage().getName());
pkqLevel = pkgLogger.getLevel();
pkgLogger.setLevel(Level.DEBUG);
ioManager = new IOManager(); ioManager = new IOManager();
} }
@AfterClass @AfterClass
public static void afterClass() { public static void afterClass() {
Logger rootLogger = Logger.getRootLogger();
rootLogger.setLevel(rootLevel);
Logger pkgLogger = rootLogger.getLoggerRepository()
.getLogger(UnilateralSortMerger.class.getPackage().getName());
pkgLogger.setLevel(pkqLevel);
} }
@Before @Before
......
pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/TestUnilateralSortMerger.java → pact/pact-runtime/src/test/java/eu/stratosphere/pact/runtime/sort/UnilateralSortMergerITCase.java
浏览文件 @ f90d0575
...@@ -22,8 +22,6 @@ import junit.framework.Assert; ...@@ -22,8 +22,6 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
...@@ -36,8 +34,6 @@ import eu.stratosphere.nephele.services.memorymanager.MemoryManager; ...@@ -36,8 +34,6 @@ import eu.stratosphere.nephele.services.memorymanager.MemoryManager;
import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager; import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager;
import eu.stratosphere.pact.common.type.KeyValuePair; import eu.stratosphere.pact.common.type.KeyValuePair;
import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory; import eu.stratosphere.pact.runtime.serialization.WritableSerializationFactory;
import eu.stratosphere.pact.runtime.sort.SortMerger;
import eu.stratosphere.pact.runtime.sort.UnilateralSortMerger;
import eu.stratosphere.pact.runtime.test.util.TestData; import eu.stratosphere.pact.runtime.test.util.TestData;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator.CreationMode; import eu.stratosphere.pact.runtime.test.util.TestData.Generator.CreationMode;
import eu.stratosphere.pact.runtime.test.util.TestData.Generator.KeyMode; import eu.stratosphere.pact.runtime.test.util.TestData.Generator.KeyMode;
...@@ -46,8 +42,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode; ...@@ -46,8 +42,8 @@ import eu.stratosphere.pact.runtime.test.util.TestData.Generator.ValueMode;
/** /**
* @author Erik Nijkamp * @author Erik Nijkamp
*/ */
public class TestUnilateralSortMerger { public class UnilateralSortMergerITCase {
private static final Log LOG = LogFactory.getLog(TestUnilateralSortMerger.class); private static final Log LOG = LogFactory.getLog(UnilateralSortMergerITCase.class);
private static final long SEED = 649180756312423613L; private static final long SEED = 649180756312423613L;
...@@ -67,30 +63,13 @@ public class TestUnilateralSortMerger { ...@@ -67,30 +63,13 @@ public class TestUnilateralSortMerger {
private MemoryManager memoryManager; private MemoryManager memoryManager;
private static Level rootLevel, pkqLevel;
@BeforeClass @BeforeClass
public static void beforeClass() { public static void beforeClass() {
Logger rootLogger = Logger.getRootLogger();
rootLevel = rootLogger.getLevel();
rootLogger.setLevel(Level.INFO);
Logger pkgLogger = rootLogger.getLoggerRepository()
.getLogger(UnilateralSortMerger.class.getPackage().getName());
pkqLevel = pkgLogger.getLevel();
pkgLogger.setLevel(Level.DEBUG);
ioManager = new IOManager(); ioManager = new IOManager();
} }
@AfterClass @AfterClass
public static void afterClass() { public static void afterClass() {
Logger rootLogger = Logger.getRootLogger();
rootLogger.setLevel(rootLevel);
Logger pkgLogger = rootLogger.getLoggerRepository()
.getLogger(UnilateralSortMerger.class.getPackage().getName());
pkgLogger.setLevel(pkqLevel);
} }
@Before @Before
...@@ -122,14 +101,14 @@ public class TestUnilateralSortMerger { ...@@ -122,14 +101,14 @@ public class TestUnilateralSortMerger {
MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>(); MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>();
// merge iterator // merge iterator
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>( SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>(
memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2, keySerialization, valSerialization, memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2, keySerialization, valSerialization,
keyComparator, reader, OFFSETS_PERCENTAGE, null); keyComparator, reader, OFFSETS_PERCENTAGE, null);
Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator(); Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator();
// emit data // emit data
LOG.info("emitting data"); LOG.debug("emitting data");
TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
ValueMode.FIX_LENGTH); ValueMode.FIX_LENGTH);
for (int i = 0; i < NUM_PAIRS; i++) { for (int i = 0; i < NUM_PAIRS; i++) {
...@@ -138,7 +117,7 @@ public class TestUnilateralSortMerger { ...@@ -138,7 +117,7 @@ public class TestUnilateralSortMerger {
reader.close(); reader.close();
// check order // check order
LOG.info("checking results"); LOG.debug("checking results");
int pairsEmitted = 0; int pairsEmitted = 0;
KeyValuePair<TestData.Key, TestData.Value> pair1 = null; KeyValuePair<TestData.Key, TestData.Value> pair1 = null;
while (iterator.hasNext()) { while (iterator.hasNext()) {
...@@ -167,14 +146,14 @@ public class TestUnilateralSortMerger { ...@@ -167,14 +146,14 @@ public class TestUnilateralSortMerger {
MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>(); MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>();
// merge iterator // merge iterator
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>( SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>(
memoryManager, ioManager, 10, 1024 * 16, 1024 * 1024 * 12, 2, keySerialization, valSerialization, memoryManager, ioManager, 10, 1024 * 16, 1024 * 1024 * 12, 2, keySerialization, valSerialization,
keyComparator, reader, 0.5f, null); keyComparator, reader, 0.5f, null);
Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator(); Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator();
// emit data // emit data
LOG.info("emitting data"); LOG.debug("emitting data");
TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
ValueMode.FIX_LENGTH); ValueMode.FIX_LENGTH);
for (int i = 0; i < NUM_PAIRS; i++) { for (int i = 0; i < NUM_PAIRS; i++) {
...@@ -183,7 +162,7 @@ public class TestUnilateralSortMerger { ...@@ -183,7 +162,7 @@ public class TestUnilateralSortMerger {
reader.close(); reader.close();
// check order // check order
LOG.info("checking results"); LOG.debug("checking results");
int pairsEmitted = 0; int pairsEmitted = 0;
KeyValuePair<TestData.Key, TestData.Value> pair1 = null; KeyValuePair<TestData.Key, TestData.Value> pair1 = null;
while (iterator.hasNext()) { while (iterator.hasNext()) {
...@@ -215,7 +194,7 @@ public class TestUnilateralSortMerger { ...@@ -215,7 +194,7 @@ public class TestUnilateralSortMerger {
MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>(); MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>();
// merge iterator // merge iterator
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>( SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>(
memoryManager, ioManager, 3, 1024 * 1024 * 64, 1024 * 1024 * 64, 16, keySerialization, valSerialization, memoryManager, ioManager, 3, 1024 * 1024 * 64, 1024 * 1024 * 64, 16, keySerialization, valSerialization,
keyComparator, reader, 0.2f, null); keyComparator, reader, 0.2f, null);
...@@ -223,14 +202,14 @@ public class TestUnilateralSortMerger { ...@@ -223,14 +202,14 @@ public class TestUnilateralSortMerger {
// emit data // emit data
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
LOG.info("emitting data"); LOG.debug("emitting data");
TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
ValueMode.RANDOM_LENGTH, CreationMode.MUTABLE); ValueMode.RANDOM_LENGTH, CreationMode.MUTABLE);
long bytesWritten = 0; long bytesWritten = 0;
for (int i = 1; i <= PAIRS; i++) { for (int i = 1; i <= PAIRS; i++) {
if (i % (PAIRS / 20) == 0 || i == PAIRS) { if (i % (PAIRS / 20) == 0 || i == PAIRS) {
long mb = bytesWritten / 1024 / 1024; long mb = bytesWritten / 1024 / 1024;
LOG.info("emitted " + (int) (100.0 * i / PAIRS) + "% (" + i + " pairs, " + mb + " mb)"); LOG.debug("emitted " + (int) (100.0 * i / PAIRS) + "% (" + i + " pairs, " + mb + " mb)");
} }
KeyValuePair<TestData.Key, TestData.Value> pair = generator.next(); KeyValuePair<TestData.Key, TestData.Value> pair = generator.next();
bytesWritten += generator.sizeOf(pair); bytesWritten += generator.sizeOf(pair);
...@@ -239,7 +218,7 @@ public class TestUnilateralSortMerger { ...@@ -239,7 +218,7 @@ public class TestUnilateralSortMerger {
reader.close(); reader.close();
// check order // check order
LOG.info("checking results"); LOG.debug("checking results");
int pairsEmitted = 0; int pairsEmitted = 0;
KeyValuePair<TestData.Key, TestData.Value> pair1 = null; KeyValuePair<TestData.Key, TestData.Value> pair1 = null;
while (iterator.hasNext()) { while (iterator.hasNext()) {
...@@ -253,7 +232,7 @@ public class TestUnilateralSortMerger { ...@@ -253,7 +232,7 @@ public class TestUnilateralSortMerger {
// log // log
if (pairsEmitted % (PAIRS / 20) == 0 || pairsEmitted == PAIRS - 1) { if (pairsEmitted % (PAIRS / 20) == 0 || pairsEmitted == PAIRS - 1) {
LOG.info("checked " + (int) (100.0 * pairsEmitted / PAIRS) + "% (" + pairsEmitted + " pairs)"); LOG.debug("checked " + (int) (100.0 * pairsEmitted / PAIRS) + "% (" + pairsEmitted + " pairs)");
} }
} }
Assert.assertTrue(PAIRS == pairsEmitted); Assert.assertTrue(PAIRS == pairsEmitted);
...@@ -263,7 +242,7 @@ public class TestUnilateralSortMerger { ...@@ -263,7 +242,7 @@ public class TestUnilateralSortMerger {
long diff = end - start; long diff = end - start;
long secs = diff / 1000; long secs = diff / 1000;
long mb = bytesWritten / 1024 / 1024; long mb = bytesWritten / 1024 / 1024;
LOG.info("sorting a workload of " + PAIRS + " pairs (" + mb + "mb) took " + secs + " seconds -> " + (1.0 * mb) LOG.debug("sorting a workload of " + PAIRS + " pairs (" + mb + "mb) took " + secs + " seconds -> " + (1.0 * mb)
/ secs + "mb/s"); / secs + "mb/s");
} }
...@@ -282,14 +261,14 @@ public class TestUnilateralSortMerger { ...@@ -282,14 +261,14 @@ public class TestUnilateralSortMerger {
MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>(); MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>> reader = new MockRecordReader<KeyValuePair<TestData.Key, TestData.Value>>();
// merge iterator // merge iterator
LOG.info("initializing sortmerger"); LOG.debug("initializing sortmerger");
SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>( SortMerger<TestData.Key, TestData.Value> merger = new UnilateralSortMerger<TestData.Key, TestData.Value>(
memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2, keySerialization, valSerialization, memoryManager, ioManager, 1, 1024 * 1024 * 4, 1024 * 1024 * 12, 2, keySerialization, valSerialization,
keyComparator, reader, LOW_OFFSETS_PERCENTAGE, null); keyComparator, reader, LOW_OFFSETS_PERCENTAGE, null);
Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator(); Iterator<KeyValuePair<TestData.Key, TestData.Value>> iterator = merger.getIterator();
// emit data // emit data
LOG.info("emitting data"); LOG.debug("emitting data");
TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
ValueMode.FIX_LENGTH); ValueMode.FIX_LENGTH);
for (int i = 0; i < NUM_PAIRS; i++) { for (int i = 0; i < NUM_PAIRS; i++) {
...@@ -298,7 +277,7 @@ public class TestUnilateralSortMerger { ...@@ -298,7 +277,7 @@ public class TestUnilateralSortMerger {
reader.close(); reader.close();
// check order // check order
LOG.info("checking results"); LOG.debug("checking results");
int pairsEmitted = 0; int pairsEmitted = 0;
KeyValuePair<TestData.Key, TestData.Value> pair1 = null; KeyValuePair<TestData.Key, TestData.Value> pair1 = null;
while (iterator.hasNext()) { while (iterator.hasNext()) {
......
pact/pact-tests/pom.xml
浏览文件 @ f90d0575
...@@ -125,26 +125,32 @@ ...@@ -125,26 +125,32 @@
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.7</version> <version>2.7</version>
<configuration> <configuration>
<systemPropertyVariables> <systemPropertyVariables>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack> <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
</systemPropertyVariables> </systemPropertyVariables>
<excludes>
<includes> <exclude>**/*TestBase*.class</exclude>
<!-- <exclude>**/*TestNepheleMiniCluster.class</exclude>
<include>**/TestNepheleMiniCluster.java</include> </excludes>
--> </configuration>
</plugin>
<include>**/ContractTests.java</include>
<include>**/PactProgramTests.java</include> <plugin>
<artifactId>maven-failsafe-plugin</artifactId>
</includes> <version>2.6</version>
</configuration> <configuration>
</plugin> <excludes>
<exclude>**/*All2AllSPITCase.class</exclude>
<exclude>**/*EnumTrianglesITCase.class</exclude>
<exclude>**/*WebLogAnalysisITCase.class</exclude>
</excludes>
</configuration>
</plugin>
</plugins> </plugins>
</build> </build>
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/CoGroupTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/CoGroupITCase.java
浏览文件 @ f90d0575
...@@ -49,12 +49,12 @@ import eu.stratosphere.pact.test.util.TestBase; ...@@ -49,12 +49,12 @@ import eu.stratosphere.pact.test.util.TestBase;
* @author Fabian Hueske * @author Fabian Hueske
*/ */
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class CoGroupTest extends TestBase public class CoGroupITCase extends TestBase
{ {
private static final Log LOG = LogFactory.getLog(CoGroupTest.class); private static final Log LOG = LogFactory.getLog(CoGroupITCase.class);
public CoGroupTest(String clusterConfig, Configuration testConfig) { public CoGroupITCase(String clusterConfig, Configuration testConfig) {
super(testConfig, clusterConfig); super(testConfig, clusterConfig);
} }
...@@ -220,6 +220,6 @@ public class CoGroupTest extends TestBase ...@@ -220,6 +220,6 @@ public class CoGroupTest extends TestBase
} }
} }
return toParameterList(CoGroupTest.class, tConfigs); return toParameterList(CoGroupITCase.class, tConfigs);
} }
} }
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/ContractTests.java 已删除 100644 → 0
浏览文件 @ 59a383cd
/***********************************************************************************************************************
*
* Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.pact.test.contracts;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses( { MapTest.class, ReduceTest.class, MatchTest.class, CrossTest.class, CoGroupTest.class })
public class ContractTests {
}
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/CrossTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/CrossITCase.java
浏览文件 @ f90d0575
...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase; ...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase;
* @author Fabian Hueske * @author Fabian Hueske
*/ */
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class CrossTest extends TestBase public class CrossITCase extends TestBase
{ {
private static final Log LOG = LogFactory.getLog(CrossTest.class); private static final Log LOG = LogFactory.getLog(CrossITCase.class);
public CrossTest(String clusterConfig, Configuration testConfig) { public CrossITCase(String clusterConfig, Configuration testConfig) {
super(testConfig, clusterConfig); super(testConfig, clusterConfig);
} }
...@@ -236,6 +236,6 @@ public class CrossTest extends TestBase ...@@ -236,6 +236,6 @@ public class CrossTest extends TestBase
} }
} }
return toParameterList(CrossTest.class, tConfigs); return toParameterList(CrossITCase.class, tConfigs);
} }
} }
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/MapTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/MapITCase.java
浏览文件 @ f90d0575
...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase; ...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase;
* @author Fabian Hueske * @author Fabian Hueske
*/ */
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class MapTest extends TestBase public class MapITCase extends TestBase
{ {
private static final Log LOG = LogFactory.getLog(MapTest.class); private static final Log LOG = LogFactory.getLog(MapITCase.class);
public MapTest(String clusterConfig, Configuration testConfig) { public MapITCase(String clusterConfig, Configuration testConfig) {
super(testConfig, clusterConfig); super(testConfig, clusterConfig);
} }
...@@ -166,6 +166,6 @@ public class MapTest extends TestBase ...@@ -166,6 +166,6 @@ public class MapTest extends TestBase
config.setInteger("MapTest#NoSubtasks", 4); config.setInteger("MapTest#NoSubtasks", 4);
testConfigs.add(config); testConfigs.add(config);
return toParameterList(MapTest.class, testConfigs); return toParameterList(MapITCase.class, testConfigs);
} }
} }
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/MatchTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/MatchITCase.java
浏览文件 @ f90d0575
...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase; ...@@ -48,12 +48,12 @@ import eu.stratosphere.pact.test.util.TestBase;
* @author Fabian Hueske * @author Fabian Hueske
*/ */
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class MatchTest extends TestBase public class MatchITCase extends TestBase
{ {
private static final Log LOG = LogFactory.getLog(MatchTest.class); private static final Log LOG = LogFactory.getLog(MatchITCase.class);
public MatchTest(String clusterConfig, Configuration testConfig) { public MatchITCase(String clusterConfig, Configuration testConfig) {
super(testConfig, clusterConfig); super(testConfig, clusterConfig);
} }
...@@ -231,7 +231,7 @@ public class MatchTest extends TestBase ...@@ -231,7 +231,7 @@ public class MatchTest extends TestBase
} }
} }
return toParameterList(MatchTest.class, tConfigs); return toParameterList(MatchITCase.class, tConfigs);
} }
} }
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/ReduceTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/contracts/ReduceITCase.java
浏览文件 @ f90d0575
...@@ -50,12 +50,12 @@ import eu.stratosphere.pact.test.util.TestBase; ...@@ -50,12 +50,12 @@ import eu.stratosphere.pact.test.util.TestBase;
* @author Fabian Hueske * @author Fabian Hueske
*/ */
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class ReduceTest extends TestBase public class ReduceITCase extends TestBase
{ {
private static final Log LOG = LogFactory.getLog(ReduceTest.class); private static final Log LOG = LogFactory.getLog(ReduceITCase.class);
public ReduceTest(String clusterConfig, Configuration testConfig) { public ReduceITCase(String clusterConfig, Configuration testConfig) {
super(testConfig, clusterConfig); super(testConfig, clusterConfig);
} }
...@@ -200,6 +200,6 @@ public class ReduceTest extends TestBase ...@@ -200,6 +200,6 @@ public class ReduceTest extends TestBase
} }
} }
return toParameterList(ReduceTest.class, tConfigs); return toParameterList(ReduceITCase.class, tConfigs);
} }
} }
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/All2AllSPTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/All2AllSPITCase.java
浏览文件 @ f90d0575
...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.graph.All2AllSP; ...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.graph.All2AllSP;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class All2AllSPTest extends TestBase { public class All2AllSPITCase extends TestBase {
private static final Log LOG = LogFactory.getLog(All2AllSPTest.class); private static final Log LOG = LogFactory.getLog(All2AllSPITCase.class);
String pathsPath = null; String pathsPath = null;
...@@ -50,7 +50,7 @@ public class All2AllSPTest extends TestBase { ...@@ -50,7 +50,7 @@ public class All2AllSPTest extends TestBase {
+ "E|D|4|H|\n" + "E|H|2| |\n" + "F|E|3| |\n" + "F|H|5|E|\n" + "G|E|4|F|\n" + "G|F|1| |\n" + "H|D|2| |\n" + "E|D|4|H|\n" + "E|H|2| |\n" + "F|E|3| |\n" + "F|H|5|E|\n" + "G|E|4|F|\n" + "G|F|1| |\n" + "H|D|2| |\n"
+ "H|E|4| |\n" + "H|F|7|D|\n"; + "H|E|4| |\n" + "H|F|7|D|\n";
public All2AllSPTest(Configuration config) { public All2AllSPITCase(Configuration config) {
super(config); super(config);
} }
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/EnumTrianglesTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/EnumTrianglesITCase.java
浏览文件 @ f90d0575
...@@ -34,23 +34,30 @@ import eu.stratosphere.pact.example.graph.EnumTriangles; ...@@ -34,23 +34,30 @@ import eu.stratosphere.pact.example.graph.EnumTriangles;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class EnumTrianglesTest extends TestBase { public class EnumTrianglesITCase extends TestBase {
private static final Log LOG = LogFactory.getLog(EnumTrianglesTest.class); private static final Log LOG = LogFactory.getLog(EnumTrianglesITCase.class);
String edgesPath = null; String edgesPath = null;
private String edges = "A|B|\n" + "A|C|\n" + "B|C|\n" + "B|D|\n" + "B|E|\n" + "B|F|\n" + "B|I|\n" + "C|D|\n" // private String edges = "A|B|\n" + "A|C|\n" + "B|C|\n" + "B|D|\n" + "B|E|\n" + "B|F|\n" + "B|I|\n" + "C|D|\n"
+ "E|F|\n" + "F|G|\n" + "F|I|\n" + "G|H|\n" + "G|J|\n" + "H|I|\n" + "H|J|\n" + "H|K|\n" + "I|K|\n"; // + "E|F|\n" + "F|G|\n" + "F|I|\n" + "G|H|\n" + "G|J|\n" + "H|I|\n" + "H|J|\n" + "H|K|\n" + "I|K|\n";
//
private String expected = "A|B|A|C|B|C|\n" + // A,B,C // private String expected = "A|B|A|C|B|C|\n" + // A,B,C
"B|C|B|D|C|D|\n" + // B,C,D // "B|C|B|D|C|D|\n" + // B,C,D
"B|E|B|F|E|F|\n" + // B,E,F // "B|E|B|F|E|F|\n" + // B,E,F
"B|F|B|I|F|I|\n" + // B,F,I // "B|F|B|I|F|I|\n" + // B,F,I
"H|I|H|K|I|K|\n" + // H,I,K // "H|I|H|K|I|K|\n" + // H,I,K
"G|H|G|J|H|J|\n"; // G,H,J // "G|H|G|J|H|J|\n"; // G,H,J
private String edges = "<a> <foaf:knows> <b>\n" + "<a> <foaf:knows> <c>\n" + "<a> <foaf:knows> <d>\n" +
"<b> <foaf:knows> <c>\n" + "<b> <foaf:knows> <e>\n" + "<b> <foaf:knows> <f>\n" +
"<c> <foaf:knows> <d>\n" + "<d> <foaf:knows> <b>\n" + "<f> <foaf:knows> <g>\n" +
"<f> <foaf:knows> <h>\n" + "<f> <foaf:knows> <i>\n" + "<g> <foaf:knows> <i>\n";
public EnumTrianglesTest(Configuration config) { private String expected = "";
public EnumTrianglesITCase(Configuration config) {
super(config); super(config);
} }
...@@ -73,7 +80,7 @@ public class EnumTrianglesTest extends TestBase { ...@@ -73,7 +80,7 @@ public class EnumTrianglesTest extends TestBase {
EnumTriangles enumTriangles = new EnumTriangles(); EnumTriangles enumTriangles = new EnumTriangles();
Plan plan = enumTriangles.getPlan( Plan plan = enumTriangles.getPlan(
config.getString("EnumTrianglesTest#NoSubtasks", "1"), config.getString("EnumTrianglesTest#NoSubtasks", "4"),
getFilesystemProvider().getURIPrefix() + edgesPath, getFilesystemProvider().getURIPrefix() + edgesPath,
getFilesystemProvider().getURIPrefix() + getFilesystemProvider().getTempDirPath() + "/triangles.txt"); getFilesystemProvider().getURIPrefix() + getFilesystemProvider().getTempDirPath() + "/triangles.txt");
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/KMeansIterationTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/KMeansIterationITCase.java
浏览文件 @ f90d0575
...@@ -36,11 +36,11 @@ import eu.stratosphere.pact.example.datamining.KMeansIteration; ...@@ -36,11 +36,11 @@ import eu.stratosphere.pact.example.datamining.KMeansIteration;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class KMeansIterationTest extends TestBase { public class KMeansIterationITCase extends TestBase {
// KMeanDataGenerator kmdg = new KMeanDataGenerator(500, 10, 2); // KMeanDataGenerator kmdg = new KMeanDataGenerator(500, 10, 2);
private static final Log LOG = LogFactory.getLog(KMeansIterationTest.class); private static final Log LOG = LogFactory.getLog(KMeansIterationITCase.class);
private final String DATAPOINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n" private final String DATAPOINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n"
+ "3|1.60|70.11|16.32|\n" + "4|2.43|19.81|89.56|\n" + "5|67.99|9.00|14.48|\n" + "6|87.80|84.49|55.83|\n" + "3|1.60|70.11|16.32|\n" + "4|2.43|19.81|89.56|\n" + "5|67.99|9.00|14.48|\n" + "6|87.80|84.49|55.83|\n"
...@@ -87,7 +87,7 @@ public class KMeansIterationTest extends TestBase { ...@@ -87,7 +87,7 @@ public class KMeansIterationTest extends TestBase {
String clusterPath = null; String clusterPath = null;
String resultPath = null; String resultPath = null;
public KMeansIterationTest(Configuration config) { public KMeansIterationITCase(Configuration config) {
super(config); super(config);
} }
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/PactProgramTests.java 已删除 100644 → 0
浏览文件 @ 59a383cd
/***********************************************************************************************************************
*
* Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.pact.test.pactPrograms;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
// @SuiteClasses( { All2AllSPTest.class, EnumTrianglesTest.class, KMeansIterationTest.class, TPCHQuery3Test.class, WebLogAnalysisTest.class, WordCountTest.class })
@SuiteClasses( { WordCountTest.class, KMeansIterationTest.class, TPCHQuery3Test.class })
public class PactProgramTests {
}
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/TPCHQuery3Test.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/TPCHQuery3ITCase.java
浏览文件 @ f90d0575
...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.relational.TPCHQuery3; ...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.relational.TPCHQuery3;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class TPCHQuery3Test extends TestBase { public class TPCHQuery3ITCase extends TestBase {
private static final Log LOG = LogFactory.getLog(TPCHQuery3Test.class); private static final Log LOG = LogFactory.getLog(TPCHQuery3ITCase.class);
private String ordersPath = null; private String ordersPath = null;
private String lineitemsPath = null; private String lineitemsPath = null;
...@@ -131,7 +131,7 @@ public class TPCHQuery3Test extends TestBase { ...@@ -131,7 +131,7 @@ public class TPCHQuery3Test extends TestBase {
String EXPECTED_RESULT = "5|0|147827|\n" + "66|0|99187|\n"; String EXPECTED_RESULT = "5|0|147827|\n" + "66|0|99187|\n";
public TPCHQuery3Test(Configuration config) { public TPCHQuery3ITCase(Configuration config) {
super(config); super(config);
} }
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/WebLogAnalysisTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/WebLogAnalysisITCase.java
浏览文件 @ f90d0575
...@@ -39,7 +39,7 @@ import eu.stratosphere.pact.example.relational.WebLogAnalysis; ...@@ -39,7 +39,7 @@ import eu.stratosphere.pact.example.relational.WebLogAnalysis;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class WebLogAnalysisTest extends TestBase { public class WebLogAnalysisITCase extends TestBase {
String docsPath = getFilesystemProvider().getTempDirPath() + "/docs"; String docsPath = getFilesystemProvider().getTempDirPath() + "/docs";
...@@ -97,7 +97,7 @@ public class WebLogAnalysisTest extends TestBase { ...@@ -97,7 +97,7 @@ public class WebLogAnalysisTest extends TestBase {
String expectedResult = "aaaaf|68|832|\n" + "aaaak|99|783|\n"; String expectedResult = "aaaaf|68|832|\n" + "aaaak|99|783|\n";
public WebLogAnalysisTest(Configuration config) { public WebLogAnalysisITCase(Configuration config) {
super(config); super(config);
} }
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/WordCountTest.java → pact/pact-tests/src/test/java/eu/stratosphere/pact/test/pactPrograms/WordCountITCase.java
浏览文件 @ f90d0575
...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.wordcount.WordCount; ...@@ -34,9 +34,9 @@ import eu.stratosphere.pact.example.wordcount.WordCount;
import eu.stratosphere.pact.test.util.TestBase; import eu.stratosphere.pact.test.util.TestBase;
@RunWith(Parameterized.class) @RunWith(Parameterized.class)
public class WordCountTest extends TestBase { public class WordCountITCase extends TestBase {
private static final Log LOG = LogFactory.getLog(WordCountTest.class); private static final Log LOG = LogFactory.getLog(WordCountITCase.class);
private final String TEXT = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n" private final String TEXT = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n"
+ "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei\n" + "Erzengel treten vor.\n" + "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei\n" + "Erzengel treten vor.\n"
...@@ -177,7 +177,7 @@ public class WordCountTest extends TestBase { ...@@ -177,7 +177,7 @@ public class WordCountTest extends TestBase {
private String textPath = null; private String textPath = null;
private String resultPath = null; private String resultPath = null;
public WordCountTest(Configuration config) { public WordCountITCase(Configuration config) {
super(config); super(config);
} }
......
pact/pact-tests/src/test/java/eu/stratosphere/pact/test/util/TestBase.java
浏览文件 @ f90d0575
...@@ -142,7 +142,7 @@ public abstract class TestBase extends TestCase { ...@@ -142,7 +142,7 @@ public abstract class TestBase extends TestCase {
p.load(new FileInputStream(configFile)); p.load(new FileInputStream(configFile));
for (String key : p.stringPropertyNames()) { for (String key : p.stringPropertyNames()) {
if (key.equals(testClassName)) { if (key.endsWith(testClassName)) {
for (String config : p.getProperty(key).split(",")) { for (String config : p.getProperty(key).split(",")) {
clusterConfigs.add(config); clusterConfigs.add(config);
} }
......
pact/pact-tests/src/test/resources/TestConfigs/eu.stratosphere.pact.test.contracts.prop
浏览文件 @ f90d0575
CoGroupTest=local1TM,local4TM CoGroupITCase=local1TM,local4TM
CrossTest=local1TM CrossITCase=local1TM
MapTest=local1TM,local4TM MapITCase=local1TM,local4TM
MatchTest=local1TM MatchITCase=local1TM
ReduceTest=local1TM ReduceITCase=local1TM
\ No newline at end of file \ No newline at end of file
pom.xml
浏览文件 @ f90d0575
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" <project
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>eu.stratosphere</groupId> <groupId>eu.stratosphere</groupId>
<artifactId>stratosphere</artifactId> <artifactId>stratosphere</artifactId>
<version>0.1</version> <version>0.1</version>
<name>stratosphere</name> <name>stratosphere</name>
<packaging>pom</packaging> <packaging>pom</packaging>
<url>http://www.stratosphere.eu</url> <url>http://www.stratosphere.eu</url>
<inceptionYear>2009</inceptionYear> <inceptionYear>2009</inceptionYear>
<licenses> <licenses>
<license> <license>
<name>The Apache Software License, Version 2.0</name> <name>The Apache Software License, Version 2.0</name>
...@@ -20,7 +21,7 @@ ...@@ -20,7 +21,7 @@
<distribution>stratosphere</distribution> <distribution>stratosphere</distribution>
</license> </license>
</licenses> </licenses>
<developers> <developers>
<developer> <developer>
<id>warneke</id> <id>warneke</id>
...@@ -32,7 +33,7 @@ ...@@ -32,7 +33,7 @@
<role>Lead Developer</role> <role>Lead Developer</role>
</roles> </roles>
</developer> </developer>
<developer> <developer>
<id>sewen</id> <id>sewen</id>
<name>Stephan Ewen</name> <name>Stephan Ewen</name>
...@@ -43,7 +44,7 @@ ...@@ -43,7 +44,7 @@
<role>Lead Developer</role> <role>Lead Developer</role>
</roles> </roles>
</developer> </developer>
<developer> <developer>
<id>fhueske</id> <id>fhueske</id>
<name>Fabian Hueske</name> <name>Fabian Hueske</name>
...@@ -58,9 +59,10 @@ ...@@ -58,9 +59,10 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8
</project.reporting.outputEncoding>
</properties> </properties>
<pluginRepositories> <pluginRepositories>
<pluginRepository> <pluginRepository>
<id>mc-release</id> <id>mc-release</id>
...@@ -82,7 +84,7 @@ ...@@ -82,7 +84,7 @@
<url>http://repository.apache.org/snapshots/</url> <url>http://repository.apache.org/snapshots/</url>
</pluginRepository> </pluginRepository>
</pluginRepositories> </pluginRepositories>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
...@@ -91,43 +93,43 @@ ...@@ -91,43 +93,43 @@
<type>jar</type> <type>jar</type>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-logging</groupId> <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId> <artifactId>commons-logging</artifactId>
<version>1.1.1</version> <version>1.1.1</version>
<type>jar</type> <type>jar</type>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>log4j</groupId> <groupId>log4j</groupId>
<artifactId>log4j</artifactId> <artifactId>log4j</artifactId>
<version>1.2.16</version> <version>1.2.16</version>
<type>jar</type> <type>jar</type>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>
<!-- <!--
this section defines the module versions that are used if nothing this section defines the module versions that are used if nothing
else is specified. else is specified.
--> -->
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId> <artifactId>hadoop-core</artifactId>
<version>0.20.2</version> <version>0.20.2</version>
<type>jar</type> <type>jar</type>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<reporting> <reporting>
<plugins> <plugins>
<plugin> <plugin>
<!-- <!--
just define the Java version to be used for compiling and plugins just define the Java version to be used for compiling and plugins
...@@ -140,27 +142,27 @@ ...@@ -140,27 +142,27 @@
<compilerArgument>-g:none -O</compilerArgument> <compilerArgument>-g:none -O</compilerArgument>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>
<!-- measure and report source code complexity --> <!-- measure and report source code complexity -->
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<artifactId>javancss-maven-plugin</artifactId> <artifactId>javancss-maven-plugin</artifactId>
<version>2.0</version> <version>2.0</version>
</plugin> </plugin>
<plugin> <plugin>
<!-- analyze dependencies in source code --> <!-- analyze dependencies in source code -->
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<artifactId>jdepend-maven-plugin</artifactId> <artifactId>jdepend-maven-plugin</artifactId>
</plugin> </plugin>
<!-- <!--
disabled because currently no SCM defined generates changelog disabled because currently no SCM defined generates changelog
<plugin> <groupId>org.apache.maven.plugins</groupId> <plugin> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-changelog-plugin</artifactId> <artifactId>maven-changelog-plugin</artifactId>
<version>2.2-SNAPSHOT</version> </plugin> <version>2.2-SNAPSHOT</version> </plugin>
--> -->
<plugin> <plugin>
<!-- report occurrences of various todo markers in code --> <!-- report occurrences of various todo markers in code -->
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
...@@ -175,19 +177,17 @@ ...@@ -175,19 +177,17 @@
</tags> </tags>
</configuration> </configuration>
</plugin> </plugin>
<!-- <!--
todo: reenable when SCM is available todo: reenable when SCM is available <plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<artifactId>scmchangelog-maven-plugin</artifactId> <artifactId>scmchangelog-maven-plugin</artifactId>
<version>1.2</version> <configuration> <version>1.2</version> <configuration>
<connectionUrl>scm:svn:https://projekte.itmc.tu-dortmund.de/svn/sla4dgrid/trunk</connectionUrl> <connectionUrl>scm:svn:https://projekte.itmc.tu-dortmund.de/svn/sla4dgrid/trunk</connectionUrl>
<tagBase>https://projekte.itmc.tu-dortmund.de/svn/sla4dgrid/tags/</tagBase> <tagBase>https://projekte.itmc.tu-dortmund.de/svn/sla4dgrid/tags/</tagBase>
<filter>.*</filter> </configuration> <filter>.*</filter> </configuration> </plugin>
</plugin>
--> -->
<plugin> <plugin>
<!-- <!--
generates cross references in code so that you can click in the generates cross references in code so that you can click in the
...@@ -199,7 +199,7 @@ ...@@ -199,7 +199,7 @@
<linkJavadoc>true</linkJavadoc> <linkJavadoc>true</linkJavadoc>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>
<!-- discovers frequent bugs in programs --> <!-- discovers frequent bugs in programs -->
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
...@@ -212,7 +212,7 @@ ...@@ -212,7 +212,7 @@
<xmlOutput>true</xmlOutput> <xmlOutput>true</xmlOutput>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>
<!-- maven source code analysis for frequent bugs--> <!-- maven source code analysis for frequent bugs-->
<artifactId>maven-pmd-plugin</artifactId> <artifactId>maven-pmd-plugin</artifactId>
...@@ -229,16 +229,14 @@ ...@@ -229,16 +229,14 @@
</reportSet> </reportSet>
</reportSets> </reportSets>
</plugin> </plugin>
<!-- <!--
<plugin> <plugin> generation of JavaDoc
generation of JavaDoc
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId> <version>2.5</version>
<version>2.5</version> </plugin>
</plugin>
--> -->
<plugin> <plugin>
<!-- style checker --> <!-- style checker -->
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
...@@ -248,14 +246,14 @@ ...@@ -248,14 +246,14 @@
<configLocation>nephele/checkstyle.xml</configLocation> <configLocation>nephele/checkstyle.xml</configLocation>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>
<!-- execution of Unit Tests --> <!-- execution of Unit Tests -->
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-report-plugin</artifactId> <artifactId>maven-surefire-report-plugin</artifactId>
<version>2.7</version> <version>2.7</version>
</plugin> </plugin>
<plugin> <plugin>
<!-- check coverage of tests --> <!-- check coverage of tests -->
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
...@@ -267,7 +265,7 @@ ...@@ -267,7 +265,7 @@
</formats> </formats>
</configuration> </configuration>
</plugin> </plugin>
<!-- <!--
Generator for QA reports, summarizes various inputs and draws Generator for QA reports, summarizes various inputs and draws
diagrams indicating improvements/deterioration <plugin> diagrams indicating improvements/deterioration <plugin>
...@@ -278,12 +276,12 @@ ...@@ -278,12 +276,12 @@
<report>report-movers-all</report> </reports> </reportSet> <report>report-movers-all</report> </reports> </reportSet>
</reportSets> </plugin> </reportSets> </plugin>
--> -->
<plugin> <plugin>
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<artifactId>dashboard-maven-plugin</artifactId> <artifactId>dashboard-maven-plugin</artifactId>
</plugin> </plugin>
</plugins> </plugins>
</reporting> </reporting>
...@@ -302,104 +300,104 @@ ...@@ -302,104 +300,104 @@
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.7</version>
<executions>
<execution>
<id>javadoc:aggregate-jar</id>
<goals>
<goal>aggregate-jar</goal>
</goals>
<phase>compile</phase>
<configuration>
<quiet>true</quiet>
</configuration>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId>
<artifactId>maven-eclipse-plugin</artifactId> <version>2.6</version>
<version>2.9-SNAPSHOT</version> <executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
<configuration> <configuration>
<classpathContainers> <systemPropertyVariables>
<classpathContainer> <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
org.eclipse.jdt.launching.JRE_CONTAINER </systemPropertyVariables>
</classpathContainer>
</classpathContainers>
</configuration> </configuration>
</plugin> </plugin>
<!--
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId>
check whether license information is included in files, can be used <artifactId>maven-surefire-plugin</artifactId>
to insert headers as well <version>2.7</version>
<groupId>com.google.code.maven-license-plugin</groupId>
<artifactId>maven-license-plugin</artifactId>
<version>1.4.0</version>
<configuration> <configuration>
<header>src/main/etc/header.txt</header> <systemPropertyVariables>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
</systemPropertyVariables>
</configuration> </configuration>
</plugin> </plugin>
-->
<!--
<plugin>
plugin that tests which fraction of the source code is covered by
JUnit tests
<groupId>org.codehaus.mojo</groupId> <plugin>
<artifactId>cobertura-maven-plugin</artifactId> <groupId>org.apache.maven.plugins</groupId>
<configuration> <artifactId>maven-javadoc-plugin</artifactId>
<instrumentation> <version>2.7</version>
<ignores>
<ignore>com.example.boringcode.*</ignore>
</ignores>
<excludes>
<exclude>com/example/dullcode/**/*.class</exclude>
<exclude>com/example/**/*Test.class</exclude>
</excludes>
</instrumentation>
</configuration>
<version>2.4</version>
<executions> <executions>
<execution> <execution>
<id>javadoc:aggregate-jar</id>
<goals> <goals>
<goal>clean</goal> <goal>aggregate-jar</goal>
</goals> </goals>
<phase>compile</phase>
<configuration>
<quiet>true</quiet>
</configuration>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
-->
<!--
<plugin> <plugin>
plugin that tests whether the code style is appropriate
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId> <artifactId>maven-eclipse-plugin</artifactId>
<version>2.3</version> <version>2.9-SNAPSHOT</version>
<dependencies> <configuration>
<dependency> <classpathContainers>
<groupId>eu.stratosphere</groupId> <classpathContainer>
<artifactId>build-tools</artifactId> org.eclipse.jdt.launching.JRE_CONTAINER
<version>0.1</version> </classpathContainer>
</dependency> </classpathContainers>
</dependencies> </configuration>
</plugin> </plugin>
-->
<!--
<plugin> check whether license information is included in files, can
be used to insert headers as well
<groupId>com.google.code.maven-license-plugin</groupId>
<artifactId>maven-license-plugin</artifactId>
<version>1.4.0</version> <configuration>
<header>src/main/etc/header.txt</header> </configuration> </plugin>
-->
<!--
<plugin> plugin that tests which fraction of the source code is
covered by JUnit tests <groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId> <configuration>
<instrumentation> <ignores>
<ignore>com.example.boringcode.*</ignore> </ignores> <excludes>
<exclude>com/example/dullcode/**/*.class</exclude>
<exclude>com/example/**/*Test.class</exclude> </excludes>
</instrumentation> </configuration> <version>2.4</version>
<executions> <execution> <goals> <goal>clean</goal> </goals>
</execution> </executions> </plugin>
-->
<!--
<plugin> plugin that tests whether the code style is appropriate
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>2.3</version> <dependencies> <dependency>
<groupId>eu.stratosphere</groupId>
<artifactId>build-tools</artifactId> <version>0.1</version>
</dependency> </dependencies> </plugin>
-->
</plugins> </plugins>
</build> </build>
<modules> <modules>
<module>nephele</module> <module>nephele</module>
<module>pact</module> <module>pact</module>
<module>build-tools</module> <module>build-tools</module>
<module>stratosphere-dist</module> <module>stratosphere-dist</module>
</modules> </modules>
</project> </project>
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或