提交 0685ca1f 编写于 作者: S StephanEwen

Adjusted generic serializer tests

上级 9050111b
......@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
......@@ -30,7 +31,7 @@ import org.junit.Test;
import eu.stratosphere.api.java.record.io.avro.generated.Colors;
import eu.stratosphere.api.java.record.io.avro.generated.User;
import eu.stratosphere.util.StringUtils;
import static org.junit.Assert.*;
......@@ -39,6 +40,47 @@ import static org.junit.Assert.*;
*/
public class EncoderDecoderTest {
@Test
public void testComplexStringsDirecty() {
try {
Random rnd = new Random(349712539451944123L);
for (int i = 0; i < 10; i++) {
String testString = StringUtils.getRandomString(rnd, 10, 100);
ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
{
DataOutputStream dataOut = new DataOutputStream(baos);
DataOutputEncoder encoder = new DataOutputEncoder();
encoder.setOut(dataOut);
encoder.writeString(testString);
dataOut.flush();
dataOut.close();
}
byte[] data = baos.toByteArray();
// deserialize
{
ByteArrayInputStream bais = new ByteArrayInputStream(data);
DataInputStream dataIn = new DataInputStream(bais);
DataInputDecoder decoder = new DataInputDecoder();
decoder.setIn(dataIn);
String deserialized = decoder.readString();
assertEquals(testString, deserialized);
}
}
}
catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test failed due to an exception: " + e.getMessage());
}
}
@Test
public void testPrimitiveTypes() {
......
......@@ -269,7 +269,30 @@ public final class StringUtils {
char[] data = new char[len];
for (int i = 0; i < data.length; i++) {
data[i] = (char) (rnd.nextInt(Character.MAX_VALUE) + 1);
data[i] = (char) (rnd.nextInt(0x7fff) + 1);
}
return new String(data);
}
/**
* Creates a random string with a length within the given interval. The string contains only characters that
* can be represented as a single code point.
*
* @param rnd The random used to create the strings.
* @param minLength The minimum string length.
* @param maxLength The maximum string length (inclusive).
* @param minValue The minimum character value to occur.
* @param maxValue The maximum character value to occur.
* @return A random String.
*/
public static String getRandomString(Random rnd, int minLength, int maxLength, char minValue, char maxValue) {
int len = rnd.nextInt(maxLength - minLength + 1) + minLength;
char[] data = new char[len];
int diff = maxValue - minValue + 1;
for (int i = 0; i < data.length; i++) {
data[i] = (char) (rnd.nextInt(diff) + minValue);
}
return new String(data);
}
......
......@@ -12,32 +12,63 @@
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.api.java.typeutils.runtime;
import eu.stratosphere.api.common.typeutils.TypeSerializer;
package eu.stratosphere.api.common.typeutils;
/**
* A test for the {@link KryoSerializer}.
*/
public class KryoStringSerializerTest extends KryoSerializerTestBase<String> {
public class SerializerTestInstance<T> extends SerializerTestBase<T> {
private final TypeSerializer<T> serializer;
private final Class<T> typeClass;
private final int length;
private final T[] testData;
// --------------------------------------------------------------------------------------------
public SerializerTestInstance(TypeSerializer<T> serializer, Class<T> typeClass, int length, T... testData) {
this.serializer = serializer;
this.typeClass = typeClass;
this.length = length;
this.testData = testData;
}
// --------------------------------------------------------------------------------------------
@Override
protected TypeSerializer<String> createSerializer() {
return new KryoSerializer<String>(String.class);
protected TypeSerializer<T> createSerializer() {
return this.serializer;
}
@Override
protected Class<String> getTypeClass() {
return String.class;
protected int getLength() {
return this.length;
}
@Override
protected int getLength() {
return -1;
protected Class<T> getTypeClass() {
return this.typeClass;
}
@Override
protected String getTestData() {
return "jbmbmner8 jhk hj \n \t üäßß@µ";
protected T[] getTestData() {
return this.testData;
}
// --------------------------------------------------------------------------------------------
public void testAll() {
testInstantiate();
testGetLength();
testCopyIntoNewElements();
testCopyIntoReusedElements();
testSerializeIndividually();
testSerializeIndividuallyReusingValues();
testSerializeAsSequence();
testSerializedCopyIndividually();
testSerializedCopyAsSequence();
testSerializabilityAndEquals();
}
}
......@@ -20,7 +20,6 @@ import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import eu.stratosphere.api.common.typeutils.TypeSerializer;
import eu.stratosphere.core.memory.DataInputView;
......@@ -29,8 +28,8 @@ import eu.stratosphere.util.InstantiationUtil;
/**
* General purpose serialization using Apache Avro's Reflect-serializers.
* For deep object copies, this class is using Kryo.
* General purpose serialization. Currently using Apache Avro's Reflect-serializers for serialization and
* Kryo for deep object copies. We want to change this to Kryo-only.
*
* @param <T> The type serialized.
*/
......@@ -40,6 +39,8 @@ public class AvroSerializer<T> extends TypeSerializer<T> {
private final Class<T> type;
private final Class<? extends T> typeToInstantiate;
private transient ReflectDatumWriter<T> writer;
private transient ReflectDatumReader<T> reader;
......@@ -48,14 +49,23 @@ public class AvroSerializer<T> extends TypeSerializer<T> {
private transient Kryo kryo;
private transient Serializer<T> serializer;
private transient T deepCopyInstance;
// --------------------------------------------------------------------------------------------
public AvroSerializer(Class<T> type) {
this(type, type);
}
public AvroSerializer(Class<T> type, Class<? extends T> typeToInstantiate) {
if (type == null || typeToInstantiate == null) {
throw new NullPointerException();
}
InstantiationUtil.checkForInstantiation(typeToInstantiate);
this.type = type;
this.typeToInstantiate = typeToInstantiate;
}
// --------------------------------------------------------------------------------------------
......@@ -72,13 +82,17 @@ public class AvroSerializer<T> extends TypeSerializer<T> {
@Override
public T createInstance() {
return InstantiationUtil.instantiate(type, Object.class);
try {
return this.typeToInstantiate.newInstance();
} catch (Exception e) {
throw new RuntimeException("Instantiation failed, even though type pre-checks passed.", e);
}
}
@Override
public T copy(T from, T reuse) {
checkKryoInitialized();
reuse = serializer.copy(this.kryo, from);
reuse = this.kryo.copy(from);
return reuse;
}
......@@ -126,13 +140,11 @@ public class AvroSerializer<T> extends TypeSerializer<T> {
}
}
@SuppressWarnings("unchecked")
private final void checkKryoInitialized() {
if (this.kryo == null) {
this.kryo = new Kryo();
this.kryo.setAsmEnabled(true);
this.kryo.register(type);
this.serializer = this.kryo.getSerializer(this.type);
}
}
......
/***********************************************************************************************************************
*
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.api.java.typeutils.runtime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.junit.Test;
import eu.stratosphere.api.common.typeutils.SerializerTestInstance;
import eu.stratosphere.util.StringUtils;
/**
* A test for the {@link AvroSerializer}.
*/
public class GenericSerializerTest {
private final Random rnd = new Random(349712539451944123L);
@Test
public void testString() {
runTests("abc", "",
StringUtils.getRandomString(new Random(289347567856686223L), 10, 100),
StringUtils.getRandomString(new Random(289347567856686223L), 1000, 5000),
StringUtils.getRandomString(new Random(289347567856686223L), 30000, 35000),
StringUtils.getRandomString(new Random(289347567856686223L), 100*1024, 105*1024));
}
@Test
public void testSimpleTypesObjects() {
SimpleTypes a = new SimpleTypes();
SimpleTypes b = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
SimpleTypes c = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
SimpleTypes d = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
SimpleTypes e = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
SimpleTypes f = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
SimpleTypes g = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
runTests(a, b, c, d, e, f, g);
}
@Test
public void testCompositeObject() {
ComplexNestedObject1 o1 = new ComplexNestedObject1(5626435);
ComplexNestedObject1 o2 = new ComplexNestedObject1(76923);
ComplexNestedObject1 o3 = new ComplexNestedObject1(-1100);
ComplexNestedObject1 o4 = new ComplexNestedObject1(0);
ComplexNestedObject1 o5 = new ComplexNestedObject1(44);
runTests(o1, o2, o3, o4, o5);
}
@Test
public void testNestedObjects() {
ComplexNestedObject2 o1 = new ComplexNestedObject2(rnd);
ComplexNestedObject2 o2 = new ComplexNestedObject2();
ComplexNestedObject2 o3 = new ComplexNestedObject2(rnd);
ComplexNestedObject2 o4 = new ComplexNestedObject2(rnd);
runTests(o1, o2, o3, o4);
}
@Test
public void testBeanStyleObjects() {
{
Book b1 = new Book(976243875L, "The Serialization Odysse", 42);
Book b2 = new Book(0L, "Debugging byte streams", 1337);
Book b3 = new Book(-1L, "Low level interfaces", 0xC0FFEE);
runTests(b1, b2, b3);
}
// object with collection
{
ArrayList<String> list = new ArrayList<String>();
list.add("A");
list.add("B");
list.add("C");
list.add("D");
list.add("E");
BookAuthor b1 = new BookAuthor(976243875L, list, "Arno Nym");
ArrayList<String> list2 = new ArrayList<String>();
BookAuthor b2 = new BookAuthor(987654321L, list2, "The Saurus");
runTests(b1, b2);
}
}
private final <T> void runTests(T... instances) {
if (instances == null || instances.length == 0) {
throw new IllegalArgumentException();
}
@SuppressWarnings("unchecked")
Class<T> clazz = (Class<T>) instances[0].getClass();
AvroSerializer<T> serializer = createSerializer(clazz);
SerializerTestInstance<T> test = new SerializerTestInstance<T>(serializer, clazz, -1, instances);
test.testAll();
}
private final <T> AvroSerializer<T> createSerializer(Class<T> type) {
return new AvroSerializer<T>(type);
}
// --------------------------------------------------------------------------------------------
// Test Objects
// --------------------------------------------------------------------------------------------
public static final class SimpleTypes {
private final int iVal;
private final long lVal;
private final byte bVal;
private final String sVal;
private final short rVal;
private final double dVal;
public SimpleTypes() {
this(0, 0, (byte) 0, "", (short) 0, 0);
}
public SimpleTypes(int iVal, long lVal, byte bVal, String sVal, short rVal, double dVal) {
this.iVal = iVal;
this.lVal = lVal;
this.bVal = bVal;
this.sVal = sVal;
this.rVal = rVal;
this.dVal = dVal;
}
@Override
public boolean equals(Object obj) {
if (obj.getClass() == SimpleTypes.class) {
SimpleTypes other = (SimpleTypes) obj;
return other.iVal == this.iVal &&
other.lVal == this.lVal &&
other.bVal == this.bVal &&
other.sVal.equals(this.sVal) &&
other.rVal == this.rVal &&
other.dVal == this.dVal;
} else {
return false;
}
}
@Override
public String toString() {
return String.format("(%d, %d, %d, %s, %d, %f)", iVal, lVal, bVal, sVal, rVal, dVal);
}
}
public static class ComplexNestedObject1 {
private double doubleValue;
private List<String> stringList;
public ComplexNestedObject1() {}
public ComplexNestedObject1(int offInit) {
this.doubleValue = 6293485.6723 + offInit;
this.stringList = new ArrayList<String>();
this.stringList.add("A" + offInit);
this.stringList.add("somewhat" + offInit);
this.stringList.add("random" + offInit);
this.stringList.add("collection" + offInit);
this.stringList.add("of" + offInit);
this.stringList.add("strings" + offInit);
}
@Override
public boolean equals(Object obj) {
if (obj.getClass() == ComplexNestedObject1.class) {
ComplexNestedObject1 other = (ComplexNestedObject1) obj;
return other.doubleValue == this.doubleValue && this.stringList.equals(other.stringList);
} else {
return false;
}
}
}
public static class ComplexNestedObject2 {
private long longValue;
private Map<String, ComplexNestedObject1> theMap = new HashMap<String, ComplexNestedObject1>();
public ComplexNestedObject2() {}
public ComplexNestedObject2(Random rnd) {
this.longValue = rnd.nextLong();
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
this.theMap.put(String.valueOf(rnd.nextLong()), new ComplexNestedObject1(rnd.nextInt()));
}
@Override
public boolean equals(Object obj) {
if (obj.getClass() == ComplexNestedObject2.class) {
ComplexNestedObject2 other = (ComplexNestedObject2) obj;
return other.longValue == this.longValue && this.theMap.equals(other.theMap);
} else {
return false;
}
}
}
public static class Book {
private long bookId;
private String title;
private long authorId;
public Book() {}
public Book(long bookId, String title, long authorId) {
this.bookId = bookId;
this.title = title;
this.authorId = authorId;
}
@Override
public boolean equals(Object obj) {
if (obj.getClass() == Book.class) {
Book other = (Book) obj;
return other.bookId == this.bookId && other.authorId == this.authorId && this.title.equals(other.title);
} else {
return false;
}
}
}
public static class BookAuthor {
private long authorId;
private List<String> bookTitles;
private String authorName;
public BookAuthor() {}
public BookAuthor(long authorId, List<String> bookTitles, String authorName) {
this.authorId = authorId;
this.bookTitles = bookTitles;
this.authorName = authorName;
}
@Override
public boolean equals(Object obj) {
if (obj.getClass() == BookAuthor.class) {
BookAuthor other = (BookAuthor) obj;
return other.authorName.equals(this.authorName) && other.authorId == this.authorId &&
other.bookTitles.equals(this.bookTitles);
} else {
return false;
}
}
}
}
/***********************************************************************************************************************
*
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.api.java.typeutils.runtime;
import static org.junit.Assert.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import org.junit.Test;
import eu.stratosphere.api.common.typeutils.TypeSerializer;
import eu.stratosphere.core.memory.DataInputView;
import eu.stratosphere.core.memory.DataOutputView;
/**
* Abstract test base for serializers.
*/
public abstract class KryoSerializerTestBase<T> {
protected abstract TypeSerializer<String> createSerializer();
protected abstract int getLength();
protected abstract Class<T> getTypeClass();
protected abstract String getTestData();
// --------------------------------------------------------------------------------------------
@Test
public void testInstantiate() {
try {
TypeSerializer<String> serializer = getSerializer();
String instance = serializer.createInstance();
assertNotNull("The created instance must not be null.", instance);
Class<T> type = getTypeClass();
assertNotNull("The test is corrupt: type class is null.", type);
assertEquals("Type of the instantiated object is wrong.", type,
instance.getClass());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Exception in test: " + e.getMessage());
}
}
@Test
public void testGetLength() {
try {
TypeSerializer<String> serializer = getSerializer();
assertEquals(getLength(), serializer.getLength());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Exception in test: " + e.getMessage());
}
}
@Test
public void testCopy() {
try {
TypeSerializer<String> serializer = getSerializer();
String testData = getData();
String copy = serializer
.copy(testData, serializer.createInstance());
assertEquals(
"Copied element is not equal to the original element.",
testData, copy);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Exception in test: " + e.getMessage());
}
}
@Test
public void testSerialize() {
try {
TypeSerializer<String> serializer = getSerializer();
String testData = getData();
TestOutputView out = new TestOutputView();
serializer.serialize(testData, out);
TestInputView in = out.getInputView();
assertTrue("No data available during deserialization.",
in.available() > 0);
String deserialized = serializer.deserialize(
serializer.createInstance(), in);
assertEquals("Deserialized value if wrong.", testData, deserialized);
assertTrue("Trailing data available after deserialization.",
in.available() == 0);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Exception in test: " + e.getMessage());
}
}
// --------------------------------------------------------------------------------------------
private TypeSerializer<String> getSerializer() {
TypeSerializer<String> serializer = createSerializer();
if (serializer == null) {
throw new RuntimeException(
"Test case corrupt. Returns null as serializer.");
}
return serializer;
}
private String getData() {
String data = getTestData();
if (data == null) {
throw new RuntimeException(
"Test case corrupt. Returns null as test data.");
}
return data;
}
// --------------------------------------------------------------------------------------------
private static final class TestOutputView extends DataOutputStream
implements DataOutputView {
public TestOutputView() {
super(new ByteArrayOutputStream(4096));
}
public TestInputView getInputView() {
ByteArrayOutputStream baos = (ByteArrayOutputStream) out;
return new TestInputView(baos.toByteArray());
}
@Override
public void skipBytesToWrite(int numBytes) throws IOException {
for (int i = 0; i < numBytes; i++) {
write(0);
}
}
@Override
public void write(DataInputView source, int numBytes)
throws IOException {
byte[] buffer = new byte[numBytes];
source.readFully(buffer);
write(buffer);
}
}
private static final class TestInputView extends DataInputStream implements
DataInputView {
public TestInputView(byte[] data) {
super(new ByteArrayInputStream(data));
}
@Override
public void skipBytesToRead(int numBytes) throws IOException {
while (numBytes > 0) {
int skipped = skipBytes(numBytes);
numBytes -= skipped;
}
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册