Skip to content

F
flink

doujutun3207 / flink
与 Fork 源项目一致

从无法访问的项目Fork

通知 24
Star 0
Fork 0
F
flink

体验新版 GitCode，发现更多精彩内容 >>

提交 0a005700 编写于 作者: V Vimal 提交者： Fabian Hueske
浏览文件
操作

[FLINK-1674] Modified test cases for nested avro type 

This closes #1091
上级 a9c29a76
隐藏空白更改
内联 并排
Showing with 79 addition and 12 deletion
flink-staging/flink-avro/src/test/java/org/apache/flink/api/avro/EncoderDecoderTest.java
浏览文件 @ 0a005700
......@@ -31,6 +31,7 @@ import java.util.Random;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.flink.api.io.avro.generated.Address;
import org.apache.flink.api.io.avro.generated.Colors;
import org.apache.flink.api.io.avro.generated.Fixed16;
import org.apache.flink.api.io.avro.generated.User;
......@@ -237,7 +238,11 @@ public class EncoderDecoderTest {
byte[] b = new byte[16];
new Random().nextBytes(b);
Fixed16 f = new Fixed16(b);
User user = new User("Freudenreich", 1337, "macintosh gray", 1234567890L, 3.1415926, null, true, strings, bools, null, Colors.GREEN, map, f, new Boolean(true));
Address addr = new Address(new Integer(239), "6th Main", "Bangalore",
"Karnataka", "560075");
User user = new User("Freudenreich", 1337, "macintosh gray",
1234567890L, 3.1415926, null, true, strings, bools, null,
Colors.GREEN, map, f, new Boolean(true), addr);
testObjectSerialization(user);
}
......
flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroPojoTest.java
浏览文件 @ 0a005700
......@@ -89,8 +89,8 @@ public class AvroPojoTest extends MultipleProgramsTestBase {
env.execute("Simple Avro read job");
expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null}\n" +
"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null}\n";
expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n" +
"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n";
}
@Test
......@@ -117,8 +117,9 @@ public class AvroPojoTest extends MultipleProgramsTestBase {
env.execute("Simple Avro read job");
expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null}\n" +
"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null}\n";
expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n" +
"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n";
}
@Test
......
flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroRecordInputFormatTest.java
浏览文件 @ 0a005700
......@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import com.esotericsoftware.kryo.Serializer;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
......@@ -43,6 +44,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.ComparatorTestBase;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.io.avro.generated.Address;
import org.apache.flink.api.io.avro.generated.Colors;
import org.apache.flink.api.io.avro.generated.User;
import org.apache.flink.api.java.io.AvroInputFormat;
......@@ -81,6 +83,13 @@ public class AvroRecordInputFormatTest {
final static long TEST_MAP_VALUE1 = 8546456L;
final static String TEST_MAP_KEY2 = "KEY 2";
final static long TEST_MAP_VALUE2 = 17554L;
final static Integer TEST_NUM = 239;
final static String TEST_STREET = "Baker Street";
final static String TEST_CITY = "London";
final static String TEST_STATE = "London";
final static String TEST_ZIP = "NW1 6XE";
private Schema userSchema = new User().getSchema();
......@@ -97,6 +106,13 @@ public class AvroRecordInputFormatTest {
HashMap<CharSequence, Long> longMap = new HashMap<CharSequence, Long>();
longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
Address addr = new Address();
addr.setNum(new Integer(TEST_NUM));
addr.setStreet(TEST_STREET);
addr.setCity(TEST_CITY);
addr.setState(TEST_STATE);
addr.setZip(TEST_ZIP);
User user1 = new User();
......@@ -109,6 +125,7 @@ public class AvroRecordInputFormatTest {
user1.setTypeArrayBoolean(booleanArray);
user1.setTypeEnum(TEST_ENUM_COLOR);
user1.setTypeMap(longMap);
user1.setTypeNested(addr);
// Construct via builder
User user2 = User.newBuilder()
......@@ -126,6 +143,10 @@ public class AvroRecordInputFormatTest {
.setTypeMap(new HashMap<CharSequence, Long>())
.setTypeFixed(null)
.setTypeUnion(null)
.setTypeNested(
Address.newBuilder().setNum(TEST_NUM).setStreet(TEST_STREET)
.setCity(TEST_CITY).setState(TEST_STATE).setZip(TEST_ZIP)
.build())
.build();
DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
......
flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/AvroSplittableInputFormatTest.java
浏览文件 @ 0a005700
......@@ -21,6 +21,7 @@ package org.apache.flink.api.io.avro;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.flink.api.io.avro.generated.Address;
import org.apache.flink.api.io.avro.generated.Colors;
import org.apache.flink.api.io.avro.generated.Fixed16;
import org.apache.flink.api.io.avro.generated.User;
......@@ -65,6 +66,12 @@ public class AvroSplittableInputFormatTest {
final static String TEST_MAP_KEY2 = "KEY 2";
final static long TEST_MAP_VALUE2 = 17554L;
final static Integer TEST_NUM = new Integer(239);
final static String TEST_STREET = "Baker Street";
final static String TEST_CITY = "London";
final static String TEST_STATE = "London";
final static String TEST_ZIP = "NW1 6XE";
final static int NUM_RECORDS = 5000;
@Before
......@@ -83,6 +90,13 @@ public class AvroSplittableInputFormatTest {
longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);
Address addr = new Address();
addr.setNum(new Integer(TEST_NUM));
addr.setStreet(TEST_STREET);
addr.setCity(TEST_CITY);
addr.setState(TEST_STATE);
addr.setZip(TEST_ZIP);
User user1 = new User();
user1.setName(TEST_NAME);
......@@ -93,6 +107,7 @@ public class AvroSplittableInputFormatTest {
user1.setTypeArrayBoolean(booleanArray);
user1.setTypeEnum(TEST_ENUM_COLOR);
user1.setTypeMap(longMap);
user1.setTypeNested(addr);
// Construct via builder
User user2 = User.newBuilder()
......@@ -110,6 +125,11 @@ public class AvroSplittableInputFormatTest {
.setTypeMap(new HashMap<CharSequence, Long>())
.setTypeFixed(new Fixed16())
.setTypeUnion(123L)
.setTypeNested(
Address.newBuilder().setNum(TEST_NUM).setStreet(TEST_STREET)
.setCity(TEST_CITY).setState(TEST_STATE).setZip(TEST_ZIP)
.build())
.build();
DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
......@@ -128,6 +148,13 @@ public class AvroSplittableInputFormatTest {
user.setTypeArrayBoolean(booleanArray);
user.setTypeEnum(TEST_ENUM_COLOR);
user.setTypeMap(longMap);
Address address = new Address();
address.setNum(new Integer(TEST_NUM));
address.setStreet(TEST_STREET);
address.setCity(TEST_CITY);
address.setState(TEST_STATE);
address.setZip(TEST_ZIP);
user.setTypeNested(address);
dataFileWriter.append(user);
}
......@@ -155,10 +182,11 @@ public class AvroSplittableInputFormatTest {
}
format.close();
}
Assert.assertEquals(1474, elementsPerSplit[0]);
Assert.assertEquals(1474, elementsPerSplit[1]);
Assert.assertEquals(1474, elementsPerSplit[2]);
Assert.assertEquals(578, elementsPerSplit[3]);
Assert.assertEquals(1539, elementsPerSplit[0]);
Assert.assertEquals(1026, elementsPerSplit[1]);
Assert.assertEquals(1539, elementsPerSplit[2]);
Assert.assertEquals(896, elementsPerSplit[3]);
Assert.assertEquals(NUM_RECORDS, elements);
format.close();
}
......
flink-staging/flink-avro/src/test/resources/avro/user.avsc
浏览文件 @ 0a005700
[
{"namespace": "org.apache.flink.api.io.avro.generated",
"type": "record",
"name": "Address",
"fields": [
{"name": "num", "type": "int"},
{"name": "street", "type": "string"},
{"name": "city", "type": "string"},
{"name": "state", "type": "string"},
{"name": "zip", "type": "string"}
]
},
{"namespace": "org.apache.flink.api.io.avro.generated",
"type": "record",
"name": "User",
......@@ -18,6 +29,7 @@
{"name": "type_fixed",
"size": 16,
"type": ["null", {"name": "Fixed16", "size": 16, "type": "fixed"}] },
{"name": "type_union", "type": ["null", "boolean", "long", "double"]}
{"name": "type_union", "type": ["null", "boolean", "long", "double"]},
{"name": "type_nested", "type": ["null", "Address"]}
]
}
\ No newline at end of file
}]
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或