未验证 提交 a7be2e18 编写于 作者: Y Yun Tang 提交者: Dawid Wysakowicz

[FLINK-9758] Fix ContinuousFileProcessingTest failure due to not setting runtimeContext

This closes #6260
上级 10ddfcaf
......@@ -19,6 +19,7 @@
package org.apache.flink.hdfstests;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.io.FileInputFormat;
import org.apache.flink.api.common.io.FilePathFilter;
import org.apache.flink.api.common.typeinfo.TypeInformation;
......@@ -55,6 +56,7 @@ import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import java.io.File;
import java.io.FileNotFoundException;
......@@ -579,8 +581,7 @@ public class ContinuousFileProcessingTest {
});
ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format,
FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_ONCE);
final FileVerifyingSourceContext context =
new FileVerifyingSourceContext(new OneShotLatch(), monitoringFunction);
......@@ -632,8 +633,7 @@ public class ContinuousFileProcessingTest {
format.setNestedFileEnumeration(true);
ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format,
FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_ONCE);
final FileVerifyingSourceContext context =
new FileVerifyingSourceContext(new OneShotLatch(), monitoringFunction);
......@@ -674,8 +674,7 @@ public class ContinuousFileProcessingTest {
FileInputSplit[] splits = format.createInputSplits(1);
ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format,
FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_ONCE);
ModTimeVerifyingSourceContext context = new ModTimeVerifyingSourceContext(modTimes);
......@@ -708,8 +707,7 @@ public class ContinuousFileProcessingTest {
format.setFilesFilter(FilePathFilter.createDefaultFilter());
final ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format,
FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_ONCE);
final FileVerifyingSourceContext context = new FileVerifyingSourceContext(latch, monitoringFunction);
......@@ -772,7 +770,7 @@ public class ContinuousFileProcessingTest {
TextInputFormat format = new TextInputFormat(new Path(testBasePath));
final ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_CONTINUOUSLY);
StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src =
new StreamSource<>(monitoringFunction);
......@@ -823,7 +821,7 @@ public class ContinuousFileProcessingTest {
testHarness.close();
final ContinuousFileMonitoringFunction<String> monitoringFunctionCopy =
new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_CONTINUOUSLY);
StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> srcCopy =
new StreamSource<>(monitoringFunctionCopy);
......@@ -857,8 +855,7 @@ public class ContinuousFileProcessingTest {
format.setFilesFilter(FilePathFilter.createDefaultFilter());
final ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format,
FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
createTestContinuousFileMonitoringFunction(format, FileProcessingMode.PROCESS_CONTINUOUSLY);
final int totalNoOfFilesToBeRead = NO_OF_FILES + 1; // 1 for the bootstrap + NO_OF_FILES
final FileVerifyingSourceContext context = new FileVerifyingSourceContext(latch,
......@@ -1055,4 +1052,14 @@ public class ContinuousFileProcessingTest {
Assert.assertTrue("No result file present", hdfs.exists(file));
return new Tuple2<>(file, str.toString());
}
/**
* Create continuous monitoring function with 1 reader-parallelism and interval: {@link #INTERVAL}.
*/
private <OUT> ContinuousFileMonitoringFunction<OUT> createTestContinuousFileMonitoringFunction(FileInputFormat<OUT> format, FileProcessingMode fileProcessingMode) {
ContinuousFileMonitoringFunction<OUT> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format, fileProcessingMode, 1, INTERVAL);
monitoringFunction.setRuntimeContext(Mockito.mock(RuntimeContext.class));
return monitoringFunction;
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册