未验证 提交 5ffcfcaf 编写于 作者: J Jack 提交者: GitHub

[FLINK-20710][hive] Hive reading fail when splits on different FileSystem nameservices

This closes #14452
上级 f5e829c4
......@@ -62,13 +62,10 @@ public class HiveSourceFileEnumerator implements FileEnumerator {
List<HiveTablePartition> partitions,
JobConf jobConf) throws IOException {
List<HiveSourceSplit> hiveSplits = new ArrayList<>();
FileSystem fs = null;
for (HiveTablePartition partition : partitions) {
StorageDescriptor sd = partition.getStorageDescriptor();
org.apache.hadoop.fs.Path inputPath = new org.apache.hadoop.fs.Path(sd.getLocation());
if (fs == null) {
fs = inputPath.getFileSystem(jobConf);
}
FileSystem fs = inputPath.getFileSystem(jobConf);
// it's possible a partition exists in metastore but the data has been removed
if (!fs.exists(inputPath)) {
continue;
......
......@@ -285,13 +285,10 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<RowData, H
JobConf jobConf) throws IOException {
List<HiveTableInputSplit> hiveSplits = new ArrayList<>();
int splitNum = 0;
FileSystem fs = null;
for (HiveTablePartition partition : partitions) {
StorageDescriptor sd = partition.getStorageDescriptor();
Path inputPath = new Path(sd.getLocation());
if (fs == null) {
fs = inputPath.getFileSystem(jobConf);
}
FileSystem fs = inputPath.getFileSystem(jobConf);
// it's possible a partition exists in metastore but the data has been removed
if (!fs.exists(inputPath)) {
continue;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册