diff --git a/docs/example_connectors.md b/docs/example_connectors.md index a971b5469811da9d5209c37a4b17837196faa4fc..69c613defae4fd938935dcb0ca1b3dbc09ecbad7 100644 --- a/docs/example_connectors.md +++ b/docs/example_connectors.md @@ -175,7 +175,6 @@ The example shows how to access an Azure table and turn data into Flink's `DataS ## Access MongoDB -_Note: This example works starting from Flink 0.5 (then called Stratosphere)_ +This [GitHub repository documents how to use MongoDB with Apache Flink (starting from 0.7-incubating)](https://github.com/okkam-it/flink-mongodb-test). -Please see this (slightly outdated) blogpost on [How to access MongoDB with Apache Flink](http://flink.incubator.apache.org/news/2014/01/28/querying_mongodb.html). diff --git a/flink-addons/flink-tachyon/pom.xml b/flink-addons/flink-tachyon/pom.xml index c03fac2d933651d9064aa4b5e8a99f512d5b7730..08046a33bc363915aa0e37a8d40973bf86a09f21 100644 --- a/flink-addons/flink-tachyon/pom.xml +++ b/flink-addons/flink-tachyon/pom.xml @@ -23,7 +23,7 @@ under the License. flink-addons org.apache.flink - 0.8-incubating-SNAPSHOT + 0.9-SNAPSHOT .. @@ -40,6 +40,7 @@ under the License. org.apache.flink flink-core ${project.version} + test org.apache.flink @@ -51,6 +52,7 @@ under the License. org.tachyonproject tachyon 0.5.0 + test org.tachyonproject diff --git a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java b/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java index 582e7e7aae70deb0e1efa42ad53cfaa671fe939f..f425a49ebef6dfce4b79bf3318d9abffec8928bd 100644 --- a/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java +++ b/flink-addons/flink-tachyon/src/test/java/org/apache/flink/tachyon/TachyonFileSystemWrapperTest.java @@ -133,11 +133,6 @@ public class TachyonFileSystemWrapperTest { WordCount.main(new String[]{input, output}); -// List files = client.listFiles("/", true); -// for(Integer file : files) { -// TachyonFile f = client.getFile(file); -// System.out.println("file = "+file+" f = "+f.getPath()); -// } // verify result TachyonFile resultFile = client.getFile("/" + TACHYON_TEST_OUT_FILE_NAME); Assert.assertNotNull("Result file has not been created", resultFile); diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java index 7980cba9e009c0f6f8f98ff9124ecbd2d54f81a1..93a9938ab70520ce291e4a26e5436f6450cacf23 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java @@ -305,12 +305,12 @@ public abstract class FileSystem { return fs; } - private static AbstractHadoopWrapper hadoopWrapper; + private static HadoopFileSystemWrapper hadoopWrapper; private static Class getHadoopWrapperClassNameForFileSystem(String scheme) { if(hadoopWrapper == null) { try { - hadoopWrapper = (AbstractHadoopWrapper) instantiateHadoopFileSystemWrapper(null); + hadoopWrapper = (HadoopFileSystemWrapper) instantiateHadoopFileSystemWrapper(null); } catch (IOException e) { throw new RuntimeException("Error creating new Hadoop wrapper", e); } diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/AbstractHadoopWrapper.java b/flink-core/src/main/java/org/apache/flink/core/fs/HadoopFileSystemWrapper.java similarity index 95% rename from flink-core/src/main/java/org/apache/flink/core/fs/AbstractHadoopWrapper.java rename to flink-core/src/main/java/org/apache/flink/core/fs/HadoopFileSystemWrapper.java index 69f37bcd513635a92171eab0fea17ec6d1f0db9d..61ba206bf91f25b8dc83eb78462226d7ad6110c5 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/AbstractHadoopWrapper.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/HadoopFileSystemWrapper.java @@ -17,7 +17,7 @@ */ package org.apache.flink.core.fs; -public interface AbstractHadoopWrapper { +public interface HadoopFileSystemWrapper { /** * Test whether the HadoopWrapper can wrap the given file system scheme. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java index e849d3290b9f6c66035d09ea42dc9479623122bb..32e3f2df89675fc9d8d17b4322bbe79434531b26 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java @@ -23,7 +23,7 @@ import java.lang.reflect.Method; import java.net.URI; import java.net.UnknownHostException; -import org.apache.flink.core.fs.AbstractHadoopWrapper; +import org.apache.flink.core.fs.HadoopFileSystemWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.flink.configuration.ConfigConstants; @@ -45,7 +45,7 @@ import org.apache.hadoop.conf.Configuration; * distributed file system (HDFS). * */ -public final class HadoopFileSystem extends FileSystem implements AbstractHadoopWrapper { +public final class HadoopFileSystem extends FileSystem implements HadoopFileSystemWrapper { private static final Logger LOG = LoggerFactory.getLogger(HadoopFileSystem.class); diff --git a/pom.xml b/pom.xml index 6c1798f0089dfc95cf3ffef47000e6643c189c1e..1376b527aaa6a2d26617ff46969b9e6bab9ece78 100644 --- a/pom.xml +++ b/pom.xml @@ -433,6 +433,44 @@ under the License. org.apache.hadoop hadoop-client ${hadoop.version} + + + asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + + + javax.servlet + servlet-api + + org.apache.hadoop @@ -564,6 +602,38 @@ under the License. org.jboss.netty netty + + asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core + @@ -575,6 +645,38 @@ under the License. org.jboss.netty netty + + asm + asm + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.mortbay.jetty + jetty-util + + + org.eclipse.jdt + core +