[FLINK-5781][docs] Generate HTML from ConfigOptions

This closes #3495.

docs/README.md

@@ -49,6 +49,21 @@ to files that are modified. Note that if you are making changes that affect
 the sidebar navigation, you'll have to build the entire site to see
 those changes reflected on every page.
 
+## Generate configuration tables
+
+Configuration descriptions are auto generated from code. To trigger the generation you need to run:
+
+```
+mvn -Pgenerate-config-docs install
+```
+
+The resulting html files will be written to `_include/generated`. Tables are regenerated each time the command is invoked.
+These tables can be directly included into the documentation:
+
+```
+{% include generated/file_name.html %}
+```
+
 # Contribute
 
 ## Markdown

flink-core/pom.xml

@@ -123,6 +123,40 @@ under the License.
 		</dependency>
     </dependencies>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.configuration" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
 	<build>
 		<plugins>
 

flink-core/src/main/java/org/apache/flink/configuration/ConfigGroup.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.lang.annotation.Target;
+import org.apache.flink.annotation.Internal;
+
+/**
+ * A class that specifies a group of {@link ConfigOption}. The name of the group will be used as the basis for the
+ * filename of the generated html file, as defined in {@link ConfigOptionsDocGenerator}.
+ * 
+ * @see ConfigGroups
+ */
+@Target({})
+@Internal
+public @interface ConfigGroup {
+	String name();
+	String keyPrefix();
+}

flink-core/src/main/java/org/apache/flink/configuration/ConfigGroups.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Annotation used on classes containing {@link ConfigOption}s that enables the separation of options into different
+ * tables based on key prefixes. A {@link ConfigOption} is assigned to a {@link ConfigGroup} if the option key matches
+ * the group prefix. If a key matches multiple prefixes the longest matching prefix takes priority. An option is never
+ * assigned to multiple groups. Options that don't match any group are implicitly added to a default group.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Internal
+public @interface ConfigGroups {
+	ConfigGroup[] groups() default {};
+}

flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java

@@ -51,6 +51,9 @@ public class ConfigOption<T> {
 	/** The default value for this option */
 	private final T defaultValue;
 
+	/** The description for this option */
+	private final String description;
+
 	// ------------------------------------------------------------------------
 
 	/**
@@ -61,6 +64,7 @@ public class ConfigOption<T> {
 	 */
 	ConfigOption(String key, T defaultValue) {
 		this.key = checkNotNull(key);
+		this.description = "";
 		this.defaultValue = defaultValue;
 		this.deprecatedKeys = EMPTY;
 	}
@@ -72,8 +76,9 @@ public class ConfigOption<T> {
 	 * @param defaultValue    The default value for this option
 	 * @param deprecatedKeys  The list of deprecated keys, in the order to be checked
 	 */
-	ConfigOption(String key, T defaultValue, String... deprecatedKeys) {
+	ConfigOption(String key, String description, T defaultValue, String... deprecatedKeys) {
 		this.key = checkNotNull(key);
+		this.description = description;
 		this.defaultValue = defaultValue;
 		this.deprecatedKeys = deprecatedKeys == null || deprecatedKeys.length == 0 ? EMPTY : deprecatedKeys;
 	}
@@ -92,7 +97,20 @@ public class ConfigOption<T> {
 	 * @return A new config options, with the given deprecated keys.
 	 */
 	public ConfigOption<T> withDeprecatedKeys(String... deprecatedKeys) {
-		return new ConfigOption<>(key, defaultValue, deprecatedKeys);
+		return new ConfigOption<>(key, description, defaultValue, deprecatedKeys);
+	}
+
+	/**
+	 * Creates a new config option, using this option's key and default value, and
+	 * adding the given description. The given description is used when generation the configuration documention.
+	 *
+	 * <p><b>NOTE:</b> You can use html to format the output of the generated cell.
+	 *
+	 * @param description The description for this option.
+	 * @return A new config option, with given description.
+	 */
+	public ConfigOption<T> withDescription(final String description) {
+		return new ConfigOption<>(key, description, defaultValue, deprecatedKeys);
 	}
 
 	// ------------------------------------------------------------------------
@@ -137,6 +155,14 @@ public class ConfigOption<T> {
 		return deprecatedKeys == EMPTY ? Collections.<String>emptyList() : Arrays.asList(deprecatedKeys);
 	}
 
+	/**
+	 * Returns the description of this option.
+	 * @return The option's description.
+	 */
+	public String description() {
+		return description;
+	}
+
 	// ------------------------------------------------------------------------
 
 	@Override
@@ -168,4 +194,4 @@ public class ConfigOption<T> {
 		return String.format("Key: '%s' , default: %s (deprecated keys: %s)",
 				key, defaultValue, Arrays.toString(deprecatedKeys));
 	}
-}
\ No newline at end of file
+}

flink-core/src/main/java/org/apache/flink/configuration/ConfigOptionsDocGenerator.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Class used for generating code based documentation of configuration parameters.
+ */
+public class ConfigOptionsDocGenerator {
+
+	/**
+	 * This method generates html tables from set of classes containing {@link ConfigOption ConfigOptions}.
+	 *
+	 * <p>For each class 1 or more html tables will be generated and placed into a separate file, depending on whether
+	 * the class is annotated with {@link ConfigGroups}. The tables contain the key, default value and description for
+	 * every {@link ConfigOption}.
+	 *
+	 * @param args first argument is output path for the generated files, second argument is full package name containing
+	 *             classes with {@link ConfigOption}
+	 */
+	public static void main(String[] args) throws IOException, ClassNotFoundException {
+		String outputPath = args[0];
+		String packageName = args[1];
+
+		Path configDir = Paths.get("../src/main/java", packageName.replaceAll("\\.", "/"));
+
+		Pattern p = Pattern.compile("(([a-zA-Z]*)(Options))\\.java");
+		try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir, "*Options.java")) {
+			for (Path entry : stream) {
+				String fileName = entry.getFileName().toString();
+				Matcher matcher = p.matcher(fileName);
+				if (!fileName.equals("ConfigOptions.java") && matcher.matches()) {
+					Class<?> optionsClass = Class.forName(packageName + "." + matcher.group(1));
+					List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
+					if (tables.size() > 0) {
+						for (Tuple2<ConfigGroup, String> group : tables) {
+
+							String name = group.f0 == null
+								? matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase()
+								: group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+
+							String outputFile = name + "_configuration.html";
+							Files.write(Paths.get(outputPath, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
+						}
+					}
+				}
+			}
+		}
+	}
+
+	@VisibleForTesting
+	static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?> optionsClass) {
+		ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
+		List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>();
+		List<ConfigOption> allOptions = extractConfigOptions(optionsClass);
+
+		if (configGroups != null) {
+			Tree tree = new Tree(configGroups.groups(), allOptions);
+
+			for (ConfigGroup group : configGroups.groups()) {
+				List<ConfigOption> configOptions = tree.findConfigOptions(group);
+				sortOptions(configOptions);
+				tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
+			}
+			List<ConfigOption> configOptions = tree.getDefaultOptions();
+			sortOptions(configOptions);
+			tables.add(Tuple2.<ConfigGroup, String>of(null, toHtmlTable(configOptions)));
+		} else {
+			sortOptions(allOptions);
+			tables.add(Tuple2.<ConfigGroup, String>of(null, toHtmlTable(allOptions)));
+		}
+		return tables;
+	}
+
+	private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
+		try {
+			List<ConfigOption> configOptions = new ArrayList<>();
+			Field[] fields = clazz.getFields();
+			for (Field field : fields) {
+				if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class) == null) {
+					configOptions.add((ConfigOption) field.get(null));
+				}
+			}
+
+			return configOptions;
+		} catch (Exception e) {
+			throw new RuntimeException("Failed to extract config options from class " + clazz + ".", e);
+		}
+	}
+
+
+	/**
+	 * Transforms this configuration group into HTML formatted table.
+	 * Options are sorted alphabetically by key.
+	 *
+	 * @param options list of options to include in this group
+	 * @return string containing HTML formatted table
+	 */
+	private static String toHtmlTable(final List<ConfigOption> options) {
+		StringBuilder htmlTable = new StringBuilder(
+			"<table class=\"table table-bordered\"><thead><tr><th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+			"<th class=\"text-left\" style=\"width: 15%\">Default Value</th><th class=\"text-left\" " +
+			"style=\"width: 65%\">Description</th></tr></thead><tbody>");
+
+		for (ConfigOption option : options) {
+			htmlTable.append(toHtmlString(option));
+		}
+
+		htmlTable.append("</tbody></table>");
+
+		return htmlTable.toString();
+	}
+
+	/**
+	 * Transforms option to table row.
+	 *
+	 * @param option option to transform
+	 * @return row with the option description
+	 */
+	private static String toHtmlString(final ConfigOption<?> option) {
+		Object defaultValue = option.defaultValue();
+		// This is a temporary hack that should be removed once FLINK-6490 is resolved.
+		// These options use System.getProperty("java.io.tmpdir") as the default.
+		// As a result the generated table contains an actual path as the default, which is simply wrong.
+		if (option == JobManagerOptions.WEB_TMP_DIR || option.key().equals("python.dc.tmp.dir")) {
+			defaultValue = null;
+		}
+		return "<tr>" +
+			"<td><h5>" + escapeCharacters(option.key()) + "</h5></td>" +
+			"<td>" + escapeCharacters(defaultValueToHtml(defaultValue)) + "</td>" +
+			"<td>" + escapeCharacters(option.description()) + "</td>" +
+			"</tr>";
+	}
+
+	private static String defaultValueToHtml(Object value) {
+		if (value instanceof String) {
+			if (((String) value).isEmpty()) {
+				return "(none)";
+			}
+			return "\"" + value + "\"";
+		}
+
+		return value == null ? "(none)" : value.toString();
+	}
+
+	private static String escapeCharacters(String value) {
+		return value
+			.replaceAll("<", "&#60;")
+			.replaceAll(">", "&#62;");
+	}
+
+	private static void sortOptions(List<ConfigOption> configOptions) {
+		Collections.sort(configOptions, new Comparator<ConfigOption>() {
+			@Override
+			public int compare(ConfigOption o1, ConfigOption o2) {
+				return o1.key().compareTo(o2.key());
+			}
+		});
+	}
+
+	/**
+	 * Data structure used to assign {@link ConfigOption ConfigOptions} to the {@link ConfigGroup} with the longest
+	 * matching prefix.
+	 */
+	private static class Tree {
+		private final Node root = new Node();
+
+		Tree(ConfigGroup[] groups, Collection<ConfigOption> options) {
+			// generate a tree based on all key prefixes
+			for (ConfigGroup group : groups) {
+				String[] keyComponents = group.keyPrefix().split("\\.");
+				Node currentNode = root;
+				for (String keyComponent : keyComponents) {
+					currentNode = currentNode.addChild(keyComponent);
+				}
+				currentNode.markAsGroupRoot();
+			}
+
+			// assign options to their corresponding group, i.e. the last group root node encountered when traversing
+			// the tree based on the option key
+			for (ConfigOption<?> option : options) {
+				findGroupRoot(option.key()).assignOption(option);
+			}
+		}
+
+		List<ConfigOption> findConfigOptions(ConfigGroup configGroup) {
+			Node groupRoot = findGroupRoot(configGroup.keyPrefix());
+			return groupRoot.getConfigOptions();
+		}
+
+		List<ConfigOption> getDefaultOptions() {
+			return root.getConfigOptions();
+		}
+
+		private Node findGroupRoot(String key) {
+			String[] keyComponents = key.split("\\.");
+			Node currentNode = root;
+			for (String keyComponent : keyComponents) {
+				currentNode = currentNode.findChild(keyComponent);
+			}
+			return currentNode.isGroupRoot() ? currentNode : root;
+		}
+
+		private static class Node {
+			private final List<ConfigOption> configOptions = new ArrayList<>();
+			private final Map<String, Node> children = new HashMap<>();
+			private boolean isGroupRoot = false;
+
+			private Node addChild(String keyComponent) {
+				Node child = children.get(keyComponent);
+				if (child == null) {
+					child = new Node();
+					children.put(keyComponent, child);
+				}
+				return child;
+			}
+
+			private Node findChild(String keyComponent) {
+				Node child = children.get(keyComponent);
+				if (child == null) {
+					return this;
+				}
+				return child;
+			}
+
+			private void assignOption(ConfigOption option) {
+				configOptions.add(option);
+			}
+
+			private boolean isGroupRoot() {
+				return isGroupRoot;
+			}
+
+			private void markAsGroupRoot() {
+				this.isGroupRoot = true;
+			}
+
+			private List<ConfigOption> getConfigOptions() {
+				return configOptions;
+			}
+		}
+	}
+
+	private ConfigOptionsDocGenerator() {
+	}
+}

flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java

@@ -342,6 +342,9 @@ public final class DelegatingConfiguration extends Configuration {
 		}
 
 		String[] deprecated = deprecatedKeys.toArray(new String[deprecatedKeys.size()]);
-		return new ConfigOption<T>(key, option.defaultValue(), deprecated);
+		return new ConfigOption<T>(key,
+			option.description(),
+			option.defaultValue(),
+			deprecated);
 	}
 }

flink-core/src/test/java/org/apache/flink/configuration/ConfigOptionsDocGeneratorTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.util.HashMap;
+import java.util.List;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ConfigOptionsDocGeneratorTest {
+
+	static class TestConfigGroup {
+		public static ConfigOption<Integer> firstOption = ConfigOptions
+			.key("first.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the first option.");
+
+		public static ConfigOption<String> secondOption = ConfigOptions
+			.key("second.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the second option.");
+	}
+
+	@Test
+	public void testCreatingDescription() throws Exception {
+		final String expectedTable = "<table class=\"table table-bordered\">" +
+			"<thead>" +
+			"<tr>" +
+			"<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+			"<th class=\"text-left\" style=\"width: 15%\">Default Value</th>" +
+			"<th class=\"text-left\" style=\"width: 65%\">Description</th>" +
+			"</tr>" +
+			"</thead>" +
+			"<tbody>" +
+			"<tr>" +
+			"<td><h5>first.option.a</h5></td>" +
+			"<td>2</td>" +
+			"<td>This is example description for the first option.</td>" +
+			"</tr>" +
+			"<tr>" +
+			"<td><h5>second.option.a</h5></td>" +
+			"<td>(none)</td>" +
+			"<td>This is long example description for the second option.</td>" +
+			"</tr>" +
+			"</tbody>" +
+			"</table>";
+		final String htmlTable = ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroup.class).get(0).f1;
+
+		assertEquals(expectedTable, htmlTable);
+	}
+
+	@ConfigGroups(groups = {
+			@ConfigGroup(name = "firstGroup", keyPrefix = "first"),
+			@ConfigGroup(name = "secondGroup", keyPrefix = "second")})
+	static class TestConfigMultipleSubGroup {
+		public static ConfigOption<Integer> firstOption = ConfigOptions
+			.key("first.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the first option.");
+
+		public static ConfigOption<String> secondOption = ConfigOptions
+			.key("second.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the second option.");
+
+		public static ConfigOption<Integer> thirdOption = ConfigOptions
+			.key("third.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the third option.");
+
+		public static ConfigOption<String> fourthOption = ConfigOptions
+			.key("fourth.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the fourth option.");
+	}
+
+	@Test
+	public void testCreatingMultipleGroups() throws Exception {
+		final List<Tuple2<ConfigGroup, String>> tables = ConfigOptionsDocGenerator.generateTablesForClass(
+			TestConfigMultipleSubGroup.class);
+
+		assertEquals(tables.size(), 3);
+		final HashMap<String, String> tablesConverted = new HashMap<>();
+		for (Tuple2<ConfigGroup, String> table : tables) {
+			tablesConverted.put(table.f0 != null ? table.f0.name() : "default", table.f1);
+		}
+
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>" +
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>" +
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>first.option.a</h5></td>" +
+		             "<td>2</td>" +
+		             "<td>This is example description for the first option.</td>" +
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("firstGroup"));
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>" +
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>" +
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>second.option.a</h5></td>" +
+		             "<td>(none)</td>" +
+		             "<td>This is long example description for the second option.</td>" +
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("secondGroup"));
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>" +
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>" +
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>fourth.option.a</h5></td>" +
+		             "<td>(none)</td>" +
+		             "<td>This is long example description for the fourth option.</td>" +
+		             "</tr>" +
+		             "<tr>" +
+		             "<td><h5>third.option.a</h5></td>" +
+		             "<td>2</td>" +
+		             "<td>This is example description for the third option.</td>" +
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("default"));
+	}
+
+}

flink-libraries/flink-python/pom.xml

@@ -51,6 +51,40 @@ under the License.
         </plugins>
     </build>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.python.api" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
     <dependencies>
         
         <!-- core dependencies -->

flink-yarn/pom.xml

@@ -121,6 +121,40 @@ under the License.
 		</dependency>
 	</dependencies>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.yarn.configuration" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
 	<build>
 		<plugins>
 			<!-- Scala Compiler -->

pom.xml

@@ -114,6 +114,7 @@ under the License.
 			to revisit the impact at that time.
 		-->
 		<minikdc.version>2.7.2</minikdc.version>
+		<generated.docs.dir>./docs/_includes/generated</generated.docs.dir>
 	</properties>
 
 	<dependencies>
@@ -991,6 +992,7 @@ under the License.
 						<exclude>docs/content/**</exclude>
 						<exclude>**/scalastyle-output.xml</exclude>
 						<exclude>build-target/**</exclude>
+						<exclude>docs/_includes/generated/**</exclude>
 						<!-- Tools: watchdog -->
 						<exclude>tools/artifacts/**</exclude>
 						<exclude>tools/flink*/**</exclude>
@@ -1222,6 +1224,35 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
+
+			<!-- generate configuration docs -->
+			<plugin>
+				<groupId>org.commonjava.maven.plugins</groupId>
+				<artifactId>directory-maven-plugin</artifactId>
+				<version>0.1</version>
+				<executions>
+					<execution>
+						<id>directories</id>
+						<goals>
+							<goal>highest-basedir</goal>
+						</goals>
+						<phase>initialize</phase>
+						<configuration>
+							<property>rootDir</property>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<artifactId>maven-clean-plugin</artifactId>
+				<configuration>
+					<filesets>
+						<fileset>
+							<directory>${generated.docs.dir}</directory>
+						</fileset>
+					</filesets>
+				</configuration>
+			</plugin>
 		</plugins>
 
 		<!-- Plugin configurations for plugins activated in sub-projects -->