[FLINK-1335] [docs] Update internals with brief description about JobManager...

[FLINK-1335] [docs] Update internals with brief description about JobManager data structures and scheduling.

This closes #281

docs/_includes/sidenav.html

@@ -17,57 +17,47 @@ specific language governing permissions and limitations
 under the License.
 -->
 <ul>
-  <li><a href="faq.html">FAQ</a></li>
-  <li>Quickstart
-    <ul>
-      <li><a href="setup_quickstart.html">Setup</a></li>
-      <li><a href="run_example_quickstart.html">Run Example</a></li>
-      <li><a href="java_api_quickstart.html">Java API</a></li>
-      <li><a href="scala_api_quickstart.html">Scala API</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category"><a href="faq.html">FAQ</a></div></li>
+  <li><div class="sidenav-category">Quickstart</div></li>
+  <li><div class="sidenav-item"><a href="setup_quickstart.html">Setup</a></div></li>
+  <li><div class="sidenav-item"><a href="run_example_quickstart.html">Run Example</a></div></li>
+  <li><div class="sidenav-item"><a href="java_api_quickstart.html">Java API</a></div></li>
+  <li><div class="sidenav-item-bottom"><a href="scala_api_quickstart.html">Scala API</a></div></li>
 
-  <li>Setup &amp; Configuration
-    <ul>
-      <li><a href="local_setup.html">Local Setup</a></li>
-      <li><a href="building.html">Build Flink</a></li>
-      <li><a href="cluster_setup.html">Cluster Setup</a></li>
-      <li><a href="yarn_setup.html">YARN Setup</a></li>
-      <li><a href="config.html">Configuration</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category">Setup &amp; Configuration</div></li>
+  <li><div class="sidenav-item"><a href="local_setup.html">Local Setup</a></div></li>
+  <li><div class="sidenav-item"><a href="building.html">Build Flink</a></div></li>
+  <li><div class="sidenav-item"><a href="cluster_setup.html">Cluster Setup</a></div></li>
+  <li><div class="sidenav-item"><a href="yarn_setup.html">YARN Setup</a></div></li>
+  <li><div class="sidenav-item-bottom"><a href="config.html">Configuration</a></div></li>
 
-  <li>Programming Guides
-    <ul>
-      <li><a href="programming_guide.html">Programming Guide</a></li>
-      <li><a href="dataset_transformations.html">DataSet Transformations</a></li>
-      <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li>
-      <li><a href="streaming_guide.html">Streaming Guide</a></li>
-      <li><a href="iterations.html">Iterations</a></li>
-      <li><a href="spargel_guide.html">Spargel Graph API</a></li>
-      <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category">Programming Guides</div></li>
+  <li><div class="sidenav-item"><a href="programming_guide.html">Programming Guide</a></div></li>
+  <li><div class="sidenav-item"><a href="dataset_transformations.html">DataSet Transformations</a></div></li>
+  <li><div class="sidenav-item"><a href="java8_programming_guide.html">Java 8 Programming Guide</a></div></li>
+  <li><div class="sidenav-item"><a href="streaming_guide.html">Streaming Guide</a></div></li>
+  <li><div class="sidenav-item"><a href="iterations.html">Iterations</a></div></li>
+  <li><div class="sidenav-item"><a href="spargel_guide.html">Spargel Graph API</a></div></li>
+  <li><div class="sidenav-item-bottom"><a href="hadoop_compatibility.html">Hadoop Compatibility</a></div></li>
 
-  <li>Examples
-    <ul>
-      <li><a href="examples.html">Bundled Examples</a></li>
-      <li><a href="example_connectors.html">Connecting to other systems</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category">Examples</div></li>
+  <li><div class="sidenav-item"><a href="examples.html">Bundled Examples</a></div></li>
+  <li><div class="sidenav-item-bottom"><a href="example_connectors.html">Connecting to other systems</a></div></li>
 
-  <li>Execution
-    <ul>
-      <li><a href="local_execution.html">Local/Debugging</a></li>
-      <li><a href="cluster_execution.html">Cluster</a></li>
-      <li><a href="cli.html">Command-Line Interface</a></li>
-      <li><a href="web_client.html">Web Interface</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category">Execution</div></li>
+  <li><div class="sidenav-item"><a href="local_execution.html">Local/Debugging</a></div></li>
+  <li><div class="sidenav-item"><a href="cluster_execution.html">Cluster</a></div></li>
+  <li><div class="sidenav-item"><a href="cli.html">Command-Line Interface</a></div></li>
+  <li><div class="sidenav-item-bottom"><a href="web_client.html">Web Interface</a></div></li>
 
-  <li>Internals
-    <ul>
-      <li><a href="internal_overview.html">Overview</a></li>
-    </ul>
-  </li>
+  <li><div class="sidenav-category">Internals</div></li>
+  <li><div class="sidenav-item"><a href="internal_general_arch.html">Architecture and Process Model</a></div></li>
+  <!-- <li><a href="internal_program_life_cycle.html">From Program to Execution</a></li> -->
+  <li><div class="sidenav-item"><a href="internal_distributed_akka.html">Distributed Communication via Akka</a></div></li>
+  <li><div class="sidenav-item"><a href="internal_job_scheduling.html">Jobs and Scheduling</a></div></li>
+  <!-- <li><a href="#">Types & Serialization</a></li> -->
+  <!-- <li><a href="#">Distributed Runtime & Data Exchange</a></li> -->
+  <!-- <li><a href="#">Runtime Algorithms and Memory Management</a></li> -->
+  <!-- <li><a href=#">Program Optimizer</a></li> -->
+  <li><div class="sidenav-item-bottom"><a href="internal_howto.html">Developer How-To's</a></div></li>
 </ul>

docs/css/main/main.css

@@ -274,6 +274,24 @@ ul {
 	font-size: 0.8em;
 }
 
+/*----------------------------------------------------------------------
+    Side navigation
+----------------------------------------------------------------------*/
+
+.sidenav-category {
+    font-weight: bold;
+    font-size: larger;
+    margin-top: 10px;
+}
+
+.sidenav-item {
+    border-top: thin solid #AAAAAA;
+}
+
+.sidenav-item-bottom {
+    border-bottom: thin solid #AAAAAA; border-top: thin solid #AAAAAA;
+}
+
 /*----------------------------------------------------------------------
     Responsive CSS 768px / 992px / 1200px
 ----------------------------------------------------------------------*/
@@ -386,4 +404,4 @@ ul {
 	.af-main-nav ul li.active a {
        background: #FA5F00;
     }
-}
\ No newline at end of file
+}

docs/internal_add_operator.md

@@ -243,4 +243,9 @@ public <R> DataSet<R> mapPartition(MapPartitionFunction<T, R> function) {
 }
 ~~~
 
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
 

docs/internal_distributed_akka.md

@@ -39,3 +39,9 @@ via Akka (http://akka.io)
 ## Failure Detection
 
 
+
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*

docs/internal_overview.md → docs/internal_howto.md

@@ -20,39 +20,20 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-This documentation provides an overview of the architecture of the Flink system
-and its components. It is intended as guide to contributors, and people
+This documentation provides an overview of the "How-To's" for
+Flink developers. It is intended as guide to contributors and people
 that are interested in the technology behind Flink.
 
-*This documentation is maintained by the contributors of the individual components.
-We kindly ask anyone that adds and changes components to eventually provide a patch
-or pull request that updates these documents as well.*
-
 
 ## Architectures and Components
 
-- [General Architecture and Process Model](internal_general_arch.html)
-
-<!--
-- [Life Cycle of a Program](program_life_cycle.html)
--->
-
-- [Distributed Communication via Akka](internal_distributed_akka.html)
-
-<!--
-- [Jobs and Scheduling](job_scheduling.html)
-
-- [Distributed Runtime](distributed_runtime.html)
-
-- [Runtime Algorithms and Memory Management](operators_and_memory.html)
+- [How-to: Adding a new Operator](internal_add_operator.html)
 
-- [Program Optimizer](optimizer.html)
--->
+- [How-to: Using logging in Flink](internal_logging.html)
 
-- [How-to: Adding a new Operator](internal_add_operator.html)
+---
 
-<!--
-- [Java API, Types, and Type Extraction](types.html)
--->
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
 
-- [How-to: Using logging in Flink](internal_logging.html)

docs/internal_job_scheduling.md

@@ -19,3 +19,64 @@ KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
+
+This document briefly describes how Flink schedules jobs and 
+how it represents and tracks job status on the JobManager.
+
+* This will be replaced by the TOC
+{:toc}
+
+
+## Scheduling
+
+Execution resources in Flink are defined through _Task Slots_. Each TaskManager will have one or more task slots,
+each of which can run one pipeline of parallel tasks. A pipeline consists of multiple successive tasks, such as the
+*n-th* parallel instance of a MapFunction together with the *n-th* parallel instance of a ReduceFunction.
+Note that Flink often executes successive tasks concurrently: For Streaming programs, that happens in any case,
+but also for batch programs, it happens frequently.
+
+The figure below illustrates that. Consider a program with a data source, a *MapFunction*, and a *ReduceFunctoin*.
+The source and MapFunction are executed with a parallelism of 4, while the ReduceFunction is executed with a
+parallism of 3. A pipeline consists of the sequence Source - Map - Reduce. On a cluster with 2 TaskManagers with
+3 slots each, the program will be executed as described below.
+
+<div style="text-align: center;">
+<img src="img/slots.svg" alt="Assigning Pipelines of Tasks to Slots" height="250px" style="text-align: center;"/>
+</div>
+
+Internally, Flink defines through [SlotSharingGroup](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/SlotSharingGroup.java) 
+and [CoLocationGroup](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/CoLocationGroup.java)
+which tasks may share a slot (permissive), respectively which tasks must be strictly placed into the same slot.
+
+
+## JobManager Data Structures
+
+During job execution, the JobManager keeps track of distributed tasks, decides when to schedule the next task (or set of tasks),
+and reacts to finished tasks or execution failures.
+
+The JobManager receives the [JobGraph](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/),
+which is a representation of the data flow consisting of operators ([JobVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java))
+and intermediate results ([IntermediateDataSet](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java)).
+Each operator has properies, like the degree of parallelism and the code that it executes.
+In addition, the JobGraph has a set of attached libraries, that are neccessary to execute the code of the operators.
+
+The JobManager transforms the JobGraph into an [ExecutionGraph](https://github.com/apache/incubator-flink/tree/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph).
+The ExecutionGraph is a parallel version of the JobGraph: For each JobVertex, it contains an [ExecutionVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java) per parallel subtask. An operator with a parallelism of 100 will have one JobVertex and 100 ExecutionVertices.
+The ExecutionVertex tracks the state of execution of a particular subtask. All ExecutionVertices from one JobVertex are held in an
+[ExecutionJobVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java),
+which tracks the status of the operator as a whole. 
+Besides the vertices, the ExecutionGraph also contains the [IntermediateResult](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java) and the [IntermediateResultPartition](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java). The former tracks the state of the *IntermediateDataSet*, the latter the state of each of its partitions.
+
+<div style="text-align: center;">
+<img src="img/job_and_execution_graph.svg" alt="JobGraph and ExecutionGraph" height="400px" style="text-align: center;"/>
+</div>
+
+During its execution, each parallel task goes through multiple stages, from *created* to *finished* or *failed*. The diagram below illustrates the 
+states and possible transitions between them. A task may be executed multiple times (for example in the course of failure recovery).
+For that reason, the execution of an ExecutionVertex is tracked in an [Execution](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java). Each ExecutionVertex has a current Execution, and prior Executions.
+
+<div style="text-align: center;">
+<img src="img/state_machine.svg" alt="States and Transitions of Task Executions" height="300px" style="text-align: center;"/>
+</div>
+
+

docs/internal_logging.md

@@ -86,4 +86,11 @@ Placeholders can also be used in conjunction with exceptions which shall be logg
 catch(Exception exception){
 	LOG.error("An {} occurred.", "error", exception);
 }
-~~~
\ No newline at end of file
+~~~
+
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
+