diff --git a/docs/_includes/sidenav.html b/docs/_includes/sidenav.html
index fdddabdfe1565a6685b9fc34f8630bf1b456b2f0..8a6585ff68a78ea3609fc903847dc97cc130b6db 100644
--- a/docs/_includes/sidenav.html
+++ b/docs/_includes/sidenav.html
@@ -17,57 +17,47 @@ specific language governing permissions and limitations
under the License.
-->
diff --git a/docs/css/main/main.css b/docs/css/main/main.css
index f30e402e96dce49fc471111beb32fc7f25ebe76f..5c744757471313d37743a7ee2d55e56b0a4a8a67 100755
--- a/docs/css/main/main.css
+++ b/docs/css/main/main.css
@@ -274,6 +274,24 @@ ul {
font-size: 0.8em;
}
+/*----------------------------------------------------------------------
+ Side navigation
+----------------------------------------------------------------------*/
+
+.sidenav-category {
+ font-weight: bold;
+ font-size: larger;
+ margin-top: 10px;
+}
+
+.sidenav-item {
+ border-top: thin solid #AAAAAA;
+}
+
+.sidenav-item-bottom {
+ border-bottom: thin solid #AAAAAA; border-top: thin solid #AAAAAA;
+}
+
/*----------------------------------------------------------------------
Responsive CSS 768px / 992px / 1200px
----------------------------------------------------------------------*/
@@ -386,4 +404,4 @@ ul {
.af-main-nav ul li.active a {
background: #FA5F00;
}
-}
\ No newline at end of file
+}
diff --git a/docs/img/ClientJmTm.svg b/docs/img/ClientJmTm.svg
old mode 100644
new mode 100755
index 99bd209018cd49f7e19227d3da59adef89c49c42..b158b7d838891b175197a9ef6c945a56c8657317
--- a/docs/img/ClientJmTm.svg
+++ b/docs/img/ClientJmTm.svg
@@ -18,6 +18,8 @@ specific language governing permissions and limitations
under the License.
-->
+
+
diff --git a/docs/img/apache-incubator-logo.png b/docs/img/apache-incubator-logo.png
deleted file mode 100644
index 81fb31ec7128939673f04d4748788ac4989c0d17..0000000000000000000000000000000000000000
Binary files a/docs/img/apache-incubator-logo.png and /dev/null differ
diff --git a/docs/img/job_and_execution_graph.svg b/docs/img/job_and_execution_graph.svg
new file mode 100755
index 0000000000000000000000000000000000000000..2f90ea1698b2dba3ef776846f3a637070b1fa4ad
--- /dev/null
+++ b/docs/img/job_and_execution_graph.svg
@@ -0,0 +1,851 @@
+
+
+
+
+
+
diff --git a/docs/img/jobgraph_executiongraph.svg b/docs/img/jobgraph_executiongraph.svg
deleted file mode 100644
index e95126fafcfb1da2149b2c47121e2ad71d7a5028..0000000000000000000000000000000000000000
--- a/docs/img/jobgraph_executiongraph.svg
+++ /dev/null
@@ -1,472 +0,0 @@
-
-
-
-
diff --git a/docs/img/slots.svg b/docs/img/slots.svg
new file mode 100755
index 0000000000000000000000000000000000000000..7d5dc2a3a726966078bfd3467926993ebd73b93e
--- /dev/null
+++ b/docs/img/slots.svg
@@ -0,0 +1,505 @@
+
+
+
+
+
+
diff --git a/docs/img/state_machine.svg b/docs/img/state_machine.svg
old mode 100644
new mode 100755
index 9bb711e9f573c9819ea61ab41daf6c3c748e71e7..8d0f57037965659559f2e767a8f08f95c363d29b
--- a/docs/img/state_machine.svg
+++ b/docs/img/state_machine.svg
@@ -26,12 +26,39 @@ under the License.
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
- version="1.0"
- width="230.41mm"
- height="154.87mm"
- id="svg2985">
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+ width="664.92505"
+ height="445.67966"
+ id="svg2"
+ version="1.1"
+ inkscape:version="0.48.5 r10040">
+
+
+ id="metadata7">
@@ -42,213 +69,250 @@ under the License.
-
-
-
- CREATED
-
-
- SCHE
- -
- DULED
-
-
- CANCEL
- -
- ED
-
-
- FAILED
-
-
-
-
- DEPLOY
- -
- ING
-
-
- FINISHED
-
-
- RUNNING
-
-
-
-
-
-
- CANCEL
- -
- ING
-
-
-
-
-
+ inkscape:label="Layer 1"
+ inkscape:groupmode="layer"
+ id="layer1"
+ transform="translate(-42.537478,-309.52235)">
+
+
+
+ CREATED
+
+
+ SCHE
+ -
+ DULED
+
+
+ CANCEL
+ -
+ ED
+
+
+ FAILED
+
+
+
+
+ DEPLOY
+ -
+ ING
+
+
+ FINISHED
+
+
+ RUNNING
+
+
+
+
+
+
+ CANCEL
+ -
+ ING
+
+
+
+
+
+
+
diff --git a/docs/internal_add_operator.md b/docs/internal_add_operator.md
index e370c1482778dfc93dcc2015af31c609c5cad747..ddcb6b5c8d0accb90886e65e4095f596fe60bf2d 100644
--- a/docs/internal_add_operator.md
+++ b/docs/internal_add_operator.md
@@ -243,4 +243,9 @@ public DataSet mapPartition(MapPartitionFunction function) {
}
~~~
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
diff --git a/docs/internal_distributed_akka.md b/docs/internal_distributed_akka.md
index f0bf8e0c2d9d44124b28f9f1861c5a70b19b0aca..49497b25a4b65ecac20987e7e66e6c830bba76f7 100644
--- a/docs/internal_distributed_akka.md
+++ b/docs/internal_distributed_akka.md
@@ -39,3 +39,9 @@ via Akka (http://akka.io)
## Failure Detection
+
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
diff --git a/docs/internal_overview.md b/docs/internal_howto.md
similarity index 66%
rename from docs/internal_overview.md
rename to docs/internal_howto.md
index 88fe6f6b8abbf0f9e549dd128291af78860c2bb3..415bda7260447aeff2dd1fe24d3e8b4bd9bb0c24 100644
--- a/docs/internal_overview.md
+++ b/docs/internal_howto.md
@@ -20,39 +20,20 @@ specific language governing permissions and limitations
under the License.
-->
-This documentation provides an overview of the architecture of the Flink system
-and its components. It is intended as guide to contributors, and people
+This documentation provides an overview of the "How-To's" for
+Flink developers. It is intended as guide to contributors and people
that are interested in the technology behind Flink.
-*This documentation is maintained by the contributors of the individual components.
-We kindly ask anyone that adds and changes components to eventually provide a patch
-or pull request that updates these documents as well.*
-
## Architectures and Components
-- [General Architecture and Process Model](internal_general_arch.html)
-
-
-
-- [Distributed Communication via Akka](internal_distributed_akka.html)
-
-
+- [How-to: Using logging in Flink](internal_logging.html)
-- [How-to: Adding a new Operator](internal_add_operator.html)
+---
-
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
-- [How-to: Using logging in Flink](internal_logging.html)
diff --git a/docs/internal_job_scheduling.md b/docs/internal_job_scheduling.md
index 1237c85911b4785940a2a074b5b4c46f362e5718..7fb450014231ee06242642c0bdd9ece05e0defac 100644
--- a/docs/internal_job_scheduling.md
+++ b/docs/internal_job_scheduling.md
@@ -19,3 +19,64 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
+
+This document briefly describes how Flink schedules jobs and
+how it represents and tracks job status on the JobManager.
+
+* This will be replaced by the TOC
+{:toc}
+
+
+## Scheduling
+
+Execution resources in Flink are defined through _Task Slots_. Each TaskManager will have one or more task slots,
+each of which can run one pipeline of parallel tasks. A pipeline consists of multiple successive tasks, such as the
+*n-th* parallel instance of a MapFunction together with the *n-th* parallel instance of a ReduceFunction.
+Note that Flink often executes successive tasks concurrently: For Streaming programs, that happens in any case,
+but also for batch programs, it happens frequently.
+
+The figure below illustrates that. Consider a program with a data source, a *MapFunction*, and a *ReduceFunctoin*.
+The source and MapFunction are executed with a parallelism of 4, while the ReduceFunction is executed with a
+parallism of 3. A pipeline consists of the sequence Source - Map - Reduce. On a cluster with 2 TaskManagers with
+3 slots each, the program will be executed as described below.
+
+
+
+
+
+Internally, Flink defines through [SlotSharingGroup](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/SlotSharingGroup.java)
+and [CoLocationGroup](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/CoLocationGroup.java)
+which tasks may share a slot (permissive), respectively which tasks must be strictly placed into the same slot.
+
+
+## JobManager Data Structures
+
+During job execution, the JobManager keeps track of distributed tasks, decides when to schedule the next task (or set of tasks),
+and reacts to finished tasks or execution failures.
+
+The JobManager receives the [JobGraph](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/),
+which is a representation of the data flow consisting of operators ([JobVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java))
+and intermediate results ([IntermediateDataSet](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java)).
+Each operator has properies, like the degree of parallelism and the code that it executes.
+In addition, the JobGraph has a set of attached libraries, that are neccessary to execute the code of the operators.
+
+The JobManager transforms the JobGraph into an [ExecutionGraph](https://github.com/apache/incubator-flink/tree/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph).
+The ExecutionGraph is a parallel version of the JobGraph: For each JobVertex, it contains an [ExecutionVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java) per parallel subtask. An operator with a parallelism of 100 will have one JobVertex and 100 ExecutionVertices.
+The ExecutionVertex tracks the state of execution of a particular subtask. All ExecutionVertices from one JobVertex are held in an
+[ExecutionJobVertex](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java),
+which tracks the status of the operator as a whole.
+Besides the vertices, the ExecutionGraph also contains the [IntermediateResult](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java) and the [IntermediateResultPartition](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java). The former tracks the state of the *IntermediateDataSet*, the latter the state of each of its partitions.
+
+
+
+
+
+During its execution, each parallel task goes through multiple stages, from *created* to *finished* or *failed*. The diagram below illustrates the
+states and possible transitions between them. A task may be executed multiple times (for example in the course of failure recovery).
+For that reason, the execution of an ExecutionVertex is tracked in an [Execution](https://github.com/apache/incubator-flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java). Each ExecutionVertex has a current Execution, and prior Executions.
+
+
+
+
+
+
diff --git a/docs/internal_logging.md b/docs/internal_logging.md
index 79e3656149e5a603af3dd5ffd6903215d6a6192a..42fd3171f42bf329f4dbcd278d428856aed2b75d 100644
--- a/docs/internal_logging.md
+++ b/docs/internal_logging.md
@@ -86,4 +86,11 @@ Placeholders can also be used in conjunction with exceptions which shall be logg
catch(Exception exception){
LOG.error("An {} occurred.", "error", exception);
}
-~~~
\ No newline at end of file
+~~~
+
+---
+
+*This documentation is maintained by the contributors of the individual components.
+We kindly ask anyone that adds and changes components to eventually provide a patch
+or pull request that updates these documents as well.*
+
+
+
+