From 5b390d5467cc040b33e6f17329dc87e0693baf5c Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Fri, 27 Jun 2014 22:25:36 +0200 Subject: [PATCH] Add first files for system internals documentation [ci skip] --- docs/internals/general_arch.md | 8 + docs/internals/overview.md | 39 ++ docs/internals/projects_dependencies.svg | 567 ++++++++++++++++++++++ docs/internals/stack.svg | 587 +++++++++++++++++++++++ 4 files changed, 1201 insertions(+) create mode 100644 docs/internals/general_arch.md create mode 100644 docs/internals/overview.md create mode 100644 docs/internals/projects_dependencies.svg create mode 100644 docs/internals/stack.svg diff --git a/docs/internals/general_arch.md b/docs/internals/general_arch.md new file mode 100644 index 00000000000..a2e5ac75dfe --- /dev/null +++ b/docs/internals/general_arch.md @@ -0,0 +1,8 @@ +--- +title: "General Architecture and Process Model" +--- + +The Flink sub-projects and their dependencies + + +The Flink component stack diff --git a/docs/internals/overview.md b/docs/internals/overview.md new file mode 100644 index 00000000000..4c639b44ea8 --- /dev/null +++ b/docs/internals/overview.md @@ -0,0 +1,39 @@ +--- +title: "Overview of Flink System Architecture & Internals" +--- + +# Overview + +This documentation provides an overview of the architecture of the Flink system +and its components. It is intended as guide to contributors, and people +that are interested in the technology behind Flink. + +*This documentation is maintained by the contributors of the individual components. +We kindly ask anyone that adds and changes components to eventually provide a patch +or pull request that updates these documents as well.* + + +### Architectures and Components + +- [General Architecture and Process Model](general_arch.html) + +- [Life Cycle of a Program](program_life_cycle.html) + +- [Jobs and Scheduling](job_scheduling.html) + +- [Distributed Runtime](distributed_runtime.html) + +- [Runtime Algorithms and Memory Management](distributed_runtime.html) + +- [Program Optimizer](optimizer.html) + +- [How-to: Adding a new Operator](add_operator.html) + + + + + diff --git a/docs/internals/projects_dependencies.svg b/docs/internals/projects_dependencies.svg new file mode 100644 index 00000000000..a9ba5af407a --- /dev/null +++ b/docs/internals/projects_dependencies.svg @@ -0,0 +1,567 @@ + + + + + + image/svg+xml + + + + + + + + + + + flink + - + core + + + flink + - + java + + + flink + - + scala + c + ore types, common API, utils, + core data structures + Scala API + Java API + Old Record API + + + flink + - + runtime + + + flink + - + compiler + + + flink + - + clients + + compiler + / + optimizer + l + ocal/distributed execution, + runtime + web monitor + local/remote execution + CLI + client + compiler web frontend + + + Needed to + write + programs + Needed + in addition for + embedded + and + distributed execution + + + flink + - + java + - + examples + + + flink + - + scala + - + examples + + + flink + - + tests + + + + + + flink + - + test + - + utils + + + Test + environments + + + + integration tests + (entire programs) + + diff --git a/docs/internals/stack.svg b/docs/internals/stack.svg new file mode 100644 index 00000000000..e45227cf42f --- /dev/null +++ b/docs/internals/stack.svg @@ -0,0 +1,587 @@ + + + + + + image/svg+xml + + + + + + + + + + + Java API + + + Flink Runtime + + + Flink Common API / Optimizer + + + Scala + API + + + SQL,Python + + + Storage + + + HDFS + + + Local + Files + + + S3 + + + Cluster + Manager + + + YARN + + + EC2 + + + Direct + + + ... + + + JDBC + JobGraphs + (generic data stream + program) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Java Program + Scala Program + + + + + + + + + + + + + + + Operator DAG + (type agnostic) + + + + -- GitLab