k8s_en.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Kubernetes &mdash; PaddlePaddle  documentation</title>
  

    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="Index"
              href="../../../genindex.html"/>
        <link rel="search" title="Search" href="../../../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../../../index.html"/>
        <link rel="up" title="Use different clusters" href="index_en.html"/>
        <link rel="next" title="Kubernetes on AWS" href="k8s_aws_en.html"/>
        <link rel="prev" title="OpenMPI" href="openmpi_en.html"/> 

  
  <script src="../../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

            <a href="../../../index_en.html" class="icon icon-home"> PaddlePaddle
          

          </a>

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
                <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../getstarted/index_en.html">GET STARTED</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../build_and_install/index_en.html">Install and Build</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../index_en.html">HOW TO</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../cmd_parameter/index_en.html">Set Command-line Parameters</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../index_en.html">Distributed Training</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../preparations_en.html">Preparations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cmd_argument_en.html">Command-line arguments</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="index_en.html">Use different clusters</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="fabric_en.html">Fabric</a></li>
<li class="toctree-l4"><a class="reference internal" href="openmpi_en.html">OpenMPI</a></li>
<li class="toctree-l4 current"><a class="current reference internal" href="#">Kubernetes</a></li>
<li class="toctree-l4"><a class="reference internal" href="k8s_aws_en.html">Kubernetes on AWS</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../rnn/index_en.html">RNN Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../optimization/gpu_profiling_en.html">Tune GPU Performance</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index_en.html">Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq/index_en.html">FAQ</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="../../../index_en.html">PaddlePaddle</a>
      </nav>


      <div class="wy-nav-content">
        <div class="rst-content">
          

<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="../../../index_en.html">Docs</a> &raquo;</li>
      
          <li><a href="../../index_en.html">HOW TO</a> &raquo;</li>
      
          <li><a href="../index_en.html">Distributed Training</a> &raquo;</li>
      
          <li><a href="index_en.html">Use different clusters</a> &raquo;</li>
      
    <li>Kubernetes</li>
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../../../_sources/howto/cluster/multi_cluster/k8s_en.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="kubernetes">
<span id="kubernetes"></span><h1>Kubernetes<a class="headerlink" href="#kubernetes" title="Permalink to this headline">¶</a></h1>
<p>In this article, we will introduce how to run PaddlePaddle training job on single CPU machine using Kubernetes. In next article, we will introduce how to run PaddlePaddle training job on distributed cluster.</p>
<div class="section" id="build-docker-image">
<span id="build-docker-image"></span><h2>Build Docker Image<a class="headerlink" href="#build-docker-image" title="Permalink to this headline">¶</a></h2>
<p>In distributed Kubernetes cluster, we will use Ceph or other distributed
storage system for storing training related data so that all processes in
PaddlePaddle training can retrieve data from Ceph. In this example, we will
only demo training job on single machine. In order to simplify the requirement
of the environment, we will directly put training data into the PaddlePaddle Docker Image,
so we need to create a PaddlePaddle Docker image that includes the training data.</p>
<p>The production Docker Image <code class="docutils literal"><span class="pre">paddlepaddle/paddle:cpu-demo-latest</span></code> has the PaddlePaddle
source code and demo. (Caution: Default PaddlePaddle Docker Image <code class="docutils literal"><span class="pre">paddlepaddle/paddle:latest</span></code> doesn&#8217;t include
the source code, PaddlePaddle&#8217;s different versions of Docker Image can be referred here:
<a class="reference external" href="http://paddlepaddle.org/docs/develop/documentation/zh/getstarted/build_and_install/docker_install_en.html">Docker Installation Guide</a>),
so we run this Docker Image and download the training data, and then commit the whole
Container to be a new Docker Image.</p>
<div class="section" id="run-docker-container">
<span id="run-docker-container"></span><h3>Run Docker Container<a class="headerlink" href="#run-docker-container" title="Permalink to this headline">¶</a></h3>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ docker run --name quick_start_data -it paddlepaddle/paddle:cpu-demo-latest
</pre></div>
</div>
</div>
<div class="section" id="download-training-data">
<span id="download-training-data"></span><h3>Download Training Data<a class="headerlink" href="#download-training-data" title="Permalink to this headline">¶</a></h3>
<p>Getting into <code class="docutils literal"><span class="pre">/root/paddle/demo/quick_start/data</span></code> Directory，using <code class="docutils literal"><span class="pre">get_data.sh</span></code> to download training data.
Then getting into <code class="docutils literal"><span class="pre">/root/paddle/demo/quick_start</span></code> Directory, using <code class="docutils literal"><span class="pre">preprocess.sh</span></code> to pre-process training data.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ root@fbd1f2bb71f4:~/paddle/demo/quick_start/data# ./get_data.sh

Downloading Amazon Electronics reviews data...
--2016-10-31 01:33:43--  http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_Electronics_5.json.gz
Resolving snap.stanford.edu (snap.stanford.edu)... 171.64.75.80
Connecting to snap.stanford.edu (snap.stanford.edu)|171.64.75.80|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 495854086 (473M) [application/x-gzip]
Saving to: &#39;reviews_Electronics_5.json.gz&#39;

 10% [=======&gt;                                         ] 874,279     64.7KB/s  eta 2h 13m
</pre></div>
</div>
</div>
<div class="section" id="modify-startup-script">
<span id="modify-startup-script"></span><h3>Modify Startup Script<a class="headerlink" href="#modify-startup-script" title="Permalink to this headline">¶</a></h3>
<p>After downloading the data，modify <code class="docutils literal"><span class="pre">/root/paddle/demo/quick_start/train.sh</span></code> file contents are as follows (one more cd cmd):</p>
<div class="highlight-default"><div class="highlight"><pre><span></span>set -e
cd /root/paddle/demo/quick_start
cfg=trainer_config.lr.py
#cfg=trainer_config.emb.py
#cfg=trainer_config.cnn.py
#cfg=trainer_config.lstm.py
#cfg=trainer_config.bidi-lstm.py
#cfg=trainer_config.db-lstm.py
paddle train \
  --config=$cfg \
  --save_dir=./output \
  --trainer_count=4 \
  --log_period=20 \
  --num_passes=15 \
  --use_gpu=false \
  --show_parameter_stats_period=100 \
  --test_all_data_in_one_period=1 \
  2&gt;&amp;1 | tee &#39;train.log&#39;
</pre></div>
</div>
</div>
<div class="section" id="commit-docker-image">
<span id="commit-docker-image"></span><h3>Commit Docker Image<a class="headerlink" href="#commit-docker-image" title="Permalink to this headline">¶</a></h3>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ docker commit quick_start_data mypaddle/paddle:quickstart
</pre></div>
</div>
</div>
</div>
<div class="section" id="use-kubernetes-for-training">
<span id="use-kubernetes-for-training"></span><h2>Use Kubernetes For Training<a class="headerlink" href="#use-kubernetes-for-training" title="Permalink to this headline">¶</a></h2>
<p>We will use Kubernetes job for training process, following steps shows how to do the training with Kubernetes.</p>
<div class="section" id="create-yaml-files">
<span id="create-yaml-files"></span><h3>Create Yaml Files<a class="headerlink" href="#create-yaml-files" title="Permalink to this headline">¶</a></h3>
<p>The output result in container will be demolished when job finished (container stopped running), so we need to mount the volume out to the local disk when creating the container to store the training result. Using our previously created image, we can create a <a class="reference external" href="http://kubernetes.io/docs/user-guide/jobs/#what-is-a-job">Kubernetes Job</a>, the yaml contents are as follows:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">apiVersion</span><span class="p">:</span> <span class="n">batch</span><span class="o">/</span><span class="n">v1</span>
<span class="n">kind</span><span class="p">:</span> <span class="n">Job</span>
<span class="n">metadata</span><span class="p">:</span>
  <span class="n">name</span><span class="p">:</span> <span class="n">quickstart</span>
<span class="n">spec</span><span class="p">:</span>
  <span class="n">parallelism</span><span class="p">:</span> <span class="mi">1</span>
  <span class="n">completions</span><span class="p">:</span> <span class="mi">1</span>
  <span class="n">template</span><span class="p">:</span>
    <span class="n">metadata</span><span class="p">:</span>
      <span class="n">name</span><span class="p">:</span> <span class="n">quickstart</span>
    <span class="n">spec</span><span class="p">:</span>
      <span class="n">volumes</span><span class="p">:</span>
      <span class="o">-</span> <span class="n">name</span><span class="p">:</span> <span class="n">output</span>
        <span class="n">hostPath</span><span class="p">:</span> 
          <span class="n">path</span><span class="p">:</span> <span class="o">/</span><span class="n">home</span><span class="o">/</span><span class="n">work</span><span class="o">/</span><span class="n">paddle_output</span>     
      <span class="n">containers</span><span class="p">:</span>
      <span class="o">-</span> <span class="n">name</span><span class="p">:</span> <span class="n">pi</span>
        <span class="n">image</span><span class="p">:</span> <span class="n">mypaddle</span><span class="o">/</span><span class="n">paddle</span><span class="p">:</span><span class="n">quickstart</span>
        <span class="n">command</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;bin/bash&quot;</span><span class="p">,</span>  <span class="s2">&quot;-c&quot;</span><span class="p">,</span> <span class="s2">&quot;/root/paddle/demo/quick_start/train.sh&quot;</span><span class="p">]</span>
        <span class="n">volumeMounts</span><span class="p">:</span>
        <span class="o">-</span> <span class="n">name</span><span class="p">:</span> <span class="n">output</span>
          <span class="n">mountPath</span><span class="p">:</span> <span class="o">/</span><span class="n">root</span><span class="o">/</span><span class="n">paddle</span><span class="o">/</span><span class="n">demo</span><span class="o">/</span><span class="n">quick_start</span><span class="o">/</span><span class="n">output</span>
      <span class="n">restartPolicy</span><span class="p">:</span> <span class="n">Never</span>
</pre></div>
</div>
</div>
<div class="section" id="start-paddlepaddle-job">
<span id="start-paddlepaddle-job"></span><h3>Start PaddlePaddle Job<a class="headerlink" href="#start-paddlepaddle-job" title="Permalink to this headline">¶</a></h3>
<p>Using the above yaml file to start the Kubernetes job.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ kubectl  create -f paddle.yaml
</pre></div>
</div>
<p>Get the detailed status of the job:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ kubectl  get job
NAME         DESIRED   SUCCESSFUL   AGE
quickstart   1         0            58s

$ kubectl  describe job quickstart
Name:       quickstart
Namespace:  default
Image(s):   registry.baidu.com/public/paddle:cpu-demo-latest
Selector:   controller-uid=f120da72-9f18-11e6-b363-448a5b355b84
Parallelism:    1
Completions:    1
Start Time: Mon, 31 Oct 2016 11:20:16 +0800
Labels:     controller-uid=f120da72-9f18-11e6-b363-448a5b355b84,job-name=quickstart
Pods Statuses:  0 Running / 1 Succeeded / 0 Failed
Volumes:
  output:
    Type:   HostPath (bare host directory volume)
    Path:   /home/work/paddle_output
Events:
  FirstSeen LastSeen    Count   From            SubobjectPath   Type        Reason          Message
  --------- --------    -----   ----            -------------   --------    ------          -------
  1m        1m      1   {job-controller }           Normal      SuccessfulCreate    Created pod: quickstart-fa0wx
</pre></div>
</div>
</div>
<div class="section" id="get-training-result">
<span id="get-training-result"></span><h3>Get Training Result<a class="headerlink" href="#get-training-result" title="Permalink to this headline">¶</a></h3>
<p>We can use kubectl command to take a look at the status of related pod.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span>$ kubectl  describe pod quickstart-fa0wx
Name:       quickstart-fa0wx
Namespace:  default
Node:       paddle-demo-let02/10.206.202.44
Start Time: Mon, 31 Oct 2016 11:20:17 +0800
Labels:     controller-uid=f120da72-9f18-11e6-b363-448a5b355b84,job-name=quickstart
Status:     Succeeded
IP:     10.0.0.9
Controllers:    Job/quickstart
Containers:
  quickstart:
    Container ID:   docker://b8561f5c79193550d64fa47418a9e67ebdd71546186e840f88de5026b8097465
    Image:      registry.baidu.com/public/paddle:cpu-demo-latest
    Image ID:       docker://18e457ce3d362ff5f3febf8e7f85ffec852f70f3b629add10aed84f930a68750
    Port:
    Command:
      bin/bash
      -c
      /root/paddle/demo/quick_start/train.sh
    QoS Tier:
      cpu:      BestEffort
      memory:       BestEffort
    State:      Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 31 Oct 2016 11:20:20 +0800
      Finished:     Mon, 31 Oct 2016 11:21:46 +0800
    Ready:      False
    Restart Count:  0
    Environment Variables:
Conditions:
  Type      Status
  Ready     False
Volumes:
  output:
    Type:   HostPath (bare host directory volume)
    Path:   /home/work/paddle_output
</pre></div>
</div>
<p>We can also ssh to Kubernetes node to take a look at the training result.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@paddle</span><span class="o">-</span><span class="n">demo</span><span class="o">-</span><span class="n">let02</span> <span class="n">paddle_output</span><span class="p">]</span><span class="c1"># ll</span>
<span class="n">total</span> <span class="mi">60</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">20</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00000</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">20</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00001</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00002</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00003</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00004</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00005</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00006</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00007</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00008</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00009</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00010</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00011</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00012</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00013</span>
<span class="n">drwxr</span><span class="o">-</span><span class="n">xr</span><span class="o">-</span><span class="n">x</span> <span class="mi">2</span> <span class="n">root</span> <span class="n">root</span> <span class="mi">4096</span> <span class="n">Oct</span> <span class="mi">31</span> <span class="mi">11</span><span class="p">:</span><span class="mi">21</span> <span class="k">pass</span><span class="o">-</span><span class="mi">00014</span>
</pre></div>
</div>
</div>
</div>
</div>


           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="k8s_aws_en.html" class="btn btn-neutral float-right" title="Kubernetes on AWS" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="openmpi_en.html" class="btn btn-neutral" title="OpenMPI" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="../../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>

  
    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>