Skip to content

P
Paddle

机器未来 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
切换分支/标签
查找文件 普通视图历史永久链接Permalink
register_grad_op.html 20.9 KB
Newer Older
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Design Doc: Gradient Operators Registration &mdash; PaddlePaddle  documentation</title>
  

  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

  
  
        <link rel="index" title="Index"
              href="../genindex.html"/>
        <link rel="search" title="Search" href="../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../index.html"/> 

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/override.css" type="text/css" />
  <script>
  var _hmt = _hmt || [];
  (function() {
    var hm = document.createElement("script");
    hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
    var s = document.getElementsByTagName("script")[0]; 
    s.parentNode.insertBefore(hm, s);
  })();
  </script>

  

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  
  <header class="site-header">
    <div class="site-logo">
      <a href="/"><img src="../_static/images/PP_w.png"></a>
    </div>
    <div class="site-nav-links">
      <div class="site-menu">
        <a class="fork-on-github" href="https://github.com/PaddlePaddle/Paddle" target="_blank"><i class="fa fa-github"></i>Fork me on Github</a>
        <div class="language-switcher dropdown">
          <a type="button" data-toggle="dropdown">
            <span>English</span>
            <i class="fa fa-angle-up"></i>
            <i class="fa fa-angle-down"></i>
          </a>
          <ul class="dropdown-menu">
            <li><a href="/doc_cn">中文</a></li>
            <li><a href="/doc">English</a></li>
          </ul>
        </div>
        <ul class="site-page-links">
          <li><a href="/">Home</a></li>
        </ul>
      </div>
      <div class="doc-module">
        
        <ul>
<li class="toctree-l1"><a class="reference internal" href="../getstarted/index_en.html">GET STARTED</a></li>
<li class="toctree-l1"><a class="reference internal" href="../howto/index_en.html">HOW TO</a></li>
<li class="toctree-l1"><a class="reference internal" href="../api/index_en.html">API</a></li>
</ul>

        
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>        
      </div>
    </div>
  </header>
  
  <div class="main-content-wrap">

    
    <nav class="doc-menu-vertical" role="navigation">
        
          
          <ul>
<li class="toctree-l1"><a class="reference internal" href="../getstarted/index_en.html">GET STARTED</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getstarted/build_and_install/index_en.html">Install and Build</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getstarted/build_and_install/docker_install_en.html">PaddlePaddle in Docker Containers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getstarted/build_and_install/build_from_source_en.html">Installing from Sources</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../howto/index_en.html">HOW TO</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../howto/usage/cmd_parameter/index_en.html">Set Command-line Parameters</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/usage/cmd_parameter/use_case_en.html">Use Case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/usage/cmd_parameter/arguments_en.html">Argument Outline</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/usage/cmd_parameter/detail_introduction_en.html">Detail Description</a></li>
</ul>
</li>
T
Deploy to GitHub Pages: 63ffe5250a120ff430469b8d000deb2b031c4881  
Travis CI 已提交
123 
<li class="toctree-l2"><a class="reference internal" href="../howto/usage/cluster/cluster_train_en.html">PaddlePaddle Distributed Training</a></li>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
124 125 126 127 
<li class="toctree-l2"><a class="reference internal" href="../howto/usage/k8s/k8s_en.html">Paddle On Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../howto/usage/k8s/k8s_aws_en.html">Distributed PaddlePaddle Training on AWS with Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../howto/dev/build_en.html">Build PaddlePaddle from Source Code and Run Unit Test</a></li>
<li class="toctree-l2"><a class="reference internal" href="../howto/dev/new_layer_en.html">Write New Layers</a></li>
T
Deploy to GitHub Pages: f122a5da2f27038b48f6ed607e296d762050e920  
Travis CI 已提交
128 
<li class="toctree-l2"><a class="reference internal" href="../howto/dev/contribute_to_paddle_en.html">Contribute Code</a></li>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 
<li class="toctree-l2"><a class="reference internal" href="../howto/deep_model/rnn/index_en.html">RNN Models</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/deep_model/rnn/rnn_config_en.html">RNN Configuration</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/optimization/gpu_profiling_en.html">Tune GPU Performance</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../api/index_en.html">API</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../api/v2/model_configs.html">Model Configuration</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/activation.html">Activation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/layer.html">Layers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/evaluators.html">Evaluators</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/optimizer.html">Optimizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/pooling.html">Pooling</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/networks.html">Networks</a></li>
<li class="toctree-l3"><a class="reference internal" href="../api/v2/config/attr.html">Parameter Attribute</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../api/v2/data.html">Data Reader Interface and DataSets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/v2/run_logic.html">Training and Inference</a></li>
</ul>
</li>
</ul>

        
    </nav>
    
    <section class="doc-content-wrap">

      

 







<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
      
    <li>Design Doc: Gradient Operators Registration</li>
  </ul>
</div>
      
      <div class="wy-nav-content" id="doc-content">
        <div class="rst-content">
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="design-doc-gradient-operators-registration">
<span id="design-doc-gradient-operators-registration"></span><h1>Design Doc: Gradient Operators Registration<a class="headerlink" href="#design-doc-gradient-operators-registration" title="Permalink to this headline"></a></h1>
<div class="section" id="the-problem-posed">
<span id="the-problem-posed"></span><h2>The Problem Posed<a class="headerlink" href="#the-problem-posed" title="Permalink to this headline"></a></h2>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
184 185 
<p>Currently, for each C++ operator class definition, a <em>gradient operator creator</em> function is registered, which takes as input a C++ operator instance and returns the corresponding gradient operator instance.</p>
<p>However, we noticed two problems with the current design:</p>
T
Deploy to GitHub Pages: 9a6dffd4d918221e324cb3c6f5abaf3a0adca737  
Travis CI 已提交
186 
<ol class="simple">
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
187 188 
<li>As we decided to separate the <em>compilation</em> and the <em>execution</em> phases, we need to change the creator to take an <code class="docutils literal"><span class="pre">OpDesc</span></code> protobuf message in a <code class="docutils literal"><span class="pre">ProgramDesc</span></code> and inserts corresponding <code class="docutils literal"><span class="pre">OpDesc</span></code> messages into the <code class="docutils literal"><span class="pre">ProgramDesc</span></code> message.</li>
<li>For some operators, the gradient computation can be written in terms of existing operators.  For example, the gradient of <em>minus</em> operator consists of two operators &#8211; an <em>identity</em> operator followed by a <em>scale</em> operator.  Hence the registration mechanism needs to support mapping from an operator to a set of operators for the gradient computation.</li>
T
Deploy to GitHub Pages: 9a6dffd4d918221e324cb3c6f5abaf3a0adca737  
Travis CI 已提交
189 
</ol>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
190 
</div>
T
Deploy to GitHub Pages: 9a6dffd4d918221e324cb3c6f5abaf3a0adca737  
Travis CI 已提交
191 192 
<div class="section" id="the-current-implementation">
<span id="the-current-implementation"></span><h2>The Current Implementation<a class="headerlink" href="#the-current-implementation" title="Permalink to this headline"></a></h2>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
193 
<p>Instances of the C++ class <code class="docutils literal"><span class="pre">OpInfo</span></code> are stored an associative map whose key is the operator type. The <code class="docutils literal"><span class="pre">grad_op_type</span></code> indicates the associated gradient operator type. An operator can create the gradient operator by invoking <code class="docutils literal"><span class="pre">OpInfo::creator_</span></code> of the gradient operator. The pseudo code is as follows</p>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">struct</span> <span class="n">OpInfo</span> <span class="p">{</span>
  <span class="n">std</span><span class="o">::</span><span class="n">function</span><span class="o">&lt;</span><span class="n">OperatorBase</span><span class="o">*</span><span class="p">(...)</span><span class="o">&gt;</span> <span class="n">creator_</span><span class="p">;</span>
  <span class="n">std</span><span class="o">::</span><span class="n">string</span> <span class="n">grad_op_type_</span><span class="p">;</span>
  <span class="p">...</span>
<span class="p">};</span>

<span class="n">map</span><span class="o">&lt;</span><span class="n">string</span><span class="p">,</span> <span class="n">OpInfo</span><span class="o">&gt;</span> <span class="n">OpInfoMap</span><span class="p">;</span>

<span class="n">OperatorBase</span><span class="o">*</span> <span class="nf">CreateGradientOperator</span><span class="p">(</span><span class="k">const</span> <span class="n">OperatorBase</span><span class="o">&amp;</span> <span class="n">op</span><span class="p">)</span> <span class="p">{</span>
  <span class="k">return</span> <span class="n">OpInfoMap</span><span class="p">.</span><span class="n">at</span><span class="p">(</span><span class="n">op</span><span class="p">.</span><span class="n">Type</span><span class="p">()).</span><span class="n">creator_</span><span class="p">(...);</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="proposed-solution">
<span id="proposed-solution"></span><h2>Proposed Solution<a class="headerlink" href="#proposed-solution" title="Permalink to this headline"></a></h2>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
210 
<p>The mapping relationship between an operator and its gradient operators is a function. The interface of this function is:</p>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
211 
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="c1">// (OpDesc) --&gt; vector&lt;OpDesc&gt;</span>
T
Deploy to GitHub Pages: b884bc33c939f811d1ceb2fa7d360cf9d7c9850c  
Travis CI 已提交
212 
<span class="n">std</span><span class="o">::</span><span class="n">function</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">OpDescBind</span><span class="o">&gt;</span><span class="p">(</span><span class="k">const</span> <span class="n">OpDescBind</span><span class="o">&amp;</span><span class="p">)</span><span class="o">&gt;</span><span class="p">;</span>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
213 214 
</pre></div>
</div>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
215 216 
<p>The function takes an <code class="docutils literal"><span class="pre">OpDescBind</span></code> of the forward operator and returns one or many gradient operator descriptions. <code class="docutils literal"><span class="pre">OpDescBind</span></code> is a C++ wrapper for  the protobuf message <code class="docutils literal"><span class="pre">OpDesc</span></code> for rapid manipulation of <code class="docutils literal"><span class="pre">OpDesc</span></code>.</p>
<p>The <code class="docutils literal"><span class="pre">GradOpDescMaker</span></code> will be registered in <code class="docutils literal"><span class="pre">OpInfo</span></code> and will replace the <code class="docutils literal"><span class="pre">grad_op_type_</span></code> field. The <code class="docutils literal"><span class="pre">OpInfo</span></code> should look like</p>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
217 
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">struct</span> <span class="n">OpInfo</span> <span class="p">{</span>
T
Deploy to GitHub Pages: b884bc33c939f811d1ceb2fa7d360cf9d7c9850c  
Travis CI 已提交
218 
  <span class="n">std</span><span class="o">::</span><span class="n">function</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">unique_ptr</span><span class="o">&lt;</span><span class="n">OpDescBind</span><span class="o">&gt;&gt;</span><span class="p">(</span><span class="k">const</span> <span class="n">OpDescBind</span><span class="o">&amp;</span><span class="p">)</span><span class="o">&gt;</span>  <span class="n">grad_op_maker_</span><span class="p">;</span>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
219 220 221 222 
  <span class="p">...</span>
<span class="p">};</span>
</pre></div>
</div>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
223 
<p>The <code class="docutils literal"><span class="pre">grad_op_maker_</span></code> is a <code class="docutils literal"><span class="pre">nullptr</span></code> if the operator does not have any associated gradient operators.</p>
T
Deploy to GitHub Pages: b884bc33c939f811d1ceb2fa7d360cf9d7c9850c  
Travis CI 已提交
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 
<p>We propose a base class called <code class="docutils literal"><span class="pre">GradOpDescMakerBase</span></code> to let operator developers generate <code class="docutils literal"><span class="pre">Gradient</span> <span class="pre">Operators</span></code> easily. The public interface of that class is</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">GradOpDescMakerBase</span> <span class="p">{</span>
<span class="k">public</span><span class="o">:</span>
  <span class="n">GradOpDescMakerBase</span><span class="p">(</span><span class="k">const</span> <span class="n">OpDescBind</span><span class="o">&amp;</span> <span class="p">);</span>
  <span class="k">virtual</span> <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">unique_ptr</span><span class="o">&lt;</span><span class="n">OpDescBind</span><span class="o">&gt;&gt;</span> <span class="k">operator</span><span class="p">()()</span><span class="k">const</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
<span class="p">};</span>
</pre></div>
</div>
<p>We can convert <code class="docutils literal"><span class="pre">GradOpDescMakerBase</span></code> to <code class="docutils literal"><span class="pre">std::function&lt;std::vector&lt;std::unique_ptr&lt;OpDescBind&gt;&gt;(const</span> <span class="pre">OpDescBind&amp;)&gt;</span></code> by</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">using</span> <span class="n">GradOpMaker</span> <span class="o">=</span> <span class="p">...;</span>
<span class="n">std</span><span class="o">::</span><span class="n">function</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">OpDescBind</span><span class="o">&gt;</span><span class="p">(</span><span class="k">const</span> <span class="n">OpDescBind</span><span class="o">&amp;</span><span class="p">)</span><span class="o">&gt;</span> <span class="n">func</span><span class="p">;</span>
<span class="n">func</span> <span class="o">=</span> <span class="p">[]</span> <span class="p">(</span><span class="k">const</span> <span class="n">OpDescBind</span><span class="o">&amp;</span> <span class="n">fwd_op</span><span class="p">)</span> <span class="p">{</span>
  <span class="n">GradOpMaker</span> <span class="n">maker</span><span class="p">(</span><span class="n">fwd_op</span><span class="p">);</span>
  <span class="k">return</span> <span class="nf">maker</span><span class="p">();</span>
<span class="p">};</span>
</pre></div>
</div>
<p>We can write many helper functions since the <code class="docutils literal"><span class="pre">GradOpDescMakerBase</span></code> is a class now. The basic helper functions get the variables of <code class="docutils literal"><span class="pre">Input</span></code>, <code class="docutils literal"><span class="pre">Output</span></code>, <code class="docutils literal"><span class="pre">InputGradient</span></code> and <code class="docutils literal"><span class="pre">OutputGradient</span></code> in the forwarding operator.</p>
T
Deploy to GitHub Pages: 43702a89d5b5311281ef92be40d1e1ce9a88abab  
Travis CI 已提交
242 
<p>We should change register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So <code class="docutils literal"><span class="pre">REGISTER_OP</span></code> just register one operator. If the <code class="docutils literal"><span class="pre">REGISTER_OPERATOR</span></code> contains <code class="docutils literal"><span class="pre">OpProtoAndCheckerMaker</span></code> and <code class="docutils literal"><span class="pre">GradOpDescMaker</span></code>, we just list them in the same macro. It can be done by a macro contains <code class="docutils literal"><span class="pre">__VA_ARGS__</span></code>.</p>
T
Deploy to GitHub Pages: f76b38c285c9396f6cd1e99242f580c64d2b0878  
Travis CI 已提交
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 
<p>The user interface should be</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="n">vector</span><span class="o">&lt;</span><span class="n">OpDesc</span><span class="o">&gt;</span> <span class="n">MinusOpGradMaker</span><span class="p">(</span><span class="n">OpDesc</span><span class="p">)</span> <span class="p">{...}</span>
<span class="n">REGISTER_OPERATOR</span><span class="p">(</span><span class="n">minus</span><span class="p">,</span> <span class="n">MinusOp</span><span class="p">,</span> <span class="n">MinusOpProtoAndCheckerMaker</span><span class="p">,</span> <span class="n">SumOpGradMaker</span><span class="p">);</span>
<span class="c1">// Developers can still manually implement gradient operator.</span>
<span class="n">REGISTER_OPERATOR</span><span class="p">(</span><span class="n">minus_grad</span><span class="p">,</span> <span class="n">MinusGradOp</span><span class="p">);</span>
</pre></div>
</div>
<p>The interface of current <code class="docutils literal"><span class="pre">REGISTER_OP</span></code> macro could not be changed. In <code class="docutils literal"><span class="pre">REGISTER_OP</span></code>, it will invoke <code class="docutils literal"><span class="pre">REGISTER_OPERATOR</span></code> two times and generate GradOpDescMaker inside.</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="n">REGISTER_OP</span><span class="p">(</span><span class="n">minus</span><span class="p">,</span> <span class="n">MinusOp</span><span class="p">,</span> <span class="n">MinusOpProtoAndCheckerMaker</span><span class="p">,</span> <span class="n">minus_grad</span><span class="p">,</span> <span class="n">MinusGradOp</span><span class="p">);</span>
</pre></div>
</div>
</div>
</div>


           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: ".txt",
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
       
  

  
  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  
  
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/js/perfect-scrollbar.jquery.min.js"></script>
  <script src="../_static/js/paddle_doc_init.js"></script> 

</body>
</html>