block.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Design Doc: Block and Scope &mdash; PaddlePaddle  文档</title>
  

    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="索引"
              href="../genindex.html"/>
        <link rel="search" title="搜索" href="../search.html"/>
    <link rel="top" title="PaddlePaddle  文档" href="../index.html"/> 
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>


  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

            <a href="../index_cn.html" class="icon icon-home"> PaddlePaddle
          

          </a>

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
<nav class="doc-menu-vertical" role="navigation">

<ul>
<li class="toctree-l1"><a class="reference internal" href="../getstarted/index_cn.html">新手入门</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getstarted/quickstart_cn.html">快速开始</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getstarted/concepts/use_concepts_cn.html">基本使用概念</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../build_and_install/index_cn.html">安装与编译</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/pip_install_cn.html">使用pip安装</a></li>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/docker_install_cn.html">使用Docker安装运行</a></li>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/build_from_source_cn.html">从源码编译</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../howto/index_cn.html">进阶使用</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../howto/cmd_parameter/index_cn.html">命令行参数设置</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/use_case_cn.html">使用案例</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/arguments_cn.html">参数概述</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/detail_introduction_cn.html">细节描述</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/cluster/index_cn.html">分布式训练</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/preparations_cn.html">环境准备</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/cmd_argument_cn.html">启动参数说明</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/multi_cluster/index_cn.html">在不同集群中运行</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/k8s_cn.html">Kubernetes单机训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/k8s_distributed_cn.html">Kubernetes分布式训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/openmpi_cn.html">在OpenMPI集群中启动训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/fabric_cn.html">使用fabric启动集群训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/k8s_aws_cn.html">Kubernetes on AWS</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/capi/index_cn.html">C-API预测库</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/capi/compile_paddle_lib_cn.html">安装与编译C-API预测库</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/capi/organization_of_the_inputs_cn.html">输入/输出数据组织</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/capi/workflow_of_capi_cn.html">C-API使用流程</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/rnn/index_cn.html">RNN模型</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/rnn_config_cn.html">RNN配置</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/recurrent_group_cn.html">Recurrent Group教程</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/hierarchical_layer_cn.html">支持双层序列作为输入的Layer</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/hrnn_rnn_api_compare_cn.html">单双层RNN API对比介绍</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/optimization/gpu_profiling_cn.html">GPU性能调优</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../dev/index_cn.html">开发标准</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../dev/contribute_to_paddle_cn.html">如何贡献代码</a></li>
<li class="toctree-l2"><a class="reference internal" href="../dev/write_docs_cn.html">如何贡献文档</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../faq/index_cn.html">FAQ</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../faq/build_and_install/index_cn.html">编译安装与单元测试</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/model/index_cn.html">模型配置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/parameter/index_cn.html">参数设置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/local/index_cn.html">本地训练与预测</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/cluster/index_cn.html">集群训练与预测</a></li>
</ul>
</li>
</ul>

</nav>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="../index_cn.html">PaddlePaddle</a>
      </nav>


      <div class="wy-nav-content">
        <div class="rst-content">
          

<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="../index_cn.html">Docs</a> &raquo;</li>
      
    <li>Design Doc: Block and Scope</li>
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../_sources/design/block.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="design-doc-block-and-scope">
<span id="design-doc-block-and-scope"></span><h1>Design Doc: Block and Scope<a class="headerlink" href="#design-doc-block-and-scope" title="永久链接至标题">¶</a></h1>
<div class="section" id="the-representation-of-computation">
<span id="the-representation-of-computation"></span><h2>The Representation of Computation<a class="headerlink" href="#the-representation-of-computation" title="永久链接至标题">¶</a></h2>
<p>Both deep learning systems and programming languages help users describe computation procedures.  These systems use various representations of computation:</p>
<ul class="simple">
<li>Caffe, Torch, and Paddle: sequences of layers.</li>
<li>TensorFlow, Caffe2, Mxnet: graph of operators.</li>
<li>PaddlePaddle: nested blocks, like C++ and Java programs.</li>
</ul>
</div>
<div class="section" id="block-in-programming-languages-and-deep-learning">
<span id="block-in-programming-languages-and-deep-learning"></span><h2>Block in Programming Languages and Deep Learning<a class="headerlink" href="#block-in-programming-languages-and-deep-learning" title="永久链接至标题">¶</a></h2>
<p>In programming languages, a block is a pair of curly braces that includes local variables definitions and a sequence of instructions or operators.</p>
<p>Blocks work with control flow structures like <code class="docutils literal"><span class="pre">if</span></code>, <code class="docutils literal"><span class="pre">else</span></code>, and <code class="docutils literal"><span class="pre">for</span></code>, which have equivalents in deep learning:</p>
<p>| programming languages | PaddlePaddle          |
|&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8211;|&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8211;|
| for, while loop       | RNN, WhileOp          |
| if, if-else, switch   | IfElseOp, SwitchOp    |
| sequential execution  | a sequence of layers  |</p>
<p>A key difference is that a C++ program describes a one pass computation, whereas a deep learning program describes both the forward and backward passes.</p>
</div>
<div class="section" id="stack-frames-and-the-scope-hierarchy">
<span id="stack-frames-and-the-scope-hierarchy"></span><h2>Stack Frames and the Scope Hierarchy<a class="headerlink" href="#stack-frames-and-the-scope-hierarchy" title="永久链接至标题">¶</a></h2>
<p>The existence of the backward pass makes the execution of a block of PaddlePaddle different from traditional programs:</p>
<p>| programming languages | PaddlePaddle                    |
|&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8211;|&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;|
| stack                 | scope hierarchy                 |
| stack frame           | scope                           |
| push at entering block| push at entering block          |
| pop at leaving block  | destroy when minibatch completes|</p>
<ol class="simple">
<li>In traditional programs:<ul>
<li>When the execution enters the left curly brace of a block, the runtime pushes a frame into the stack, where it realizes local variables.</li>
<li>After the execution leaves the right curly brace, the runtime pops the frame.</li>
<li>The maximum number of frames in the stack is the maximum depth of nested blocks.</li>
</ul>
</li>
<li>In PaddlePaddle<ul>
<li>When the execution enters a block, PaddlePaddle adds a new scope, where it realizes variables.</li>
<li>PaddlePaddle doesn&#8217;t pop a scope after the execution of the block because variables therein are used by the backward pass.  So it has a stack forest known as a <em>scope hierarchy</em>.</li>
<li>The height of the highest tree is the maximum depth of nested blocks.</li>
<li>After the processing of a minibatch, PaddlePaddle destroys the scope hierarchy.</li>
</ul>
</li>
</ol>
</div>
<div class="section" id="use-blocks-in-c-and-paddlepaddle-programs">
<span id="use-blocks-in-c-and-paddlepaddle-programs"></span><h2>Use Blocks in C++ and PaddlePaddle Programs<a class="headerlink" href="#use-blocks-in-c-and-paddlepaddle-programs" title="永久链接至标题">¶</a></h2>
<p>Let us consolidate the discussion by presenting some examples.</p>
<div class="section" id="blocks-with-if-else-and-ifelseop">
<span id="blocks-with-if-else-and-ifelseop"></span><h3>Blocks with <code class="docutils literal"><span class="pre">if-else</span></code> and <code class="docutils literal"><span class="pre">IfElseOp</span></code><a class="headerlink" href="#blocks-with-if-else-and-ifelseop" title="永久链接至标题">¶</a></h3>
<p>The following C++ programs shows how blocks are used with the <code class="docutils literal"><span class="pre">if-else</span></code> structure:</p>
<div class="highlight-c++"><div class="highlight"><pre><span></span><span class="k">namespace</span> <span class="n">pd</span> <span class="o">=</span> <span class="n">paddle</span><span class="p">;</span>

<span class="kt">int</span> <span class="n">x</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
<span class="kt">int</span> <span class="n">y</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span>
<span class="kt">int</span> <span class="n">z</span> <span class="o">=</span> <span class="mi">10</span><span class="p">;</span>
<span class="kt">bool</span> <span class="n">cond</span> <span class="o">=</span> <span class="nb">false</span><span class="p">;</span>
<span class="kt">int</span> <span class="n">o1</span><span class="p">,</span> <span class="n">o2</span><span class="p">;</span>
<span class="k">if</span> <span class="p">(</span><span class="n">cond</span><span class="p">)</span> <span class="p">{</span>
  <span class="kt">int</span> <span class="n">z</span> <span class="o">=</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span><span class="p">;</span>
  <span class="n">o1</span> <span class="o">=</span> <span class="n">z</span><span class="p">;</span>
  <span class="n">o2</span> <span class="o">=</span> <span class="n">pd</span><span class="o">::</span><span class="n">layer</span><span class="o">::</span><span class="n">softmax</span><span class="p">(</span><span class="n">z</span><span class="p">);</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
  <span class="kt">int</span> <span class="n">d</span> <span class="o">=</span> <span class="n">pd</span><span class="o">::</span><span class="n">layer</span><span class="o">::</span><span class="n">fc</span><span class="p">(</span><span class="n">z</span><span class="p">);</span>
  <span class="n">o1</span> <span class="o">=</span> <span class="n">d</span><span class="p">;</span>
  <span class="n">o2</span> <span class="o">=</span> <span class="n">d</span><span class="o">+</span><span class="mi">1</span><span class="p">;</span>
<span class="p">}</span>
</pre></div>
</div>
<p>An equivalent PaddlePaddle program from the design doc of the <a class="reference internal" href="if_else_op.html"><span class="doc">IfElseOp operator</span></a> is as follows:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">paddle</span> <span class="kn">as</span> <span class="nn">pd</span>

<span class="n">x</span> <span class="o">=</span> <span class="n">minibatch</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span> <span class="c1"># shape=[None, 1]</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">var</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="c1"># shape=[1], value=1</span>
<span class="n">z</span> <span class="o">=</span> <span class="n">minibatch</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span> <span class="c1"># shape=[None, 1]</span>
<span class="n">cond</span> <span class="o">=</span> <span class="n">larger_than</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="mi">15</span><span class="p">)</span> <span class="c1"># [false, true, true]</span>

<span class="n">ie</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">ifelse</span><span class="p">()</span>
<span class="k">with</span> <span class="n">ie</span><span class="o">.</span><span class="n">true_block</span><span class="p">():</span>
    <span class="n">d</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">layer</span><span class="o">.</span><span class="n">add_scalar</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
    <span class="n">ie</span><span class="o">.</span><span class="n">output</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">layer</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">d</span><span class="p">))</span>
<span class="k">with</span> <span class="n">ie</span><span class="o">.</span><span class="n">false_block</span><span class="p">():</span>
    <span class="n">d</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">layer</span><span class="o">.</span><span class="n">fc</span><span class="p">(</span><span class="n">z</span><span class="p">)</span>
    <span class="n">ie</span><span class="o">.</span><span class="n">output</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="n">d</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span>
<span class="n">o1</span><span class="p">,</span> <span class="n">o2</span> <span class="o">=</span> <span class="n">ie</span><span class="p">(</span><span class="n">cond</span><span class="p">)</span>
</pre></div>
</div>
<p>In both examples, the left branch computes <code class="docutils literal"><span class="pre">x+y</span></code> and <code class="docutils literal"><span class="pre">softmax(x+y)</span></code>, the right branch computes <code class="docutils literal"><span class="pre">fc(x)</span></code> and <code class="docutils literal"><span class="pre">x+1</span></code> .</p>
<p>The difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances.</p>
</div>
<div class="section" id="blocks-with-for-and-rnnop">
<span id="blocks-with-for-and-rnnop"></span><h3>Blocks with <code class="docutils literal"><span class="pre">for</span></code> and <code class="docutils literal"><span class="pre">RNNOp</span></code><a class="headerlink" href="#blocks-with-for-and-rnnop" title="永久链接至标题">¶</a></h3>
<p>The following RNN model in PaddlePaddle from the <a class="reference external" href="design/rnn.md">RNN design doc</a> :</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="n">sequence</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span> <span class="c1"># shape=[None, 1]</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">var</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># shape=[1]</span>
<span class="n">W</span> <span class="o">=</span> <span class="n">var</span><span class="p">(</span><span class="mf">0.314</span><span class="p">,</span> <span class="n">param</span><span class="o">=</span><span class="n">true</span><span class="p">)</span> <span class="c1"># shape=[1]</span>
<span class="n">U</span> <span class="o">=</span> <span class="n">var</span><span class="p">(</span><span class="mf">0.375</span><span class="p">,</span> <span class="n">param</span><span class="o">=</span><span class="n">true</span><span class="p">)</span> <span class="c1"># shape=[1]</span>

<span class="n">rnn</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">rnn</span><span class="p">()</span>
<span class="k">with</span> <span class="n">rnn</span><span class="o">.</span><span class="n">step</span><span class="p">():</span>
  <span class="n">h</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">memory</span><span class="p">(</span><span class="n">init</span> <span class="o">=</span> <span class="n">m</span><span class="p">)</span>
  <span class="n">h_prev</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">previous_memory</span><span class="p">(</span><span class="n">h</span><span class="p">)</span>
  <span class="n">a</span> <span class="o">=</span> <span class="n">layer</span><span class="o">.</span><span class="n">fc</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
  <span class="n">b</span> <span class="o">=</span> <span class="n">layer</span><span class="o">.</span><span class="n">fc</span><span class="p">(</span><span class="n">U</span><span class="p">,</span> <span class="n">h_prev</span><span class="p">)</span>  
  <span class="n">s</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
  <span class="n">act</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
  <span class="n">rnn</span><span class="o">.</span><span class="n">update_memory</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">act</span><span class="p">)</span>
  <span class="n">rnn</span><span class="o">.</span><span class="n">output</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
<span class="n">o1</span><span class="p">,</span> <span class="n">o2</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">()</span>
</pre></div>
</div>
<p>has its equivalent C++ program as follows</p>
<div class="highlight-c++"><div class="highlight"><pre><span></span><span class="kt">int</span><span class="o">*</span> <span class="n">x</span> <span class="o">=</span> <span class="p">{</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">};</span>
<span class="kt">int</span><span class="o">*</span> <span class="n">m</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">};</span>
<span class="kt">int</span><span class="o">*</span> <span class="n">W</span> <span class="o">=</span> <span class="p">{</span><span class="mf">0.314</span><span class="p">};</span>
<span class="kt">int</span><span class="o">*</span> <span class="n">U</span> <span class="o">=</span> <span class="p">{</span><span class="mf">0.375</span><span class="p">};</span>

<span class="kt">int</span> <span class="n">mem</span><span class="p">[</span><span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">+</span> <span class="mi">1</span><span class="p">];</span>
<span class="kt">int</span> <span class="n">o1</span><span class="p">[</span><span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">+</span> <span class="mi">1</span><span class="p">];</span>
<span class="kt">int</span> <span class="n">o2</span><span class="p">[</span><span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">/</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">+</span> <span class="mi">1</span><span class="p">];</span>
<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span> <span class="n">i</span> <span class="o">&lt;=</span> <span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">/</span><span class="k">sizeof</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]);</span> <span class="o">++</span><span class="n">i</span><span class="p">)</span> <span class="p">{</span>
  <span class="kt">int</span> <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">];</span>
  <span class="k">if</span> <span class="p">(</span><span class="n">i</span> <span class="o">==</span> <span class="mi">1</span><span class="p">)</span> <span class="n">mem</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="n">m</span><span class="p">;</span>
  <span class="kt">int</span> <span class="n">a</span> <span class="o">=</span> <span class="n">W</span> <span class="o">*</span> <span class="n">x</span><span class="p">;</span>
  <span class="kt">int</span> <span class="n">b</span> <span class="o">=</span> <span class="n">Y</span> <span class="o">*</span> <span class="n">mem</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">];</span>
  <span class="kt">int</span> <span class="n">s</span> <span class="o">=</span> <span class="n">fc_out</span> <span class="o">+</span> <span class="n">hidden_out</span><span class="p">;</span>
  <span class="kt">int</span> <span class="n">act</span> <span class="o">=</span> <span class="n">sigmoid</span><span class="p">(</span><span class="n">sum</span><span class="p">);</span>
  <span class="n">mem</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">act</span><span class="p">;</span>
  <span class="n">o1</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">act</span><span class="p">;</span>
  <span class="n">o2</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">hidden_out</span><span class="p">;</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="compilation-and-execution">
<span id="compilation-and-execution"></span><h2>Compilation and Execution<a class="headerlink" href="#compilation-and-execution" title="永久链接至标题">¶</a></h2>
<p>Like TensorFlow, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest executes the message for training or inference.</p>
<p>The generation of this protobuf message is similar to how a compiler generates a binary executable file. The execution of the message is similar to how the OS executes the binary file.</p>
</div>
<div class="section" id="the-binary-executable-file-format">
<span id="the-binary-executable-file-format"></span><h2>The &#8220;Binary Executable File Format&#8221;<a class="headerlink" href="#the-binary-executable-file-format" title="永久链接至标题">¶</a></h2>
<p>The definition of the protobuf message is as follows:</p>
<div class="highlight-protobuf"><div class="highlight"><pre><span></span><span class="kd">message</span> <span class="nc">BlockDesc</span> <span class="p">{</span>
  <span class="k">repeated</span> <span class="n">VarDesc</span> <span class="na">vars</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span>
  <span class="k">repeated</span> <span class="n">OpDesc</span> <span class="na">ops</span> <span class="o">=</span> <span class="mi">2</span><span class="p">;</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The step net in above RNN example would look like</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">BlockDesc</span> <span class="p">{</span>
  <span class="nb">vars</span> <span class="o">=</span> <span class="p">{</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">x</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">h</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">fc_out</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">hidden_out</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="nb">sum</span>
    <span class="n">VarDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">act</span>
  <span class="p">}</span>
  <span class="n">ops</span> <span class="o">=</span> <span class="p">{</span>
    <span class="n">OpDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">matmul</span>
    <span class="n">OpDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">add_two</span>
    <span class="n">OpDesc</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span> <span class="o">//</span> <span class="n">sigmoid</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p>Also, the RNN operator in above example is serialized into a protobuf message of type <code class="docutils literal"><span class="pre">OpDesc</span></code> and would look like:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">OpDesc</span> <span class="p">{</span>
  <span class="n">inputs</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">}</span> <span class="o">//</span> <span class="n">the</span> <span class="n">index</span> <span class="n">of</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">vars</span> <span class="n">of</span> <span class="n">BlockDesc</span> <span class="n">above</span>
  <span class="n">outputs</span> <span class="o">=</span> <span class="p">{</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">}</span> <span class="o">//</span> <span class="n">indices</span> <span class="n">of</span> <span class="n">act</span> <span class="ow">and</span> <span class="n">hidden_out</span> <span class="ow">in</span> <span class="nb">vars</span> <span class="n">of</span> <span class="n">BlockDesc</span> <span class="n">above</span>
  <span class="n">attrs</span> <span class="p">{</span>
    <span class="s2">&quot;states&quot;</span> <span class="p">:</span> <span class="p">{</span><span class="mi">1</span><span class="p">}</span> <span class="o">//</span> <span class="n">the</span> <span class="n">index</span> <span class="n">of</span> <span class="n">h</span>
    <span class="s2">&quot;step_net&quot;</span> <span class="p">:</span> <span class="o">&lt;</span><span class="n">above</span> <span class="n">step</span> <span class="n">net</span><span class="o">&gt;</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p>This <code class="docutils literal"><span class="pre">OpDesc</span></code> value is in the <code class="docutils literal"><span class="pre">ops</span></code> field of the <code class="docutils literal"><span class="pre">BlockDesc</span></code> value representing the global block.</p>
</div>
<div class="section" id="the-compilation-of-blocks">
<span id="the-compilation-of-blocks"></span><h2>The Compilation of Blocks<a class="headerlink" href="#the-compilation-of-blocks" title="永久链接至标题">¶</a></h2>
<p>During the generation of the Protobuf message, the Block should store VarDesc (the Protobuf message which describes Variable) and OpDesc (the Protobuf message which describes Operator).</p>
<p>VarDesc in a block should have its name scope to avoid local variables affecting parent block&#8217;s name scope.
Child block&#8217;s name scopes should inherit the parent&#8217;s so that OpDesc in child block can reference a VarDesc that is stored in the parent block. For example:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">a</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">20</span><span class="p">])</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">fc</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;fc.w&quot;</span><span class="p">,</span> <span class="s2">&quot;fc.b&quot;</span><span class="p">])</span>

<span class="n">rnn</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">create_rnn</span><span class="p">()</span>
<span class="k">with</span> <span class="n">rnn</span><span class="o">.</span><span class="n">stepnet</span><span class="p">():</span>
    <span class="n">x</span> <span class="o">=</span> <span class="n">a</span><span class="o">.</span><span class="n">as_step_input</span><span class="p">()</span>
    <span class="c1"># reuse fc&#39;s parameter</span>
    <span class="n">fc_without_b</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="s2">&quot;fc.w&quot;</span><span class="p">)</span>
    <span class="n">rnn</span><span class="o">.</span><span class="n">output</span><span class="p">(</span><span class="n">fc_without_b</span><span class="p">)</span>

<span class="n">out</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">()</span>
</pre></div>
</div>
<p>The method <code class="docutils literal"><span class="pre">pd.get_variable</span></code> can help retrieve a Variable by the name. The Variable may be stored in a parent block, but might be retrieved in a child block, so block should have a variable scope that supports inheritance.</p>
<p>In compiler design, the symbol table is a data structure created and maintained by compilers to store information about the occurrence of various entities such as variable names, function names, classes, etc.</p>
<p>To store the definition of variables and operators, we define a C++ class <code class="docutils literal"><span class="pre">SymbolTable</span></code>, like the one used in compilers.</p>
<p><code class="docutils literal"><span class="pre">SymbolTable</span></code> can do the following:</p>
<ul class="simple">
<li>store the definitions (some names and attributes) of variables and operators,</li>
<li>verify if a variable was declared,</li>
<li>make it possible to implement type checking (offer Protobuf message pointers to <code class="docutils literal"><span class="pre">InferShape</span></code> handlers).</li>
</ul>
<div class="highlight-c++"><div class="highlight"><pre><span></span><span class="c1">// Information in SymbolTable is enough to trace the dependency graph. So maybe</span>
<span class="c1">// the Eval() interface takes a SymbolTable is enough.</span>
<span class="k">class</span> <span class="nc">SymbolTable</span> <span class="p">{</span>
 <span class="k">public</span><span class="o">:</span>
  <span class="n">SymbolTable</span><span class="p">(</span><span class="n">SymbolTable</span><span class="o">*</span> <span class="n">parent</span><span class="p">)</span> <span class="o">:</span> <span class="n">parent_</span><span class="p">(</span><span class="n">parent</span><span class="p">)</span> <span class="p">{}</span>

  <span class="n">OpDesc</span><span class="o">*</span> <span class="n">NewOp</span><span class="p">(</span><span class="k">const</span> <span class="n">string</span><span class="o">&amp;</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;&quot;</span><span class="p">);</span>

  <span class="c1">// TODO determine whether name is generated by python or C++.</span>
  <span class="c1">// Currently assume that a unique name will be generated by C++ if the</span>
  <span class="c1">// argument name is left default.</span>
  <span class="n">VarDesc</span><span class="o">*</span> <span class="nf">Var</span><span class="p">(</span><span class="k">const</span> <span class="n">string</span><span class="o">&amp;</span> <span class="n">name</span><span class="o">=</span><span class="s">&quot;&quot;</span><span class="p">);</span>

  <span class="c1">// find a VarDesc by name, if recursive is true, find parent&#39;s SymbolTable</span>
  <span class="c1">// recursively.</span>
  <span class="c1">// this interface is introduced to support InferShape, find protobuf messages</span>
  <span class="c1">// of variables and operators, pass pointers into InferShape.</span>
  <span class="c1">//</span>
  <span class="c1">// NOTE maybe some C++ classes such as VarDescBuilder and OpDescBuilder should</span>
  <span class="c1">// be proposed and embedded into pybind to enable python operation on C++ pointers.</span>
  <span class="n">VarDesc</span><span class="o">*</span> <span class="nf">FindVar</span><span class="p">(</span><span class="k">const</span> <span class="n">string</span><span class="o">&amp;</span> <span class="n">name</span><span class="p">,</span> <span class="kt">bool</span> <span class="n">recursive</span><span class="o">=</span><span class="nb">true</span><span class="p">);</span>

  <span class="n">OpDesc</span><span class="o">*</span> <span class="nf">FindOp</span><span class="p">(</span><span class="k">const</span> <span class="n">string</span><span class="o">&amp;</span> <span class="n">name</span><span class="p">);</span>

  <span class="n">BlockDesc</span> <span class="nf">Compile</span><span class="p">()</span> <span class="k">const</span><span class="p">;</span>

 <span class="k">private</span><span class="o">:</span>
  <span class="n">SymbolTable</span><span class="o">*</span> <span class="n">parent_</span><span class="p">;</span>

  <span class="n">map</span><span class="o">&lt;</span><span class="n">string</span><span class="p">,</span> <span class="n">OpDesc</span><span class="o">&gt;</span> <span class="n">ops_</span><span class="p">;</span>
  <span class="n">map</span><span class="o">&lt;</span><span class="n">string</span><span class="p">,</span> <span class="n">VarDesc</span><span class="o">&gt;</span> <span class="n">vars_</span><span class="p">;</span>
<span class="p">};</span>
</pre></div>
</div>
<p>After all the description of variables and operators is added into SymbolTable,
the block has enough information to run.</p>
<p>The <code class="docutils literal"><span class="pre">Block</span></code> class takes a <code class="docutils literal"><span class="pre">BlockDesc</span></code> as input, and provides <code class="docutils literal"><span class="pre">Run</span></code> and <code class="docutils literal"><span class="pre">InferShape</span></code> functions.</p>
<div class="highlight-c++"><div class="highlight"><pre><span></span><span class="k">namespace</span> <span class="p">{</span>

<span class="k">class</span> <span class="nc">Block</span> <span class="o">:</span> <span class="n">OperatorBase</span> <span class="p">{</span>
<span class="k">public</span><span class="o">:</span>
  <span class="n">Block</span><span class="p">(</span><span class="k">const</span> <span class="n">BlockDesc</span><span class="o">&amp;</span> <span class="n">desc</span><span class="p">)</span> <span class="n">desc_</span><span class="p">(</span><span class="n">desc</span><span class="p">)</span> <span class="p">{}</span>

  <span class="kt">void</span> <span class="n">InferShape</span><span class="p">(</span><span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">Scope</span><span class="o">&amp;</span> <span class="n">scope</span><span class="p">)</span> <span class="k">const</span> <span class="k">override</span> <span class="p">{</span>
    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">symbols_ready_</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">CreateVariables</span><span class="p">(</span><span class="n">scope</span><span class="p">);</span>
      <span class="n">CreateOperators</span><span class="p">();</span>
    <span class="p">}</span>
    <span class="c1">// should run InferShape first.</span>
    <span class="k">for</span> <span class="p">(</span><span class="k">auto</span><span class="o">&amp;</span> <span class="nl">op</span> <span class="p">:</span> <span class="n">runtime_table_</span><span class="p">.</span><span class="n">ops</span><span class="p">())</span> <span class="p">{</span>
      <span class="n">op</span><span class="o">-&gt;</span><span class="n">InferShape</span><span class="p">(</span><span class="n">scope</span><span class="p">);</span>
    <span class="p">}</span>
  <span class="p">}</span>

  <span class="kt">void</span> <span class="n">Run</span><span class="p">(</span><span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">Scope</span><span class="o">&amp;</span> <span class="n">scope</span><span class="p">,</span>
           <span class="k">const</span> <span class="n">platform</span><span class="o">::</span><span class="n">Place</span><span class="o">&amp;</span> <span class="n">place</span><span class="p">)</span> <span class="k">const</span> <span class="k">override</span> <span class="p">{</span>
    <span class="n">PADDLE_ENFORCE</span><span class="p">(</span><span class="n">symbols_ready_</span><span class="p">,</span> <span class="s">&quot;operators and variables should be created first.&quot;</span><span class="p">);</span>
    <span class="k">for</span> <span class="p">(</span><span class="k">auto</span><span class="o">&amp;</span> <span class="nl">op</span> <span class="p">:</span> <span class="n">runtime_table_</span><span class="p">.</span><span class="n">ops</span><span class="p">())</span> <span class="p">{</span>
      <span class="n">op</span><span class="o">-&gt;</span><span class="n">Run</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">place</span><span class="p">);</span>
    <span class="p">}</span>
  <span class="p">}</span>

  <span class="kt">void</span> <span class="n">CreateVariables</span><span class="p">(</span><span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">Scope</span><span class="o">&amp;</span> <span class="n">scope</span><span class="p">);</span>
  <span class="kt">void</span> <span class="nf">CreateOperators</span><span class="p">();</span>

  <span class="c1">// some other necessary interfaces of NetOp are listed below</span>
  <span class="c1">// ...</span>

<span class="k">private</span><span class="o">:</span>
  <span class="n">BlockDesc</span> <span class="n">desc_</span><span class="p">;</span>
  <span class="kt">bool</span> <span class="n">symbols_ready_</span><span class="p">{</span><span class="nb">false</span><span class="p">};</span>
<span class="p">};</span>
</pre></div>
</div>
</div>
<div class="section" id="the-execution-of-blocks">
<span id="the-execution-of-blocks"></span><h2>The Execution of Blocks<a class="headerlink" href="#the-execution-of-blocks" title="永久链接至标题">¶</a></h2>
<p>Block inherits from OperatorBase, which has a Run method.
Block&#8217;s Run method will run its operators sequentially.</p>
<p>There is another important interface called <code class="docutils literal"><span class="pre">Eval</span></code>, which takes some arguments called targets and generates a minimal graph which treats targets as the end points and creates a new Block. After <code class="docutils literal"><span class="pre">Run</span></code>, <code class="docutils literal"><span class="pre">Eval</span></code> will get the latest value and return the targets.</p>
<p>The definition of Eval is as follows:</p>
<div class="highlight-c++"><div class="highlight"><pre><span></span><span class="c1">// clean a block description by targets using the corresponding dependency graph.</span>
<span class="c1">// return a new BlockDesc with minimal number of operators.</span>
<span class="c1">// NOTE: The return type is not a Block but the block&#39;s description so that this can be distributed</span>
<span class="c1">// to a cluster.</span>
<span class="n">BlockDesc</span> <span class="nf">Prune</span><span class="p">(</span><span class="k">const</span> <span class="n">BlockDesc</span><span class="o">&amp;</span> <span class="n">desc</span><span class="p">,</span> <span class="n">vector</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;</span> <span class="n">targets</span><span class="p">);</span>

<span class="kt">void</span> <span class="n">Block</span><span class="o">::</span><span class="n">Eval</span><span class="p">(</span><span class="k">const</span> <span class="n">vector</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;&amp;</span> <span class="n">targets</span><span class="p">,</span>
                 <span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">Scope</span><span class="o">&amp;</span> <span class="n">scope</span><span class="p">,</span>
                 <span class="k">const</span> <span class="n">platform</span><span class="o">::</span><span class="n">DeviceContext</span><span class="o">&amp;</span> <span class="n">dev_ctx</span><span class="p">)</span> <span class="p">{</span>
  <span class="n">BlockDesc</span> <span class="n">min_desc</span> <span class="o">=</span> <span class="n">Prune</span><span class="p">(</span><span class="n">desc_</span><span class="p">,</span> <span class="n">targets</span><span class="p">);</span>
  <span class="n">Block</span> <span class="nf">min_block</span><span class="p">(</span><span class="n">min_desc</span><span class="p">);</span>
  <span class="n">min_block</span><span class="p">.</span><span class="n">Run</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">dev_ctx</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
</div>


           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>
      <script type="text/javascript" src="../_static/translations.js"></script>
      <script type="text/javascript" src="https://cdn.bootcss.com/mathjax/2.7.0/MathJax.js"></script>

  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>