rnn.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>RNNOp design &mdash; PaddlePaddle  文档</title>
  

    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="索引"
              href="../../genindex.html"/>
        <link rel="search" title="搜索" href="../../search.html"/>
    <link rel="top" title="PaddlePaddle  文档" href="../../index.html"/> 
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>


  <script src="../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

            <a href="../../index_cn.html" class="icon icon-home"> PaddlePaddle
          

          </a>

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
<nav class="doc-menu-vertical" role="navigation">

<ul>
<li class="toctree-l1"><a class="reference internal" href="../../getstarted/index_cn.html">新手入门</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/quickstart_cn.html">快速开始</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/concepts/use_concepts_cn.html">基本使用概念</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../build_and_install/index_cn.html">安装与编译</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/pip_install_cn.html">使用pip安装</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/docker_install_cn.html">使用Docker安装运行</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/build_from_source_cn.html">从源码编译</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index_cn.html">进阶使用</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../howto/cmd_parameter/index_cn.html">命令行参数设置</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/use_case_cn.html">使用案例</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/arguments_cn.html">参数概述</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/detail_introduction_cn.html">细节描述</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/cluster/index_cn.html">分布式训练</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/preparations_cn.html">环境准备</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/cmd_argument_cn.html">启动参数说明</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/multi_cluster/index_cn.html">在不同集群中运行</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_cn.html">Kubernetes单机训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_distributed_cn.html">Kubernetes分布式训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/openmpi_cn.html">在OpenMPI集群中启动训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/fabric_cn.html">使用fabric启动集群训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_aws_cn.html">Kubernetes on AWS</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/capi/index_cn.html">C-API预测库</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/compile_paddle_lib_cn.html">安装与编译C-API预测库</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/organization_of_the_inputs_cn.html">输入/输出数据组织</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/workflow_of_capi_cn.html">C-API使用流程</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/rnn/index_cn.html">RNN模型</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/rnn_config_cn.html">RNN配置</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/recurrent_group_cn.html">Recurrent Group教程</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/hierarchical_layer_cn.html">支持双层序列作为输入的Layer</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/hrnn_rnn_api_compare_cn.html">单双层RNN API对比介绍</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/optimization/gpu_profiling_cn.html">GPU性能调优</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index_cn.html">开发标准</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../dev/contribute_to_paddle_cn.html">如何贡献代码</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../dev/write_docs_cn.html">如何贡献文档</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../faq/index_cn.html">FAQ</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../faq/build_and_install/index_cn.html">编译安装与单元测试</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/model/index_cn.html">模型配置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/parameter/index_cn.html">参数设置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/local/index_cn.html">本地训练与预测</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/cluster/index_cn.html">集群训练与预测</a></li>
</ul>
</li>
</ul>

</nav>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="../../index_cn.html">PaddlePaddle</a>
      </nav>


      <div class="wy-nav-content">
        <div class="rst-content">
          

<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="../../index_cn.html">Docs</a> &raquo;</li>
      
    <li>RNNOp design</li>
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../../_sources/design/ops/rnn.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="rnnop-design">
<span id="rnnop-design"></span><h1>RNNOp design<a class="headerlink" href="#rnnop-design" title="永久链接至标题">¶</a></h1>
<p>This document describes the RNN (Recurrent Neural Network) operator and how it is implemented in PaddlePaddle. The RNN op requires that all instances in a mini-batch have the same length. We will have a more flexible dynamic RNN operator in the future.</p>
<div class="section" id="rnn-algorithm-implementation">
<span id="rnn-algorithm-implementation"></span><h2>RNN Algorithm Implementation<a class="headerlink" href="#rnn-algorithm-implementation" title="永久链接至标题">¶</a></h2>
<p align="center">
<img src="./images/rnn.jpg"/>
</p><p>The above diagram shows an RNN unrolled into a full network.</p>
<p>There are several important concepts here:</p>
<ul class="simple">
<li><em>step-net</em>: the sub-graph that runs at each step.</li>
<li><em>memory</em>, $h_t$, the state of the current step.</li>
<li><em>ex-memory</em>, $h_{t-1}$, the state of the previous step.</li>
<li><em>initial memory value</em>, the memory of the first (initial) step.</li>
</ul>
<div class="section" id="step-scope">
<span id="step-scope"></span><h3>Step-scope<a class="headerlink" href="#step-scope" title="永久链接至标题">¶</a></h3>
<p>There could be local variables defined in each step-net.  PaddlePaddle runtime realizes these variables in <em>step-scopes</em> which are created for each step.</p>
<p align="center">
<img src="./images/rnn.png"/><br/>
Figure 2 illustrates the RNN's data flow
</p><p>Please be aware that every step runs the same step-net.  Each step does the following:</p>
<ol class="simple">
<li>Creates the step-scope.</li>
<li>Initializes the local variables including step-outputs, in the step-scope.</li>
<li>Runs the step-net, which uses the above mentioned variables.</li>
</ol>
<p>The RNN operator will compose its output from step outputs in each of the step scopes.</p>
</div>
<div class="section" id="memory-and-ex-memory">
<span id="memory-and-ex-memory"></span><h3>Memory and Ex-memory<a class="headerlink" href="#memory-and-ex-memory" title="永久链接至标题">¶</a></h3>
<p>Let&#8217;s give more details about memory and ex-memory using a simple example:</p>
<p>$$
h_t = U h_{t-1} + W x_t
$$,</p>
<p>where $h_t$ and $h_{t-1}$ are the memory and ex-memory (previous memory) of step $t$ respectively.</p>
<p>In the implementation, we can make an ex-memory variable either &#8220;refer to&#8221; the memory variable of the previous step,
or copy the memory value of the previous step to the current ex-memory variable.</p>
</div>
<div class="section" id="usage-in-python">
<span id="usage-in-python"></span><h3>Usage in Python<a class="headerlink" href="#usage-in-python" title="永久链接至标题">¶</a></h3>
<p>For more information on Block, please refer to the <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.md">design doc</a>.</p>
<p>We can define an RNN&#8217;s step-net using a Block:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">paddle</span> <span class="kn">as</span> <span class="nn">pd</span>

<span class="n">X</span> <span class="o">=</span> <span class="n">some_op</span><span class="p">()</span> <span class="c1"># x is some operator&#39;s output and is a LoDTensor</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">some_op</span><span class="p">()</span>

<span class="c1"># declare parameters</span>
<span class="n">W</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>
<span class="n">U</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>

<span class="n">rnn</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">create_rnn_op</span><span class="p">(</span><span class="n">output_num</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="k">with</span> <span class="n">rnn</span><span class="o">.</span><span class="n">stepnet</span><span class="p">():</span>
    <span class="n">x</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_input</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
    <span class="c1"># declare a memory (rnn&#39;s step)</span>
    <span class="n">h</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_memory</span><span class="p">(</span><span class="n">init</span><span class="o">=</span><span class="n">a</span><span class="p">)</span>
    <span class="c1"># h.pre_state(), the previous memory of rnn</span>
    <span class="n">new_state</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">add_two</span><span class="p">(</span> <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span> <span class="o">+</span> <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">U</span><span class="p">,</span> <span class="n">h</span><span class="o">.</span><span class="n">pre_state</span><span class="p">()))</span>
    <span class="c1"># update current memory</span>
    <span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">new_state</span><span class="p">)</span>
    <span class="c1"># indicate that h variables in all step scopes should be merged</span>
    <span class="n">rnn</span><span class="o">.</span><span class="n">add_outputs</span><span class="p">(</span><span class="n">h</span><span class="p">)</span>

<span class="n">out</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">()</span>
</pre></div>
</div>
<p>Python API functions in above example:</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">rnn.add_input</span></code>: indicates that the parameter is a variable that will be segmented into step-inputs.</li>
<li><code class="docutils literal"><span class="pre">rnn.add_memory</span></code>: creates a variable used as the memory.</li>
<li><code class="docutils literal"><span class="pre">rnn.add_outputs</span></code>: marks the variables that will be concatenated across steps into the RNN output.</li>
</ul>
</div>
<div class="section" id="nested-rnn-and-lodtensor">
<span id="nested-rnn-and-lodtensor"></span><h3>Nested RNN and LoDTensor<a class="headerlink" href="#nested-rnn-and-lodtensor" title="永久链接至标题">¶</a></h3>
<p>An RNN whose step-net includes other RNN operators is known as an <em>nested RNN</em>.</p>
<p>For example, we could have a 2-level RNN, where the top level corresponds to paragraphs, and the lower level corresponds to sentences. Each step of the higher level RNN also receives an input from the corresponding step of the lower level, and additionally the output from the previous time step at the same level.</p>
<p>The following figure illustrates feeding in text into the lower level, one sentence at a step, and the feeding in step outputs to the top level. The final top level output is about the whole text.</p>
<p align="center">
<img src="./images/2_level_rnn.png"/>
</p><div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">paddle</span> <span class="kn">as</span> <span class="nn">pd</span>

<span class="n">W</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>
<span class="n">U</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>

<span class="n">W0</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>
<span class="n">U0</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>

<span class="c1"># a is output of some op</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">some_op</span><span class="p">()</span>

<span class="c1"># chapter_data is a set of 128-dim word vectors</span>
<span class="c1"># the first level of LoD is sentence</span>
<span class="c1"># the second level of LoD is a chapter</span>
<span class="n">chapter_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Variable</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="bp">None</span><span class="p">,</span> <span class="mi">128</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">lod_tensor</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>

<span class="k">def</span> <span class="nf">lower_level_rnn</span><span class="p">(</span><span class="n">paragraph</span><span class="p">):</span>
    <span class="sd">&#39;&#39;&#39;</span>
<span class="sd">    x: the input</span>
<span class="sd">    &#39;&#39;&#39;</span>
    <span class="n">rnn</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">create_rnn_op</span><span class="p">(</span><span class="n">output_num</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">rnn</span><span class="o">.</span><span class="n">stepnet</span><span class="p">():</span>
        <span class="n">sentence</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_input</span><span class="p">(</span><span class="n">paragraph</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
        <span class="n">h</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_memory</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">])</span>
        <span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
            <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">W</span><span class="p">,</span> <span class="n">sentence</span><span class="p">)</span> <span class="o">+</span> <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">U</span><span class="p">,</span> <span class="n">h</span><span class="o">.</span><span class="n">pre_state</span><span class="p">()))</span>
        <span class="c1"># get the last state as sentence&#39;s info</span>
        <span class="n">rnn</span><span class="o">.</span><span class="n">add_outputs</span><span class="p">(</span><span class="n">h</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">rnn</span>

<span class="n">top_level_rnn</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">create_rnn_op</span><span class="p">(</span><span class="n">output_num</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="k">with</span> <span class="n">top_level_rnn</span><span class="o">.</span><span class="n">stepnet</span><span class="p">():</span>
    <span class="n">paragraph_data</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_input</span><span class="p">(</span><span class="n">chapter_data</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
    <span class="n">low_rnn</span> <span class="o">=</span> <span class="n">lower_level_rnn</span><span class="p">(</span><span class="n">paragraph_data</span><span class="p">)</span>
    <span class="n">paragraph_out</span> <span class="o">=</span> <span class="n">low_rnn</span><span class="p">()</span>

    <span class="n">h</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">add_memory</span><span class="p">(</span><span class="n">init</span><span class="o">=</span><span class="n">a</span><span class="p">)</span>
    <span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
        <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">W0</span><span class="p">,</span> <span class="n">paragraph_data</span><span class="p">)</span> <span class="o">+</span> <span class="n">pd</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">U0</span><span class="p">,</span> <span class="n">h</span><span class="o">.</span><span class="n">pre_state</span><span class="p">()))</span>
    <span class="n">top_level_rnn</span><span class="o">.</span><span class="n">add_outputs</span><span class="p">(</span><span class="n">h</span><span class="p">)</span>

<span class="c1"># output the last step</span>
<span class="n">chapter_out</span> <span class="o">=</span> <span class="n">top_level_rnn</span><span class="p">(</span><span class="n">output_all_steps</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
</pre></div>
</div>
<p>In the above example, the construction of the <code class="docutils literal"><span class="pre">top_level_rnn</span></code> calls  <code class="docutils literal"><span class="pre">lower_level_rnn</span></code>.  The input is an LoD Tensor. The top level RNN segments input text data into paragraphs, and the lower level RNN segments each paragraph into sentences.</p>
<p>By default, the <code class="docutils literal"><span class="pre">RNNOp</span></code> will concatenate the outputs from all the time steps.
If the <code class="docutils literal"><span class="pre">output_all_steps</span></code> is set to False, it will only output the final time step.</p>
<p align="center">
<img src="images/rnn_2level_data.png"/>
</p></div>
</div>
</div>


           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../_static/doctools.js"></script>
      <script type="text/javascript" src="../../_static/translations.js"></script>
      <script type="text/javascript" src="https://cdn.bootcss.com/mathjax/2.7.0/MathJax.js"></script>

  
    <script type="text/javascript" src="../../_static/js/theme.js"></script>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>