new_op_en.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>How to write a new operator &mdash; PaddlePaddle  documentation</title>
  

    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="Index"
              href="../genindex.html"/>
        <link rel="search" title="Search" href="../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../index.html"/> 
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>


  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

            <a href="../index_en.html" class="icon icon-home"> PaddlePaddle
          

          </a>

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
<nav class="doc-menu-vertical" role="navigation">

<ul>
<li class="toctree-l1"><a class="reference internal" href="../getstarted/index_en.html">GET STARTED</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getstarted/quickstart_en.html">Quick Start</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getstarted/concepts/use_concepts_en.html">Basic Concept</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../build_and_install/index_en.html">Install and Build</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/pip_install_en.html">Install using pip</a></li>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/docker_install_en.html">Run in Docker Containers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../build_and_install/build_from_source_en.html">Build from Sources</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../howto/index_en.html">HOW TO</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../howto/cmd_parameter/index_en.html">Set Command-line Parameters</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/use_case_en.html">Use Case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/arguments_en.html">Argument Outline</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cmd_parameter/detail_introduction_en.html">Detail Description</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/cluster/index_en.html">Distributed Training</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/preparations_en.html">Preparations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/cmd_argument_en.html">Command-line arguments</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/cluster/multi_cluster/index_en.html">Use different clusters</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/fabric_en.html">Fabric</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/openmpi_en.html">OpenMPI</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/k8s_en.html">Kubernetes</a></li>
<li class="toctree-l4"><a class="reference internal" href="../howto/cluster/multi_cluster/k8s_aws_en.html">Kubernetes on AWS</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/rnn/index_en.html">RNN Models</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/rnn_config_en.html">RNN Configuration</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/recurrent_group_en.html">Recurrent Group Tutorial</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/hierarchical_layer_en.html">Layers supporting hierarchical sequence as input</a></li>
<li class="toctree-l3"><a class="reference internal" href="../howto/rnn/hrnn_rnn_api_compare_en.html">API comparision between RNN and hierarchical RNN</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../howto/optimization/gpu_profiling_en.html">Tune GPU Performance</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="index_en.html">Development</a><ul>
<li class="toctree-l2"><a class="reference internal" href="contribute_to_paddle_en.html">Contribute Code</a></li>
<li class="toctree-l2"><a class="reference internal" href="write_docs_en.html">Contribute Documentation</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../faq/index_en.html">FAQ</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../faq/build_and_install/index_en.html">Install, Build and Unit test</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/model/index_en.html">Model Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/parameter/index_en.html">Parameter Setting</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/local/index_en.html">Local Training and Prediction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../faq/cluster/index_en.html">Cluster Training and Prediction</a></li>
</ul>
</li>
</ul>

</nav>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="../index_en.html">PaddlePaddle</a>
      </nav>


      <div class="wy-nav-content">
        <div class="rst-content">
          

<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="../index_en.html">Docs</a> &raquo;</li>
      
    <li>How to write a new operator</li>
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../_sources/dev/new_op_en.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="how-to-write-a-new-operator">
<span id="how-to-write-a-new-operator"></span><h1>How to write a new operator<a class="headerlink" href="#how-to-write-a-new-operator" title="Permalink to this headline">¶</a></h1>
<ul class="simple">
<li><a class="reference external" href="#background">Background</a></li>
<li><a class="reference external" href="#implementing-c-types">Implementing C++ Types</a><ul>
<li><a class="reference external" href="#defining-protomaker">Defining ProtoMaker</a></li>
<li><a class="reference external" href="#defining-operator">Defining Operator</a></li>
<li><a class="reference external" href="#defining-opkernel">Defining OpKernel</a></li>
<li><a class="reference external" href="#registering-operator-and-opkernel">Registering Operator and OpKernel</a></li>
<li><a class="reference external" href="#compilation">Compilation</a></li>
</ul>
</li>
<li><a class="reference external" href="#python-binding">Python Binding</a></li>
<li><a class="reference external" href="#unit-tests">Unit Tests</a><ul>
<li><a class="reference external" href="#testing-forward-operators">Testing Forward Operators</a></li>
<li><a class="reference external" href="#testing-backward-operators">Testing Backward Operators</a></li>
<li><a class="reference external" href="#compiling-and-running">Compiling and Running</a></li>
</ul>
</li>
<li><a class="reference external" href="#remarks">Remarks</a></li>
</ul>
<div class="section" id="background">
<span id="background"></span><h2>Background<a class="headerlink" href="#background" title="Permalink to this headline">¶</a></h2>
<p>Here are the base types needed. For details, please refer to the design docs.</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">class</span> <span class="pre">OpProtoAndCheckerMaker</span></code>: Describes an Operator&#8217;s input, output, attributes and description, mainly used to interface with Python API.</li>
<li><code class="docutils literal"><span class="pre">framework::OperatorBase</span></code>: Operator (Op)base class.</li>
<li><code class="docutils literal"><span class="pre">framework::OpKernel</span></code>: Base class for Op computation kernel.</li>
<li><code class="docutils literal"><span class="pre">framework::OperatorWithKernel</span></code>: Inherited from OperatorBase, describing an operator with computation kernels.</li>
</ul>
<p>Operators can be categorized into two groups: operator with kernel(s) and operator without kernel(s). An operator with kernel(s) inherits from <code class="docutils literal"><span class="pre">OperatorWithKernel</span></code> while the one without kernel(s) inherits from <code class="docutils literal"><span class="pre">OperatorBase</span></code>. This tutorial focuses on implementing operators with kernels. In short, an operator includes the following information:</p>
<p>Information           | Where is it defined
&#8212;&#8212;&#8212;&#8212;&#8211;  | :&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;&#8212;-
OpProtoMake definition  | <code class="docutils literal"><span class="pre">.cc</span></code>files, Backward Op does not need an OpProtoMake interface.
Op definition           | <code class="docutils literal"><span class="pre">.cc</span></code> files
Kernel implementation       | The kernel methods shared between CPU and CUDA are defined in <code class="docutils literal"><span class="pre">.h</span></code> files. CPU-specific kernels live in <code class="docutils literal"><span class="pre">.cc</span></code> files, while CUDA-specific kernels are implemented in <code class="docutils literal"><span class="pre">.cu</span></code>files.
Registering the Op           | Ops are registered in <code class="docutils literal"><span class="pre">.cc</span></code> files; For Kernel registration, <code class="docutils literal"><span class="pre">.cc</span></code> files contain the CPU implementation, while <code class="docutils literal"><span class="pre">.cu</span></code> files contain the CUDA implementation.</p>
<p>New Operator implementations are added to the list <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators">paddle/operators</a>, with file names in the format <code class="docutils literal"><span class="pre">*_op.h</span></code> (if applicable), <code class="docutils literal"><span class="pre">*_op.cc</span></code>, <code class="docutils literal"><span class="pre">*_op.cu</span></code> (if applicable).** The system will use the naming scheme to automatically build operators and their corresponding Python extensions.**</p>
<p>Let&#8217;s take matrix multiplication operator, <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/mul_op.cc">MulOp</a>, as an example to introduce the writing of an Operator with Kernel.</p>
</div>
<div class="section" id="implementing-c-types">
<span id="implementing-c-types"></span><h2>Implementing C++ Types<a class="headerlink" href="#implementing-c-types" title="Permalink to this headline">¶</a></h2>
<div class="section" id="defining-protomaker">
<span id="defining-protomaker"></span><h3>Defining ProtoMaker<a class="headerlink" href="#defining-protomaker" title="Permalink to this headline">¶</a></h3>
<p>Matrix Multiplication can be written as $Out = X * Y$, meaning that the operation consists of two inputs and pne output.</p>
<p>First, define <code class="docutils literal"><span class="pre">ProtoMaker</span></code> to describe the Operator&#8217;s input, output, and additional comments:</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MulOpMaker</span> <span class="o">:</span> <span class="k">public</span> <span class="n">framework</span><span class="o">::</span><span class="n">OpProtoAndCheckerMaker</span> <span class="p">{</span>
 <span class="k">public</span><span class="o">:</span>
  <span class="n">MulOpMaker</span><span class="p">(</span><span class="n">OpProto</span> <span class="o">*</span><span class="n">proto</span><span class="p">,</span> <span class="n">OpAttrChecker</span> <span class="o">*</span><span class="n">op_checker</span><span class="p">)</span>
      <span class="o">:</span> <span class="n">OpProtoAndCheckerMaker</span><span class="p">(</span><span class="n">proto</span><span class="p">,</span> <span class="n">op_checker</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">AddInput</span><span class="p">(</span><span class="s">&quot;X&quot;</span><span class="p">,</span> <span class="s">&quot;(Tensor), 2D tensor of size (M x K)&quot;</span><span class="p">);</span>
    <span class="n">AddInput</span><span class="p">(</span><span class="s">&quot;Y&quot;</span><span class="p">,</span> <span class="s">&quot;(Tensor), 2D tensor of size (K x N)&quot;</span><span class="p">);</span>
    <span class="n">AddOutput</span><span class="p">(</span><span class="s">&quot;Out&quot;</span><span class="p">,</span> <span class="s">&quot;(Tensor), 2D tensor of size (M x N)&quot;</span><span class="p">);</span>
    <span class="n">AddComment</span><span class="p">(</span><span class="sa">R</span><span class="s">&quot;</span><span class="dl">DOC(</span><span class="s"></span>
<span class="s">Two Element Mul Operator.</span>
<span class="s">The equation is: Out = X * Y</span>
<span class="dl">)DOC</span><span class="s">&quot;</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p><a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/mul_op.cc#L43"><code class="docutils literal"><span class="pre">MulOpMaker</span></code></a>is inherited from<code class="docutils literal"><span class="pre">framework::OpProtoAndCheckerMaker</span></code>, consisting of 2 variables in the constructor：</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">framework::OpProto</span></code> stores Operator input and variable attribute, used for generating Python API interfaces.</li>
<li><code class="docutils literal"><span class="pre">framework::OpAttrChecker</span></code> is used to validate variable attributes.</li>
</ul>
<p>The constructor utilizes <code class="docutils literal"><span class="pre">AddInput</span></code>, <code class="docutils literal"><span class="pre">AddOutput</span></code>, and <code class="docutils literal"><span class="pre">AddComment</span></code>, so that the corresponding information will be added to <code class="docutils literal"><span class="pre">OpProto</span></code>.</p>
<p>The code above adds two inputs <code class="docutils literal"><span class="pre">X</span></code> and <code class="docutils literal"><span class="pre">Y</span></code> to <code class="docutils literal"><span class="pre">MulOp</span></code>, an output <code class="docutils literal"><span class="pre">Out</span></code>, and their corresponding descriptions, in accordance to Paddle&#8217;s <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/name_convention.md">naming convention</a>.</p>
<p>An additional example <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37"><code class="docutils literal"><span class="pre">ScaleOp</span></code></a> is implemented as follows:</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">template</span> <span class="o">&lt;</span><span class="k">typename</span> <span class="n">AttrType</span><span class="o">&gt;</span>
<span class="k">class</span> <span class="nc">ScaleOpMaker</span> <span class="o">:</span> <span class="k">public</span> <span class="n">framework</span><span class="o">::</span><span class="n">OpProtoAndCheckerMaker</span> <span class="p">{</span>
 <span class="k">public</span><span class="o">:</span>
  <span class="n">ScaleOpMaker</span><span class="p">(</span><span class="n">OpProto</span> <span class="o">*</span><span class="n">proto</span><span class="p">,</span> <span class="n">OpAttrChecker</span> <span class="o">*</span><span class="n">op_checker</span><span class="p">)</span>
      <span class="o">:</span> <span class="n">OpProtoAndCheckerMaker</span><span class="p">(</span><span class="n">proto</span><span class="p">,</span> <span class="n">op_checker</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">AddInput</span><span class="p">(</span><span class="s">&quot;X&quot;</span><span class="p">,</span> <span class="s">&quot;The input tensor of scale operator.&quot;</span><span class="p">).</span><span class="n">NotInGradient</span><span class="p">();</span>
    <span class="n">AddOutput</span><span class="p">(</span><span class="s">&quot;Out&quot;</span><span class="p">,</span> <span class="s">&quot;The output tensor of scale operator.&quot;</span><span class="p">).</span><span class="n">NotInGradient</span><span class="p">();</span>
    <span class="n">AddComment</span><span class="p">(</span><span class="sa">R</span><span class="s">&quot;</span><span class="dl">DOC(</span><span class="s">Scale operator</span>
<span class="s">The equation is: Out = scale*X</span>
<span class="dl">)DOC</span><span class="s">&quot;</span><span class="p">);</span>
    <span class="n">AddAttr</span><span class="o">&lt;</span><span class="n">AttrType</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;scale&quot;</span><span class="p">,</span> <span class="s">&quot;scale of scale operator.&quot;</span><span class="p">).</span><span class="n">SetDefault</span><span class="p">(</span><span class="mf">1.0</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p>There are two changes in this example:</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">AddInput(&quot;X&quot;,&quot;...&quot;).NotInGradient()</span></code> expresses that input <code class="docutils literal"><span class="pre">X</span></code> is not involved in <code class="docutils literal"><span class="pre">ScaleOp</span></code>&#8216;s corresponding computation. If an input to an operator is not participating in back-propagation, please explicitly set <code class="docutils literal"><span class="pre">.NotInGradient()</span></code>.</li>
<li><code class="docutils literal"><span class="pre">AddAttr&lt;AttrType&gt;(&quot;scale&quot;,</span> <span class="pre">&quot;...&quot;).SetDefault(1.0);</span></code>  adds <code class="docutils literal"><span class="pre">scale</span></code>constant as an attribute, and sets the default value to 1.0.</li>
</ul>
</div>
<div class="section" id="defining-operator">
<span id="defining-operator"></span><h3>Defining Operator<a class="headerlink" href="#defining-operator" title="Permalink to this headline">¶</a></h3>
<p>The following code defines the interface for MulOp:</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MulOp</span> <span class="o">:</span> <span class="k">public</span> <span class="n">framework</span><span class="o">::</span><span class="n">OperatorWithKernel</span> <span class="p">{</span>
 <span class="k">public</span><span class="o">:</span>
  <span class="k">using</span> <span class="n">framework</span><span class="o">::</span><span class="n">OperatorWithKernel</span><span class="o">::</span><span class="n">OperatorWithKernel</span><span class="p">;</span>

 <span class="k">protected</span><span class="o">:</span>
  <span class="kt">void</span> <span class="n">InferShape</span><span class="p">(</span><span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">InferShapeContext</span> <span class="o">&amp;</span><span class="n">ctx</span><span class="p">)</span> <span class="k">const</span> <span class="k">override</span> <span class="p">{</span>
    <span class="k">auto</span> <span class="n">dim0</span> <span class="o">=</span> <span class="n">ctx</span><span class="p">.</span><span class="n">Input</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;X&quot;</span><span class="p">)</span><span class="o">-&gt;</span><span class="n">dims</span><span class="p">();</span>
    <span class="k">auto</span> <span class="n">dim1</span> <span class="o">=</span> <span class="n">ctx</span><span class="p">.</span><span class="n">Input</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;Y&quot;</span><span class="p">)</span><span class="o">-&gt;</span><span class="n">dims</span><span class="p">();</span>
    <span class="n">PADDLE_ENFORCE_EQ</span><span class="p">(</span><span class="n">dim0</span><span class="p">.</span><span class="n">size</span><span class="p">(),</span> <span class="mi">2</span><span class="p">,</span>
                      <span class="s">&quot;input X(%s) should be a tensor with 2 dims, a matrix&quot;</span><span class="p">,</span>
                      <span class="n">ctx</span><span class="p">.</span><span class="n">op_</span><span class="p">.</span><span class="n">Input</span><span class="p">(</span><span class="s">&quot;X&quot;</span><span class="p">));</span>
    <span class="n">PADDLE_ENFORCE_EQ</span><span class="p">(</span><span class="n">dim1</span><span class="p">.</span><span class="n">size</span><span class="p">(),</span> <span class="mi">2</span><span class="p">,</span>
                      <span class="s">&quot;input Y(%s) should be a tensor with 2 dims, a matrix&quot;</span><span class="p">,</span>
                      <span class="n">ctx</span><span class="p">.</span><span class="n">op_</span><span class="p">.</span><span class="n">Input</span><span class="p">(</span><span class="s">&quot;Y&quot;</span><span class="p">));</span>
    <span class="n">PADDLE_ENFORCE_EQ</span><span class="p">(</span>
        <span class="n">dim0</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">dim1</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
        <span class="s">&quot;First matrix&#39;s width must be equal with second matrix&#39;s height.&quot;</span><span class="p">);</span>
    <span class="n">ctx</span><span class="p">.</span><span class="n">Output</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;Out&quot;</span><span class="p">)</span><span class="o">-&gt;</span><span class="n">Resize</span><span class="p">({</span><span class="n">dim0</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">dim1</span><span class="p">[</span><span class="mi">1</span><span class="p">]});</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p><a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/mul_op.cc#L22"><code class="docutils literal"><span class="pre">MulOp</span></code></a> is inherited from <code class="docutils literal"><span class="pre">OperatorWithKernel</span></code>. Its <code class="docutils literal"><span class="pre">public</span></code> member</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">using</span> <span class="n">framework</span><span class="o">::</span><span class="n">OperatorWithKernel</span><span class="o">::</span><span class="n">OperatorWithKernel</span><span class="p">;</span>
</pre></div>
</div>
<p>expresses an operator constructor using base class <code class="docutils literal"><span class="pre">OperatorWithKernel</span></code>, alternatively written as</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="n">MulOp</span><span class="p">(</span><span class="k">const</span> <span class="n">std</span><span class="o">::</span><span class="n">string</span> <span class="o">&amp;</span><span class="n">type</span><span class="p">,</span> <span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">VariableNameMap</span> <span class="o">&amp;</span><span class="n">inputs</span><span class="p">,</span>
      <span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">VariableNameMap</span> <span class="o">&amp;</span><span class="n">outputs</span><span class="p">,</span>
      <span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">AttributeMap</span> <span class="o">&amp;</span><span class="n">attrs</span><span class="p">)</span>
  <span class="o">:</span> <span class="n">OperatorWithKernel</span><span class="p">(</span><span class="n">type</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">outputs</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span> <span class="p">{}</span>
</pre></div>
</div>
<p><code class="docutils literal"><span class="pre">InferShape</span></code> interface needs to be re-written.<code class="docutils literal"><span class="pre">InferShape</span></code> is a constant method and cannot modify Op&#8217;s member variables, its constant member <code class="docutils literal"><span class="pre">const</span> <span class="pre">framework::InferShapeContext</span> <span class="pre">&amp;ctx</span></code> can be used to extract input, output, and attributes. It functions to</p>
<ul class="simple">
<li>1). validate and error out early: it checks input data dimensions and types.</li>
<li>2). configures the tensor shape in the output.</li>
</ul>
<p>Usually <code class="docutils literal"><span class="pre">OpProtoMaker</span></code> and <code class="docutils literal"><span class="pre">Op</span></code>&#8216;s type definitions are written in <code class="docutils literal"><span class="pre">.cc</span></code> files, which also include the registration methods introduced later.</p>
</div>
<div class="section" id="defining-opkernel">
<span id="defining-opkernel"></span><h3>Defining OpKernel<a class="headerlink" href="#defining-opkernel" title="Permalink to this headline">¶</a></h3>
<p><code class="docutils literal"><span class="pre">MulKernel</span></code> inherits <code class="docutils literal"><span class="pre">framework::OpKernel</span></code>, which includes the following templates:</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">typename</span> <span class="pre">DeviceContext</span></code> denotes device context type. When different devices, namely the CPUDeviceContext and the CUDADeviceContext, share the same kernel, this template needs to be added. If they don&#8217;t share kernels, this must not be added. An example of a non-sharing kernel is <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43"><code class="docutils literal"><span class="pre">OnehotCrossEntropyOpKernel</span></code></a>.</li>
<li><code class="docutils literal"><span class="pre">typename</span> <span class="pre">T</span></code> denotes data type, such as <code class="docutils literal"><span class="pre">float</span></code> or <code class="docutils literal"><span class="pre">double</span></code>.</li>
</ul>
<p><code class="docutils literal"><span class="pre">MulKernel</span></code> types need to rewrite the interface for <code class="docutils literal"><span class="pre">Compute</span></code>.</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">Compute</span></code> takes one input parameter: <code class="docutils literal"><span class="pre">const</span> <span class="pre">framework::ExecutionContext&amp;</span> <span class="pre">context</span></code>.</li>
<li>Compared with <code class="docutils literal"><span class="pre">InferShapeContext</span></code>, <code class="docutils literal"><span class="pre">ExecutionContext</span></code> includes device types, and can similarly extract input, output, and attribute variables.</li>
<li><code class="docutils literal"><span class="pre">Compute</span></code> implements the computation logics of an <code class="docutils literal"><span class="pre">OpKernel</span></code>.</li>
</ul>
<p><code class="docutils literal"><span class="pre">MulKernel</span></code>&#8216;s implementation of <code class="docutils literal"><span class="pre">Compute</span></code> is as follows:</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">template</span> <span class="o">&lt;</span><span class="k">typename</span> <span class="n">DeviceContext</span><span class="p">,</span> <span class="k">typename</span> <span class="n">T</span><span class="o">&gt;</span>
<span class="k">class</span> <span class="nc">MulKernel</span> <span class="o">:</span> <span class="k">public</span> <span class="n">framework</span><span class="o">::</span><span class="n">OpKernel</span> <span class="p">{</span>
<span class="k">public</span><span class="o">:</span>
<span class="kt">void</span> <span class="n">Compute</span><span class="p">(</span><span class="k">const</span> <span class="n">framework</span><span class="o">::</span><span class="n">ExecutionContext</span><span class="o">&amp;</span> <span class="n">context</span><span class="p">)</span> <span class="k">const</span> <span class="k">override</span> <span class="p">{</span>
  <span class="k">auto</span><span class="o">*</span> <span class="n">X</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="n">Input</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;X&quot;</span><span class="p">);</span>
  <span class="k">auto</span><span class="o">*</span> <span class="n">Y</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="n">Input</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;Y&quot;</span><span class="p">);</span>
  <span class="k">auto</span><span class="o">*</span> <span class="n">Z</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="n">Output</span><span class="o">&lt;</span><span class="n">Tensor</span><span class="o">&gt;</span><span class="p">(</span><span class="s">&quot;Out&quot;</span><span class="p">);</span>
  <span class="n">Z</span><span class="o">-&gt;</span><span class="n">mutable_data</span><span class="o">&lt;</span><span class="n">T</span><span class="o">&gt;</span><span class="p">(</span><span class="n">context</span><span class="p">.</span><span class="n">GetPlace</span><span class="p">());</span>
  <span class="k">auto</span><span class="o">&amp;</span> <span class="n">device_context</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="k">template</span> <span class="n">device_context</span><span class="o">&lt;</span><span class="n">DeviceContext</span><span class="o">&gt;</span><span class="p">();</span>
  <span class="n">math</span><span class="o">::</span><span class="n">matmul</span><span class="o">&lt;</span><span class="n">DeviceContext</span><span class="p">,</span> <span class="n">T</span><span class="o">&gt;</span><span class="p">(</span><span class="o">*</span><span class="n">X</span><span class="p">,</span> <span class="nb">false</span><span class="p">,</span> <span class="o">*</span><span class="n">Y</span><span class="p">,</span> <span class="nb">false</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">Z</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">device_context</span><span class="p">);</span>
<span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p>Note that <strong>different devices (CPU, CUDA)share one Op definition; whether or not they share the same <code class="docutils literal"><span class="pre">OpKernel</span></code> depends on whether <code class="docutils literal"><span class="pre">Compute</span></code> calls functions can support both devices.</strong></p>
<p><code class="docutils literal"><span class="pre">MulOp</span></code>&#8216;s CPU and CUDA share the same <code class="docutils literal"><span class="pre">Kernel</span></code>. A non-sharing  <code class="docutils literal"><span class="pre">OpKernel</span></code> example can be seen in <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43"><code class="docutils literal"><span class="pre">OnehotCrossEntropyOpKernel</span></code></a>.</p>
<p>To ease the writing of <code class="docutils literal"><span class="pre">OpKernel</span></code> compute, and for reusing code cross-device, <a class="reference external" href="https://bitbucket.org/eigen/eigen/src/default/unsupported/Eigen/CXX11/src/Tensor/README.md?fileviewer=file-view-default"><code class="docutils literal"><span class="pre">Eigen-unsupported</span> <span class="pre">Tensor</span></code></a> module is used to implement <code class="docutils literal"><span class="pre">Compute</span></code> interface. To learn about how the Eigen library is used in PaddlePaddle, please see <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md">usage document</a>.</p>
<p>This concludes the forward implementation of an operator. Next its operation and kernel need to be registered in a <code class="docutils literal"><span class="pre">.cc</span></code> file.</p>
<p>The definition of its corresponding backward operator, if applicable, is similar to that of an forward operator. <strong>Note that a backward operator does not include a <code class="docutils literal"><span class="pre">ProtoMaker</span></code></strong>.</p>
</div>
<div class="section" id="registering-operator-and-opkernel">
<span id="registering-operator-and-opkernel"></span><h3>Registering Operator and OpKernel<a class="headerlink" href="#registering-operator-and-opkernel" title="Permalink to this headline">¶</a></h3>
<ul>
<li><p class="first">In <code class="docutils literal"><span class="pre">.cc</span></code> files, register forward and backward operator classes and the CPU kernel.</p>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="k">namespace</span> <span class="n">ops</span> <span class="o">=</span> <span class="n">paddle</span><span class="o">::</span><span class="n">operators</span><span class="p">;</span>
<span class="n">REGISTER_OP</span><span class="p">(</span><span class="n">mul</span><span class="p">,</span> <span class="n">ops</span><span class="o">::</span><span class="n">MulOp</span><span class="p">,</span> <span class="n">ops</span><span class="o">::</span><span class="n">MulOpMaker</span><span class="p">,</span> <span class="n">mul_grad</span><span class="p">,</span> <span class="n">ops</span><span class="o">::</span><span class="n">MulOpGrad</span><span class="p">);</span>

<span class="n">REGISTER_OP_CPU_KERNEL</span><span class="p">(</span><span class="n">mul</span><span class="p">,</span> <span class="n">ops</span><span class="o">::</span><span class="n">MulKernel</span><span class="o">&lt;</span><span class="n">paddle</span><span class="o">::</span><span class="n">platform</span><span class="o">::</span><span class="n">CPUDeviceContext</span><span class="p">,</span> <span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span>
<span class="n">REGISTER_OP_CPU_KERNEL</span><span class="p">(</span><span class="n">mul_grad</span><span class="p">,</span>
              <span class="n">ops</span><span class="o">::</span><span class="n">MulGradKernel</span><span class="o">&lt;</span><span class="n">paddle</span><span class="o">::</span><span class="n">platform</span><span class="o">::</span><span class="n">CPUDeviceContext</span><span class="p">,</span> <span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span>
</pre></div>
</div>
<p>In that code block,</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">REGISTER_OP</span></code> registers the <code class="docutils literal"><span class="pre">ops::MulOp</span></code> class, type named <code class="docutils literal"><span class="pre">mul</span></code>, its type <code class="docutils literal"><span class="pre">ProtoMaker</span></code> is <code class="docutils literal"><span class="pre">ops::MulOpMaker</span></code>, registering <code class="docutils literal"><span class="pre">ops::MulOpGrad</span></code> as <code class="docutils literal"><span class="pre">mul_grad</span></code>.</li>
<li><code class="docutils literal"><span class="pre">REGISTER_OP_WITHOUT_GRADIENT</span></code> registers an operator without gradient.</li>
<li><code class="docutils literal"><span class="pre">REGISTER_OP_CPU_KERNEL</span></code> registers <code class="docutils literal"><span class="pre">ops::MulKernel</span></code> class and specialized template types <code class="docutils literal"><span class="pre">paddle::platform::CPUPlace</span></code> and <code class="docutils literal"><span class="pre">float</span></code>, which also registers <code class="docutils literal"><span class="pre">ops::MulGradKernel</span></code>.</li>
</ul>
</li>
</ul>
<ul>
<li><p class="first">Registering CUDA Kernel in <code class="docutils literal"><span class="pre">.cu</span></code> files</p>
<ul class="simple">
<li>Note that if CUDA Kernel is implemented using the <code class="docutils literal"><span class="pre">Eigen</span> <span class="pre">unsupported</span></code> module, then on top of <code class="docutils literal"><span class="pre">.cu</span></code>, a macro definition <code class="docutils literal"><span class="pre">#define</span> <span class="pre">EIGEN_USE_GPU</span></code> is needed, such as</li>
</ul>
<div class="highlight-cpp"><div class="highlight"><pre><span></span><span class="c1">// if use Eigen unsupported module before include head files</span>
<span class="cp">#define EIGEN_USE_GPU</span>

<span class="k">namespace</span> <span class="n">ops</span> <span class="o">=</span> <span class="n">paddle</span><span class="o">::</span><span class="n">operators</span><span class="p">;</span>
<span class="n">REGISTER_OP_CUDA_KERNEL</span><span class="p">(</span><span class="n">mul</span><span class="p">,</span> <span class="n">ops</span><span class="o">::</span><span class="n">MulKernel</span><span class="o">&lt;</span><span class="n">paddle</span><span class="o">::</span><span class="n">platform</span><span class="o">::</span><span class="n">CUDADeviceContext</span><span class="p">,</span> <span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span>
<span class="n">REGISTER_OP_CUDA_KERNEL</span><span class="p">(</span><span class="n">mul_grad</span><span class="p">,</span>
                       <span class="n">ops</span><span class="o">::</span><span class="n">MulGradKernel</span><span class="o">&lt;</span><span class="n">paddle</span><span class="o">::</span><span class="n">platform</span><span class="o">::</span><span class="n">CUDADeviceContext</span><span class="p">,</span> <span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span>
</pre></div>
</div>
</li>
</ul>
</div>
<div class="section" id="compilation">
<span id="compilation"></span><h3>Compilation<a class="headerlink" href="#compilation" title="Permalink to this headline">¶</a></h3>
<p>Run the following commands to compile.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="c1"># maybe you need to rerun cmake</span>
<span class="n">make</span> <span class="n">mul_op</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="python-binding">
<span id="python-binding"></span><h2>Python Binding<a class="headerlink" href="#python-binding" title="Permalink to this headline">¶</a></h2>
<p>The system will automatically bind to Python and link it to a generated library.</p>
</div>
<div class="section" id="unit-tests">
<span id="unit-tests"></span><h2>Unit Tests<a class="headerlink" href="#unit-tests" title="Permalink to this headline">¶</a></h2>
<p>Unit tests for an operator include</p>
<ol class="simple">
<li>comparing a forward operator&#8217;s implementations on different devices,</li>
<li>comparing a backward operator&#8217;s implementation on different devices, and</li>
<li>a scaling test for the backward operator.</li>
</ol>
<p>Here, we introduce the <a class="reference external" href="https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py">unit tests for <code class="docutils literal"><span class="pre">MulOp</span></code></a>.</p>
<div class="section" id="testing-forward-operators">
<span id="testing-forward-operators"></span><h3>Testing Forward Operators<a class="headerlink" href="#testing-forward-operators" title="Permalink to this headline">¶</a></h3>
<p>A forward operator unit test inherits <code class="docutils literal"><span class="pre">unittest.TestCase</span></code> and defines metaclass <code class="docutils literal"><span class="pre">__metaclass__</span> <span class="pre">=</span> <span class="pre">OpTestMeta</span></code>. More concrete tests are performed in <code class="docutils literal"><span class="pre">OpTestMeta</span></code>. Testing a forward operator requires the following:</p>
<ol class="simple">
<li>Defining input, output and relevant attributes in <code class="docutils literal"><span class="pre">setUp</span></code> method.</li>
<li>Generating random input data.</li>
<li>Implementing the same computation logic in a Python script.</li>
<li>Call check gradient function to check the backward operator.</li>
</ol>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">unittest</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">op_test</span> <span class="kn">import</span> <span class="n">OpTest</span>


<span class="k">class</span> <span class="nc">TestMulOp</span><span class="p">(</span><span class="n">OpTest</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">op_type</span> <span class="o">=</span> <span class="s2">&quot;mul&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inputs</span> <span class="o">=</span> <span class="p">{</span>
            <span class="s1">&#39;X&#39;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">((</span><span class="mi">32</span><span class="p">,</span> <span class="mi">84</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">),</span>
            <span class="s1">&#39;Y&#39;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">((</span><span class="mi">84</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">)</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">outputs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;Out&#39;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">inputs</span><span class="p">[</span><span class="s1">&#39;X&#39;</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">inputs</span><span class="p">[</span><span class="s1">&#39;Y&#39;</span><span class="p">])}</span>

    <span class="k">def</span> <span class="nf">test_check_output</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">check_output</span><span class="p">()</span>
        
    <span class="k">def</span> <span class="nf">test_check_grad_normal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">check_grad</span><span class="p">([</span><span class="s1">&#39;X&#39;</span><span class="p">,</span> <span class="s1">&#39;Y&#39;</span><span class="p">],</span> <span class="s1">&#39;Out&#39;</span><span class="p">,</span> <span class="n">max_relative_error</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">test_check_grad_ingore_x</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">check_grad</span><span class="p">(</span>
            <span class="p">[</span><span class="s1">&#39;Y&#39;</span><span class="p">],</span> <span class="s1">&#39;Out&#39;</span><span class="p">,</span> <span class="n">max_relative_error</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">no_grad_set</span><span class="o">=</span><span class="nb">set</span><span class="p">(</span><span class="s2">&quot;X&quot;</span><span class="p">))</span>

    <span class="k">def</span> <span class="nf">test_check_grad_ingore_y</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">check_grad</span><span class="p">(</span>
            <span class="p">[</span><span class="s1">&#39;X&#39;</span><span class="p">],</span> <span class="s1">&#39;Out&#39;</span><span class="p">,</span> <span class="n">max_relative_error</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">no_grad_set</span><span class="o">=</span><span class="nb">set</span><span class="p">(</span><span class="s1">&#39;Y&#39;</span><span class="p">))</span>
</pre></div>
</div>
<p>Get its output, and compare it with the forward operator&#8217;s own output.</p>
<p>The code above first loads required packages. In addition, we have</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">self.op_type</span> <span class="pre">=</span> <span class="pre">&quot;mul&quot;</span></code> defines the type that is identical to what the operator&#8217;s registered type.</li>
<li><code class="docutils literal"><span class="pre">self.inputs</span></code> defines input, with type <code class="docutils literal"><span class="pre">numpy.array</span></code> and initializes it.</li>
<li><code class="docutils literal"><span class="pre">self.outputs</span></code> defines output and completes the same operator computation in the Python script, and returns its result from the Python script.</li>
</ul>
</div>
<div class="section" id="testing-backward-operators">
<span id="testing-backward-operators"></span><h3>Testing Backward Operators<a class="headerlink" href="#testing-backward-operators" title="Permalink to this headline">¶</a></h3>
<p>Some key points in checking gradient above include:</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">test_normal</span></code> calls <code class="docutils literal"><span class="pre">check_grad</span></code> to validate scaling tests&#8217; correctness and stability through numeric methods.<ul>
<li>The first variable <code class="docutils literal"><span class="pre">[&quot;X&quot;,</span> <span class="pre">&quot;Y&quot;]</span></code> appoints <code class="docutils literal"><span class="pre">X</span></code> and <code class="docutils literal"><span class="pre">Y</span></code> to be scale tested.</li>
<li>The second variable <code class="docutils literal"><span class="pre">&quot;Out&quot;</span></code> points to the network&#8217;s final output target <code class="docutils literal"><span class="pre">Out</span></code>.</li>
<li>The third variable <code class="docutils literal"><span class="pre">max_relative_error</span></code> points to the maximum relative tolerance error during scaling tests.</li>
</ul>
</li>
<li><code class="docutils literal"><span class="pre">test_check_grad_ingore_x</span></code> and <code class="docutils literal"><span class="pre">test_check_grad_ingore_y</span></code>branches test the cases where there is only one scaling input.</li>
</ul>
</div>
<div class="section" id="compiling-and-running">
<span id="compiling-and-running"></span><h3>Compiling and Running<a class="headerlink" href="#compiling-and-running" title="Permalink to this headline">¶</a></h3>
<p>Any new unit testing file of the format <code class="docutils literal"><span class="pre">test_*.py</span></code>  added to the director <code class="docutils literal"><span class="pre">python/paddle/v2/framework/tests</span></code> is automatically added to the project to compile.</p>
<p>Note that <strong>unlike the compile test for Ops, running unit tests requires compiling the entire project</strong> and requires compiling with flag <code class="docutils literal"><span class="pre">WITH_TESTING</span></code> on i.e. <code class="docutils literal"><span class="pre">cmake</span> <span class="pre">paddle_dir</span> <span class="pre">-DWITH_TESTING=ON</span></code>.</p>
<p>After successfully compiling the project, run the following command to run unit tests:</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>make <span class="nb">test</span> <span class="nv">ARGS</span><span class="o">=</span><span class="s2">&quot;-R test_mul_op -V&quot;</span>
</pre></div>
</div>
<p>Or,</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>ctest -R test_mul_op
</pre></div>
</div>
</div>
</div>
<div class="section" id="remarks">
<span id="remarks"></span><h2>Remarks<a class="headerlink" href="#remarks" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li>Every <code class="docutils literal"><span class="pre">*_op.h</span></code> (if applicable), <code class="docutils literal"><span class="pre">*_op.cc</span></code>, and <code class="docutils literal"><span class="pre">*_op.cu</span></code> (if applicable) must be created for a unique Op. Compiling will fail if multiple operators are included per file.</li>
<li>The type with which an operator is registered needs to be identical to the Op&#8217;s name. Registering <code class="docutils literal"><span class="pre">REGISTER_OP(B,</span> <span class="pre">...)</span></code> in <code class="docutils literal"><span class="pre">A_op.cc</span></code> will cause unit testing failures.</li>
<li>If the operator does not implement a CUDA kernel, please refrain from creating an empty <code class="docutils literal"><span class="pre">*_op.cu</span></code> file, or else unit tests will fail.</li>
<li>If multiple operators rely on some shared methods, a file NOT named <code class="docutils literal"><span class="pre">*_op.*</span></code> can be created to store them, such as <code class="docutils literal"><span class="pre">gather.h</span></code>.</li>
</ul>
</div>
</div>


           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>

  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>