optimizer.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Optimizer &mdash; PaddlePaddle  documentation</title>
  

    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="Index"
              href="../../../genindex.html"/>
        <link rel="search" title="Search" href="../../../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../../../index.html"/>
        <link rel="up" title="Fluid" href="../fluid.html"/>
        <link rel="next" title="ParamAttr" href="param_attr.html"/>
        <link rel="prev" title="Nets" href="nets.html"/> 

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/override.css" type="text/css" />
  <script>
  var _hmt = _hmt || [];
  (function() {
    var hm = document.createElement("script");
    hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
    var s = document.getElementsByTagName("script")[0]; 
    s.parentNode.insertBefore(hm, s);
  })();
  </script>

  
  <script src="../../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  
  <header class="site-header">
    <div class="site-logo">
      <a href="/"><img src="../../../_static/images/PP_w.png"></a>
    </div>
    <div class="site-nav-links">
      <div class="site-menu">
        <a class="fork-on-github" href="https://github.com/PaddlePaddle/Paddle" target="_blank"><i class="fa fa-github"></i>Fork me on Github</a>
        <div class="language-switcher dropdown">
          <a type="button" data-toggle="dropdown">
            <span>English</span>
            <i class="fa fa-angle-up"></i>
            <i class="fa fa-angle-down"></i>
          </a>
          <ul class="dropdown-menu">
            <li><a href="/doc_cn">中文</a></li>
            <li><a href="/doc">English</a></li>
          </ul>
        </div>
        <ul class="site-page-links">
          <li><a href="/">Home</a></li>
        </ul>
      </div>
      <div class="doc-module">
        
        <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../getstarted/index_en.html">GET STARTED</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../howto/index_en.html">HOW TO</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../index_en.html">API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../mobile/index_en.html">MOBILE</a></li>
</ul>

        
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>        
      </div>
    </div>
  </header>
  
  <div class="main-content-wrap">

    
    <nav class="doc-menu-vertical" role="navigation">
        
          
          <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../getstarted/index_en.html">GET STARTED</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../getstarted/build_and_install/index_en.html">Install and Build</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../../getstarted/build_and_install/pip_install_en.html">Install Using pip</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../getstarted/build_and_install/docker_install_en.html">Run in Docker Containers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../howto/dev/build_en.html">Build using Docker</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../getstarted/build_and_install/build_from_source_en.html">Build from Sources</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../howto/index_en.html">HOW TO</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/usage/cmd_parameter/index_en.html">Set Command-line Parameters</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../../howto/usage/cmd_parameter/use_case_en.html">Use Case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../howto/usage/cmd_parameter/arguments_en.html">Argument Outline</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../howto/usage/cmd_parameter/detail_introduction_en.html">Detail Description</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/usage/cluster/cluster_train_en.html">PaddlePaddle Distributed Training</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/usage/k8s/k8s_en.html">Paddle On Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/usage/k8s/k8s_aws_en.html">Distributed PaddlePaddle Training on AWS with Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/dev/new_layer_en.html">Write New Layers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/dev/contribute_to_paddle_en.html">Contribute Code</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/dev/write_docs_en.html">Contribute Documentation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/deep_model/rnn/index_en.html">RNN Models</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../../howto/deep_model/rnn/rnn_config_en.html">RNN Configuration</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../howto/optimization/gpu_profiling_en.html">Tune GPU Performance</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="../../index_en.html">API</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../model_configs.html">Model Configuration</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../config/activation.html">Activation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/layer.html">Layers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/evaluators.html">Evaluators</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/optimizer.html">Optimizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/pooling.html">Pooling</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/networks.html">Networks</a></li>
<li class="toctree-l3"><a class="reference internal" href="../config/attr.html">Parameter Attribute</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data.html">Data Reader Interface and DataSets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data/data_reader.html">Data Reader Interface</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data/image.html">Image Interface</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data/dataset.html">Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../run_logic.html">Training and Inference</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../fluid.html">Fluid</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="layers.html">Layers</a></li>
<li class="toctree-l3"><a class="reference internal" href="data_feeder.html">DataFeeder</a></li>
<li class="toctree-l3"><a class="reference internal" href="executor.html">Executor</a></li>
<li class="toctree-l3"><a class="reference internal" href="initializer.html">Initializer</a></li>
<li class="toctree-l3"><a class="reference internal" href="evaluator.html">Evaluator</a></li>
<li class="toctree-l3"><a class="reference internal" href="nets.html">Nets</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Optimizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="param_attr.html">ParamAttr</a></li>
<li class="toctree-l3"><a class="reference internal" href="profiler.html">Profiler</a></li>
<li class="toctree-l3"><a class="reference internal" href="regularizer.html">Regularizer</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../mobile/index_en.html">MOBILE</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../mobile/cross_compiling_for_android_en.html">Build PaddlePaddle for Android</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../mobile/cross_compiling_for_raspberry_en.html">Build PaddlePaddle for Raspberry Pi</a></li>
</ul>
</li>
</ul>

        
    </nav>
    
    <section class="doc-content-wrap">

      
<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
      
        <li><a href="../../index_en.html">API</a> > </li>
      
        <li><a href="../fluid.html">Fluid</a> > </li>
      
    <li>Optimizer</li>
  </ul>
</div>
      
      <div class="wy-nav-content" id="doc-content">
        <div class="rst-content">
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="optimizer">
<h1>Optimizer<a class="headerlink" href="#optimizer" title="Permalink to this headline">¶</a></h1>
<div class="section" id="id1">
<h2>Optimizer<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">Optimizer</code><span class="sig-paren">(</span><em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Optimizer Base class.</p>
<p>Define the common interface of an optimizer.
User should not use this class directly,
but need to use one of it&#8217;s implementation.</p>
<dl class="method">
<dt>
<code class="descname">create_optimization_pass</code><span class="sig-paren">(</span><em>parameters_and_grads</em>, <em>loss</em>, <em>startup_program=None</em><span class="sig-paren">)</span></dt>
<dd><p>Add optimization operators to update gradients to variables.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>loss</strong> &#8211; the target that this optimization is for.</li>
<li><strong>parameters_and_grads</strong> &#8211; a list of (variable, gradient) pair to update.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">a list of operators that will complete one step of
optimization. This will include parameter update ops, global step
update ops and any other custom ops required by subclasses to manage
their internal state.
:param startup_program:</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">return_op_list</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="method">
<dt>
<code class="descname">minimize</code><span class="sig-paren">(</span><em>loss</em>, <em>startup_program=None</em>, <em>parameter_list=None</em>, <em>no_grad_set=None</em><span class="sig-paren">)</span></dt>
<dd><p>Add operations to minimize <cite>loss</cite> by updating <cite>parameter_list</cite>.</p>
<p>This method combines interface <cite>append_backward_ops()</cite> and
<cite>create_optimization_pass()</cite> into one.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="sgdoptimizer">
<h2>SGDOptimizer<a class="headerlink" href="#sgdoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">SGDOptimizer</code><span class="sig-paren">(</span><em>learning_rate</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Simple SGD optimizer without any state.</p>
</dd></dl>

</div>
<div class="section" id="momentumoptimizer">
<h2>MomentumOptimizer<a class="headerlink" href="#momentumoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">MomentumOptimizer</code><span class="sig-paren">(</span><em>learning_rate</em>, <em>momentum</em>, <em>use_nesterov=False</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Simple Momentum optimizer with velocity state</p>
</dd></dl>

</div>
<div class="section" id="adagradoptimizer">
<h2>AdagradOptimizer<a class="headerlink" href="#adagradoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">AdagradOptimizer</code><span class="sig-paren">(</span><em>learning_rate</em>, <em>epsilon=1e-06</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Simple Adagrad optimizer with moment state</p>
</dd></dl>

</div>
<div class="section" id="adamoptimizer">
<h2>AdamOptimizer<a class="headerlink" href="#adamoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">AdamOptimizer</code><span class="sig-paren">(</span><em>learning_rate=0.001</em>, <em>beta1=0.9</em>, <em>beta2=0.999</em>, <em>epsilon=1e-08</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Implements the Adam Optimizer</p>
</dd></dl>

</div>
<div class="section" id="adamaxoptimizer">
<h2>AdamaxOptimizer<a class="headerlink" href="#adamaxoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">AdamaxOptimizer</code><span class="sig-paren">(</span><em>learning_rate=0.001</em>, <em>beta1=0.9</em>, <em>beta2=0.999</em>, <em>epsilon=1e-08</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Implements the Adamax Optimizer</p>
</dd></dl>

</div>
<div class="section" id="decayedadagradoptimizer">
<h2>DecayedAdagradOptimizer<a class="headerlink" href="#decayedadagradoptimizer" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.fluid.optimizer.</code><code class="descname">DecayedAdagradOptimizer</code><span class="sig-paren">(</span><em>learning_rate</em>, <em>decay=0.95</em>, <em>epsilon=1e-06</em>, <em>global_step=None</em><span class="sig-paren">)</span></dt>
<dd><p>Simple Decayed Adagrad optimizer with moment state</p>
</dd></dl>

</div>
</div>


           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="param_attr.html" class="btn btn-neutral float-right" title="ParamAttr" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="nets.html" class="btn btn-neutral" title="Nets" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: ".txt",
        };
    </script>
      <script type="text/javascript" src="../../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
       
  
    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
  
  
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/js/perfect-scrollbar.jquery.min.js"></script>
  <script src="../../../_static/js/paddle_doc_init.js"></script> 

</body>
</html>