data_reader.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Data Reader Interface &mdash; PaddlePaddle  documentation</title>
  

    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="Index"
              href="../../genindex.html"/>
        <link rel="search" title="Search" href="../../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../../index.html"/>
        <link rel="up" title="Data Reader Interface and DataSets" href="../data.html"/>
        <link rel="next" title="Image Interface" href="image.html"/>
        <link rel="prev" title="Data Reader Interface and DataSets" href="../data.html"/> 

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/override.css" type="text/css" />
  <script>
  var _hmt = _hmt || [];
  (function() {
    var hm = document.createElement("script");
    hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
    var s = document.getElementsByTagName("script")[0]; 
    s.parentNode.insertBefore(hm, s);
  })();
  </script>

  
  <script src="../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  
  <header class="site-header">
    <div class="site-logo">
      <a href="/"><img src="../../_static/images/PP_w.png"></a>
    </div>
    <div class="site-nav-links">
      <div class="site-menu">
        <a class="fork-on-github" href="https://github.com/PaddlePaddle/Paddle" target="_blank"><i class="fa fa-github"></i>Fork me on Github</a>
        <div class="language-switcher dropdown">
          <a type="button" data-toggle="dropdown">
            <span>English</span>
            <i class="fa fa-angle-up"></i>
            <i class="fa fa-angle-down"></i>
          </a>
          <ul class="dropdown-menu">
            <li><a href="/doc_cn">中文</a></li>
            <li><a href="/doc">English</a></li>
          </ul>
        </div>
        <ul class="site-page-links">
          <li><a href="/">Home</a></li>
        </ul>
      </div>
      <div class="doc-module">
        
        <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../overview.html">V2 API Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_configs.html">Model Configuration</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../data.html">Data Reader Interface and DataSets</a></li>
<li class="toctree-l1"><a class="reference internal" href="../run_logic.html">Training and Inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../fluid.html">Fluid</a></li>
</ul>

        
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>        
      </div>
    </div>
  </header>
  
  <div class="main-content-wrap">

    
    <nav class="doc-menu-vertical" role="navigation">
        
          
          <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../overview.html">V2 API Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_configs.html">Model Configuration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../config/activation.html">Activation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/layer.html">Layers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/evaluators.html">Evaluators</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/optimizer.html">Optimizer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/pooling.html">Pooling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/networks.html">Networks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../config/attr.html">Parameter Attribute</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="../data.html">Data Reader Interface and DataSets</a><ul class="current">
<li class="toctree-l2 current"><a class="current reference internal" href="#">Data Reader Interface</a></li>
<li class="toctree-l2"><a class="reference internal" href="image.html">Image Interface</a></li>
<li class="toctree-l2"><a class="reference internal" href="dataset.html">Dataset</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../run_logic.html">Training and Inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../fluid.html">Fluid</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../fluid/layers.html">layers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/data_feeder.html">data_feeder</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/executor.html">executor</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/initializer.html">initializer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/evaluator.html">evaluator</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/nets.html">nets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/optimizer.html">optimizer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/param_attr.html">param_attr</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/profiler.html">profiler</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/regularizer.html">regularizer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../fluid/io.html">io</a></li>
</ul>
</li>
</ul>

        
    </nav>
    
    <section class="doc-content-wrap">

      
<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
      
        <li><a href="../data.html">Data Reader Interface and DataSets</a> > </li>
      
    <li>Data Reader Interface</li>
  </ul>
</div>
      
      <div class="wy-nav-content" id="doc-content">
        <div class="rst-content">
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="data-reader-interface">
<h1>Data Reader Interface<a class="headerlink" href="#data-reader-interface" title="Permalink to this headline">¶</a></h1>
<div class="section" id="datatypes">
<h2>DataTypes<a class="headerlink" href="#datatypes" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">dense_array</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Dense Array. It means the input feature is dense array with float type.
For example, if the input is an image with 28*28 pixels, the input of
Paddle neural network could be a dense vector with dimension 784 or a
numpy array with shape (28, 28).</p>
<p>For the 2-D convolution operation, each sample in one mini-batch must have
the similarly size in PaddlePaddle now. But, it supports variable-dimension
feature across mini-batch. For the variable-dimension, the param dim is not
used. While the data reader must yield numpy array and the data feeder will
set the data shape correctly.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">dense_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Dense Array. It means the input feature is dense array with float type.
For example, if the input is an image with 28*28 pixels, the input of
Paddle neural network could be a dense vector with dimension 784 or a
numpy array with shape (28, 28).</p>
<p>For the 2-D convolution operation, each sample in one mini-batch must have
the similarly size in PaddlePaddle now. But, it supports variable-dimension
feature across mini-batch. For the variable-dimension, the param dim is not
used. While the data reader must yield numpy array and the data feeder will
set the data shape correctly.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">dense_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of dense vector.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of dense vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">integer_value</code><span class="sig-paren">(</span><em>value_range</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of integer.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
<li><strong>value_range</strong> (<em>int</em>) &#8211; range of this integer.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">integer_value_sequence</code><span class="sig-paren">(</span><em>value_range</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of integer.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>value_range</strong> (<em>int</em>) &#8211; range of each element.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_binary_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse binary vector. It means the input feature is a sparse vector and the
every element in this vector is either zero or one.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_binary_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><dl class="docutils">
<dt>Data type of a sequence of sparse vector, which every element is either zero</dt>
<dd>or one.</dd>
</dl>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of sparse vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_float_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse vector. It means the input feature is a sparse vector. Most of the
elements in this vector are zero, others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_float_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of sparse vector, which most elements are zero,
others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of sparse vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_non_value_slot</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse binary vector. It means the input feature is a sparse vector and the
every element in this vector is either zero or one.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_value_slot</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse vector. It means the input feature is a sparse vector. Most of the
elements in this vector are zero, others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.data_type.</code><code class="descname">InputType</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type</em>, <em>tp</em><span class="sig-paren">)</span></dt>
<dd><p>InputType is the base class for paddle input types.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">this is a base class, and should never be used by user.</p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of input. If the input is an integer, it means the
value range. Otherwise, it means the size of layer.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of input. 0 means it is not a sequence. 1
means it is a variable length sequence. 2 means it is a
nested sequence.</li>
<li><strong>type</strong> (<em>int</em>) &#8211; data type of input.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="datafeeder">
<h2>DataFeeder<a class="headerlink" href="#datafeeder" title="Permalink to this headline">¶</a></h2>
</div>
<div class="section" id="reader">
<h2>Reader<a class="headerlink" href="#reader" title="Permalink to this headline">¶</a></h2>
<p>At training and testing time, PaddlePaddle programs need to read data. To ease
the users&#8217; work to write data reading code, we define that</p>
<ul class="simple">
<li>A <em>reader</em> is a function that reads data (from file, network, random number
generator, etc) and yields data items.</li>
<li>A <em>reader creator</em> is a function that returns a reader function.</li>
<li>A <em>reader decorator</em> is a function, which accepts one or more readers, and
returns a reader.</li>
<li>A <em>batch reader</em> is a function that reads data (from <em>reader</em>, file, network,
random number generator, etc) and yields a batch of data items.</li>
</ul>
<div class="section" id="id1">
<h3>Data Reader Interface<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
<p>Indeed, <em>data reader</em> doesn&#8217;t have to be a function that reads and yields data
items. It can be any function with no parameter that creates a iterable
(anything can be used in <code class="code docutils literal"><span class="pre">for</span> <span class="pre">x</span> <span class="pre">in</span> <span class="pre">iterable</span></code>):</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">iterable</span> <span class="o">=</span> <span class="n">data_reader</span><span class="p">()</span>
</pre></div>
</div>
<p>Element produced from the iterable should be a <strong>single</strong> entry of data,
<strong>not</strong> a mini batch. That entry of data could be a single item, or a tuple of
items.
Item should be of <a class="reference external" href="http://www.paddlepaddle.org/doc/ui/data_provider/pydataprovider2.html?highlight=dense_vector#input-types">supported type</a> (e.g., numpy 1d
array of float32, int, list of int)</p>
<p>An example implementation for single item data reader creator:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">reader_creator_random_image</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">reader</span><span class="p">():</span>
        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">numpy</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">width</span><span class="o">*</span><span class="n">height</span><span class="p">)</span>
<span class="k">return</span> <span class="n">reader</span>
</pre></div>
</div>
<p>An example implementation for multiple item data reader creator:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">reader_creator_random_image_and_label</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">label</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">reader</span><span class="p">():</span>
        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">numpy</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">width</span><span class="o">*</span><span class="n">height</span><span class="p">),</span> <span class="n">label</span>
<span class="k">return</span> <span class="n">reader</span>
</pre></div>
</div>
<p>TODO(yuyang18): Should we add whole design doc here?</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">map_readers</code><span class="sig-paren">(</span><em>func</em>, <em>*readers</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader that outputs return value of function using
output of each data readers as arguments.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>func</strong> &#8211; function to use. The type of func should be (Sample) =&gt; Sample</li>
<li><strong>readers</strong> &#8211; readers whose outputs will be used as arguments of func.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Type:</th><td class="field-body"><p class="first">callable</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the created data reader.</p>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">buffered</code><span class="sig-paren">(</span><em>reader</em>, <em>size</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a buffered data reader.</p>
<p>The buffered data reader will read and save data entries into a
buffer. Reading from the buffered data reader will proceed as long
as the buffer is not empty.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>size</strong> (<em>int</em>) &#8211; max buffer size.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">the buffered data reader.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">compose</code><span class="sig-paren">(</span><em>*readers</em>, <em>**kwargs</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose output is the combination of input readers.</p>
<p>If input readers output following data entries:
(1, 2)    3    (4, 5)
The composed reader will output:
(1, 2, 3, 4, 5)</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>readers</strong> &#8211; readers that will be composed together.</li>
<li><strong>check_alignment</strong> (<em>bool</em>) &#8211; if True, will check if input readers are aligned
correctly. If False, will not check alignment and trailing outputs
will be discarded. Defaults to True.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the new data reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><strong>ComposeNotAligned</strong> &#8211; outputs of readers are not aligned.
Will not raise when check_alignment is set to False.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">chain</code><span class="sig-paren">(</span><em>*readers</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose output is the outputs of input data
readers chained together.</p>
<p>If input readers output following data entries:
[0, 0, 0]
[1, 1, 1]
[2, 2, 2]
The chained reader will output:
[0, 0, 0, 1, 1, 1, 2, 2, 2]</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>readers</strong> &#8211; input readers.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">the new data reader.</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">callable</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">shuffle</code><span class="sig-paren">(</span><em>reader</em>, <em>buf_size</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose data output is shuffled.</p>
<p>Output from the iterator that created by original reader will be
buffered into shuffle buffer, and then shuffled. The size of shuffle buffer
is determined by argument buf_size.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the original reader whose output will be shuffled.</li>
<li><strong>buf_size</strong> (<em>int</em>) &#8211; shuffle buffer size.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the new reader whose output is shuffled.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">firstn</code><span class="sig-paren">(</span><em>reader</em>, <em>n</em><span class="sig-paren">)</span></dt>
<dd><p>Limit the max number of samples that reader could return.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>n</strong> (<em>int</em>) &#8211; the max number of samples that return.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the decorated reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">xmap_readers</code><span class="sig-paren">(</span><em>mapper</em>, <em>reader</em>, <em>process_num</em>, <em>buffer_size</em>, <em>order=False</em><span class="sig-paren">)</span></dt>
<dd><p>Use multiprocess to map samples from reader by a mapper defined by user.
And this function contains a buffered decorator.
:param mapper:  a function to map sample.
:type mapper: callable
:param reader: the data reader to read from
:type reader: callable
:param process_num: process number to handle original sample
:type process_num: int
:param buffer_size: max buffer size
:type buffer_size: int
:param order: keep the order of reader
:type order: bool
:return: the decarated reader
:rtype: callable</p>
</dd></dl>

<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.reader.</code><code class="descname">PipeReader</code><span class="sig-paren">(</span><em>command</em>, <em>bufsize=8192</em>, <em>file_type='plain'</em><span class="sig-paren">)</span></dt>
<dd><p>PipeReader read data by stream from a command, take it&#8217;s
stdout into a pipe buffer and redirect it to the parser to
parse, then yield data as your desired format.</p>
<p>You can using standard linux command or call another program
to read data, from HDFS, Ceph, URL, AWS S3 etc:</p>
<p>An example:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">example_reader</span><span class="p">():</span>
    <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">myfiles</span><span class="p">:</span>
        <span class="n">pr</span> <span class="o">=</span> <span class="n">PipeReader</span><span class="p">(</span><span class="s2">&quot;cat </span><span class="si">%s</span><span class="s2">&quot;</span><span class="o">%</span><span class="n">f</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">pr</span><span class="o">.</span><span class="n">get_line</span><span class="p">():</span>
            <span class="n">sample</span> <span class="o">=</span> <span class="n">l</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">)</span>
            <span class="k">yield</span> <span class="n">sample</span>
</pre></div>
</div>
<dl class="method">
<dt>
<code class="descname">get_line</code><span class="sig-paren">(</span><em>cut_lines=True</em>, <em>line_break='\n'</em><span class="sig-paren">)</span></dt>
<dd><blockquote>
<div><table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name" colspan="2">param cut_lines:</th></tr>
<tr class="field-odd field"><td>&#160;</td><td class="field-body">cut buffer to lines</td>
</tr>
<tr class="field-even field"><th class="field-name">type cut_lines:</th><td class="field-body">bool</td>
</tr>
<tr class="field-odd field"><th class="field-name" colspan="2">param line_break:</th></tr>
<tr class="field-odd field"><td>&#160;</td><td class="field-body">line break of the file, like</td>
</tr>
</tbody>
</table>
</div></blockquote>
<dl class="docutils">
<dt>or</dt>
<dd><table class="first last docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name" colspan="2">type line_break:</th></tr>
<tr class="field-odd field"><td>&#160;</td><td class="field-body">string</td>
</tr>
<tr class="field-even field"><th class="field-name">return:</th><td class="field-body">one line or a buffer of bytes</td>
</tr>
<tr class="field-odd field"><th class="field-name">rtype:</th><td class="field-body">string</td>
</tr>
</tbody>
</table>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<p>Creator package contains some simple reader creator, which could
be used in user program.</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.creator.</code><code class="descname">np_array</code><span class="sig-paren">(</span><em>x</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a reader that yields elements of x, if it is a
numpy vector. Or rows of x, if it is a numpy matrix.
Or any sub-hyperplane indexed by the highest dimension.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>x</strong> &#8211; the numpy array to create reader from.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">data reader created from x.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.creator.</code><code class="descname">text_file</code><span class="sig-paren">(</span><em>path</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader that outputs text line by line from given text file.
Trailing new line (&#8216;\n&#8217;) of each line will be removed.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Path:</th><td class="field-body">path of the text file.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">data reader of text file</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.creator.</code><code class="descname">cloud_reader</code><span class="sig-paren">(</span><em>paths</em>, <em>etcd_endpoints</em>, <em>timeout_sec=5</em>, <em>buf_size=64</em><span class="sig-paren">)</span></dt>
<dd><dl class="docutils">
<dt>Create a data reader that yield a record one by one from</dt>
<dd>the paths:</dd>
</dl>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Paths:</th><td class="field-body">path of recordio files, can be a string or a string list.</td>
</tr>
<tr class="field-even field"><th class="field-name">Etcd_endpoints:</th><td class="field-body">the endpoints for etcd cluster</td>
</tr>
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">data reader of recordio files.</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="minibatch">
<h2>minibatch<a class="headerlink" href="#minibatch" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.minibatch.</code><code class="descname">batch</code><span class="sig-paren">(</span><em>reader</em>, <em>batch_size</em><span class="sig-paren">)</span></dt>
<dd><p>Create a batched reader.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>batch_size</strong> (<em>int</em>) &#8211; size of each mini-batch</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the batched reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
</div>


           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="image.html" class="btn btn-neutral float-right" title="Image Interface" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="../data.html" class="btn btn-neutral" title="Data Reader Interface and DataSets" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: ".txt",
        };
    </script>
      <script type="text/javascript" src="../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
       
  
    <script type="text/javascript" src="../../_static/js/theme.js"></script>
  
  
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/js/perfect-scrollbar.jquery.min.js"></script>
  <script src="../../_static/js/paddle_doc_init.js"></script> 

</body>
</html>