data.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Datasets &mdash; PaddlePaddle  documentation</title>
  

    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="Index"
              href="../../genindex.html"/>
        <link rel="search" title="Search" href="../../search.html"/>
    <link rel="top" title="PaddlePaddle  documentation" href="../../index.html"/>
        <link rel="up" title="API" href="../index_en.html"/>
        <link rel="next" title="Training and Inference" href="run_logic.html"/>
        <link rel="prev" title="Parameter Attribute" href="config/attr.html"/> 

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/override.css" type="text/css" />
  <script>
  var _hmt = _hmt || [];
  (function() {
    var hm = document.createElement("script");
    hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
    var s = document.getElementsByTagName("script")[0]; 
    s.parentNode.insertBefore(hm, s);
  })();
  </script>

  
  <script src="../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  
  <header class="site-header">
    <div class="site-logo">
      <a href="/"><img src="../../_static/images/PP_w.png"></a>
    </div>
    <div class="site-nav-links">
      <div class="site-menu">
        <a class="fork-on-github" href="https://github.com/PaddlePaddle/Paddle" target="_blank"><i class="fa fa-github"></i>Folk me on Github</a>
        <div class="language-switcher dropdown">
          <a type="button" data-toggle="dropdown">
            <span>English</span>
            <i class="fa fa-angle-up"></i>
            <i class="fa fa-angle-down"></i>
          </a>
          <ul class="dropdown-menu">
            <li><a href="/doc_cn">中文</a></li>
            <li><a href="/doc">English</a></li>
          </ul>
        </div>
        <ul class="site-page-links">
          <li><a>Home</a></li>
          <li><a>Get Started</a></li>
          <li class="active"><a>Documentation</a></li>
          <li><a>About Us</a></li>
        </ul>
      </div>
      <div class="doc-module">
        
        <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../getstarted/index_en.html">GET STARTED</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorials/index_en.html">TUTORIALS</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index_en.html">HOW TO</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../index_en.html">API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../about/index_en.html">ABOUT</a></li>
</ul>

        
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>        
      </div>
    </div>
  </header>
  
  <div class="main-content-wrap">

    
    <nav class="doc-menu-vertical" role="navigation">
        
          
          <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../getstarted/index_en.html">GET STARTED</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/build_and_install/index_en.html">Install and Build</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../getstarted/build_and_install/docker_install_en.html">PaddlePaddle in Docker Containers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../getstarted/build_and_install/ubuntu_install_en.html">Debian Package installation guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../getstarted/build_and_install/build_from_source_en.html">Installing from Sources</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/basic_usage/index_en.html">Simple Linear Regression</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorials/index_en.html">TUTORIALS</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/quick_start/index_en.html">Quick Start</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/rec/ml_regression_en.html">MovieLens Regression</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/image_classification/index_en.html">Image Classification</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/sentiment_analysis/index_en.html">Sentiment Analysis</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/semantic_role_labeling/index_en.html">Semantic Role Labeling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/text_generation/index_en.html">Text Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/gan/index_en.html">Image Auto-Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/imagenet_model/resnet_model_en.html">ImageNet: ResNet</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tutorials/embedding_model/index_en.html">Embedding: Chinese Word</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index_en.html">HOW TO</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../howto/usage/cmd_parameter/index_en.html">Set Command-line Parameters</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/usage/cmd_parameter/use_case_en.html">Use Case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/usage/cmd_parameter/arguments_en.html">Argument Outline</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/usage/cmd_parameter/detail_introduction_en.html">Detail Description</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/usage/cluster/cluster_train_en.html">Run Distributed Training</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/usage/k8s/k8s_en.html">Paddle On Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/usage/k8s/k8s_aws_en.html">Distributed PaddlePaddle Training on AWS with Kubernetes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/dev/new_layer_en.html">Write New Layers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/dev/contribute_to_paddle_en.html">Contribute Code</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/deep_model/rnn/index_en.html">RNN Models</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/deep_model/rnn/rnn_config_en.html">RNN Configuration</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/optimization/gpu_profiling_en.html">Tune GPU Performance</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="../index_en.html">API</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="model_configs.html">Model Configuration</a><ul>
<li class="toctree-l3"><a class="reference internal" href="config/activation.html">Activation</a></li>
<li class="toctree-l3"><a class="reference internal" href="config/layer.html">Layers</a></li>
<li class="toctree-l3"><a class="reference internal" href="config/optimizer.html">Optimizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="config/pooling.html">Pooling</a></li>
<li class="toctree-l3"><a class="reference internal" href="config/networks.html">Networks</a></li>
<li class="toctree-l3"><a class="reference internal" href="config/attr.html">Parameter Attribute</a></li>
</ul>
</li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="run_logic.html">Training and Inference</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../about/index_en.html">ABOUT</a></li>
</ul>

        
    </nav>
    
    <nav class="local-toc"><ul>
<li><a class="reference internal" href="#">Datasets</a><ul>
<li><a class="reference internal" href="#datatypes">DataTypes</a></li>
<li><a class="reference internal" href="#datafeeder">DataFeeder</a></li>
<li><a class="reference internal" href="#reader">Reader</a><ul>
<li><a class="reference internal" href="#data-reader-interface">Data Reader Interface</a></li>
</ul>
</li>
<li><a class="reference internal" href="#minibatch">minibatch</a></li>
<li><a class="reference internal" href="#dataset">Dataset</a><ul>
<li><a class="reference internal" href="#mnist">mnist</a></li>
<li><a class="reference internal" href="#cifar">cifar</a></li>
<li><a class="reference internal" href="#conll05">conll05</a></li>
<li><a class="reference internal" href="#imdb">imdb</a></li>
<li><a class="reference internal" href="#imikolov">imikolov</a></li>
<li><a class="reference internal" href="#movielens">movielens</a></li>
<li><a class="reference internal" href="#sentiment">sentiment</a></li>
<li><a class="reference internal" href="#uci-housing">uci_housing</a></li>
<li><a class="reference internal" href="#wmt14">wmt14</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav>
    
    <section class="doc-content-wrap">

      
<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
      
        <li><a href="../index_en.html">API</a> > </li>
      
    <li>Datasets</li>
  </ul>
</div>
      
      <div class="wy-nav-content" id="doc-content">
        <div class="rst-content">
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="datasets">
<h1>Datasets<a class="headerlink" href="#datasets" title="Permalink to this headline">¶</a></h1>
<div class="section" id="datatypes">
<h2>DataTypes<a class="headerlink" href="#datatypes" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">dense_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Dense Vector. It means the input feature is dense float vector. For example,
if the input is an image with 28*28 pixels, the input of Paddle neural
network should be a dense vector with dimension 784.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">dense_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of dense vector.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of dense vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">integer_value</code><span class="sig-paren">(</span><em>value_range</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of integer.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
<li><strong>value_range</strong> (<em>int</em>) &#8211; range of this integer.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">integer_value_sequence</code><span class="sig-paren">(</span><em>value_range</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of integer.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>value_range</strong> (<em>int</em>) &#8211; range of each element.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_binary_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse binary vector. It means the input feature is a sparse vector and the
every element in this vector is either zero or one.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_binary_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><dl class="docutils">
<dt>Data type of a sequence of sparse vector, which every element is either zero</dt>
<dd>or one.</dd>
</dl>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of sparse vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_non_value_slot</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse binary vector. It means the input feature is a sparse vector and the
every element in this vector is either zero or one.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_value_slot</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse vector. It means the input feature is a sparse vector. Most of the
elements in this vector are zero, others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_vector</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type=0</em><span class="sig-paren">)</span></dt>
<dd><p>Sparse vector. It means the input feature is a sparse vector. Most of the
elements in this vector are zero, others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of this vector.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of this input.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An input type object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">InputType</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.data_type.</code><code class="descname">sparse_vector_sequence</code><span class="sig-paren">(</span><em>dim</em><span class="sig-paren">)</span></dt>
<dd><p>Data type of a sequence of sparse vector, which most elements are zero,
others could be any float value.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dim</strong> (<em>int</em>) &#8211; dimension of sparse vector.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An input type object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">InputType</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.data_type.</code><code class="descname">InputType</code><span class="sig-paren">(</span><em>dim</em>, <em>seq_type</em>, <em>tp</em><span class="sig-paren">)</span></dt>
<dd><p>InputType is the base class for paddle input types.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">this is a base class, and should never be used by user.</p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>dim</strong> (<em>int</em>) &#8211; dimension of input. If the input is an integer, it means the
value range. Otherwise, it means the size of layer.</li>
<li><strong>seq_type</strong> (<em>int</em>) &#8211; sequence type of input. 0 means it is not a sequence. 1
means it is a variable length sequence. 2 means it is a
nested sequence.</li>
<li><strong>type</strong> (<em>int</em>) &#8211; data type of input.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="datafeeder">
<h2>DataFeeder<a class="headerlink" href="#datafeeder" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="descclassname">paddle.v2.data_feeder.</code><code class="descname">DataFeeder</code><span class="sig-paren">(</span><em>data_types</em>, <em>feeding=None</em><span class="sig-paren">)</span></dt>
<dd><p>DataFeeder converts the data returned by paddle.reader into a data structure
of Arguments which is defined in the API. The paddle.reader usually returns
a list of mini-batch data entries. Each data entry in the list is one sample.
Each sample is a list or a tuple with one feature or multiple features.
DataFeeder converts this mini-batch data entries into Arguments in order
to feed it to C++ interface.</p>
<p>The example usage:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">data_types</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;image&#39;</span><span class="p">,</span> <span class="n">paddle</span><span class="o">.</span><span class="n">data_type</span><span class="o">.</span><span class="n">dense_vector</span><span class="p">(</span><span class="mi">784</span><span class="p">)),</span>
              <span class="p">(</span><span class="s1">&#39;label&#39;</span><span class="p">,</span> <span class="n">paddle</span><span class="o">.</span><span class="n">data_type</span><span class="o">.</span><span class="n">integer_value</span><span class="p">(</span><span class="mi">10</span><span class="p">))]</span>
<span class="n">reader_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;image&#39;</span><span class="p">:</span><span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;label&#39;</span><span class="p">:</span><span class="mi">1</span><span class="p">}</span>
<span class="n">feeder</span> <span class="o">=</span> <span class="n">DataFeeder</span><span class="p">(</span><span class="n">data_types</span><span class="o">=</span><span class="n">data_types</span><span class="p">,</span> <span class="n">reader_dict</span><span class="o">=</span><span class="n">reader_dict</span><span class="p">)</span>
<span class="n">minibatch_data</span> <span class="o">=</span> <span class="p">[</span>
                   <span class="p">(</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">,</span><span class="mf">3.0</span><span class="p">,</span><span class="mf">4.0</span><span class="p">],</span> <span class="mi">5</span><span class="p">,</span> <span class="p">[</span><span class="mi">6</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">8</span><span class="p">]</span> <span class="p">),</span>  <span class="c1"># first sample</span>
                   <span class="p">(</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">,</span><span class="mf">3.0</span><span class="p">,</span><span class="mf">4.0</span><span class="p">],</span> <span class="mi">5</span><span class="p">,</span> <span class="p">[</span><span class="mi">6</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">8</span><span class="p">]</span> <span class="p">)</span>   <span class="c1"># second sample</span>
                 <span class="p">]</span>
<span class="c1"># or minibatch_data = [</span>
<span class="c1">#                       [ [1.0,2.0,3.0,4.0], 5, [6,7,8] ],  # first sample</span>
<span class="c1">#                       [ [1.0,2.0,3.0,4.0], 5, [6,7,8] ]   # second sample</span>
<span class="c1">#                     ]</span>
<span class="n">arg</span> <span class="o">=</span> <span class="n">feeder</span><span class="p">(</span><span class="n">minibatch_data</span><span class="p">)</span>
</pre></div>
</div>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">This module is for internal use only. Users should use the <cite>reader</cite>
interface.</p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>data_types</strong> (<em>list</em>) &#8211; A list to specify data name and type. Each item is
a tuple of (data_name, data_type).</li>
<li><strong>reader_dict</strong> &#8211; A dictionary to specify the position of each data
in the input data.</li>
</ul>
</td>
</tr>
</tbody>
</table>
<dl class="method">
<dt>
<code class="descname">convert</code><span class="sig-paren">(</span><em>dat</em>, <em>argument=None</em><span class="sig-paren">)</span></dt>
<dd><table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>dat</strong> (<em>list</em>) &#8211; A list of mini-batch data. Each sample is a list or tuple
one feature or multiple features.</li>
<li><strong>argument</strong> (<em>py_paddle.swig_paddle.Arguments</em>) &#8211; An Arguments object contains this mini-batch data with
one or multiple features. The Arguments definition is
in the API.</li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="reader">
<h2>Reader<a class="headerlink" href="#reader" title="Permalink to this headline">¶</a></h2>
<p>At training and testing time, PaddlePaddle programs need to read data. To ease
the users&#8217; work to write data reading code, we define that</p>
<ul class="simple">
<li>A <em>reader</em> is a function that reads data (from file, network, random number
generator, etc) and yields data items.</li>
<li>A <em>reader creator</em> is a function that returns a reader function.</li>
<li>A <em>reader decorator</em> is a function, which accepts one or more readers, and
returns a reader.</li>
<li>A <em>batch reader</em> is a function that reads data (from <em>reader</em>, file, network,
random number generator, etc) and yields a batch of data items.</li>
</ul>
<div class="section" id="data-reader-interface">
<h3>Data Reader Interface<a class="headerlink" href="#data-reader-interface" title="Permalink to this headline">¶</a></h3>
<p>Indeed, <em>data reader</em> doesn&#8217;t have to be a function that reads and yields data
items. It can be any function with no parameter that creates a iterable
(anything can be used in <code class="code docutils literal"><span class="pre">for</span> <span class="pre">x</span> <span class="pre">in</span> <span class="pre">iterable</span></code>):</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">iterable</span> <span class="o">=</span> <span class="n">data_reader</span><span class="p">()</span>
</pre></div>
</div>
<p>Element produced from the iterable should be a <strong>single</strong> entry of data,
<strong>not</strong> a mini batch. That entry of data could be a single item, or a tuple of
items.
Item should be of <a class="reference external" href="http://www.paddlepaddle.org/doc/ui/data_provider/pydataprovider2.html?highlight=dense_vector#input-types">supported type</a> (e.g., numpy 1d
array of float32, int, list of int)</p>
<p>An example implementation for single item data reader creator:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">reader_creator_random_image</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">reader</span><span class="p">():</span>
        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">numpy</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">width</span><span class="o">*</span><span class="n">height</span><span class="p">)</span>
<span class="k">return</span> <span class="n">reader</span>
</pre></div>
</div>
<p>An example implementation for multiple item data reader creator:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">reader_creator_random_image_and_label</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">label</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">reader</span><span class="p">():</span>
        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">numpy</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">width</span><span class="o">*</span><span class="n">height</span><span class="p">),</span> <span class="n">label</span>
<span class="k">return</span> <span class="n">reader</span>
</pre></div>
</div>
<p>TODO(yuyang18): Should we add whole design doc here?</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">map_readers</code><span class="sig-paren">(</span><em>func</em>, <em>*readers</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader that outputs return value of function using
output of each data readers as arguments.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>func</strong> &#8211; function to use. The type of func should be (Sample) =&gt; Sample</li>
<li><strong>readers</strong> &#8211; readers whose outputs will be used as arguments of func.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Type:</th><td class="field-body"><p class="first">callable</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the created data reader.</p>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">buffered</code><span class="sig-paren">(</span><em>reader</em>, <em>size</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a buffered data reader.</p>
<p>The buffered data reader will read and save data entries into a
buffer. Reading from the buffered data reader will proceed as long
as the buffer is not empty.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>size</strong> (<em>int</em>) &#8211; max buffer size.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">the buffered data reader.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">compose</code><span class="sig-paren">(</span><em>*readers</em>, <em>**kwargs</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose output is the combination of input readers.</p>
<p>If input readers output following data entries:
(1, 2)    3    (4, 5)
The composed reader will output:
(1, 2, 3, 4, 5)</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>readers</strong> &#8211; readers that will be composed together.</li>
<li><strong>check_alignment</strong> (<em>bool</em>) &#8211; if True, will check if input readers are aligned
correctly. If False, will not check alignment and trailing outputs
will be discarded. Defaults to True.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the new data reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><strong>ComposeNotAligned</strong> &#8211; outputs of readers are not aligned.
Will not raise when check_alignment is set to False.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">chain</code><span class="sig-paren">(</span><em>*readers</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose output is the outputs of input data
readers chained together.</p>
<p>If input readers output following data entries:
[0, 0, 0]
[1, 1, 1]
[2, 2, 2]
The chained reader will output:
[0, 0, 0, 1, 1, 1, 2, 2, 2]</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>readers</strong> &#8211; input readers.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">the new data reader.</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">callable</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">shuffle</code><span class="sig-paren">(</span><em>reader</em>, <em>buf_size</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader whose data output is shuffled.</p>
<p>Output from the iterator that created by original reader will be
buffered into shuffle buffer, and then shuffled. The size of shuffle buffer
is determined by argument buf_size.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the original reader whose output will be shuffled.</li>
<li><strong>buf_size</strong> (<em>int</em>) &#8211; shuffle buffer size.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the new reader whose output is shuffled.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.</code><code class="descname">firstn</code><span class="sig-paren">(</span><em>reader</em>, <em>n</em><span class="sig-paren">)</span></dt>
<dd><p>Limit the max number of samples that reader could return.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>n</strong> (<em>int</em>) &#8211; the max number of samples that return.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the decorated reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<p>Creator package contains some simple reader creator, which could be used in user
program.</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.creator.</code><code class="descname">np_array</code><span class="sig-paren">(</span><em>x</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a reader that yields elements of x, if it is a
numpy vector. Or rows of x, if it is a numpy matrix.
Or any sub-hyperplane indexed by the highest dimension.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>x</strong> &#8211; the numpy array to create reader from.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">data reader created from x.</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.reader.creator.</code><code class="descname">text_file</code><span class="sig-paren">(</span><em>path</em><span class="sig-paren">)</span></dt>
<dd><p>Creates a data reader that outputs text line by line from given text file.
Trailing new line (&#8216;\n&#8217;) of each line will be removed.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Path:</th><td class="field-body">path of the text file.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">data reader of text file</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="minibatch">
<h2>minibatch<a class="headerlink" href="#minibatch" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.minibatch.</code><code class="descname">batch</code><span class="sig-paren">(</span><em>reader</em>, <em>batch_size</em><span class="sig-paren">)</span></dt>
<dd><p>Create a batched reader.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>reader</strong> (<em>callable</em>) &#8211; the data reader to read from.</li>
<li><strong>batch_size</strong> (<em>int</em>) &#8211; size of each mini-batch</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">the batched reader.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">callable</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="dataset">
<h2>Dataset<a class="headerlink" href="#dataset" title="Permalink to this headline">¶</a></h2>
<p>Dataset package.</p>
<div class="section" id="mnist">
<h3>mnist<a class="headerlink" href="#mnist" title="Permalink to this headline">¶</a></h3>
<p>MNIST dataset.</p>
<p>This module will download dataset from <a class="reference external" href="http://yann.lecun.com/exdb/mnist/">http://yann.lecun.com/exdb/mnist/</a> and
parse train set and test set into paddle reader creators.</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.dataset.mnist.</code><code class="descname">train</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>MNIST train set creator.</p>
<p>It returns a reader creator, each sample in the reader is image pixels in
[0, 1] and label in [0, 9].</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">Train reader creator</td>
</tr>
<tr class="field-even field"><th class="field-name">Return type:</th><td class="field-body">callable</td>
</tr>
</tbody>
</table>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.dataset.mnist.</code><code class="descname">test</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>MNIST test set cretor.</p>
<p>It returns a reader creator, each sample in the reader is image pixels in
[0, 1] and label in [0, 9].</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">Test reader creator.</td>
</tr>
<tr class="field-even field"><th class="field-name">Return type:</th><td class="field-body">callable</td>
</tr>
</tbody>
</table>
</dd></dl>

</div>
<div class="section" id="cifar">
<h3>cifar<a class="headerlink" href="#cifar" title="Permalink to this headline">¶</a></h3>
<p>CIFAR dataset: <a class="reference external" href="https://www.cs.toronto.edu/~kriz/cifar.html">https://www.cs.toronto.edu/~kriz/cifar.html</a></p>
<p>TODO(yuyang18): Complete the comments.</p>
</div>
<div class="section" id="conll05">
<h3>conll05<a class="headerlink" href="#conll05" title="Permalink to this headline">¶</a></h3>
</div>
<div class="section" id="imdb">
<h3>imdb<a class="headerlink" href="#imdb" title="Permalink to this headline">¶</a></h3>
<p>IMDB dataset: <a class="reference external" href="http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz">http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz</a></p>
<p>TODO(yuyang18): Complete comments.</p>
</div>
<div class="section" id="imikolov">
<h3>imikolov<a class="headerlink" href="#imikolov" title="Permalink to this headline">¶</a></h3>
<p>imikolov&#8217;s simple dataset: <a class="reference external" href="http://www.fit.vutbr.cz/~imikolov/rnnlm/">http://www.fit.vutbr.cz/~imikolov/rnnlm/</a></p>
<p>Complete comments.</p>
</div>
<div class="section" id="movielens">
<h3>movielens<a class="headerlink" href="#movielens" title="Permalink to this headline">¶</a></h3>
<p>Movielens 1-M dataset.</p>
<p>TODO(yuyang18): Complete comments.</p>
</div>
<div class="section" id="sentiment">
<h3>sentiment<a class="headerlink" href="#sentiment" title="Permalink to this headline">¶</a></h3>
<p>The script fetch and preprocess movie_reviews data set that provided by NLTK</p>
<p>TODO(yuyang18): Complete dataset.</p>
<dl class="function">
<dt>
<code class="descclassname">paddle.v2.dataset.sentiment.</code><code class="descname">get_word_dict</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Sorted the words by the frequency of words which occur in sample
:return:</p>
<blockquote>
<div>words_freq_sorted</div></blockquote>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.dataset.sentiment.</code><code class="descname">train</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Default train set reader creator</p>
</dd></dl>

<dl class="function">
<dt>
<code class="descclassname">paddle.v2.dataset.sentiment.</code><code class="descname">test</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Default test set reader creator</p>
</dd></dl>

</div>
<div class="section" id="uci-housing">
<h3>uci_housing<a class="headerlink" href="#uci-housing" title="Permalink to this headline">¶</a></h3>
<p>UCI Housing dataset.</p>
<p>TODO(yuyang18): Complete comments.</p>
</div>
<div class="section" id="wmt14">
<h3>wmt14<a class="headerlink" href="#wmt14" title="Permalink to this headline">¶</a></h3>
<p>UCI Housing dataset.</p>
<p>TODO(yuyang18): Complete comments.</p>
</div>
</div>
</div>


           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="run_logic.html" class="btn btn-neutral float-right" title="Training and Inference" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="config/attr.html" class="btn btn-neutral" title="Parameter Attribute" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
       
  
    <script type="text/javascript" src="../../_static/js/theme.js"></script>
  
  
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/js/perfect-scrollbar.jquery.min.js"></script>
  <script src="../../_static/js/paddle_doc_init.js"></script> 

</body>
</html>