README.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Required CMake Function &mdash; PaddlePaddle  文档</title>
  

    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  

        <link rel="index" title="索引"
              href="../../genindex.html"/>
        <link rel="search" title="搜索" href="../../search.html"/>
    <link rel="top" title="PaddlePaddle  文档" href="../../index.html"/> 

  <link rel="stylesheet" href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/override.css" type="text/css" />
  <script>
  var _hmt = _hmt || [];
  (function() {
    var hm = document.createElement("script");
    hm.src = "//hm.baidu.com/hm.js?b9a314ab40d04d805655aab1deee08ba";
    var s = document.getElementsByTagName("script")[0]; 
    s.parentNode.insertBefore(hm, s);
  })();
  </script>

  
  <script src="../../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  
  <header class="site-header">
    <div class="site-logo">
      <a href="/"><img src="../../_static/images/PP_w.png"></a>
    </div>
    <div class="site-nav-links">
      <div class="site-menu">
        <a class="fork-on-github" href="https://github.com/PaddlePaddle/Paddle" target="_blank"><i class="fa fa-github"></i>Fork me on Github</a>
        <div class="language-switcher dropdown">
          <a type="button" data-toggle="dropdown">
            <span>English</span>
            <i class="fa fa-angle-up"></i>
            <i class="fa fa-angle-down"></i>
          </a>
          <ul class="dropdown-menu">
            <li><a href="/doc_cn">中文</a></li>
            <li><a href="/doc">English</a></li>
          </ul>
        </div>
        <ul class="site-page-links">
          <li><a href="/">Home</a></li>
        </ul>
      </div>
      <div class="doc-module">
        
        <ul>
<li class="toctree-l1"><a class="reference internal" href="../../getstarted/index_cn.html">新手入门</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../build_and_install/index_cn.html">安装与编译</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index_cn.html">进阶使用</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index_cn.html">开发标准</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq/index_cn.html">FAQ</a></li>
</ul>

        
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>        
      </div>
    </div>
  </header>
  
  <div class="main-content-wrap">

    
    <nav class="doc-menu-vertical" role="navigation">
        
          
          <ul>
<li class="toctree-l1"><a class="reference internal" href="../../getstarted/index_cn.html">新手入门</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/quickstart_cn.html">快速开始</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../getstarted/concepts/use_concepts_cn.html">基本使用概念</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../build_and_install/index_cn.html">安装与编译</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/pip_install_cn.html">使用pip安装</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/docker_install_cn.html">使用Docker安装运行</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/build_cn.html">用Docker编译和测试PaddlePaddle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../build_and_install/build_from_source_cn.html">从源码编译</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index_cn.html">进阶使用</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../howto/cmd_parameter/index_cn.html">命令行参数设置</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/use_case_cn.html">使用案例</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/arguments_cn.html">参数概述</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cmd_parameter/detail_introduction_cn.html">细节描述</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/cluster/index_cn.html">分布式训练</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/preparations_cn.html">环境准备</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/cmd_argument_cn.html">启动参数说明</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/cluster/multi_cluster/index_cn.html">在不同集群中运行</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/fabric_cn.html">使用fabric启动集群训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/openmpi_cn.html">在OpenMPI集群中提交训练作业</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_cn.html">Kubernetes单机训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_distributed_cn.html">Kubernetes分布式训练</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../howto/cluster/multi_cluster/k8s_aws_cn.html">Distributed PaddlePaddle Training on AWS with Kubernetes</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/capi/index_cn.html">C-API预测库</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/compile_paddle_lib_cn.html">安装与编译C-API预测库</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/organization_of_the_inputs_cn.html">输入/输出数据组织</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/capi/workflow_of_capi_cn.html">C-API使用流程</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/rnn/index_cn.html">RNN模型</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/rnn_config_cn.html">RNN配置</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/recurrent_group_cn.html">Recurrent Group教程</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/hierarchical_layer_cn.html">支持双层序列作为输入的Layer</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../howto/rnn/hrnn_rnn_api_compare_cn.html">单双层RNN API对比介绍</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../howto/optimization/gpu_profiling_cn.html">GPU性能调优</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index_cn.html">开发标准</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../dev/contribute_to_paddle_cn.html">如何贡献代码</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../dev/write_docs_cn.html">如何贡献文档</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../faq/index_cn.html">FAQ</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../faq/build_and_install/index_cn.html">编译安装与单元测试</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/model/index_cn.html">模型配置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/parameter/index_cn.html">参数设置</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/local/index_cn.html">本地训练与预测</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq/cluster/index_cn.html">集群训练与预测</a></li>
</ul>
</li>
</ul>

        
    </nav>
    
    <section class="doc-content-wrap">

      
<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
      
    <li>Required CMake Function</li>
  </ul>
</div>
      
      <div class="wy-nav-content" id="doc-content">
        <div class="rst-content">
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <p>A few months ago when we were trying to replace CMake with Bazel, &#64;emailweixu suggested that we rewrite those handy Bazel functions using CMake. Now it seems that it&#8217;s the right time to get this done, as we are facing problems from the porting of Majel and the development of new the parameter server using Go and C++.</p>
<p>Here are some initial thoughts. Your comments are welcome!</p>
<div class="section" id="required-cmake-function">
<span id="required-cmake-function"></span><h1>Required CMake Function<a class="headerlink" href="#required-cmake-function" title="永久链接至标题">¶</a></h1>
<p>I think we need only the following few CMake functions to make a project description mean and clean:</p>
<p>| C++ | CUDA C++ | Go |
|&#8212;|&#8212;|&#8212;|
| cc_library | nv_library | go_library |
| cc_binary | nv_binary | go_binary |
| cc_test | nv_test | go_test |</p>
<ul class="simple">
<li>The <code class="docutils literal"><span class="pre">_library</span></code> functions generate  .a files from source code.</li>
<li>The <code class="docutils literal"><span class="pre">_binary</span></code> functions generate executable binary files.</li>
<li>The <code class="docutils literal"><span class="pre">_test</span></code> functions generate executable unit test files. They work like <code class="docutils literal"><span class="pre">_binary</span></code> but links <code class="docutils literal"><span class="pre">-lgtest</span></code> and <code class="docutils literal"><span class="pre">-lgtest_main</span></code>.</li>
</ul>
<p>The difference between <code class="docutils literal"><span class="pre">nv_</span></code> functions and <code class="docutils literal"><span class="pre">cc_</span></code> functions is that the former use <code class="docutils literal"><span class="pre">nvcc</span></code> instead of the system-default C++ compiler.</p>
<p>Both <code class="docutils literal"><span class="pre">nv_</span></code> and <code class="docutils literal"><span class="pre">cc_</span></code> functions enables C++11 (-std=c++11).</p>
<p>Also,</p>
<ul class="simple">
<li>to describe external dependencies, we need <code class="docutils literal"><span class="pre">external_library</span></code>.</li>
<li>to build shared libraries, we need <code class="docutils literal"><span class="pre">shared_library</span></code>.</li>
</ul>
</div>
<div class="section" id="an-example-project">
<span id="an-example-project"></span><h1>An Example Project<a class="headerlink" href="#an-example-project" title="永久链接至标题">¶</a></h1>
<p>Suppose that we have aforementioned functions defined in our <code class="docutils literal"><span class="pre">/cmake</span></code> directory.  The following example <code class="docutils literal"><span class="pre">CMakeLists.txt</span></code> describes a project including the following source files:</p>
<ul class="simple">
<li>tensor.h</li>
<li>tensor.cc</li>
<li>tensor_test.cc</li>
<li>ops.h</li>
<li>ops.cu</li>
<li>ops_test.cu</li>
<li>api.go</li>
<li>api_test.go</li>
</ul>
<p>Suppose that ops.cu depends on CUDNN.</p>
<div class="highlight-cmake"><div class="highlight"><pre><span></span><span class="c"># cc_binary parses tensor.cc and figures out that target also depend</span>
<span class="c"># on tensor.h.</span>
<span class="nb">cc_binary</span><span class="p">(</span><span class="s">tensor</span>
  <span class="s">SRCS</span>
  <span class="s">tensor.cc</span><span class="p">)</span>

<span class="c"># The dependency to target tensor implies that if any of</span>
<span class="c"># tensor{.h,.cc,_test.cc} is changed, tensor_test need to be re-built.</span>
<span class="nb">cc_test</span><span class="p">(</span><span class="s">tensor_test</span>
  <span class="s">SRCS</span>
  <span class="s">tensor_test.cc</span>
  <span class="s">DEPS</span>
  <span class="s">tensor</span><span class="p">)</span>

<span class="c"># I don&#39;t have a clear idea what parameters external_library need to</span>
<span class="c"># have.  @gangliao as a CMake expert would have better ideas.</span>
<span class="nb">external_library</span><span class="p">(</span><span class="s">cudnn</span>
  <span class="s">....</span><span class="p">)</span>

<span class="c"># Suppose that ops.cu depends on external target CUDNN.  Also, ops.cu</span>
<span class="c"># include global functions that take Tensor as their parameters, so</span>
<span class="c"># ops depend on tensor.  This implies that if any of tensor.{h.cc},</span>
<span class="c"># ops.{h,cu} is changed, ops need to be re-built.</span>
<span class="nb">nv_library</span><span class="p">(</span><span class="s">ops</span>
  <span class="s">SRCS</span>
  <span class="s">ops.cu</span>
  <span class="s">DEPS</span>
  <span class="s">tensor</span>
  <span class="s">cudnn</span><span class="p">)</span>  <span class="c"># cudnn is defined later.</span>

<span class="nb">nv_test</span><span class="p">(</span><span class="s">ops_test</span>
  <span class="s">SRCS</span>
  <span class="s">ops_test.cu</span>
  <span class="s">DEPS</span>
  <span class="s">ops</span><span class="p">)</span>

<span class="c"># Because api.go defines a GO wrapper to ops and tensor, it depends on</span>
<span class="c"># both.  This implies that if any of tensor.{h,cc}, ops.{h,cu}, or</span>
<span class="c"># api.go is changed, api need to be re-built.</span>
<span class="nb">go_library</span><span class="p">(</span><span class="s">api</span>
  <span class="s">SRCS</span>
  <span class="s">api.go</span>
  <span class="s">DEPS</span>
  <span class="s">tensor</span> <span class="c"># Because ops depend on tensor, this line is optional.</span>
  <span class="s">ops</span><span class="p">)</span>

<span class="nb">go_test</span><span class="p">(</span><span class="s">api_test</span>
  <span class="s">SRCS</span>
  <span class="s">api_test.go</span>
  <span class="s">DEPS</span>
  <span class="s">api</span><span class="p">)</span>


<span class="c"># This builds libapi.so.  shared_library might use CMake target</span>
<span class="c"># api_shared so to distinguish it from above target api.</span>
<span class="nb">shared_library</span><span class="p">(</span><span class="s">api</span>
  <span class="s">DEPS</span>
  <span class="s">api</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="implementation">
<span id="implementation"></span><h1>Implementation<a class="headerlink" href="#implementation" title="永久链接至标题">¶</a></h1>
<p>As above example CMakeLists.txt executes, each function invocation adds &#8220;nodes&#8221; to a dependency graph.  It also use this graph to generate CMake commands including <code class="docutils literal"><span class="pre">add_executable</span></code>, <code class="docutils literal"><span class="pre">add_dependencies</span></code>, <code class="docutils literal"><span class="pre">target_link_libraries</span></code>, and <code class="docutils literal"><span class="pre">add_test</span></code>.</p>
</div>
<div class="section" id="using-package-manager-for-go">
<span id="using-package-manager-for-go"></span><h1>Using Package Manager For Go<a class="headerlink" href="#using-package-manager-for-go" title="永久链接至标题">¶</a></h1>
<p>Building Go binaries and libraries need to satisfy their dependencies, generally
we can do <code class="docutils literal"><span class="pre">go</span> <span class="pre">get</span> <span class="pre">./...</span></code> to download and compile all external dependencies. The
problems are:</p>
<ol class="simple">
<li><code class="docutils literal"><span class="pre">go</span> <span class="pre">get</span></code> will always get the latest code from the default branch of the
remote repo, so changes of dependents might break the build. This is very
different with what we already have in <code class="docutils literal"><span class="pre">cmake/external</span></code> which download a
specific version or commit id of the dependency.</li>
<li>Some locations can not access external dependencies through the internet, as mentioned
in https://github.com/PaddlePaddle/Paddle/issues/2605. Using package management
tools can package the dependencies as a &#8220;vendor&#8221; package, which can be mirrored
at many cloud file hosting, so users what to compile paddle by themselves can
download this &#8220;vendor&#8221; package from a mirror site.</li>
</ol>
<div class="section" id="choose-a-suitable-tool">
<span id="choose-a-suitable-tool"></span><h2>Choose A Suitable Tool<a class="headerlink" href="#choose-a-suitable-tool" title="永久链接至标题">¶</a></h2>
<p>As mentioned by &#64;wangkuiyi, <a class="reference external" href="https://github.com/golang/go/wiki/PackageManagementTools">Here</a>
list dozens of Go package managers. We choose the tool using following principles:</p>
<ul class="simple">
<li>Most &#8220;active&#8221; projects with more stars, more pull requests or commits</li>
<li>Widely used project</li>
</ul>
<p>After comparing all these projects, we shall choose between the most popular
tools: Godep and Glide.</p>
<p>Here&#8217;s a brief comparison between Godep and Glide
: https://github.com/Masterminds/glide/wiki/Go-Package-Manager-Comparison. There are
also many complaints about using <code class="docutils literal"><span class="pre">Godep</span></code>. There&#8217;s also a new &#8220;official&#8221; pakcage
management tool has been started at: https://github.com/golang/dep to resolve
such problems, but it&#8217;s currently at Alpha stage. So the best choice now is
glide obviously.</p>
</div>
<div class="section" id="manage-go-packages">
<span id="manage-go-packages"></span><h2>Manage Go Packages<a class="headerlink" href="#manage-go-packages" title="永久链接至标题">¶</a></h2>
<ul class="simple">
<li>Dependencies: <code class="docutils literal"><span class="pre">go/glide.yaml</span></code> will store the dependencies and their versions which
is directly imported by paddle. <code class="docutils literal"><span class="pre">go/glide.lock</span></code> will store all dependencies recursively
with their commit id. Builds will &#8220;lock&#8221; to these packages if we don&#8217;t <code class="docutils literal"><span class="pre">glide</span> <span class="pre">up</span></code>
them</li>
<li>Vendor package: <code class="docutils literal"><span class="pre">go/vendor</span></code> directory will generated when running <code class="docutils literal"><span class="pre">cmake</span></code> command. <code class="docutils literal"><span class="pre">cmake</span></code>
will download the code corresponding to <code class="docutils literal"><span class="pre">go/glide.lock</span></code>. If we put a vendor folder
under <code class="docutils literal"><span class="pre">go/</span></code>, cmake will just check the commit id to the packages under the folder,
if commit id matches, there will be no download at all.</li>
</ul>
</div>
</div>


           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2016, PaddlePaddle developers.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../../',
            VERSION:'',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: ".txt",
        };
    </script>
      <script type="text/javascript" src="../../_static/jquery.js"></script>
      <script type="text/javascript" src="../../_static/underscore.js"></script>
      <script type="text/javascript" src="../../_static/doctools.js"></script>
      <script type="text/javascript" src="../../_static/translations.js"></script>
      <script type="text/javascript" src="https://cdn.bootcss.com/mathjax/2.7.0/MathJax.js"></script>
       
  
    <script type="text/javascript" src="../../_static/js/theme.js"></script>
  
  
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/js/perfect-scrollbar.jquery.min.js"></script>
  <script src="../../_static/js/paddle_doc_init.js"></script> 

</body>
</html>