Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into trt_stack_op

README.md

- 
+
 <p align="center">
 <img align="center" src="doc/imgs/logo.png", width=1600>
 <p>

paddle/fluid/framework/distributed_strategy.proto

@@ -43,6 +43,12 @@ message GradientMergeConfig {
   optional bool avg = 2 [ default = true ];
 }
 
+message DGCConfig {
+  optional int32 rampup_begin_step = 1 [ default = 0 ];
+  optional int32 rampup_step = 2 [ default = 1 ];
+  repeated float sparsity = 3;
+}
+
 message LarsConfig {
   optional float lars_coeff = 1 [ default = 0.001 ];
   optional float lars_weight_decay = 2 [ default = 0.0005 ];
@@ -114,6 +120,7 @@ message DistributedStrategy {
   optional AMPConfig amp_configs = 102;
   optional LocalSGDConfig localsgd_configs = 103;
   optional GradientMergeConfig gradient_merge_configs = 104;
+  optional DGCConfig dgc_configs = 105;
   optional PipelineConfig pipeline_configs = 106;
   optional AsyncConfig a_sync_configs = 107;
   optional LarsConfig lars_configs = 108;

paddle/fluid/operators/p_norm_op.cu

@@ -99,39 +99,25 @@ __global__ void PnormGradient(const T* x, const T* x_norm, const T* y_grad,
                               const float porder, const int pre,
                               const int axis_n, const int post, const T eps,
                               T* x_grad) {
-  typedef cub::BlockReduce<T, BlockDim> BlockReduce;
-  __shared__ typename BlockReduce::TempStorage temp_storage_sum;
   // dx = (x/pnorm_broadcast).pow(p-1) * norm_dy.broadcast * sign(x)
   int num = pre * post;
   auto porder_grad = static_cast<T>(porder - 1.0f);
   for (int i = blockIdx.x; i < num; i += gridDim.x) {
-    T sum = 0.0;
-    __shared__ T row_sum;
-    __shared__ T row_sqrt_norm;
-    __shared__ T row_norm;
+    __shared__ T pnorm_i;
+    __shared__ T yout_i;
 
     auto base = (i / post) * post * axis_n + (i % post);
 
-    for (int j = threadIdx.x; j < axis_n; j += blockDim.x) {
-      int index = base + j * post;
-      sum += x[index] * y_grad[index];
-    }
-    T reduce_result = BlockReduce(temp_storage_sum).Sum(sum);
-
     if (threadIdx.x == 0) {
-      row_sum = reduce_result;
-      row_sqrt_norm = x_norm[i];
-      row_norm = row_sqrt_norm * row_sqrt_norm;
+      pnorm_i = x_norm[i];
+      yout_i = y_grad[i];
     }
-    __syncthreads();
 
-    const T pnorm_i = x_norm[i];
-    const T yout_i = y_grad[i];
+    __syncthreads();
 
     for (int j = threadIdx.x; j < axis_n; j += blockDim.x) {
       int index = base + j * post;
       const T x_ij = inline_abs(x[index]);
-      const T dy_ij = y_grad[index];
       x_grad[index] = inline_pow(x_ij, porder_grad) /
                       (inline_pow(pnorm_i, porder_grad) + eps) * yout_i *
                       inline_sign(x[index]);

python/paddle/fleet/base/distributed_strategy.py

@@ -604,6 +604,15 @@ class DistributedStrategy(object):
         else:
             print("WARNING: lars should have value of bool type")
 
+    @property
+    def lars_configs(self):
+        return get_msg_dict(self.strategy.lars_configs)
+
+    @lars_configs.setter
+    def lars_configs(self, configs):
+        check_configs_key(self.strategy.lars_configs, configs, "lars_configs")
+        assign_configs_value(self.strategy.lars_configs, configs)
+
     @property
     def lamb(self):
         return self.strategy.lamb

python/paddle/fleet/base/meta_optimizer_factory.py

@@ -19,6 +19,7 @@ from ..meta_optimizers import GraphExecutionOptimizer
 from ..meta_optimizers import PipelineOptimizer
 from ..meta_optimizers import LocalSGDOptimizer
 from ..meta_optimizers import LarsOptimizer
+from ..meta_optimizers import DGCOptimizer
 
 __all__ = ["MetaOptimizerFactory"]
 
@@ -30,6 +31,7 @@ meta_optimizer_names = [
     "PipelineOptimizer",
     "LocalSGDOptimizer",
     "LarsOptimizer",
+    "DGCOptimizer",
 ]
 
 

python/paddle/fleet/meta_optimizers/__init__.py

@@ -18,6 +18,7 @@ from .graph_execution_optimizer import GraphExecutionOptimizer
 from .pipeline_optimizer import PipelineOptimizer
 from .localsgd_optimizer import LocalSGDOptimizer
 from .lars_optimizer import LarsOptimizer
+from .dgc_optimizer import DGCOptimizer
 
 __all__ = [
     'AMPOptimizer',
@@ -26,4 +27,5 @@ __all__ = [
     'PipelineOptimizer',
     'LocalSGDOptimizer',
     'LarsOptimizer',
+    'DGCOptimizer',
 ]

python/paddle/fleet/meta_optimizers/dgc_optimizer.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+from paddle.fluid.optimizer import Momentum, DGCMomentumOptimizer
+from .meta_optimizer_base import MetaOptimizerBase
+import logging
+
+__all__ = ["DGCOptimizer"]
+
+
+class DGCOptimizer(MetaOptimizerBase):
+    def __init__(self, optimizer):
+        super(DGCOptimizer, self).__init__(optimizer)
+        self.inner_opt = optimizer
+        self.dgc_opt = None
+        # we do not allow meta optimizer to be inner optimizer currently
+        self.meta_optimizers_white_list = []
+
+    def _set_basic_info(self, loss, role_maker, user_defined_optimizer,
+                        user_defined_strategy):
+        super(DGCOptimizer, self)._set_basic_info(
+            loss, role_maker, user_defined_optimizer, user_defined_strategy)
+
+        opt = self.inner_opt
+        if not isinstance(opt, Momentum):
+            return
+
+        configs = self.user_defined_strategy.dgc_configs
+        if len(configs['sparsity']) == 0:
+            # default is [0.999]
+            configs['sparsity'] = [0.999]
+
+        self.dgc_opt = DGCMomentumOptimizer(
+            learning_rate=opt._learning_rate,
+            momentum=opt._momentum,
+            rampup_begin_step=configs['rampup_begin_step'],
+            rampup_step=configs['rampup_step'],
+            sparsity=configs['sparsity'],
+            parameter_list=opt._parameter_list,
+            use_nesterov=opt._use_nesterov,
+            num_trainers=self.role_maker.worker_num(),
+            regularization=opt.regularization,
+            grad_clip=opt._grad_clip,
+            name=opt._name)
+
+    def _can_apply(self):
+        if self.user_defined_strategy.dgc:
+            if not isinstance(self.inner_opt, Momentum):
+                logging.warn("dgc only works on Momentum optimizer")
+                return False
+            if self.role_maker.worker_num() <= 1:
+                logging.warn("dgc only works on multi cards")
+                return False
+
+            return True
+
+        return False
+
+    def _disable_strategy(self, dist_strategy):
+        dist_strategy.dgc = False
+        dist_strategy.dgc_configs = {
+            'rampup_begin_step': 0,
+            'rampup_step': 1,
+            'sparsity': [0.999]
+        }
+
+    def backward(self,
+                 loss,
+                 startup_program=None,
+                 parameter_list=None,
+                 no_grad_set=None,
+                 callbacks=None):
+        return self.dgc_opt.backward(loss, startup_program, parameter_list,
+                                     no_grad_set, callbacks)
+
+    def minimize_impl(self,
+                      loss,
+                      startup_program=None,
+                      parameter_list=None,
+                      no_grad_set=None):
+        optimize_ops, params_grads = \
+            self.dgc_opt.minimize(loss, startup_program,
+                                      parameter_list, no_grad_set)
+        return optimize_ops, params_grads

python/paddle/fleet/meta_optimizers/meta_optimizer_base.py

@@ -40,7 +40,8 @@ class MetaOptimizerBase(object):
             return True
 
     def _disable_strategy(self, dist_strategy):
-        raise NotImplementedError("you should implement disable strategy")
+        raise NotImplementedError("you should implement disable strategy in {}".
+                                  format(type(self).__name__))
 
     def minimize_impl(self,
                       loss,

python/paddle/fluid/optimizer.py

@@ -47,9 +47,8 @@ __all__ = [
     'AdamOptimizer', 'AdamaxOptimizer', 'DpsgdOptimizer',
     'DecayedAdagradOptimizer', 'RMSPropOptimizer', 'FtrlOptimizer', 'Adadelta',
     'AdadeltaOptimizer', 'ModelAverage', 'LarsMomentum',
-    'LarsMomentumOptimizer', 'DGCMomentumOptimizer', 'LambOptimizer',
-    'ExponentialMovingAverage', 'PipelineOptimizer', 'LookaheadOptimizer',
-    'RecomputeOptimizer'
+    'LarsMomentumOptimizer', 'LambOptimizer', 'ExponentialMovingAverage',
+    'PipelineOptimizer', 'LookaheadOptimizer', 'RecomputeOptimizer'
 ]
 
 

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -39,6 +39,7 @@ list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_meta_optimizer)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_gradient_merge_meta_optimizer)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_localsgd_meta_optimizer)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_lars_meta_optimizer)
+list(APPEND MIXED_DIST_TEST_OPS test_fleet_dgc_meta_optimizer)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_private_function)
 list(APPEND MIXED_DIST_TEST_OPS test_fleet_graph_executor)
 foreach(TEST_OP ${MIXED_DIST_TEST_OPS})
@@ -388,6 +389,7 @@ if(WITH_DISTRIBUTE)
         py_test_modules(test_dgc_op MODULES test_dgc_op)
         py_test_modules(test_dgc_momentum_op MODULES test_dgc_momentum_op)
         py_test_modules(test_dgc_optimizer MODULES test_dgc_optimizer)
+        py_test_modules(test_fleet_dgc_meta_optimizer MODULES test_fleet_dgc_meta_optimizer)
     else()
         # if not with dgc, must close all dgc tests
         list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")

python/paddle/fluid/tests/unittests/test_dist_mnist_dgc_nccl.py

@@ -39,7 +39,6 @@ class TestDistMnistNCCL2DGC(TestDistBase):
         self._nccl2_mode = True
         self._use_dgc = True
 
-    @unittest.skip(reason="Skip unstable ut")
     def test_dist_train(self):
         import paddle.fluid as fluid
         if fluid.core.is_compiled_with_cuda():
@@ -69,7 +68,6 @@ class TestDistMnistNCCL2DGCMultiCards(TestDistBase):
         self._nccl2_mode = True
         self._use_dgc = True
 
-    @unittest.skip(reason="Skip unstable ut")
     def test_dist_train(self):
         import paddle.fluid as fluid
         if fluid.core.is_compiled_with_cuda():

python/paddle/fluid/tests/unittests/test_fleet_dgc_meta_optimizer.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import paddle
+import os
+import paddle.fleet as fleet
+import paddle.fluid.incubate.fleet.base.role_maker as role_maker
+
+
+class TestFleetDGCOptimizer(unittest.TestCase):
+    def setUp(self):
+        os.environ["PADDLE_TRAINER_ID"] = "1"
+        os.environ[
+            "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002"
+
+    def net(self):
+        role = role_maker.PaddleCloudRoleMaker(is_collective=True)
+        fleet.init(role)
+        input_x = paddle.fluid.layers.data(
+            name="x", shape=[32], dtype='float32')
+        input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64')
+
+        fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh')
+        fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh')
+        prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax')
+        cost = paddle.fluid.layers.cross_entropy(
+            input=prediction, label=input_y)
+        avg_cost = paddle.fluid.layers.mean(x=cost)
+
+        strategy = paddle.fleet.DistributedStrategy()
+        strategy.dgc = True
+        strategy.dgc_configs = {
+            "rampup_begin_step": 128,
+            "rampup_step": 100,
+            "sparsity": [0.996, 0.999]
+        }
+        return avg_cost, strategy
+
+    def test_dgc_optimizer(self):
+        avg_cost, strategy = self.net()
+        optimizer = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
+        optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+        optimizer.minimize(avg_cost)
+
+        ops = [op.type for op in avg_cost.block.ops]
+        self.assertIn('dgc', ops)
+        self.assertIn('dgc_momentum', ops)
+
+    def test_dgc_not_apply_with_adam(self):
+        avg_cost, strategy = self.net()
+        optimizer = paddle.optimizer.Adam(learning_rate=0.01)
+        optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+        optimizer.minimize(avg_cost)
+
+        ops = [op.type for op in avg_cost.block.ops]
+        self.assertNotIn('dgc', ops)
+        self.assertNotIn('dgc_momentum', ops)
+
+    def test_dgc_not_apply_with_one_worker(self):
+        os.environ["PADDLE_TRAINER_ID"] = "0"
+        os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001"
+
+        avg_cost, strategy = self.net()
+        optimizer = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
+        optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+        optimizer.minimize(avg_cost)
+
+        ops = [op.type for op in avg_cost.block.ops]
+        self.assertNotIn('dgc', ops)
+        self.assertNotIn('dgc_momentum', ops)
+
+
+if __name__ == "__main__":
+    unittest.main()

python/paddle/fluid/tests/unittests/test_sort_op.py

@@ -37,7 +37,7 @@ class TestSortOnCPU(unittest.TestCase):
                 [[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]],
                  [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]],
                 dtype='float32')
-            result, = exe.run(feed={'input': data}, fetch_list=[output[0]])
+            result, = exe.run(feed={'input': data}, fetch_list=[output])
             np_result = np.sort(result)
             self.assertEqual((result == np_result).all(), True)
 
@@ -50,7 +50,7 @@ class TestSortOnCPU(unittest.TestCase):
                 [[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]],
                  [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]],
                 dtype='float32')
-            result, = exe.run(feed={'input': data}, fetch_list=[output[0]])
+            result, = exe.run(feed={'input': data}, fetch_list=[output])
             np_result = np.sort(result, axis=1)
             self.assertEqual((result == np_result).all(), True)
 
@@ -75,7 +75,7 @@ class TestSortDygraph(unittest.TestCase):
         with imperative.guard(self.place):
             var_x = imperative.to_variable(self.input_data)
             out = paddle.sort(var_x)
-            self.assertEqual((np.sort(self.input_data) == out[0].numpy()).all(),
+            self.assertEqual((np.sort(self.input_data) == out.numpy()).all(),
                              True)
 
     def test_api_1(self):
@@ -84,5 +84,4 @@ class TestSortDygraph(unittest.TestCase):
             out = paddle.sort(var_x, axis=-1)
             self.assertEqual(
                 (np.sort(
-                    self.input_data, axis=-1) == out[0].numpy()).all(),
-                True)
+                    self.input_data, axis=-1) == out.numpy()).all(), True)

python/paddle/incubate/hapi/__init__.py

@@ -25,6 +25,7 @@ from . import datasets
 from . import distributed
 from . import vision
 from . import text
+from . import utils
 
 from . import device
 from .device import *
@@ -41,6 +42,7 @@ __all__ = [
     'metrics',
     'vision',
     'text',
+    'utils',
 ] + model.__all__ + device.__all__
 
 monkey_patch_layer()

python/paddle/incubate/hapi/tests/test_uncombined_weight2state_dict.py

+# copyright (c) 2020 paddlepaddle authors. all rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+
+import numpy as np
+import shutil
+import tempfile
+
+from paddle import fluid
+from paddle.nn import Conv2D, Pool2D, Linear, ReLU, Sequential
+
+from paddle.incubate.hapi.utils import uncombined_weight_to_state_dict
+
+
+class LeNetDygraph(fluid.dygraph.Layer):
+    def __init__(self, num_classes=10, classifier_activation='softmax'):
+        super(LeNetDygraph, self).__init__()
+        self.num_classes = num_classes
+        self.features = Sequential(
+            Conv2D(
+                1, 6, 3, stride=1, padding=1),
+            ReLU(),
+            Pool2D(2, 'max', 2),
+            Conv2D(
+                6, 16, 5, stride=1, padding=0),
+            ReLU(),
+            Pool2D(2, 'max', 2))
+
+        if num_classes > 0:
+            self.fc = Sequential(
+                Linear(400, 120),
+                Linear(120, 84),
+                Linear(
+                    84, 10, act=classifier_activation))
+
+    def forward(self, inputs):
+        x = self.features(inputs)
+
+        if self.num_classes > 0:
+            x = fluid.layers.flatten(x, 1)
+            x = self.fc(x)
+        return x
+
+
+class TestUncombinedWeight2StateDict(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.save_dir = tempfile.mkdtemp()
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.save_dir)
+
+    def test_infer(self):
+        start_prog = fluid.Program()
+        train_prog = fluid.Program()
+
+        x = fluid.data(name='x', shape=[None, 1, 28, 28], dtype='float32')
+
+        with fluid.program_guard(train_prog, start_prog):
+            with fluid.unique_name.guard():
+                x = fluid.data(
+                    name='x', shape=[None, 1, 28, 28], dtype='float32')
+                model = LeNetDygraph()
+                output = model.forward(x)
+
+        excutor = fluid.Executor()
+        excutor.run(start_prog)
+
+        test_prog = train_prog.clone(for_test=True)
+
+        fluid.io.save_params(excutor, self.save_dir, test_prog)
+
+        rand_x = np.random.rand(1, 1, 28, 28).astype('float32')
+        out = excutor.run(program=test_prog,
+                          feed={'x': rand_x},
+                          fetch_list=[output.name],
+                          return_numpy=True)
+
+        state_dict = uncombined_weight_to_state_dict(self.save_dir)
+
+        key2key_dict = {
+            'features.0.weight': 'conv2d_0.w_0',
+            'features.0.bias': 'conv2d_0.b_0',
+            'features.3.weight': 'conv2d_1.w_0',
+            'features.3.bias': 'conv2d_1.b_0',
+            'fc.0.weight': 'linear_0.w_0',
+            'fc.0.bias': 'linear_0.b_0',
+            'fc.1.weight': 'linear_1.w_0',
+            'fc.1.bias': 'linear_1.b_0',
+            'fc.2.weight': 'linear_2.w_0',
+            'fc.2.bias': 'linear_2.b_0'
+        }
+
+        fluid.enable_imperative()
+        dygraph_model = LeNetDygraph()
+
+        converted_state_dict = dygraph_model.state_dict()
+        for k1, k2 in key2key_dict.items():
+            converted_state_dict[k1] = state_dict[k2]
+
+        dygraph_model.set_dict(converted_state_dict)
+
+        dygraph_model.eval()
+        dy_out = dygraph_model(fluid.dygraph.to_variable(rand_x))
+
+        np.testing.assert_allclose(dy_out.numpy(), out[0], atol=1e-5)
+
+
+if __name__ == '__main__':
+    unittest.main()

python/paddle/incubate/hapi/utils.py

@@ -12,13 +12,171 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import inspect
 import numpy as np
 
+from collections import OrderedDict
 from paddle import fluid
 from paddle.fluid.framework import Variable
 from paddle.fluid.executor import global_scope
 
+__all__ = ['uncombined_weight_to_state_dict']
+
+
+def uncombined_weight_to_state_dict(weight_dir):
+    """
+    Convert uncombined weight which getted by using `fluid.io.save_params` or `fluid.io.save_persistables` to state_dict
+
+    Args:
+        weight_dir (str): weight direcotory path.
+
+    Returns:
+        OrderDict: weight dict.
+
+    Examples:
+        .. code-block:: python
+
+            import os
+
+            from paddle import fluid
+            from paddle.nn import Conv2D, Pool2D, Linear, ReLU, Sequential
+            from paddle.incubate.hapi.utils import uncombined_weight_to_state_dict
+
+
+            class LeNetDygraph(fluid.dygraph.Layer):
+                def __init__(self, num_classes=10, classifier_activation='softmax'):
+                    super(LeNetDygraph, self).__init__()
+                    self.num_classes = num_classes
+                    self.features = Sequential(
+                        Conv2D(
+                            1, 6, 3, stride=1, padding=1),
+                        ReLU(),
+                        Pool2D(2, 'max', 2),
+                        Conv2D(
+                            6, 16, 5, stride=1, padding=0),
+                        ReLU(),
+                        Pool2D(2, 'max', 2))
+
+                    if num_classes > 0:
+                        self.fc = Sequential(
+                            Linear(400, 120),
+                            Linear(120, 84),
+                            Linear(
+                                84, 10, act=classifier_activation))
+
+                def forward(self, inputs):
+                    x = self.features(inputs)
+
+                    if self.num_classes > 0:
+                        x = fluid.layers.flatten(x, 1)
+                        x = self.fc(x)
+                    return x
+
+            # save weight use fluid.io.save_params
+            save_dir = 'temp'
+            if not os.path.exists(save_dir):
+                os.makedirs(save_dir)
+
+            start_prog = fluid.Program()
+            train_prog = fluid.Program()
+
+            x = fluid.data(name='x', shape=[None, 1, 28, 28], dtype='float32')
+
+            with fluid.program_guard(train_prog, start_prog):
+                with fluid.unique_name.guard():
+                    x = fluid.data(
+                        name='x', shape=[None, 1, 28, 28], dtype='float32')
+                    model = LeNetDygraph()
+                    output = model.forward(x)
+
+            excutor = fluid.Executor()
+            excutor.run(start_prog)
+
+            test_prog = train_prog.clone(for_test=True)
+
+            fluid.io.save_params(excutor, save_dir, test_prog)
+
+            # convert uncombined weight to state dict
+            state_dict = uncombined_weight_to_state_dict(save_dir)
+
+            key2key_dict = {
+                'features.0.weight': 'conv2d_0.w_0',
+                'features.0.bias': 'conv2d_0.b_0',
+                'features.3.weight': 'conv2d_1.w_0',
+                'features.3.bias': 'conv2d_1.b_0',
+                'fc.0.weight': 'linear_0.w_0',
+                'fc.0.bias': 'linear_0.b_0',
+                'fc.1.weight': 'linear_1.w_0',
+                'fc.1.bias': 'linear_1.b_0',
+                'fc.2.weight': 'linear_2.w_0',
+                'fc.2.bias': 'linear_2.b_0'
+            }
+
+            fluid.enable_imperative()
+            dygraph_model = LeNetDygraph()
+
+            converted_state_dict = dygraph_model.state_dict()
+            for k1, k2 in key2key_dict.items():
+                converted_state_dict[k1] = state_dict[k2]
+
+            # dygraph model load state dict which converted from uncombined weight
+            dygraph_model.set_dict(converted_state_dict)
+    """
+
+    def _get_all_params_name(dir):
+        params_name = []
+        dir = os.path.expanduser(dir)
+
+        dir_len = len(dir)
+        for root, _, fnames in sorted(os.walk(dir, followlinks=True)):
+            for fname in sorted(fnames):
+                path = os.path.join(root[dir_len:], fname)
+                params_name.append(path)
+
+        return params_name
+
+    class Load(fluid.dygraph.Layer):
+        def __init__(self):
+            super(Load, self).__init__()
+
+        def forward(self, filename):
+            weight = self.create_parameter(
+                shape=[1],
+                dtype='float32',
+                default_initializer=fluid.initializer.ConstantInitializer(0.0))
+            self._helper.append_op(
+                type='load',
+                inputs={},
+                outputs={'Out': [weight]},
+                attrs={'file_path': filename})
+            return weight
+
+    params_name_list = _get_all_params_name(weight_dir)
+    if not fluid.in_dygraph_mode():
+        dygraph_enabled = False
+        fluid.enable_imperative()
+    else:
+        dygraph_enabled = True
+
+    load = Load()
+    state_dict = OrderedDict()
+
+    for param_name in params_name_list:
+        param_path = os.path.join(weight_dir, param_name)
+        weight = load(param_path)
+        try:
+            weight = weight.numpy()
+        except Exception as e:
+            print(e)
+
+        state_dict[param_name] = weight
+
+    if not dygraph_enabled:
+        fluid.disable_imperative()
+
+    return state_dict
+
 
 def to_list(value):
     if value is None:

python/paddle/tensor/search.py

@@ -46,8 +46,7 @@ def argsort(x, axis=-1, descending=False, name=None):
 	:alias_main: paddle.argsort
 	:alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort
 
-    This OP sorts the input along the given axis, and returns sorted output
-    data Varibale and its corresponding index Variable with the same shape as ``x``.
+    This OP sorts the input along the given axis, and returns the corresponding index tensor for the sorted output values. The default sort algorithm is ascending, if you want the sort algorithm to be descending, you must set the :attr:`descending` as True.
 
     Args:
         x(Tensor): An input N-D Tensor with type float32, float64, int16,
@@ -84,26 +83,26 @@ def argsort(x, axis=-1, descending=False, name=None):
             out2 = paddle.argsort(x=x, axis=0)
             out3 = paddle.argsort(x=x, axis=1)
             print(out1.numpy())
-	    #[[[0 3 1 2]
-	    #  [0 1 2 3]
-	    #  [2 3 0 1]]
+            #[[[0 3 1 2]
+            #  [0 1 2 3]
+            #  [2 3 0 1]]
             # [[1 3 2 0]
-	    #  [0 1 2 3]
-	    #  [2 0 3 1]]]
+            #  [0 1 2 3]
+            #  [2 0 3 1]]]
             print(out2.numpy())
-	    #[[[0 1 1 1]
-	    #  [0 0 0 0]
-	    #  [1 1 1 0]]
-	    # [[1 0 0 0]
-	    #  [1 1 1 1]
-	    #  [0 0 0 1]]]
+            #[[[0 1 1 1]
+            #  [0 0 0 0]
+            #  [1 1 1 0]]
+            # [[1 0 0 0]
+            #  [1 1 1 1]
+            #  [0 0 0 1]]]
             print(out3.numpy())
-	    #[[[1 1 1 2]
-	    #  [0 0 2 0]
-	    #  [2 2 0 1]]
-	    # [[2 0 2 0]
-	    #  [1 1 0 2]
-	    #  [0 2 1 1]]]
+            #[[[1 1 1 2]
+            #  [0 0 2 0]
+            #  [2 2 0 1]]
+            # [[2 0 2 0]
+            #  [1 1 0 2]
+            #  [0 2 1 1]]]
     """
     if in_dygraph_mode():
         _, ids = core.ops.argsort(x, 'axis', axis, 'descending', descending)
@@ -381,8 +380,7 @@ def sort(x, axis=-1, descending=False, name=None):
 	:alias_main: paddle.sort
 	:alias: paddle.sort,paddle.tensor.sort,paddle.tensor.search.sort
 
-    This OP sorts the input along the given axis, and returns sorted output
-    data Tensor and its corresponding index Tensor with the same shape as ``x``.
+    This OP sorts the input along the given axis, and returns the sorted output tensor. The default sort algorithm is ascending, if you want the sort algorithm to be descending, you must set the :attr:`descending` as True.
 
     Args:
         x(Tensor): An input N-D Tensor with type float32, float64, int16,
@@ -397,9 +395,7 @@ def sort(x, axis=-1, descending=False, name=None):
             need for user to set this property. For more information, please
             refer to :ref:`api_guide_Name`.
     Returns:
-        tuple: A tuple of sorted data tensor(with the same shape and data
-        type as ``x``) and the sorted indices(with the same shape as ``x``
-        and with data type int64).
+        Tensor: sorted tensor(with the same shape and data type as ``x``).
     Examples:
         .. code-block:: python
             import paddle
@@ -417,38 +413,31 @@ def sort(x, axis=-1, descending=False, name=None):
             out1 = paddle.sort(x=x, axis=-1)
             out2 = paddle.sort(x=x, axis=0)
             out3 = paddle.sort(x=x, axis=1)
-            print(out1[0].numpy())
-	    #[[[5. 5. 8. 9.]
-	    #  [0. 0. 1. 7.]
-	    #  [2. 4. 6. 9.]]
-	    # [[2. 2. 4. 5.]
-	    #  [4. 7. 7. 9.]
-	    #  [0. 1. 6. 7.]]]
-            print(out1[1].numpy())
-	    #[[[0 3 1 2]
-	    #  [0 1 2 3]
-	    #  [2 3 0 1]]
-            # [[1 3 2 0]
-	    #  [0 1 2 3]
-	    #  [2 0 3 1]]]
-            print(out2[0].numpy())
+            print(out1.numpy())
+            #[[[5. 5. 8. 9.]
+            #  [0. 0. 1. 7.]
+            #  [2. 4. 6. 9.]]
+            # [[2. 2. 4. 5.]
+            #  [4. 7. 7. 9.]
+            #  [0. 1. 6. 7.]]]
+            print(out2.numpy())
             #[[[5. 2. 4. 2.]
-	    #  [0. 0. 1. 7.]
-	    #  [1. 7. 0. 4.]]
-	    # [[5. 8. 9. 5.]
-	    #  [4. 7. 7. 9.]
-	    #  [6. 9. 2. 6.]]]
-            print(out3[0].numpy())
+            #  [0. 0. 1. 7.]
+            #  [1. 7. 0. 4.]]
+            # [[5. 8. 9. 5.]
+            #  [4. 7. 7. 9.]
+            #  [6. 9. 2. 6.]]]
+            print(out3.numpy())
             #[[[0. 0. 1. 4.]
-	    #  [5. 8. 2. 5.]
-	    #  [6. 9. 9. 7.]]
-	    # [[1. 2. 0. 2.]
-	    #  [4. 7. 4. 6.]
-	    #  [5. 7. 7. 9.]]]
+            #  [5. 8. 2. 5.]
+            #  [6. 9. 9. 7.]]
+            # [[1. 2. 0. 2.]
+            #  [4. 7. 4. 6.]
+            #  [5. 7. 7. 9.]]]
     """
     if in_dygraph_mode():
-        out, ids = core.ops.argsort(x, 'axis', axis, 'descending', descending)
-        return out, ids
+        out, _ = core.ops.argsort(x, 'axis', axis, 'descending', descending)
+        return out
     helper = LayerHelper("sort", **locals())
     out = helper.create_variable_for_type_inference(
         dtype=x.dtype, stop_gradient=False)
@@ -461,7 +450,7 @@ def sort(x, axis=-1, descending=False, name=None):
                  'Indices': ids},
         attrs={'axis': axis,
                'descending': descending})
-    return out, ids
+    return out
 
 
 def where(condition, x, y, name=None):

tools/dockerfile/Dockerfile.centos

@@ -13,7 +13,7 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
 ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
 ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
-RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
+RUN yum install -y gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
 COPY build_scripts /build_scripts
 RUN bash build_scripts/build.sh
 RUN bash build_scripts/install_nccl2.sh && \
@@ -22,6 +22,13 @@ RUN rm -rf build_scripts
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 
+# git 2.17.1
+RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
+  tar -xvf git-2.17.1.tar.gz && \
+  cd git-2.17.1 && \
+  ./configure --prefix=/usr/local && \
+  make -j8 && make install 
+
 # for paddle
 RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \
     tar -xz -C /usr/local && \

tools/dockerfile/ci_dockerfile.sh

@@ -3,9 +3,9 @@ function make_ubuntu_dockerfile(){
   dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16"
   sed 's/<baseimg>/10.1-cudnn7-devel-ubuntu16.04/g' ./Dockerfile.ubuntu >${dockerfile_name}
   sed -i 's#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g' ${dockerfile_name} 
+  dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
   sed -i "${dockerfile_line}i RUN wget --no-check-certificate  -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \
      tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} 
-  dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
   sed -i 's#<install_gcc>#WORKDIR /usr/bin \
       COPY tools/dockerfile/build_scripts /build_scripts \
       RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \
@@ -24,6 +24,9 @@ function make_centos_dockerfile(){
   sed 's/<baseimg>/9.0-cudnn7-devel-centos6/g' Dockerfile.centos >${dockerfile_name}
   sed -i 's#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g' ${dockerfile_name} 
   dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
+  sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so && \
+     ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ && \
+     rm -rf /usr/include/NvInfer*" ${dockerfile_name}
   sed -i "${dockerfile_line}i RUN wget --no-check-certificate  -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \
      tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
 }