diff --git a/README.md b/README.md
index b07709facd528114a1d69513a487d201f1dfc160..4196811e37f73f84b0327f5cbf1996aaaf7e6dcc 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-
+
diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto
index d17e68276cd1ce576029cf306a18469aef2ffdb0..05b7a16f1594f370cbf73ab7fdb4c98e3bb76024 100755
--- a/paddle/fluid/framework/distributed_strategy.proto
+++ b/paddle/fluid/framework/distributed_strategy.proto
@@ -43,6 +43,12 @@ message GradientMergeConfig {
optional bool avg = 2 [ default = true ];
}
+message DGCConfig {
+ optional int32 rampup_begin_step = 1 [ default = 0 ];
+ optional int32 rampup_step = 2 [ default = 1 ];
+ repeated float sparsity = 3;
+}
+
message LarsConfig {
optional float lars_coeff = 1 [ default = 0.001 ];
optional float lars_weight_decay = 2 [ default = 0.0005 ];
@@ -114,6 +120,7 @@ message DistributedStrategy {
optional AMPConfig amp_configs = 102;
optional LocalSGDConfig localsgd_configs = 103;
optional GradientMergeConfig gradient_merge_configs = 104;
+ optional DGCConfig dgc_configs = 105;
optional PipelineConfig pipeline_configs = 106;
optional AsyncConfig a_sync_configs = 107;
optional LarsConfig lars_configs = 108;
diff --git a/paddle/fluid/operators/p_norm_op.cu b/paddle/fluid/operators/p_norm_op.cu
index a0b3df6d6e74e870651f0b91041293c2956c2ba9..d9ac98ff880bcf42e0af5bb75b080464c5211671 100644
--- a/paddle/fluid/operators/p_norm_op.cu
+++ b/paddle/fluid/operators/p_norm_op.cu
@@ -99,39 +99,25 @@ __global__ void PnormGradient(const T* x, const T* x_norm, const T* y_grad,
const float porder, const int pre,
const int axis_n, const int post, const T eps,
T* x_grad) {
- typedef cub::BlockReduce BlockReduce;
- __shared__ typename BlockReduce::TempStorage temp_storage_sum;
// dx = (x/pnorm_broadcast).pow(p-1) * norm_dy.broadcast * sign(x)
int num = pre * post;
auto porder_grad = static_cast(porder - 1.0f);
for (int i = blockIdx.x; i < num; i += gridDim.x) {
- T sum = 0.0;
- __shared__ T row_sum;
- __shared__ T row_sqrt_norm;
- __shared__ T row_norm;
+ __shared__ T pnorm_i;
+ __shared__ T yout_i;
auto base = (i / post) * post * axis_n + (i % post);
- for (int j = threadIdx.x; j < axis_n; j += blockDim.x) {
- int index = base + j * post;
- sum += x[index] * y_grad[index];
- }
- T reduce_result = BlockReduce(temp_storage_sum).Sum(sum);
-
if (threadIdx.x == 0) {
- row_sum = reduce_result;
- row_sqrt_norm = x_norm[i];
- row_norm = row_sqrt_norm * row_sqrt_norm;
+ pnorm_i = x_norm[i];
+ yout_i = y_grad[i];
}
- __syncthreads();
- const T pnorm_i = x_norm[i];
- const T yout_i = y_grad[i];
+ __syncthreads();
for (int j = threadIdx.x; j < axis_n; j += blockDim.x) {
int index = base + j * post;
const T x_ij = inline_abs(x[index]);
- const T dy_ij = y_grad[index];
x_grad[index] = inline_pow(x_ij, porder_grad) /
(inline_pow(pnorm_i, porder_grad) + eps) * yout_i *
inline_sign(x[index]);
diff --git a/python/paddle/fleet/base/distributed_strategy.py b/python/paddle/fleet/base/distributed_strategy.py
index 4cc7beadd80a071f7b22bb46f0b157bdffbd74f2..43e50ca0bee6b324655f7dcfb5e5da2ebc0e85a8 100644
--- a/python/paddle/fleet/base/distributed_strategy.py
+++ b/python/paddle/fleet/base/distributed_strategy.py
@@ -604,6 +604,15 @@ class DistributedStrategy(object):
else:
print("WARNING: lars should have value of bool type")
+ @property
+ def lars_configs(self):
+ return get_msg_dict(self.strategy.lars_configs)
+
+ @lars_configs.setter
+ def lars_configs(self, configs):
+ check_configs_key(self.strategy.lars_configs, configs, "lars_configs")
+ assign_configs_value(self.strategy.lars_configs, configs)
+
@property
def lamb(self):
return self.strategy.lamb
diff --git a/python/paddle/fleet/base/meta_optimizer_factory.py b/python/paddle/fleet/base/meta_optimizer_factory.py
index 89ebb0ec601e249c58fd43995df1530f44940af4..802f6c4dab7f3a98cc11d9bb1956db5ee33b2746 100755
--- a/python/paddle/fleet/base/meta_optimizer_factory.py
+++ b/python/paddle/fleet/base/meta_optimizer_factory.py
@@ -19,6 +19,7 @@ from ..meta_optimizers import GraphExecutionOptimizer
from ..meta_optimizers import PipelineOptimizer
from ..meta_optimizers import LocalSGDOptimizer
from ..meta_optimizers import LarsOptimizer
+from ..meta_optimizers import DGCOptimizer
__all__ = ["MetaOptimizerFactory"]
@@ -30,6 +31,7 @@ meta_optimizer_names = [
"PipelineOptimizer",
"LocalSGDOptimizer",
"LarsOptimizer",
+ "DGCOptimizer",
]
diff --git a/python/paddle/fleet/meta_optimizers/__init__.py b/python/paddle/fleet/meta_optimizers/__init__.py
index aa6708e758a78cf2cb10f8ebda81d50ac796b548..718805c5aadaf3476fa1fc495a355395fec6396d 100755
--- a/python/paddle/fleet/meta_optimizers/__init__.py
+++ b/python/paddle/fleet/meta_optimizers/__init__.py
@@ -18,6 +18,7 @@ from .graph_execution_optimizer import GraphExecutionOptimizer
from .pipeline_optimizer import PipelineOptimizer
from .localsgd_optimizer import LocalSGDOptimizer
from .lars_optimizer import LarsOptimizer
+from .dgc_optimizer import DGCOptimizer
__all__ = [
'AMPOptimizer',
@@ -26,4 +27,5 @@ __all__ = [
'PipelineOptimizer',
'LocalSGDOptimizer',
'LarsOptimizer',
+ 'DGCOptimizer',
]
diff --git a/python/paddle/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/fleet/meta_optimizers/dgc_optimizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9a28fdaf11dd0d4d45cfd3fb1904b80dc136711
--- /dev/null
+++ b/python/paddle/fleet/meta_optimizers/dgc_optimizer.py
@@ -0,0 +1,94 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+from paddle.fluid.optimizer import Momentum, DGCMomentumOptimizer
+from .meta_optimizer_base import MetaOptimizerBase
+import logging
+
+__all__ = ["DGCOptimizer"]
+
+
+class DGCOptimizer(MetaOptimizerBase):
+ def __init__(self, optimizer):
+ super(DGCOptimizer, self).__init__(optimizer)
+ self.inner_opt = optimizer
+ self.dgc_opt = None
+ # we do not allow meta optimizer to be inner optimizer currently
+ self.meta_optimizers_white_list = []
+
+ def _set_basic_info(self, loss, role_maker, user_defined_optimizer,
+ user_defined_strategy):
+ super(DGCOptimizer, self)._set_basic_info(
+ loss, role_maker, user_defined_optimizer, user_defined_strategy)
+
+ opt = self.inner_opt
+ if not isinstance(opt, Momentum):
+ return
+
+ configs = self.user_defined_strategy.dgc_configs
+ if len(configs['sparsity']) == 0:
+ # default is [0.999]
+ configs['sparsity'] = [0.999]
+
+ self.dgc_opt = DGCMomentumOptimizer(
+ learning_rate=opt._learning_rate,
+ momentum=opt._momentum,
+ rampup_begin_step=configs['rampup_begin_step'],
+ rampup_step=configs['rampup_step'],
+ sparsity=configs['sparsity'],
+ parameter_list=opt._parameter_list,
+ use_nesterov=opt._use_nesterov,
+ num_trainers=self.role_maker.worker_num(),
+ regularization=opt.regularization,
+ grad_clip=opt._grad_clip,
+ name=opt._name)
+
+ def _can_apply(self):
+ if self.user_defined_strategy.dgc:
+ if not isinstance(self.inner_opt, Momentum):
+ logging.warn("dgc only works on Momentum optimizer")
+ return False
+ if self.role_maker.worker_num() <= 1:
+ logging.warn("dgc only works on multi cards")
+ return False
+
+ return True
+
+ return False
+
+ def _disable_strategy(self, dist_strategy):
+ dist_strategy.dgc = False
+ dist_strategy.dgc_configs = {
+ 'rampup_begin_step': 0,
+ 'rampup_step': 1,
+ 'sparsity': [0.999]
+ }
+
+ def backward(self,
+ loss,
+ startup_program=None,
+ parameter_list=None,
+ no_grad_set=None,
+ callbacks=None):
+ return self.dgc_opt.backward(loss, startup_program, parameter_list,
+ no_grad_set, callbacks)
+
+ def minimize_impl(self,
+ loss,
+ startup_program=None,
+ parameter_list=None,
+ no_grad_set=None):
+ optimize_ops, params_grads = \
+ self.dgc_opt.minimize(loss, startup_program,
+ parameter_list, no_grad_set)
+ return optimize_ops, params_grads
diff --git a/python/paddle/fleet/meta_optimizers/meta_optimizer_base.py b/python/paddle/fleet/meta_optimizers/meta_optimizer_base.py
index 1a3cfda94b98c9514208433dfcf5947caea8537c..9ba184fb0089589a86d6444d12cf402b9687b041 100644
--- a/python/paddle/fleet/meta_optimizers/meta_optimizer_base.py
+++ b/python/paddle/fleet/meta_optimizers/meta_optimizer_base.py
@@ -40,7 +40,8 @@ class MetaOptimizerBase(object):
return True
def _disable_strategy(self, dist_strategy):
- raise NotImplementedError("you should implement disable strategy")
+ raise NotImplementedError("you should implement disable strategy in {}".
+ format(type(self).__name__))
def minimize_impl(self,
loss,
diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index e66f640665e2ba9ca9aab51af3f65b50169de404..c84d2ac3796efe9d16641552f1be939a666aa4cf 100644
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -47,9 +47,8 @@ __all__ = [
'AdamOptimizer', 'AdamaxOptimizer', 'DpsgdOptimizer',
'DecayedAdagradOptimizer', 'RMSPropOptimizer', 'FtrlOptimizer', 'Adadelta',
'AdadeltaOptimizer', 'ModelAverage', 'LarsMomentum',
- 'LarsMomentumOptimizer', 'DGCMomentumOptimizer', 'LambOptimizer',
- 'ExponentialMovingAverage', 'PipelineOptimizer', 'LookaheadOptimizer',
- 'RecomputeOptimizer'
+ 'LarsMomentumOptimizer', 'LambOptimizer', 'ExponentialMovingAverage',
+ 'PipelineOptimizer', 'LookaheadOptimizer', 'RecomputeOptimizer'
]
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 7696839843b41d21eb6fdea0664ca69b917d8a0e..d73b9511b76ed6585c662264e99fe41f3354bc29 100755
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -39,6 +39,7 @@ list(APPEND MIXED_DIST_TEST_OPS test_fleet_amp_meta_optimizer)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_gradient_merge_meta_optimizer)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_localsgd_meta_optimizer)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_lars_meta_optimizer)
+list(APPEND MIXED_DIST_TEST_OPS test_fleet_dgc_meta_optimizer)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_private_function)
list(APPEND MIXED_DIST_TEST_OPS test_fleet_graph_executor)
foreach(TEST_OP ${MIXED_DIST_TEST_OPS})
@@ -388,6 +389,7 @@ if(WITH_DISTRIBUTE)
py_test_modules(test_dgc_op MODULES test_dgc_op)
py_test_modules(test_dgc_momentum_op MODULES test_dgc_momentum_op)
py_test_modules(test_dgc_optimizer MODULES test_dgc_optimizer)
+ py_test_modules(test_fleet_dgc_meta_optimizer MODULES test_fleet_dgc_meta_optimizer)
else()
# if not with dgc, must close all dgc tests
list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")
diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_dgc_nccl.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_dgc_nccl.py
index 07746dd9f6cff297feacfa2dac24d89b2af876ab..0b9b85d5d52c38f748679a92a99ec61c3dec7903 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_mnist_dgc_nccl.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_dgc_nccl.py
@@ -39,7 +39,6 @@ class TestDistMnistNCCL2DGC(TestDistBase):
self._nccl2_mode = True
self._use_dgc = True
- @unittest.skip(reason="Skip unstable ut")
def test_dist_train(self):
import paddle.fluid as fluid
if fluid.core.is_compiled_with_cuda():
@@ -69,7 +68,6 @@ class TestDistMnistNCCL2DGCMultiCards(TestDistBase):
self._nccl2_mode = True
self._use_dgc = True
- @unittest.skip(reason="Skip unstable ut")
def test_dist_train(self):
import paddle.fluid as fluid
if fluid.core.is_compiled_with_cuda():
diff --git a/python/paddle/fluid/tests/unittests/test_fleet_dgc_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_dgc_meta_optimizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0590650bd02f5535b9c35bae187e77bc7274901c
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_fleet_dgc_meta_optimizer.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import paddle
+import os
+import paddle.fleet as fleet
+import paddle.fluid.incubate.fleet.base.role_maker as role_maker
+
+
+class TestFleetDGCOptimizer(unittest.TestCase):
+ def setUp(self):
+ os.environ["PADDLE_TRAINER_ID"] = "1"
+ os.environ[
+ "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002"
+
+ def net(self):
+ role = role_maker.PaddleCloudRoleMaker(is_collective=True)
+ fleet.init(role)
+ input_x = paddle.fluid.layers.data(
+ name="x", shape=[32], dtype='float32')
+ input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64')
+
+ fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh')
+ fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh')
+ prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax')
+ cost = paddle.fluid.layers.cross_entropy(
+ input=prediction, label=input_y)
+ avg_cost = paddle.fluid.layers.mean(x=cost)
+
+ strategy = paddle.fleet.DistributedStrategy()
+ strategy.dgc = True
+ strategy.dgc_configs = {
+ "rampup_begin_step": 128,
+ "rampup_step": 100,
+ "sparsity": [0.996, 0.999]
+ }
+ return avg_cost, strategy
+
+ def test_dgc_optimizer(self):
+ avg_cost, strategy = self.net()
+ optimizer = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
+ optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+ optimizer.minimize(avg_cost)
+
+ ops = [op.type for op in avg_cost.block.ops]
+ self.assertIn('dgc', ops)
+ self.assertIn('dgc_momentum', ops)
+
+ def test_dgc_not_apply_with_adam(self):
+ avg_cost, strategy = self.net()
+ optimizer = paddle.optimizer.Adam(learning_rate=0.01)
+ optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+ optimizer.minimize(avg_cost)
+
+ ops = [op.type for op in avg_cost.block.ops]
+ self.assertNotIn('dgc', ops)
+ self.assertNotIn('dgc_momentum', ops)
+
+ def test_dgc_not_apply_with_one_worker(self):
+ os.environ["PADDLE_TRAINER_ID"] = "0"
+ os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001"
+
+ avg_cost, strategy = self.net()
+ optimizer = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
+ optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
+ optimizer.minimize(avg_cost)
+
+ ops = [op.type for op in avg_cost.block.ops]
+ self.assertNotIn('dgc', ops)
+ self.assertNotIn('dgc_momentum', ops)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/paddle/fluid/tests/unittests/test_sort_op.py b/python/paddle/fluid/tests/unittests/test_sort_op.py
index 990c7a8b2dfb68ef7d2365a8f2918cd68692a216..087586aa89607a58493c2d4427cbb6d30b31f0da 100644
--- a/python/paddle/fluid/tests/unittests/test_sort_op.py
+++ b/python/paddle/fluid/tests/unittests/test_sort_op.py
@@ -37,7 +37,7 @@ class TestSortOnCPU(unittest.TestCase):
[[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]],
[[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]],
dtype='float32')
- result, = exe.run(feed={'input': data}, fetch_list=[output[0]])
+ result, = exe.run(feed={'input': data}, fetch_list=[output])
np_result = np.sort(result)
self.assertEqual((result == np_result).all(), True)
@@ -50,7 +50,7 @@ class TestSortOnCPU(unittest.TestCase):
[[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]],
[[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]],
dtype='float32')
- result, = exe.run(feed={'input': data}, fetch_list=[output[0]])
+ result, = exe.run(feed={'input': data}, fetch_list=[output])
np_result = np.sort(result, axis=1)
self.assertEqual((result == np_result).all(), True)
@@ -75,7 +75,7 @@ class TestSortDygraph(unittest.TestCase):
with imperative.guard(self.place):
var_x = imperative.to_variable(self.input_data)
out = paddle.sort(var_x)
- self.assertEqual((np.sort(self.input_data) == out[0].numpy()).all(),
+ self.assertEqual((np.sort(self.input_data) == out.numpy()).all(),
True)
def test_api_1(self):
@@ -84,5 +84,4 @@ class TestSortDygraph(unittest.TestCase):
out = paddle.sort(var_x, axis=-1)
self.assertEqual(
(np.sort(
- self.input_data, axis=-1) == out[0].numpy()).all(),
- True)
+ self.input_data, axis=-1) == out.numpy()).all(), True)
diff --git a/python/paddle/incubate/hapi/__init__.py b/python/paddle/incubate/hapi/__init__.py
index 94b6d1c6333f0fa0b5e19f5711ed5b3bc3e2bdaa..a6b5faef57ca95188f0759f53753177e4f5946f3 100644
--- a/python/paddle/incubate/hapi/__init__.py
+++ b/python/paddle/incubate/hapi/__init__.py
@@ -25,6 +25,7 @@ from . import datasets
from . import distributed
from . import vision
from . import text
+from . import utils
from . import device
from .device import *
@@ -41,6 +42,7 @@ __all__ = [
'metrics',
'vision',
'text',
+ 'utils',
] + model.__all__ + device.__all__
monkey_patch_layer()
diff --git a/python/paddle/incubate/hapi/tests/test_uncombined_weight2state_dict.py b/python/paddle/incubate/hapi/tests/test_uncombined_weight2state_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2035a8b5c5958d54c79d6ee0ff6df654bb35d51
--- /dev/null
+++ b/python/paddle/incubate/hapi/tests/test_uncombined_weight2state_dict.py
@@ -0,0 +1,126 @@
+# copyright (c) 2020 paddlepaddle authors. all rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+
+import numpy as np
+import shutil
+import tempfile
+
+from paddle import fluid
+from paddle.nn import Conv2D, Pool2D, Linear, ReLU, Sequential
+
+from paddle.incubate.hapi.utils import uncombined_weight_to_state_dict
+
+
+class LeNetDygraph(fluid.dygraph.Layer):
+ def __init__(self, num_classes=10, classifier_activation='softmax'):
+ super(LeNetDygraph, self).__init__()
+ self.num_classes = num_classes
+ self.features = Sequential(
+ Conv2D(
+ 1, 6, 3, stride=1, padding=1),
+ ReLU(),
+ Pool2D(2, 'max', 2),
+ Conv2D(
+ 6, 16, 5, stride=1, padding=0),
+ ReLU(),
+ Pool2D(2, 'max', 2))
+
+ if num_classes > 0:
+ self.fc = Sequential(
+ Linear(400, 120),
+ Linear(120, 84),
+ Linear(
+ 84, 10, act=classifier_activation))
+
+ def forward(self, inputs):
+ x = self.features(inputs)
+
+ if self.num_classes > 0:
+ x = fluid.layers.flatten(x, 1)
+ x = self.fc(x)
+ return x
+
+
+class TestUncombinedWeight2StateDict(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.save_dir = tempfile.mkdtemp()
+
+ @classmethod
+ def tearDownClass(cls):
+ shutil.rmtree(cls.save_dir)
+
+ def test_infer(self):
+ start_prog = fluid.Program()
+ train_prog = fluid.Program()
+
+ x = fluid.data(name='x', shape=[None, 1, 28, 28], dtype='float32')
+
+ with fluid.program_guard(train_prog, start_prog):
+ with fluid.unique_name.guard():
+ x = fluid.data(
+ name='x', shape=[None, 1, 28, 28], dtype='float32')
+ model = LeNetDygraph()
+ output = model.forward(x)
+
+ excutor = fluid.Executor()
+ excutor.run(start_prog)
+
+ test_prog = train_prog.clone(for_test=True)
+
+ fluid.io.save_params(excutor, self.save_dir, test_prog)
+
+ rand_x = np.random.rand(1, 1, 28, 28).astype('float32')
+ out = excutor.run(program=test_prog,
+ feed={'x': rand_x},
+ fetch_list=[output.name],
+ return_numpy=True)
+
+ state_dict = uncombined_weight_to_state_dict(self.save_dir)
+
+ key2key_dict = {
+ 'features.0.weight': 'conv2d_0.w_0',
+ 'features.0.bias': 'conv2d_0.b_0',
+ 'features.3.weight': 'conv2d_1.w_0',
+ 'features.3.bias': 'conv2d_1.b_0',
+ 'fc.0.weight': 'linear_0.w_0',
+ 'fc.0.bias': 'linear_0.b_0',
+ 'fc.1.weight': 'linear_1.w_0',
+ 'fc.1.bias': 'linear_1.b_0',
+ 'fc.2.weight': 'linear_2.w_0',
+ 'fc.2.bias': 'linear_2.b_0'
+ }
+
+ fluid.enable_imperative()
+ dygraph_model = LeNetDygraph()
+
+ converted_state_dict = dygraph_model.state_dict()
+ for k1, k2 in key2key_dict.items():
+ converted_state_dict[k1] = state_dict[k2]
+
+ dygraph_model.set_dict(converted_state_dict)
+
+ dygraph_model.eval()
+ dy_out = dygraph_model(fluid.dygraph.to_variable(rand_x))
+
+ np.testing.assert_allclose(dy_out.numpy(), out[0], atol=1e-5)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/incubate/hapi/utils.py b/python/paddle/incubate/hapi/utils.py
index 4182d81d9fc1a593252ceeed7ba8943373a239a9..d9708f2927912870218f41103df5b0f94609cd88 100644
--- a/python/paddle/incubate/hapi/utils.py
+++ b/python/paddle/incubate/hapi/utils.py
@@ -12,13 +12,171 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import inspect
import numpy as np
+from collections import OrderedDict
from paddle import fluid
from paddle.fluid.framework import Variable
from paddle.fluid.executor import global_scope
+__all__ = ['uncombined_weight_to_state_dict']
+
+
+def uncombined_weight_to_state_dict(weight_dir):
+ """
+ Convert uncombined weight which getted by using `fluid.io.save_params` or `fluid.io.save_persistables` to state_dict
+
+ Args:
+ weight_dir (str): weight direcotory path.
+
+ Returns:
+ OrderDict: weight dict.
+
+ Examples:
+ .. code-block:: python
+
+ import os
+
+ from paddle import fluid
+ from paddle.nn import Conv2D, Pool2D, Linear, ReLU, Sequential
+ from paddle.incubate.hapi.utils import uncombined_weight_to_state_dict
+
+
+ class LeNetDygraph(fluid.dygraph.Layer):
+ def __init__(self, num_classes=10, classifier_activation='softmax'):
+ super(LeNetDygraph, self).__init__()
+ self.num_classes = num_classes
+ self.features = Sequential(
+ Conv2D(
+ 1, 6, 3, stride=1, padding=1),
+ ReLU(),
+ Pool2D(2, 'max', 2),
+ Conv2D(
+ 6, 16, 5, stride=1, padding=0),
+ ReLU(),
+ Pool2D(2, 'max', 2))
+
+ if num_classes > 0:
+ self.fc = Sequential(
+ Linear(400, 120),
+ Linear(120, 84),
+ Linear(
+ 84, 10, act=classifier_activation))
+
+ def forward(self, inputs):
+ x = self.features(inputs)
+
+ if self.num_classes > 0:
+ x = fluid.layers.flatten(x, 1)
+ x = self.fc(x)
+ return x
+
+ # save weight use fluid.io.save_params
+ save_dir = 'temp'
+ if not os.path.exists(save_dir):
+ os.makedirs(save_dir)
+
+ start_prog = fluid.Program()
+ train_prog = fluid.Program()
+
+ x = fluid.data(name='x', shape=[None, 1, 28, 28], dtype='float32')
+
+ with fluid.program_guard(train_prog, start_prog):
+ with fluid.unique_name.guard():
+ x = fluid.data(
+ name='x', shape=[None, 1, 28, 28], dtype='float32')
+ model = LeNetDygraph()
+ output = model.forward(x)
+
+ excutor = fluid.Executor()
+ excutor.run(start_prog)
+
+ test_prog = train_prog.clone(for_test=True)
+
+ fluid.io.save_params(excutor, save_dir, test_prog)
+
+ # convert uncombined weight to state dict
+ state_dict = uncombined_weight_to_state_dict(save_dir)
+
+ key2key_dict = {
+ 'features.0.weight': 'conv2d_0.w_0',
+ 'features.0.bias': 'conv2d_0.b_0',
+ 'features.3.weight': 'conv2d_1.w_0',
+ 'features.3.bias': 'conv2d_1.b_0',
+ 'fc.0.weight': 'linear_0.w_0',
+ 'fc.0.bias': 'linear_0.b_0',
+ 'fc.1.weight': 'linear_1.w_0',
+ 'fc.1.bias': 'linear_1.b_0',
+ 'fc.2.weight': 'linear_2.w_0',
+ 'fc.2.bias': 'linear_2.b_0'
+ }
+
+ fluid.enable_imperative()
+ dygraph_model = LeNetDygraph()
+
+ converted_state_dict = dygraph_model.state_dict()
+ for k1, k2 in key2key_dict.items():
+ converted_state_dict[k1] = state_dict[k2]
+
+ # dygraph model load state dict which converted from uncombined weight
+ dygraph_model.set_dict(converted_state_dict)
+ """
+
+ def _get_all_params_name(dir):
+ params_name = []
+ dir = os.path.expanduser(dir)
+
+ dir_len = len(dir)
+ for root, _, fnames in sorted(os.walk(dir, followlinks=True)):
+ for fname in sorted(fnames):
+ path = os.path.join(root[dir_len:], fname)
+ params_name.append(path)
+
+ return params_name
+
+ class Load(fluid.dygraph.Layer):
+ def __init__(self):
+ super(Load, self).__init__()
+
+ def forward(self, filename):
+ weight = self.create_parameter(
+ shape=[1],
+ dtype='float32',
+ default_initializer=fluid.initializer.ConstantInitializer(0.0))
+ self._helper.append_op(
+ type='load',
+ inputs={},
+ outputs={'Out': [weight]},
+ attrs={'file_path': filename})
+ return weight
+
+ params_name_list = _get_all_params_name(weight_dir)
+ if not fluid.in_dygraph_mode():
+ dygraph_enabled = False
+ fluid.enable_imperative()
+ else:
+ dygraph_enabled = True
+
+ load = Load()
+ state_dict = OrderedDict()
+
+ for param_name in params_name_list:
+ param_path = os.path.join(weight_dir, param_name)
+ weight = load(param_path)
+ try:
+ weight = weight.numpy()
+ except Exception as e:
+ print(e)
+
+ state_dict[param_name] = weight
+
+ if not dygraph_enabled:
+ fluid.disable_imperative()
+
+ return state_dict
+
def to_list(value):
if value is None:
diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py
index cffaae6153cf79b90e22afa103fcd11d8bfaa402..1cb775c9d4b73beaf0f2167fe7fc9909e91d116d 100644
--- a/python/paddle/tensor/search.py
+++ b/python/paddle/tensor/search.py
@@ -46,8 +46,7 @@ def argsort(x, axis=-1, descending=False, name=None):
:alias_main: paddle.argsort
:alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort
- This OP sorts the input along the given axis, and returns sorted output
- data Varibale and its corresponding index Variable with the same shape as ``x``.
+ This OP sorts the input along the given axis, and returns the corresponding index tensor for the sorted output values. The default sort algorithm is ascending, if you want the sort algorithm to be descending, you must set the :attr:`descending` as True.
Args:
x(Tensor): An input N-D Tensor with type float32, float64, int16,
@@ -84,26 +83,26 @@ def argsort(x, axis=-1, descending=False, name=None):
out2 = paddle.argsort(x=x, axis=0)
out3 = paddle.argsort(x=x, axis=1)
print(out1.numpy())
- #[[[0 3 1 2]
- # [0 1 2 3]
- # [2 3 0 1]]
+ #[[[0 3 1 2]
+ # [0 1 2 3]
+ # [2 3 0 1]]
# [[1 3 2 0]
- # [0 1 2 3]
- # [2 0 3 1]]]
+ # [0 1 2 3]
+ # [2 0 3 1]]]
print(out2.numpy())
- #[[[0 1 1 1]
- # [0 0 0 0]
- # [1 1 1 0]]
- # [[1 0 0 0]
- # [1 1 1 1]
- # [0 0 0 1]]]
+ #[[[0 1 1 1]
+ # [0 0 0 0]
+ # [1 1 1 0]]
+ # [[1 0 0 0]
+ # [1 1 1 1]
+ # [0 0 0 1]]]
print(out3.numpy())
- #[[[1 1 1 2]
- # [0 0 2 0]
- # [2 2 0 1]]
- # [[2 0 2 0]
- # [1 1 0 2]
- # [0 2 1 1]]]
+ #[[[1 1 1 2]
+ # [0 0 2 0]
+ # [2 2 0 1]]
+ # [[2 0 2 0]
+ # [1 1 0 2]
+ # [0 2 1 1]]]
"""
if in_dygraph_mode():
_, ids = core.ops.argsort(x, 'axis', axis, 'descending', descending)
@@ -381,8 +380,7 @@ def sort(x, axis=-1, descending=False, name=None):
:alias_main: paddle.sort
:alias: paddle.sort,paddle.tensor.sort,paddle.tensor.search.sort
- This OP sorts the input along the given axis, and returns sorted output
- data Tensor and its corresponding index Tensor with the same shape as ``x``.
+ This OP sorts the input along the given axis, and returns the sorted output tensor. The default sort algorithm is ascending, if you want the sort algorithm to be descending, you must set the :attr:`descending` as True.
Args:
x(Tensor): An input N-D Tensor with type float32, float64, int16,
@@ -397,9 +395,7 @@ def sort(x, axis=-1, descending=False, name=None):
need for user to set this property. For more information, please
refer to :ref:`api_guide_Name`.
Returns:
- tuple: A tuple of sorted data tensor(with the same shape and data
- type as ``x``) and the sorted indices(with the same shape as ``x``
- and with data type int64).
+ Tensor: sorted tensor(with the same shape and data type as ``x``).
Examples:
.. code-block:: python
import paddle
@@ -417,38 +413,31 @@ def sort(x, axis=-1, descending=False, name=None):
out1 = paddle.sort(x=x, axis=-1)
out2 = paddle.sort(x=x, axis=0)
out3 = paddle.sort(x=x, axis=1)
- print(out1[0].numpy())
- #[[[5. 5. 8. 9.]
- # [0. 0. 1. 7.]
- # [2. 4. 6. 9.]]
- # [[2. 2. 4. 5.]
- # [4. 7. 7. 9.]
- # [0. 1. 6. 7.]]]
- print(out1[1].numpy())
- #[[[0 3 1 2]
- # [0 1 2 3]
- # [2 3 0 1]]
- # [[1 3 2 0]
- # [0 1 2 3]
- # [2 0 3 1]]]
- print(out2[0].numpy())
+ print(out1.numpy())
+ #[[[5. 5. 8. 9.]
+ # [0. 0. 1. 7.]
+ # [2. 4. 6. 9.]]
+ # [[2. 2. 4. 5.]
+ # [4. 7. 7. 9.]
+ # [0. 1. 6. 7.]]]
+ print(out2.numpy())
#[[[5. 2. 4. 2.]
- # [0. 0. 1. 7.]
- # [1. 7. 0. 4.]]
- # [[5. 8. 9. 5.]
- # [4. 7. 7. 9.]
- # [6. 9. 2. 6.]]]
- print(out3[0].numpy())
+ # [0. 0. 1. 7.]
+ # [1. 7. 0. 4.]]
+ # [[5. 8. 9. 5.]
+ # [4. 7. 7. 9.]
+ # [6. 9. 2. 6.]]]
+ print(out3.numpy())
#[[[0. 0. 1. 4.]
- # [5. 8. 2. 5.]
- # [6. 9. 9. 7.]]
- # [[1. 2. 0. 2.]
- # [4. 7. 4. 6.]
- # [5. 7. 7. 9.]]]
+ # [5. 8. 2. 5.]
+ # [6. 9. 9. 7.]]
+ # [[1. 2. 0. 2.]
+ # [4. 7. 4. 6.]
+ # [5. 7. 7. 9.]]]
"""
if in_dygraph_mode():
- out, ids = core.ops.argsort(x, 'axis', axis, 'descending', descending)
- return out, ids
+ out, _ = core.ops.argsort(x, 'axis', axis, 'descending', descending)
+ return out
helper = LayerHelper("sort", **locals())
out = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=False)
@@ -461,7 +450,7 @@ def sort(x, axis=-1, descending=False, name=None):
'Indices': ids},
attrs={'axis': axis,
'descending': descending})
- return out, ids
+ return out
def where(condition, x, y, name=None):
diff --git a/tools/dockerfile/Dockerfile.centos b/tools/dockerfile/Dockerfile.centos
index bb620e6822aff6b6632752b4d468715c4f35bf44..049621b9388997deaeab618c09c579858a60d47e 100644
--- a/tools/dockerfile/Dockerfile.centos
+++ b/tools/dockerfile/Dockerfile.centos
@@ -13,7 +13,7 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
-RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
+RUN yum install -y gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
COPY build_scripts /build_scripts
RUN bash build_scripts/build.sh
RUN bash build_scripts/install_nccl2.sh && \
@@ -22,6 +22,13 @@ RUN rm -rf build_scripts
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+# git 2.17.1
+RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
+ tar -xvf git-2.17.1.tar.gz && \
+ cd git-2.17.1 && \
+ ./configure --prefix=/usr/local && \
+ make -j8 && make install
+
# for paddle
RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \
tar -xz -C /usr/local && \
diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh
index f088c5728a5d549c56fb4ba9ea092e7188115099..3716084487e6ab8ba89cb25698e209a1e1859a8b 100644
--- a/tools/dockerfile/ci_dockerfile.sh
+++ b/tools/dockerfile/ci_dockerfile.sh
@@ -3,9 +3,9 @@ function make_ubuntu_dockerfile(){
dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16"
sed 's//10.1-cudnn7-devel-ubuntu16.04/g' ./Dockerfile.ubuntu >${dockerfile_name}
sed -i 's#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g' ${dockerfile_name}
+ dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
- dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
sed -i 's##WORKDIR /usr/bin \
COPY tools/dockerfile/build_scripts /build_scripts \
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \
@@ -24,6 +24,9 @@ function make_centos_dockerfile(){
sed 's//9.0-cudnn7-devel-centos6/g' Dockerfile.centos >${dockerfile_name}
sed -i 's#COPY build_scripts /build_scripts#COPY tools/dockerfile/build_scripts ./build_scripts#g' ${dockerfile_name}
dockerfile_line=`wc -l ${dockerfile_name}|awk '{print $1}'`
+ sed -i "${dockerfile_line}i RUN ln -s /usr/lib64/libz.so /usr/local/lib/libz.so && \
+ ln -s /usr/local/lib/libnccl.so /usr/local/cuda/lib64/ && \
+ rm -rf /usr/include/NvInfer*" ${dockerfile_name}
sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz && \
tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
}