Skip to content

P
Paddle

Crayon鑫 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
提交 09d32b06 编写于 作者: W wanghaoshuang
浏览文件
操作

Add unitest and comments.

上级 c2dd75be
隐藏空白更改
内联 并排
Showing with 212 addition and 78 deletion
paddle/operators/nce_op.cc
浏览文件 @ 09d32b06
......@@ -23,57 +23,87 @@ class NCEOp : public framework::OperatorWithKernel {
public:
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"));
PADDLE_ENFORCE(ctx->HasInput("Input"));
PADDLE_ENFORCE(ctx->HasInput("Label"));
PADDLE_ENFORCE(ctx->HasInput("W"));
PADDLE_ENFORCE(ctx->HasOutput("Out"));
PADDLE_ENFORCE(ctx->HasInput("Weight"));
PADDLE_ENFORCE(ctx->HasOutput("Cost"));
PADDLE_ENFORCE(ctx->HasOutput("SampleLogits"));
PADDLE_ENFORCE(ctx->HasOutput("SampleLabels"));
auto x_dims = ctx->GetInputDim("X");
auto x_dims = ctx->GetInputDim("Input");
auto label_dims = ctx->GetInputDim("Label");
PADDLE_ENFORCE_EQ(x_dims[0], label_dims[0]);
if (ctx->HasInput("B")) {
PADDLE_ENFORCE_EQ(ctx->GetInputDim("W")[0], ctx->GetInputDim("B")[0]);
int num_true_classes = label_dims.size() == 2 ? label_dims[1] : 1;
if (ctx->HasInput("Bias")) {
PADDLE_ENFORCE_EQ(ctx->GetInputDim("Weight")[0],
ctx->GetInputDim("Bias")[0]);
}
int num_sampled_classes = ctx->Attrs().Get<int>("num_sampled_classes");
int num_classes = ctx->Attrs().Get<int>("num_classes");
PADDLE_ENFORCE_EQ(num_classes, ctx->GetInputDim("W")[0]);
auto num_sampled_classes = ctx->Attrs().Get<int>("num_sampled_classes");
auto num_classes = ctx->Attrs().Get<int>("num_classes");
std::vector<int> sampled_labels =
ctx->Attrs().Get<std::vector<int>>("sampled_labels");
PADDLE_ENFORCE_EQ(num_classes, ctx->GetInputDim("Weight")[0]);
PADDLE_ENFORCE_LT(num_sampled_classes, num_classes);
if (sampled_labels.size() > 0) {
PADDLE_ENFORCE_EQ(sampled_labels.size(),
static_cast<size_t>(num_sampled_classes));
}
// set dims of output(Out)
std::vector<int64_t> out_dims(1);
std::vector<int64_t> out_dims;
out_dims.push_back(x_dims[0]);
ctx->SetOutputDim("Out", framework::make_ddim(out_dims));
ctx->SetOutputDim("Cost", framework::make_ddim(out_dims));
// set dims of output(SampleOut)
std::vector<int64_t> sample_out_dims(2);
std::vector<int64_t> sample_out_dims;
sample_out_dims.push_back(x_dims[0]);
sample_out_dims.push_back(num_sampled_classes + 1);
sample_out_dims.push_back(num_sampled_classes + num_true_classes);
ctx->SetOutputDim("SampleLogits", framework::make_ddim(sample_out_dims));
ctx->SetOutputDim("SampleLabels", framework::make_ddim(sample_out_dims));
}
protected:
framework::OpKernelType GetKernelType(
const framework::ExecutionContext& ctx) const override {
return framework::OpKernelType(
framework::ToDataType(ctx.Input<Tensor>("Input")->type()),
ctx.device_context());
}
};
class NCEOpMaker : public framework::OpProtoAndCheckerMaker {
public:
NCEOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) {
AddInput("X", "");
AddInput("Label", "");
AddInput("W", "");
AddInput("B", "");
AddInput("SampleWeight", "");
AddOutput("Out", "");
AddOutput("SampleLogits", "");
AddOutput("SampleLabels", "");
AddAttr<int>("num_classes", "");
AddAttr<int>("num_sampled_classes", "").SetDefault(10);
AddInput("Input", "(Tensor) A tensor of shape [batch_size, dim].");
AddInput("Label",
"(Tensor) A tensor of shape [batch_size, num_true_class]. "
"'num_true_class' is the number of target class in each sample.");
AddInput("Weight",
"(Tensor) A tensor of shape [num_class, dim]. 'num_class' is the "
"total number of class.");
AddInput("Bias",
"(Tensor) A tensor of shape [num_class]. 'num_class' is the total "
"number of class. It is a dispensable input.")
.AsDispensable();
AddInput("SampleWeight",
"(Tensor) A tensor of shape [batch_size] storing a weight for "
"each sample. And it is a dispensable input. The default value of "
"sample is 1.")
.AsDispensable();
AddOutput("Cost",
"(Tensor) A tensor of shape [batch_size]. Cost of samples.");
AddOutput("SampleLogits", "An intermediate tensor.").AsIntermediate();
AddOutput("SampleLabels", "An intermediate tensor.").AsIntermediate();
AddAttr<int>("num_classes", "Total number of classes.");
AddAttr<int>("num_sampled_classes", "The number of negative classes.")
.SetDefault(10);
AddAttr<std::vector<int>>("sampled_labels", "");
AddComment(R"DOC(
Expand input(X) according to LOD of input(Y).
Computes and returns the noise-contrastive estimation training loss.
See [Noise-contrastive estimation: A new estimation principle for unnormalized statistical models](http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf).
By default this uses a uniform distribution for sampling.
The number of target classes per example should be same. If you have a variable number of target classes, you can pad them out to a constant number by either repeating them or by padding with an otherwise unused class.
)DOC");
}
};
......@@ -82,32 +112,41 @@ class NCEOpGrad : public framework::OperatorWithKernel {
public:
using framework::OperatorWithKernel::OperatorWithKernel;
protected:
void InferShape(framework::InferShapeContext* ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("X"));
PADDLE_ENFORCE(ctx->HasInput("W"));
PADDLE_ENFORCE(ctx->HasInput("Out"));
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
PADDLE_ENFORCE(ctx->HasInput("Input"));
PADDLE_ENFORCE(ctx->HasInput("Weight"));
PADDLE_ENFORCE(ctx->HasInput("Cost"));
PADDLE_ENFORCE(ctx->HasInput("SampleLogits"));
PADDLE_ENFORCE(ctx->HasInput("SampleLabels"));
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Cost")),
"The input(Out@GRAD) should not be null");
auto x_dims = ctx->GetInputDim("X");
auto x_grad_name = framework::GradVarName("X");
auto x_dims = ctx->GetInputDim("Input");
auto x_grad_name = framework::GradVarName("Input");
if (ctx->HasOutput(x_grad_name)) {
ctx->SetOutputDim(x_grad_name, x_dims);
}
auto w_dims = ctx->GetInputDim("W");
auto w_grad_name = framework::GradVarName("W");
auto w_dims = ctx->GetInputDim("Weight");
auto w_grad_name = framework::GradVarName("Weight");
if (ctx->HasOutput(w_grad_name)) {
ctx->SetOutputDim(w_grad_name, w_dims);
}
auto bias_grad_name = framework::GradVarName("B");
auto bias_grad_name = framework::GradVarName("Bias");
if (ctx->HasOutput(bias_grad_name)) {
auto bias_dims = ctx->GetInputDim("B");
auto bias_dims = ctx->GetInputDim("Bias");
ctx->SetOutputDim(bias_grad_name, bias_dims);
}
}
protected:
framework::OpKernelType GetKernelType(
const framework::ExecutionContext& ctx) const override {
return framework::OpKernelType(
framework::ToDataType(ctx.Input<Tensor>("Input")->type()),
ctx.device_context());
}
};
} // namespace operators
......
paddle/operators/nce_op.h
浏览文件 @ 09d32b06
......@@ -14,12 +14,11 @@
#pragma once
#include <math.h>
#include <random>
#include "paddle/framework/eigen.h"
#include "paddle/framework/op_registry.h"
#include "paddle/memory/memcpy.h"
#include "unsupported/Eigen/CXX11/Tensor"
namespace paddle {
namespace operators {
......@@ -32,9 +31,12 @@ using EigenMatrix = framework::EigenMatrix<T, MajorType, IndexType>;
template <typename Place, typename T>
void PrepareSamples(const framework::ExecutionContext& context) {
auto label = context.Input<Tensor>("Label");
const T* label_data = label->data<T>();
const int64_t* label_data = label->data<int64_t>();
auto label_dims = label->dims();
int num_classes = context.Attr<int>("num_classes");
// for unitest
std::vector<int> sampled_labels =
context.Attr<std::vector<int>>("sampled_labels");
// random machine
std::random_device rd;
std::mt19937 rng(rd());
......@@ -42,19 +44,24 @@ void PrepareSamples(const framework::ExecutionContext& context) {
auto sample_labels = context.Output<Tensor>("SampleLabels");
auto sample_labels_dims = sample_labels->dims();
int* sample_labels_data =
sample_labels->mutable_data<int>(context.GetPlace());
int64_t* sample_labels_data =
sample_labels->mutable_data<int64_t>(context.GetPlace());
int num_label = label_dims.size() == 2 ? label_dims[1] : 1;
int index = 0;
for (size_t i = 0; i < label_dims[0]; ++i) {
int j = 0;
for (; j < num_label; ++j) {
sample_labels_data[sample_labels_dims[1] * i + j] =
label_data[i * num_label + j];
sample_labels_data[index++] = label_data[i * num_label + j];
}
for (; j < sample_labels_dims[1]; ++j) {
int id = rand(rng);
sample_labels_data[sample_labels_dims[1] * i + j] = id;
if (sampled_labels.size() > 0) {
for (auto label : sampled_labels) {
sample_labels_data[index++] = label;
}
} else {
for (; j < sample_labels_dims[1]; ++j) {
sample_labels_data[index++] = rand(rng);
}
}
}
}
......@@ -65,7 +72,7 @@ class NCEKernel : public framework::OpKernel<T> {
void Compute(const framework::ExecutionContext& context) const override {
PrepareSamples<Place, T>(context);
auto sample_labels = context.Output<Tensor>("SampleLabels");
const int* sample_labels_data = sample_labels->data<int>();
const int64_t* sample_labels_data = sample_labels->data<int64_t>();
auto sample_out = context.Output<Tensor>("SampleLogits");
T* sample_out_data = sample_out->mutable_data<T>(context.GetPlace());
auto label = context.Input<Tensor>("Label");
......@@ -74,7 +81,7 @@ class NCEKernel : public framework::OpKernel<T> {
if (sample_weight != nullptr) {
sample_weight_data = sample_weight->data<T>();
}
auto out = context.Output<Tensor>("Out");
auto out = context.Output<Tensor>("Cost");
T* out_data = out->mutable_data<T>(context.GetPlace());
int num_smalped_classes = context.Attr<int>("num_sampled_classes");
int num_classes = context.Attr<int>("num_classes");
......@@ -83,9 +90,8 @@ class NCEKernel : public framework::OpKernel<T> {
num_true_class = label->dims()[1];
}
T b = 1. / num_classes * num_smalped_classes;
// forward bias
auto bias = context.Input<Tensor>("B");
auto bias = context.Input<Tensor>("Bias");
if (bias != nullptr) {
const T* bias_data = bias->data<T>();
for (size_t i = 0; i < sample_labels->numel(); ++i) {
......@@ -96,27 +102,23 @@ class NCEKernel : public framework::OpKernel<T> {
sample_out_data[i] = 0;
}
}
// forward mul
auto input_mat = EigenMatrix<T>::From(*(context.Input<Tensor>("X")));
auto weight_mat = EigenMatrix<T>::From(*(context.Input<Tensor>("W")));
auto input_mat = EigenMatrix<T>::From(*(context.Input<Tensor>("Input")));
auto weight_mat = EigenMatrix<T>::From(*(context.Input<Tensor>("Weight")));
for (size_t i = 0; i < sample_labels->numel(); ++i) {
// sample_out_data[i] += (input_mat.chip((int)(i /
// sample_labels->dims()[1]), 0) * weight_mat.chip(sample_labels_data[i],
// 0)).sum();
Eigen::Tensor<float, 0, Eigen::RowMajor, Eigen::DenseIndex> result =
(input_mat.chip((int)(i / sample_labels->dims()[1]), 0) *
weight_mat.chip(sample_labels_data[i], 0))
.sum();
sample_out_data[i] += result(0);
// activation_->forward
sample_out_data[i] = (1 / 1 + (sample_out_data[i]));
sample_out_data[i] = (1. / (1. + exp(-sample_out_data[i])));
}
// forward cost
for (size_t i = 0; i < sample_labels->dims()[0]; ++i) {
size_t j = 0;
T w = sample_weight == nullptr ? 1 : sample_weight_data[i];
out_data[i] = 0;
T w = sample_weight == nullptr ? 1. : sample_weight_data[i];
// for true classes
for (; j < num_true_class; ++j) {
T o = sample_out_data[i * sample_out->dims()[1] + j];
......@@ -137,11 +139,13 @@ template <typename Place, typename T>
class NCEGradKernel : public framework::OpKernel<T> {
public:
void Compute(const framework::ExecutionContext& context) const override {
auto d_out = context.Input<Tensor>(framework::GradVarName("Cost"));
const T* d_out_data = d_out->data<T>();
auto label = context.Input<Tensor>("Label");
auto sample_out = context.Input<Tensor>("SampleLogits");
const T* sample_out_data = sample_out->data<T>();
auto sample_labels = context.Input<Tensor>("SampleLabels");
const int* sample_labels_data = sample_labels->data<int>();
const int64_t* sample_labels_data = sample_labels->data<int64_t>();
auto sample_weight = context.Input<Tensor>("SampleWeight");
const T* sample_weight_data = nullptr;
if (sample_weight != nullptr) {
......@@ -154,11 +158,9 @@ class NCEGradKernel : public framework::OpKernel<T> {
num_true_class = label->dims()[1];
}
T b = 1. / num_classes * num_smalped_classes;
Tensor sample_grad; // tmp tensor
T* sample_grad_data =
sample_grad.mutable_data<T>(sample_labels->dims(), context.GetPlace());
// backward cost
for (size_t i = 0; i < sample_labels->numel(); ++i) {
T o = sample_out_data[i];
......@@ -166,15 +168,12 @@ class NCEGradKernel : public framework::OpKernel<T> {
? 1
: sample_weight_data[i / sample_labels->dims()[1]];
sample_grad_data[i] = (i % sample_labels->dims()[1]) < num_true_class
? -w * b / (o * (o + b))
: w / (o + b);
// sigmoid->backward
sample_grad_data[i] =
(o > 0) ? sample_grad_data[i] : ((o < 0) ? -sample_grad_data[i] : 0);
? w * (b / (o + b)) * (o - 1)
: w * (o * (1 - o) / (o + b));
sample_grad_data[i] *= d_out_data[i / sample_labels->dims()[1]];
}
// get d_bias
auto d_bias = context.Output<Tensor>(framework::GradVarName("B"));
auto d_bias = context.Output<Tensor>(framework::GradVarName("Bias"));
if (d_bias != nullptr) {
T* d_bias_data = d_bias->mutable_data<T>(context.GetPlace());
for (size_t i = 0; i < sample_labels->numel(); ++i) {
......@@ -182,22 +181,23 @@ class NCEGradKernel : public framework::OpKernel<T> {
}
}
// get d_w
auto d_w = context.Output<Tensor>(framework::GradVarName("W"));
auto d_w = context.Output<Tensor>(framework::GradVarName("Weight"));
if (d_w != nullptr) {
d_w->mutable_data<T>(context.GetPlace());
auto d_w_matrix = EigenMatrix<T>::From(*d_w);
auto x_matrix = EigenMatrix<T>::From(*(context.Input<Tensor>("X")));
auto x_matrix = EigenMatrix<T>::From(*(context.Input<Tensor>("Input")));
for (size_t i = 0; i < sample_labels->numel(); ++i) {
d_w_matrix.chip(sample_labels_data[i], 0) =
d_w_matrix.chip(sample_labels_data[i], 0) +=
x_matrix.chip((int)(i / sample_labels->dims()[1]), 0) *
sample_grad_data[i];
}
}
// get d_x
auto d_x = context.Output<Tensor>(framework::GradVarName("X"));
auto d_x = context.Output<Tensor>(framework::GradVarName("Input"));
if (d_x != nullptr) {
d_x->mutable_data<T>(context.GetPlace());
auto d_x_matrix = EigenMatrix<T>::From(*d_x);
auto w_matrix = EigenMatrix<T>::From(*(context.Input<Tensor>("W")));
auto w_matrix = EigenMatrix<T>::From(*(context.Input<Tensor>("Weight")));
for (size_t i = 0; i < sample_labels->numel(); ++i) {
d_x_matrix.chip((int)(i / sample_labels->dims()[1]), 0) +=
w_matrix.chip(sample_labels_data[i], 0) * sample_grad_data[i];
......@@ -205,6 +205,5 @@ class NCEGradKernel : public framework::OpKernel<T> {
}
}
};
} // namespace operators
} // namespace paddle
python/paddle/v2/framework/tests/test_nce.py 0 → 100644
浏览文件 @ 09d32b06
import unittest
import numpy as np
from op_test import OpTest
def nce(input, weight, bias, sample_weight, labels, num_classes,
num_sample_class):
samples = []
sample_labels = []
batch_size = input.shape[0]
num_true_class = labels.shape[1]
for i in range(batch_size):
w = 1 if sample_weight is None else sample_weight[i]
for label in labels[i]:
samples.append((i, label, True, w))
sample_labels.append(label)
for num in range(num_sample_class):
samples.append((i, num, False, w))
sample_labels.append(num)
# forward bias
sampleOut = np.zeros(len(samples)).astype(np.float32)
if bias is not None:
for i in range(len(samples)):
sampleOut[i] = bias[samples[i][1]]
# forward weight
for i in range(len(samples)):
sampleOut[i] += np.dot(input[samples[i][0]], weight[samples[i][1]])
# forward activation
sampleOut = 1.0 / (1.0 + np.exp(-sampleOut))
# forward cost
out = np.zeros(batch_size).astype(np.float32)
b = 1.0 / num_classes * num_sample_class
for i in range(len(samples)):
o = sampleOut[i]
cost = -np.log(o / (o + b)) if samples[i][2] else -np.log(b / (o + b))
out[samples[i][0]] += cost * samples[i][3]
return (out, np.array(sampleOut).reshape(batch_size,
num_sample_class + num_true_class),
np.array(sample_labels).reshape(batch_size,
num_sample_class + num_true_class))
class TestNCE(OpTest):
def generate_data(self, dim, batch_size, num_classes, num_true_class,
num_sampled_classes):
input = np.random.randn(batch_size, dim).astype(np.float32)
weight = np.random.randn(num_classes, dim).astype(np.float32)
bias = np.random.randn(num_classes).astype(np.float32)
sample_weight = np.random.randn(batch_size).astype(np.float32)
labels = np.random.randint(0, num_classes, (batch_size, num_true_class))
self.attrs = {
'num_classes': num_classes,
'num_sampled_classes': num_sampled_classes,
'sampled_labels': range(num_sampled_classes)
}
self.inputs = {
'X': input,
'Label': labels,
'W': weight,
'B': bias,
'SampleWeight': sample_weight
}
def set_data(self):
self.generate_data(5, 5, 4, 1, 2)
def compute(self):
out = nce(self.inputs['X'], self.inputs['W'], self.inputs['B'],
self.inputs['SampleWeight'], self.inputs['Label'],
self.attrs['num_classes'], self.attrs['num_sampled_classes'])
self.outputs = {
'Out': out[0],
'SampleLogits': out[1],
'SampleLabels': out[2]
}
def setUp(self):
self.op_type = 'nce'
self.set_data()
self.compute()
def test_check_output(self):
self.check_output()
def test_check_grad(self):
self.check_grad(["X", "W", "B"], "Out", max_relative_error=0.02)
class TestNCECase1(TestNCE):
def set_data(self):
self.generate_data(10, 20, 10, 2, 5)
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或