提交 ec96bbf2 编写于 作者: B baiyfbupt

adabert multi-gpus

上级 491c3489
import paddle.fluid as fluid
from paddleslim.teachers.bert.reader.cls import *
from paddleslim.nas.darts.search_space import AdaBERTClassifier
from paddleslim.nas.darts import DARTSearch
def main():
place = fluid.CUDAPlace(0)
BERT_BASE_PATH = "./data/pretrained_models/uncased_L-12_H-768_A-12/"
bert_config_path = BERT_BASE_PATH + "/bert_config.json"
vocab_path = BERT_BASE_PATH + "/vocab.txt"
data_dir = "./data/glue_data/MNLI/"
max_seq_len = 512
do_lower_case = True
batch_size = 32
epoch = 30
processor = MnliProcessor(
data_dir=data_dir,
vocab_path=vocab_path,
max_seq_len=max_seq_len,
do_lower_case=do_lower_case,
in_tokens=False)
train_reader = processor.data_generator(
batch_size=batch_size,
phase='train',
epoch=epoch,
dev_count=1,
shuffle=True)
val_reader = processor.data_generator(
batch_size=batch_size,
phase='train',
epoch=epoch,
dev_count=1,
shuffle=True)
with fluid.dygraph.guard(place):
model = AdaBERTClassifier(
3,
teacher_model="/work/PaddleSlim/demo/bert_1/checkpoints/steps_23000"
)
searcher = DARTSearch(
model,
train_reader,
val_reader,
batchsize=batch_size,
num_epochs=epoch,
log_freq=10)
searcher.train()
if __name__ == '__main__':
main()
...@@ -3,7 +3,6 @@ from itertools import izip ...@@ -3,7 +3,6 @@ from itertools import izip
import paddle.fluid as fluid import paddle.fluid as fluid
from paddleslim.teachers.bert.reader.cls import * from paddleslim.teachers.bert.reader.cls import *
from paddleslim.nas.darts.search_space import AdaBERTClassifier from paddleslim.nas.darts.search_space import AdaBERTClassifier
from paddleslim.nas.darts.architect_for_bert import Architect
import logging import logging
from paddleslim.common import AvgrageMeter, get_logger from paddleslim.common import AvgrageMeter, get_logger
...@@ -18,79 +17,94 @@ def count_parameters_in_MB(all_params): ...@@ -18,79 +17,94 @@ def count_parameters_in_MB(all_params):
return parameters_number / 1e6 return parameters_number / 1e6
def model_loss(model, data_ids): def train_one_epoch(model, train_loader, valid_loader, optimizer,
# src_ids = data_ids[0] arch_optimizer, epoch, use_data_parallel, log_freq):
# position_ids = data_ids[1] total_losses = AvgrageMeter()
# sentence_ids = data_ids[2]
# input_mask = data_ids[3]
labels = data_ids[4]
labels.stop_gradient = True
enc_output = model(data_ids)
ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
logits=enc_output, label=labels, return_softmax=True)
loss = fluid.layers.mean(x=ce_loss)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs)
return loss, accuracy
def train_one_epoch(model, architect, train_loader, valid_loader, optimizer,
epoch, use_data_parallel, log_freq):
ce_losses = AvgrageMeter()
accs = AvgrageMeter() accs = AvgrageMeter()
ce_losses = AvgrageMeter()
kd_losses = AvgrageMeter()
val_accs = AvgrageMeter()
model.train() model.train()
step_id = 0 step_id = 0
for train_data, valid_data in izip(train_loader(), valid_loader): for train_data, valid_data in izip(train_loader(), valid_loader()):
architect.step(train_data, valid_data) #for train_data in train_loader():
loss, acc = model_loss(model, train_data) batch_size = train_data[0].shape[0]
try:
total_loss, acc, ce_loss, kd_loss, _ = model._layers.loss(
train_data, epoch)
except:
total_loss, acc, ce_loss, kd_loss, _ = model.loss(train_data,
epoch)
if use_data_parallel: if use_data_parallel:
loss = model.scale_loss(loss) total_loss = model.scale_loss(total_loss)
loss.backward() total_loss.backward()
model.apply_collective_grads() model.apply_collective_grads()
else: else:
loss.backward() total_loss.backward()
optimizer.minimize(total_loss)
optimizer.minimize(loss)
model.clear_gradients() model.clear_gradients()
total_losses.update(total_loss.numpy(), batch_size)
batch_size = train_data[0].shape[0]
ce_losses.update(loss.numpy(), batch_size)
accs.update(acc.numpy(), batch_size) accs.update(acc.numpy(), batch_size)
ce_losses.update(ce_loss.numpy(), batch_size)
kd_losses.update(kd_loss.numpy(), batch_size)
try:
arch_loss, _, _, _, arch_logits = model._layers.loss(valid_data,
epoch)
except:
arch_loss, _, _, _, arch_logits = model.loss(valid_data, epoch)
if use_data_parallel:
arch_loss = model.scale_loss(arch_loss)
arch_loss.backward()
model.apply_collective_grads()
else:
arch_loss.backward()
arch_optimizer.minimize(arch_loss)
arch_optimizer.clear_gradients()
probs = fluid.layers.softmax(arch_logits[-1])
val_acc = fluid.layers.accuracy(input=probs, label=valid_data[4])
val_accs.update(val_acc.numpy(), batch_size)
if step_id % log_freq == 0: if step_id % log_freq == 0:
logger.info( logger.info(
"Train Epoch {}, Step {}, Lr {:.6f} loss {:.6f}; acc: {:.6f};". "Train Epoch {}, Step {}, Lr {:.6f} total_loss {:.6f}; ce_loss {:.6f}, kd_loss {:.6f}, train_acc {:.6f}, valid_acc {:.6f};".
format(epoch, step_id, format(epoch, step_id,
optimizer.current_step_lr(), ce_losses.avg[0], accs.avg[ optimizer.current_step_lr(), total_losses.avg[
0])) 0], ce_losses.avg[0], kd_losses.avg[0], accs.avg[0],
val_accs.avg[0]))
step_id += 1 step_id += 1
def valid_one_epoch(model, valid_loader, epoch, log_freq): def valid_one_epoch(model, valid_loader, epoch, log_freq):
ce_losses = AvgrageMeter()
accs = AvgrageMeter() accs = AvgrageMeter()
ce_losses = AvgrageMeter()
model.eval() model.eval()
step_id = 0 step_id = 0
for valid_data in valid_loader(): for valid_data in valid_loader():
loss, acc = model_loss(model, valid_data) try:
loss, acc, ce_loss, _, _ = model._layers.loss(valid_data, epoch)
except:
loss, acc, ce_loss, _, _ = model.loss(valid_data, epoch)
batch_size = valid_data[0].shape[0] batch_size = valid_data[0].shape[0]
ce_losses.update(loss.numpy(), batch_size) ce_losses.update(ce_loss.numpy(), batch_size)
accs.update(acc.numpy(), batch_size) accs.update(acc.numpy(), batch_size)
if step_id % log_freq == 0: # if step_id % log_freq == 0:
logger.info("Valid Epoch {}, Step {}, loss {:.6f}; acc: {:.6f};". # logger.info("Valid Epoch {}, Step {}, ce_loss {:.6f}; acc: {:.6f};".
format(epoch, step_id, ce_losses.avg[0], accs.avg[0])) # format(epoch, step_id, ce_losses.avg[0], accs.avg[0]))
step_id += 1 step_id += 1
return ce_losses.avg[0], accs.avg[0]
def main(): def main():
use_data_parallel = False # whether use multi-gpus
use_data_parallel = True
place = fluid.CUDAPlace(fluid.dygraph.parallel.Env( place = fluid.CUDAPlace(fluid.dygraph.parallel.Env(
).dev_id) if use_data_parallel else fluid.CUDAPlace(0) ).dev_id) if use_data_parallel else fluid.CUDAPlace(0)
...@@ -98,44 +112,31 @@ def main(): ...@@ -98,44 +112,31 @@ def main():
bert_config_path = BERT_BASE_PATH + "/bert_config.json" bert_config_path = BERT_BASE_PATH + "/bert_config.json"
vocab_path = BERT_BASE_PATH + "/vocab.txt" vocab_path = BERT_BASE_PATH + "/vocab.txt"
data_dir = "./data/glue_data/MNLI/" data_dir = "./data/glue_data/MNLI/"
teacher_model_dir = "./teacher_model/steps_23000" teacher_model_dir = "./data/teacher_model/steps_23000"
num_samples = 392702
max_seq_len = 128
do_lower_case = True do_lower_case = True
batch_size = 128 #num_samples = 392702
num_samples = 8016987
max_seq_len = 128
batch_size = 64
hidden_size = 768 hidden_size = 768
emb_size = 768 emb_size = 768
max_layer = 8 max_layer = 8
epoch = 80 epoch = 80
log_freq = 10 log_freq = 10
use_fixed_gumbel = True
device_num = fluid.dygraph.parallel.Env().nranks
processor = MnliProcessor( search = True
data_dir=data_dir,
vocab_path=vocab_path, if search:
max_seq_len=max_seq_len, use_fixed_gumbel = False
do_lower_case=do_lower_case, train_phase = "search_train"
in_tokens=False) val_phase = "search_valid"
step_per_epoch = int(num_samples / ((batch_size * 0.5) * device_num))
train_reader = processor.data_generator( else:
batch_size=batch_size, use_fixed_gumbel = True
phase='search_train', train_phase = "train"
epoch=1, val_phase = "dev"
dev_count=1, step_per_epoch = int(num_samples / (batch_size * device_num))
shuffle=True)
val_reader = processor.data_generator(
batch_size=batch_size,
phase='search_valid',
epoch=1,
dev_count=1,
shuffle=True)
if use_data_parallel:
train_reader = fluid.contrib.reader.distributed_batch_reader(
train_reader)
valid_reader = fluid.contrib.reader.distributed_batch_reader(
valid_reader)
with fluid.dygraph.guard(place): with fluid.dygraph.guard(place):
model = AdaBERTClassifier( model = AdaBERTClassifier(
...@@ -147,47 +148,106 @@ def main(): ...@@ -147,47 +148,106 @@ def main():
data_dir=data_dir, data_dir=data_dir,
use_fixed_gumbel=use_fixed_gumbel) use_fixed_gumbel=use_fixed_gumbel)
if use_data_parallel:
strategy = fluid.dygraph.parallel.prepare_context()
model = fluid.dygraph.parallel.DataParallel(model, strategy)
device_num = fluid.dygraph.parallel.Env().nranks
step_per_epoch = int(num_samples / (batch_size * device_num))
learning_rate = fluid.dygraph.CosineDecay(2e-2, step_per_epoch, epoch) learning_rate = fluid.dygraph.CosineDecay(2e-2, step_per_epoch, epoch)
model_parameters = [ model_parameters = []
p for p in model.parameters() for p in model.parameters():
if p.name not in [a.name for a in model.arch_parameters()] if (p.name not in [a.name for a in model.arch_parameters()] and
] p.name not in
[a.name for a in model.teacher.parameters()]):
model_parameters.append(p)
clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) #clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0)
optimizer = fluid.optimizer.MomentumOptimizer( optimizer = fluid.optimizer.MomentumOptimizer(
learning_rate, learning_rate,
0.9, 0.9,
regularization=fluid.regularizer.L2DecayRegularizer(3e-4), regularization=fluid.regularizer.L2DecayRegularizer(3e-4),
parameter_list=model_parameters, parameter_list=model_parameters)
grad_clip=clip) # grad_clip=clip)
arch_optimizer = fluid.optimizer.Adam(
3e-4,
0.5,
0.999,
regularization=fluid.regularizer.L2Decay(1e-3),
parameter_list=model.arch_parameters())
processor = MnliProcessor(
data_dir=data_dir,
vocab_path=vocab_path,
max_seq_len=max_seq_len,
do_lower_case=do_lower_case,
in_tokens=False)
train_reader = processor.data_generator(
batch_size=batch_size,
phase=train_phase,
epoch=1,
dev_count=1,
shuffle=True)
valid_reader = processor.data_generator(
batch_size=batch_size,
phase=val_phase,
epoch=1,
dev_count=1,
shuffle=True)
print("train_data nums:", processor.get_num_examples(train_phase))
print("valid_data nums:", processor.get_num_examples(val_phase))
print("dev_data nums:", processor.get_num_examples("dev"))
if use_data_parallel:
train_reader = fluid.contrib.reader.distributed_batch_reader(
train_reader)
valid_reader = fluid.contrib.reader.distributed_batch_reader(
valid_reader)
dev_reader = processor.data_generator(
batch_size=batch_size,
phase="dev",
epoch=1,
dev_count=1,
shuffle=False)
train_loader = fluid.io.DataLoader.from_generator( train_loader = fluid.io.DataLoader.from_generator(
capacity=1024, capacity=512,
use_double_buffer=True, use_double_buffer=True,
iterable=True, iterable=True,
return_list=True) return_list=True,
use_multiprocess=True)
valid_loader = fluid.io.DataLoader.from_generator( valid_loader = fluid.io.DataLoader.from_generator(
capacity=1024, capacity=512,
use_double_buffer=True,
iterable=True,
return_list=True,
use_multiprocess=True)
dev_loader = fluid.io.DataLoader.from_generator(
capacity=512,
use_double_buffer=True, use_double_buffer=True,
iterable=True, iterable=True,
return_list=True) return_list=True,
use_multiprocess=True)
train_loader.set_batch_generator(train_reader, places=place) train_loader.set_batch_generator(train_reader, places=place)
valid_loader.set_batch_generator(val_reader, places=place) valid_loader.set_batch_generator(valid_reader, places=place)
dev_loader.set_batch_generator(dev_reader, places=place)
architect = Architect(model, learning_rate, 3e-4, place, False) if use_data_parallel:
strategy = fluid.dygraph.parallel.prepare_context()
model = fluid.dygraph.parallel.DataParallel(model, strategy)
for epoch_id in range(epoch): for epoch_id in range(epoch):
train_one_epoch(model, architect, train_loader, valid_loader, train_one_epoch(model, train_loader, valid_loader, optimizer,
optimizer, epoch_id, use_data_parallel, log_freq) arch_optimizer, epoch_id, use_data_parallel,
valid_one_epoch(model, valid_loader, epoch_id, log_freq) log_freq)
print(model.student._encoder.alphas.numpy()) loss, acc = valid_one_epoch(model, dev_loader, epoch_id, log_freq)
logger.info("Valid set2, ce_loss {:.6f}; acc: {:.6f};".format(loss,
acc))
try:
print(model.student._encoder.alphas.numpy())
except:
print(model._layers.student._encoder.alphas.numpy())
print("=" * 100) print("=" * 100)
......
...@@ -18,7 +18,6 @@ from __future__ import print_function ...@@ -18,7 +18,6 @@ from __future__ import print_function
import copy import copy
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.nn.functional as F
class DML(fluid.dygraph.Layer): class DML(fluid.dygraph.Layer):
...@@ -70,7 +69,7 @@ class DML(fluid.dygraph.Layer): ...@@ -70,7 +69,7 @@ class DML(fluid.dygraph.Layer):
cur_kl_loss = 0 cur_kl_loss = 0
for j in range(self.model_num): for j in range(self.model_num):
if i != j: if i != j:
x = F.log_softmax(logits[i], axis=1) x = fluid.layers.log_softmax(logits[i], axis=1)
y = fluid.layers.softmax(logits[j], axis=1) y = fluid.layers.softmax(logits[j], axis=1)
cur_kl_loss += fluid.layers.kldiv_loss( cur_kl_loss += fluid.layers.kldiv_loss(
x, y, reduction='batchmean') x, y, reduction='batchmean')
......
...@@ -49,17 +49,17 @@ class Architect(object): ...@@ -49,17 +49,17 @@ class Architect(object):
self.network_weight_decay), self.network_weight_decay),
parameter_list=self.unrolled_model_params) parameter_list=self.unrolled_model_params)
def step(self, train_data, valid_data): def step(self, train_data, valid_data, epoch):
if self.unrolled: if self.unrolled:
params_grads = self._backward_step_unrolled(train_data, valid_data) params_grads = self._backward_step_unrolled(train_data, valid_data)
self.optimizer.apply_gradients(params_grads) self.optimizer.apply_gradients(params_grads)
else: else:
loss = self._backward_step(valid_data) loss = self._backward_step(valid_data, epoch)
self.optimizer.minimize(loss) self.optimizer.minimize(loss)
self.optimizer.clear_gradients() self.optimizer.clear_gradients()
def _backward_step(self, valid_data): def _backward_step(self, valid_data, epoch):
loss = self.model.loss(valid_data) loss = self.model.loss(valid_data, epoch)
loss[0].backward() loss[0].backward()
return loss[0] return loss[0]
......
...@@ -31,6 +31,7 @@ import multiprocessing ...@@ -31,6 +31,7 @@ import multiprocessing
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid.dygraph import to_variable, Layer, Linear from paddle.fluid.dygraph import to_variable, Layer, Linear
from paddle.fluid.dygraph.base import to_variable
from .reader.cls import * from .reader.cls import *
from .model.bert import BertModelLayer from .model.bert import BertModelLayer
from .optimization import Optimizer from .optimization import Optimizer
...@@ -65,14 +66,17 @@ class AdaBERTClassifier(Layer): ...@@ -65,14 +66,17 @@ class AdaBERTClassifier(Layer):
self._teacher_model = teacher_model self._teacher_model = teacher_model
self._data_dir = data_dir self._data_dir = data_dir
self.use_fixed_gumbel = use_fixed_gumbel self.use_fixed_gumbel = use_fixed_gumbel
#print( self.T = 1.0
# "----------------------load teacher model and test----------------------------------------" print(
#) "----------------------load teacher model and test----------------------------------------"
#self.teacher = BERTClassifier(num_labels, model_path=self._teacher_model) )
self.teacher = BERTClassifier(
num_labels, model_path=self._teacher_model)
self.teacher.eval()
#self.teacher.test(self._data_dir) #self.teacher.test(self._data_dir)
#print( print(
# "----------------------finish load teacher model and test----------------------------------------" "----------------------finish load teacher model and test----------------------------------------"
#) )
self.student = BertModelLayer( self.student = BertModelLayer(
n_layer=self._n_layer, n_layer=self._n_layer,
emb_size=self._emb_size, emb_size=self._emb_size,
...@@ -81,46 +85,84 @@ class AdaBERTClassifier(Layer): ...@@ -81,46 +85,84 @@ class AdaBERTClassifier(Layer):
search_layer=self._search_layer, search_layer=self._search_layer,
use_fixed_gumbel=self.use_fixed_gumbel) use_fixed_gumbel=self.use_fixed_gumbel)
self.cls_fc = list() fix_emb = False
for i in range(self._n_layer): for s_emb, t_emb in zip(self.student.emb_names(),
fc = Linear( self.teacher.emb_names()):
input_dim=self._hidden_size, t_emb.stop_gradient = True
output_dim=self._num_labels, if fix_emb:
param_attr=fluid.ParamAttr( s_emb.stop_gradient = True
name="s_cls_out_%d_w" % i, print(
initializer=fluid.initializer.TruncatedNormal(scale=0.02)), "Assigning embedding[{}] from teacher to embedding[{}] in student.".
bias_attr=fluid.ParamAttr( format(t_emb.name, s_emb.name))
name="s_cls_out_%d_b" % i, fluid.layers.assign(input=t_emb, output=s_emb)
initializer=fluid.initializer.Constant(0.))) print(
fc = self.add_sublayer("cls_fc_%d" % i, fc) "Assigned embedding[{}] from teacher to embedding[{}] in student.".
self.cls_fc.append(fc) format(t_emb.name, s_emb.name))
def forward(self, data_ids): def forward(self, data_ids, epoch):
src_ids = data_ids[0] src_ids = data_ids[0]
position_ids = data_ids[1] position_ids = data_ids[1]
sentence_ids = data_ids[2] sentence_ids = data_ids[2]
return self.student(src_ids, position_ids, sentence_ids) return self.student(src_ids, position_ids, sentence_ids, epoch)
def arch_parameters(self): def arch_parameters(self):
return self.student.arch_parameters() return self.student.arch_parameters()
def genotype(self): def ce(self, logits):
return self.arch_parameters() logits = np.exp(logits - np.max(logits))
logits = logits / logits.sum(axis=0)
return logits
def loss(self, data_ids): def loss(self, data_ids, epoch):
src_ids = data_ids[0] src_ids = data_ids[0]
position_ids = data_ids[1] position_ids = data_ids[1]
sentence_ids = data_ids[2] sentence_ids = data_ids[2]
input_mask = data_ids[3] input_mask = data_ids[3]
labels = data_ids[4] labels = data_ids[4]
enc_output = self.student( s_logits = self.student(src_ids, position_ids, sentence_ids, epoch)
src_ids, position_ids, sentence_ids, flops=[], model_size=[])
t_enc_outputs, t_logits, t_losses, t_accs, _ = self.teacher(data_ids)
#define kd loss
kd_weights = []
for i in range(len(s_logits)):
j = int(np.ceil(i * (float(len(t_logits)) / len(s_logits))))
kd_weights.append(t_losses[j].numpy())
kd_weights = np.array(kd_weights)
kd_weights = np.squeeze(kd_weights)
kd_weights = to_variable(kd_weights)
kd_weights = fluid.layers.softmax(-kd_weights)
kd_losses = []
for i in range(len(s_logits)):
j = int(np.ceil(i * (float(len(t_logits)) / len(s_logits))))
t_logit = t_logits[j]
s_logit = s_logits[i]
t_logit.stop_gradient = True
t_probs = fluid.layers.softmax(t_logit) # P_j^T
s_probs = fluid.layers.softmax(s_logit / self.T) #P_j^S
#kd_loss = -t_probs * fluid.layers.log(s_probs)
kd_loss = fluid.layers.cross_entropy(
input=s_probs, label=t_probs, soft_label=True)
kd_loss = fluid.layers.reduce_mean(kd_loss)
kd_loss = fluid.layers.scale(kd_loss, scale=kd_weights[i])
kd_losses.append(kd_loss)
kd_loss = fluid.layers.sum(kd_losses)
losses = []
for logit in s_logits:
ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
logits=logit, label=labels, return_softmax=True)
loss = fluid.layers.mean(x=ce_loss)
losses.append(loss)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(
input=probs, label=labels, total=num_seqs)
ce_loss = fluid.layers.sum(losses)
total_loss = (1 - self._gamma) * ce_loss + self._gamma * kd_loss
ce_loss, probs = fluid.layers.softmax_with_cross_entropy( return total_loss, accuracy, ce_loss, kd_loss, s_logits
logits=enc_output, label=labels, return_softmax=True)
loss = fluid.layers.mean(x=ce_loss)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(
input=probs, label=labels, total=num_seqs)
return loss, accuracy
...@@ -56,9 +56,9 @@ class BertModelLayer(Layer): ...@@ -56,9 +56,9 @@ class BertModelLayer(Layer):
self.use_fixed_gumbel = use_fixed_gumbel self.use_fixed_gumbel = use_fixed_gumbel
self._word_emb_name = "word_embedding" self._word_emb_name = "s_word_embedding"
self._pos_emb_name = "pos_embedding" self._pos_emb_name = "s_pos_embedding"
self._sent_emb_name = "sent_embedding" self._sent_emb_name = "s_sent_embedding"
self._dtype = "float16" if use_fp16 else "float32" self._dtype = "float16" if use_fp16 else "float32"
self._conv_type = conv_type self._conv_type = conv_type
...@@ -95,6 +95,10 @@ class BertModelLayer(Layer): ...@@ -95,6 +95,10 @@ class BertModelLayer(Layer):
search_layer=self._search_layer, search_layer=self._search_layer,
use_fixed_gumbel=self.use_fixed_gumbel) use_fixed_gumbel=self.use_fixed_gumbel)
def emb_names(self):
return self._src_emb.parameters() + self._pos_emb.parameters(
) + self._sent_emb.parameters()
def max_flops(self): def max_flops(self):
return self._encoder.max_flops return self._encoder.max_flops
...@@ -108,6 +112,7 @@ class BertModelLayer(Layer): ...@@ -108,6 +112,7 @@ class BertModelLayer(Layer):
src_ids, src_ids,
position_ids, position_ids,
sentence_ids, sentence_ids,
epoch,
flops=[], flops=[],
model_size=[]): model_size=[]):
""" """
...@@ -150,9 +155,9 @@ class BertModelLayer(Layer): ...@@ -150,9 +155,9 @@ class BertModelLayer(Layer):
src_emb_1 = self._src_emb(ids1) src_emb_1 = self._src_emb(ids1)
emb_out_0 = self._emb_fac(src_emb_0) emb_out_0 = self._emb_fac(src_emb_0)
emb_out_1 = self._emb_fac(src_emb_1) emb_out_1 = self._emb_fac(src_emb_1)
# (bs, seq_len, 768) # (bs, seq_len, hidden_size)
enc_output = self._encoder( enc_outputs = self._encoder(
emb_out_0, emb_out_1, flops=flops, model_size=model_size) emb_out_0, emb_out_1, epoch, flops=flops, model_size=model_size)
return enc_output return enc_outputs
...@@ -52,11 +52,6 @@ class MixedOp(fluid.dygraph.Layer): ...@@ -52,11 +52,6 @@ class MixedOp(fluid.dygraph.Layer):
def __init__(self, n_channel, name=None): def __init__(self, n_channel, name=None):
super(MixedOp, self).__init__() super(MixedOp, self).__init__()
PRIMITIVES = ConvBN_PRIMITIVES PRIMITIVES = ConvBN_PRIMITIVES
# ops = [
# OPS[primitive](n_channel, name
# if name is None else name + "/" + primitive)
# for primitive in PRIMITIVES
# ]
ops = [] ops = []
for primitive in PRIMITIVES: for primitive in PRIMITIVES:
op = OPS[primitive](n_channel, name op = OPS[primitive](n_channel, name
...@@ -74,21 +69,19 @@ class MixedOp(fluid.dygraph.Layer): ...@@ -74,21 +69,19 @@ class MixedOp(fluid.dygraph.Layer):
self._ops = fluid.dygraph.LayerList(ops) self._ops = fluid.dygraph.LayerList(ops)
def forward(self, x, weights): def forward(self, x, weights, index):
#out = weights[0] * self._ops[0](x) out = fluid.layers.sums(
# out = fluid.layers.sums( [weights[i] * op(x) for i, op in enumerate(self._ops)])
# [weights[i] * op(x) for i, op in enumerate(self._ops)]) return out
# return out
for i in range(len(self._ops)): # causebug in multi-gpus
if weights[i].numpy() != 0: #for i in range(len(self._ops)):
return self._ops[i](x) * weights[i] # if weights[i].numpy() != 0:
# return self._ops[i](x) * weights[i]
def gumbel_softmax(logits, temperature=1, hard=True, eps=1e-10): def gumbel_softmax(logits, epoch, temperature=1.0, hard=True, eps=1e-10):
#U = np.random.uniform(0, 1, logits.shape) temperature = temperature * (0.98**epoch)
#U = - to_variable(
# np.log(-np.log(U + eps) + eps).astype("float32"))
U = np.random.gumbel(0, 1, logits.shape).astype("float32") U = np.random.gumbel(0, 1, logits.shape).astype("float32")
logits = logits + to_variable(U) logits = logits + to_variable(U)
...@@ -98,13 +91,13 @@ def gumbel_softmax(logits, temperature=1, hard=True, eps=1e-10): ...@@ -98,13 +91,13 @@ def gumbel_softmax(logits, temperature=1, hard=True, eps=1e-10):
if hard: if hard:
maxes = fluid.layers.reduce_max(logits, dim=1, keep_dim=True) maxes = fluid.layers.reduce_max(logits, dim=1, keep_dim=True)
hard = fluid.layers.cast((logits == maxes), logits.dtype) hard = fluid.layers.cast((logits == maxes), logits.dtype)
# out = hard - logits.detach() + logits index = np.argmax(hard.numpy(), axis=1)
tmp = hard - logits out = hard - logits.detach() + logits
tmp.stop_gradient = True # tmp.stop_gradient = True
out = tmp + logits # out = tmp + logits
else: else:
out = logits out = logits
return out return out, index
class Zero(fluid.dygraph.Layer): class Zero(fluid.dygraph.Layer):
...@@ -135,8 +128,6 @@ class ReluConvBN(fluid.dygraph.Layer): ...@@ -135,8 +128,6 @@ class ReluConvBN(fluid.dygraph.Layer):
use_cudnn=True, use_cudnn=True,
name=None): name=None):
super(ReluConvBN, self).__init__() super(ReluConvBN, self).__init__()
#conv_std = (2.0 /
# (filter_size[0] * filter_size[1] * out_c * in_c))**0.5
conv_param = fluid.ParamAttr( conv_param = fluid.ParamAttr(
name=name if name is None else (name + "_conv.weights"), name=name if name is None else (name + "_conv.weights"),
initializer=fluid.initializer.MSRA()) initializer=fluid.initializer.MSRA())
...@@ -184,7 +175,7 @@ class Cell(fluid.dygraph.Layer): ...@@ -184,7 +175,7 @@ class Cell(fluid.dygraph.Layer):
ops.append(op) ops.append(op)
self._ops = fluid.dygraph.LayerList(ops) self._ops = fluid.dygraph.LayerList(ops)
def forward(self, s0, s1, weights): def forward(self, s0, s1, weights, index):
s0 = self.preprocess0(s0) s0 = self.preprocess0(s0)
s1 = self.preprocess1(s1) s1 = self.preprocess1(s1)
...@@ -192,7 +183,8 @@ class Cell(fluid.dygraph.Layer): ...@@ -192,7 +183,8 @@ class Cell(fluid.dygraph.Layer):
offset = 0 offset = 0
for i in range(self._steps): for i in range(self._steps):
s = fluid.layers.sums([ s = fluid.layers.sums([
self._ops[offset + j](h, weights[offset + j]) self._ops[offset + j](h, weights[offset + j],
index[offset + j])
for j, h in enumerate(states) for j, h in enumerate(states)
]) ])
offset += len(states) offset += len(states)
...@@ -216,12 +208,27 @@ class EncoderLayer(Layer): ...@@ -216,12 +208,27 @@ class EncoderLayer(Layer):
super(EncoderLayer, self).__init__() super(EncoderLayer, self).__init__()
self._n_layer = n_layer self._n_layer = n_layer
self._hidden_size = hidden_size self._hidden_size = hidden_size
self._n_channel = 256 self._n_channel = 128
self._steps = 3 self._steps = 3
self._n_ops = len(ConvBN_PRIMITIVES) self._n_ops = len(ConvBN_PRIMITIVES)
self.use_fixed_gumbel = use_fixed_gumbel self.use_fixed_gumbel = use_fixed_gumbel
self.stem = fluid.dygraph.Sequential( self.stem0 = fluid.dygraph.Sequential(
Conv2D(
num_channels=1,
num_filters=self._n_channel,
filter_size=[3, self._hidden_size],
padding=[1, 0],
param_attr=fluid.ParamAttr(initializer=MSRA()),
bias_attr=False),
BatchNorm(
num_channels=self._n_channel,
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=1)),
bias_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=0))))
self.stem1 = fluid.dygraph.Sequential(
Conv2D( Conv2D(
num_channels=1, num_channels=1,
num_filters=self._n_channel, num_filters=self._n_channel,
...@@ -254,11 +261,7 @@ class EncoderLayer(Layer): ...@@ -254,11 +261,7 @@ class EncoderLayer(Layer):
default_initializer=NormalInitializer( default_initializer=NormalInitializer(
loc=0.0, scale=1e-3)) loc=0.0, scale=1e-3))
# self.k = fluid.layers.create_parameter( self.pool2d_avg = Pool2D(pool_type='avg', global_pooling=True)
# shape=[1, self._n_layer],
# dtype="float32",
# default_initializer=NormalInitializer(
# loc=0.0, scale=1e-3))
self.BN = BatchNorm( self.BN = BatchNorm(
num_channels=self._n_channel, num_channels=self._n_channel,
...@@ -269,38 +272,58 @@ class EncoderLayer(Layer): ...@@ -269,38 +272,58 @@ class EncoderLayer(Layer):
initializer=fluid.initializer.Constant(value=0), initializer=fluid.initializer.Constant(value=0),
trainable=False)) trainable=False))
self.pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) self.bns = []
self.outs = []
self.out = Linear( for i in range(self._n_layer):
self._n_channel, bn = BatchNorm(
3, num_channels=self._n_channel,
param_attr=ParamAttr(initializer=MSRA()), param_attr=fluid.ParamAttr(
bias_attr=ParamAttr(initializer=MSRA())) initializer=fluid.initializer.Constant(value=1),
trainable=False),
bias_attr=fluid.ParamAttr(
initializer=fluid.initializer.Constant(value=0),
trainable=False))
out = Linear(
self._n_channel,
3,
param_attr=ParamAttr(initializer=MSRA()),
bias_attr=ParamAttr(initializer=MSRA()))
self.bns.append(bn)
self.outs.append(out)
self._bns = fluid.dygraph.LayerList(self.bns)
self._outs = fluid.dygraph.LayerList(self.outs)
self.pooled_fc = Linear(
input_dim=self._n_channel,
output_dim=self._hidden_size,
param_attr=fluid.ParamAttr(
name=self.full_name() + "pooled_fc.w_0",
initializer=fluid.initializer.TruncatedNormal(scale=1.0)),
bias_attr=fluid.ParamAttr(name=self.full_name() + "pooled_fc.b_0"),
act="tanh")
self.use_fixed_gumbel = use_fixed_gumbel self.use_fixed_gumbel = use_fixed_gumbel
self.gumbel_alphas = gumbel_softmax(self.alphas).detach() self.gumbel_alphas = gumbel_softmax(self.alphas, 0)[0].detach()
#print("gumbel_alphas: \n", self.gumbel_alphas.numpy())
def forward(self, enc_input_0, enc_input_1, flops=[], model_size=[]): def forward(self, enc_input_0, enc_input_1, epoch, flops=[],
alphas = self.gumbel_alphas if self.use_fixed_gumbel else gumbel_softmax( model_size=[]):
self.alphas) alphas, index = self.gumbel_alphas if self.use_fixed_gumbel else gumbel_softmax(
self.alphas, epoch)
s0 = fluid.layers.reshape( s0 = fluid.layers.unsqueeze(enc_input_0, [1])
enc_input_0, [-1, 1, enc_input_0.shape[1], enc_input_0.shape[2]]) s1 = fluid.layers.unsqueeze(enc_input_1, [1])
s1 = fluid.layers.reshape( s0 = self.stem0(s0)
enc_input_1, [-1, 1, enc_input_1.shape[1], enc_input_1.shape[2]]) s1 = self.stem1(s1)
# (bs, 1, seq_len, hidden_size)
s0 = self.stem(s0)
s1 = self.stem(s1)
# (bs, n_channel, seq_len, 1)
enc_outputs = []
for i in range(self._n_layer): for i in range(self._n_layer):
s0, s1 = s1, self._cells[i](s0, s1, alphas) s0, s1 = s1, self._cells[i](s0, s1, alphas, index)
# (bs, n_channel, seq_len, 1) # (bs, n_channel, seq_len, 1)
tmp = self._bns[i](s1)
s1 = self.BN(s1) tmp = self.pool2d_avg(tmp)
tmp = fluid.layers.reshape(tmp, shape=[-1, 0])
outputs = self.pool2d_avg(s1) tmp = self._outs[i](tmp)
outputs = fluid.layers.reshape(outputs, shape=[-1, 0]) enc_outputs.append(tmp)
outputs = self.out(outputs)
return outputs return enc_outputs
...@@ -58,7 +58,8 @@ class BERTClassifier(Layer): ...@@ -58,7 +58,8 @@ class BERTClassifier(Layer):
num_labels, num_labels,
task_name="mnli", task_name="mnli",
model_path=None, model_path=None,
use_cuda=True): use_cuda=True,
return_pooled_out=True):
super(BERTClassifier, self).__init__() super(BERTClassifier, self).__init__()
self.task_name = task_name.lower() self.task_name = task_name.lower()
BERT_BASE_PATH = "./data/pretrained_models/uncased_L-12_H-768_A-12/" BERT_BASE_PATH = "./data/pretrained_models/uncased_L-12_H-768_A-12/"
...@@ -84,7 +85,7 @@ class BERTClassifier(Layer): ...@@ -84,7 +85,7 @@ class BERTClassifier(Layer):
} }
self.cls_model = ClsModelLayer( self.cls_model = ClsModelLayer(
self.bert_config, num_labels, return_pooled_out=True) self.bert_config, num_labels, return_pooled_out=return_pooled_out)
if model_path is not None: if model_path is not None:
#restore the model #restore the model
...@@ -101,6 +102,9 @@ class BERTClassifier(Layer): ...@@ -101,6 +102,9 @@ class BERTClassifier(Layer):
"You should load pretrained model for training this teacher model." "You should load pretrained model for training this teacher model."
) )
def emb_names(self):
return self.cls_model.emb_names()
def forward(self, input): def forward(self, input):
return self.cls_model(input) return self.cls_model(input)
......
...@@ -122,6 +122,10 @@ class BertModelLayer(Layer): ...@@ -122,6 +122,10 @@ class BertModelLayer(Layer):
postprocess_cmd="dan", postprocess_cmd="dan",
param_initializer=self._param_initializer) param_initializer=self._param_initializer)
def emb_names(self):
return self._src_emb.parameters() + self._pos_emb.parameters(
) + self._sent_emb.parameters()
def forward(self, src_ids, position_ids, sentence_ids, input_mask): def forward(self, src_ids, position_ids, sentence_ids, input_mask):
""" """
forward forward
......
...@@ -46,6 +46,7 @@ class ClsModelLayer(Layer): ...@@ -46,6 +46,7 @@ class ClsModelLayer(Layer):
self.use_fp16 = use_fp16 self.use_fp16 = use_fp16
self.loss_scaling = loss_scaling self.loss_scaling = loss_scaling
self.n_layers = config['num_hidden_layers'] self.n_layers = config['num_hidden_layers']
self.return_pooled_out = return_pooled_out
self.bert_layer = BertModelLayer( self.bert_layer = BertModelLayer(
config=self.config, return_pooled_out=True, use_fp16=self.use_fp16) config=self.config, return_pooled_out=True, use_fp16=self.use_fp16)
...@@ -64,6 +65,9 @@ class ClsModelLayer(Layer): ...@@ -64,6 +65,9 @@ class ClsModelLayer(Layer):
fc = self.add_sublayer("cls_fc_%d" % i, fc) fc = self.add_sublayer("cls_fc_%d" % i, fc)
self.cls_fc.append(fc) self.cls_fc.append(fc)
def emb_names(self):
return self.bert_layer.emb_names()
def forward(self, data_ids): def forward(self, data_ids):
""" """
forward forward
...@@ -76,11 +80,23 @@ class ClsModelLayer(Layer): ...@@ -76,11 +80,23 @@ class ClsModelLayer(Layer):
enc_outputs, next_sent_feats = self.bert_layer( enc_outputs, next_sent_feats = self.bert_layer(
src_ids, position_ids, sentence_ids, input_mask) src_ids, position_ids, sentence_ids, input_mask)
if not self.return_pooled_out:
cls_feat = fluid.layers.dropout(
x=next_sent_feats[-1],
dropout_prob=0.1,
dropout_implementation="upscale_in_train")
logits = self.cls_fc[-1](cls_feat)
probs = fluid.layers.softmax(logits)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(
input=probs, label=labels, total=num_seqs)
return enc_outputs, logits, accuracy, num_seqs
logits = [] logits = []
losses = [] losses = []
accuracys = [] accuracys = []
for next_sent_feat, fc in zip(next_sent_feats, self.cls_fc): for next_sent_feat, fc in zip(next_sent_feats, self.cls_fc):
cls_feat = fluid.layers.dropout( cls_feat = fluid.layers.dropout(
x=next_sent_feat, x=next_sent_feat,
dropout_prob=0.1, dropout_prob=0.1,
......
...@@ -16,6 +16,7 @@ import io ...@@ -16,6 +16,7 @@ import io
import os import os
import types import types
import csv import csv
import random
import numpy as np import numpy as np
from . import tokenization from . import tokenization
from .batching import prepare_batch_data from .batching import prepare_batch_data
...@@ -110,9 +111,9 @@ class DataProcessor(object): ...@@ -110,9 +111,9 @@ class DataProcessor(object):
def get_num_examples(self, phase): def get_num_examples(self, phase):
"""Get number of examples for train, dev or test.""" """Get number of examples for train, dev or test."""
if phase not in ['train', 'dev', 'test']: #if phase not in ['train', 'dev', 'test']:
raise ValueError( # raise ValueError(
"Unknown phase, which should be in ['train', 'dev', 'test'].") # "Unknown phase, which should be in ['train', 'dev', 'test'].")
return self.num_examples[phase] return self.num_examples[phase]
def get_train_progress(self): def get_train_progress(self):
...@@ -135,6 +136,8 @@ class DataProcessor(object): ...@@ -135,6 +136,8 @@ class DataProcessor(object):
epoch: int. Total epoches to generate data. epoch: int. Total epoches to generate data.
shuffle: bool. Whether to shuffle examples. shuffle: bool. Whether to shuffle examples.
""" """
search_examples = self.get_train_examples(self.data_dir)
random.shuffle(search_examples)
if phase == 'train': if phase == 'train':
examples = self.get_train_examples(self.data_dir) examples = self.get_train_examples(self.data_dir)
self.num_examples['train'] = len(examples) self.num_examples['train'] = len(examples)
...@@ -145,13 +148,13 @@ class DataProcessor(object): ...@@ -145,13 +148,13 @@ class DataProcessor(object):
examples = self.get_test_examples(self.data_dir) examples = self.get_test_examples(self.data_dir)
self.num_examples['test'] = len(examples) self.num_examples['test'] = len(examples)
elif phase == 'search_train': elif phase == 'search_train':
examples = self.get_train_examples(self.data_dir) #examples = self.get_train_examples(self.data_dir)
self.num_examples['search_train'] = len(examples) / 2 self.num_examples['search_train'] = len(search_examples) / 2
examples = examples[:self.num_examples['search_train']] examples = search_examples[:self.num_examples['search_train']]
elif phase == 'search_valid': elif phase == 'search_valid':
examples = self.get_train_examples(self.data_dir) #examples = self.get_train_examples(self.data_dir)
self.num_examples['search_valid'] = len(examples) / 2 self.num_examples['search_valid'] = len(search_examples) / 2
examples = examples[self.num_examples['search_train']:] examples = search_examples[self.num_examples['search_valid']:]
else: else:
raise ValueError( raise ValueError(
"Unknown phase, which should be in ['train', 'dev', 'test'].") "Unknown phase, which should be in ['train', 'dev', 'test'].")
...@@ -340,7 +343,7 @@ class MnliProcessor(DataProcessor): ...@@ -340,7 +343,7 @@ class MnliProcessor(DataProcessor):
def get_train_examples(self, data_dir): def get_train_examples(self, data_dir):
"""See base class.""" """See base class."""
return self._create_examples( return self._create_examples(
self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "train")
def get_dev_examples(self, data_dir): def get_dev_examples(self, data_dir):
"""See base class.""" """See base class."""
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册