Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
OpenDocCN
d2l-zh
提交
d02e0125
D
d2l-zh
项目概览
OpenDocCN
/
d2l-zh
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
d2l-zh
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d02e0125
编写于
5月 28, 2018
作者:
A
Aston Zhang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
nlp code
上级
893f2752
变更
5
展开全部
显示空白变更内容
内联
并排
Showing
5 changed file
with
223 addition
and
313 deletion
+223
-313
chapter_natural-language-processing/nmt.md
chapter_natural-language-processing/nmt.md
+105
-147
chapter_natural-language-processing/pretrained-embedding.md
chapter_natural-language-processing/pretrained-embedding.md
+10
-16
chapter_natural-language-processing/sentiment-analysis.md
chapter_natural-language-processing/sentiment-analysis.md
+103
-146
chapter_recurrent-neural-networks/rnn-gluon.md
chapter_recurrent-neural-networks/rnn-gluon.md
+3
-2
chapter_recurrent-neural-networks/rnn-scratch.md
chapter_recurrent-neural-networks/rnn-scratch.md
+2
-2
未找到文件。
chapter_natural-language-processing/nmt.md
浏览文件 @
d02e0125
此差异已折叠。
点击以展开。
chapter_natural-language-processing/pretrained-embedding.md
浏览文件 @
d02e0125
...
...
@@ -10,9 +10,9 @@
我们先载入需要的包。
```
{.python .input n=1}
from mxnet import gluon
from mxnet import nd
from mxnet.contrib import text
from mxnet.gluon import nn
```
## 由数据集建立词典和载入词向量——以fastText为例
...
...
@@ -70,17 +70,17 @@ my_embedding.get_vecs_by_tokens(['hello', 'world'])[:, :5]
my_embedding.to_indices(['hello', 'world'])
```
### 使用预训练词向量初始化
gluon.
nn.Embedding
### 使用预训练词向量初始化nn.Embedding
我们可以使用预训练的词向量初始化
`
gluon.
nn.Embedding`
。
我们可以使用预训练的词向量初始化
`nn.Embedding`
。
```
{.python .input n=8}
layer =
gluon.
nn.Embedding(len(my_embedding), my_embedding.vec_len)
layer = nn.Embedding(len(my_embedding), my_embedding.vec_len)
layer.initialize()
layer.weight.set_data(my_embedding.idx_to_vec)
```
使用词典中“hello”和“world”两个词在词典中的索引,我们可以通过
`
gluon.
nn.Embedding`
得到它们的词向量,并向神经网络的下一层传递。
使用词典中“hello”和“world”两个词在词典中的索引,我们可以通过
`nn.Embedding`
得到它们的词向量,并向神经网络的下一层传递。
```
{.python .input n=9}
layer(nd.array([2, 1]))[:, :5]
...
...
@@ -110,12 +110,8 @@ print(len(glove_6b50d))
我们可以访问词向量的属性。
```
{.python .input n=12}
# 词到索引。
print(glove_6b50d.token_to_idx['beautiful'])
# 索引到词。
print(glove_6b50d.idx_to_token[3367])
# 词向量长度。
print(glove_6b50d.vec_len)
# 词到索引,索引到词。
glove_6b50d.token_to_idx['beautiful'], glove_6b50d.idx_to_token[3367]
```
## 预训练词向量的应用——以GloVe为例
...
...
@@ -123,9 +119,8 @@ print(glove_6b50d.vec_len)
为了应用预训练词向量,我们需要定义余弦相似度。它可以比较两个向量之间的相似度。
```
{.python .input n=13}
from mxnet import nd
def cos_sim(x, y):
return nd.dot(x, y) / (
nd.norm(x) * nd.norm(y
))
return nd.dot(x, y) / (
x.norm() * y.norm(
))
```
余弦相似度的值域在-1到1之间。余弦相似度值越大,两个向量越接近。
...
...
@@ -135,8 +130,7 @@ x = nd.array([1, 2])
y = nd.array([10, 20])
z = nd.array([-1, -2])
print(cos_sim(x, y))
print(cos_sim(x, z))
cos_sim(x, y), cos_sim(x, z)
```
### 求近似词
...
...
@@ -145,7 +139,7 @@ print(cos_sim(x, z))
```
{.python .input}
def norm_vecs_by_row(x):
return x / nd.s
qrt(nd.sum(x * x, axis=1)).reshape((-1,
1))
return x / nd.s
um(x * x, axis=1).sqrt().reshape((-1,
1))
def get_knn(token_embedding, k, word):
word_vec = token_embedding.get_vecs_by_tokens([word]).reshape((-1, 1))
...
...
chapter_natural-language-processing/sentiment-analysis.md
浏览文件 @
d02e0125
...
...
@@ -12,15 +12,17 @@
首先,我们当然需要加载MXNet和Gluon。
```
{.python .input n=1}
from collections import Counter
import sys
sys.path.append('..')
import collections
import gluonbook as gb
import mxnet as mx
from mxnet import autograd, gluon, init, metric, nd
from mxnet.gluon import loss as gloss, nn, rnn
from mxnet.contrib import text
import os
import random
import time
import zipfile
import mxnet as mx
from mxnet import autograd, gluon, init, nd
from mxnet.contrib import text
```
### 读取IMDb数据集
...
...
@@ -50,31 +52,30 @@ if demo:
```
```
{.python .input}
def readIMDB(dir_url, seg
=
'train'):
pos_or_neg = ['pos','neg']
data
set
= []
for l
b
in pos_or_neg:
files = os.listdir('../data/' + dir_url + '/' + seg + '/' + l
b
+ '/')
def readIMDB(dir_url, seg
=
'train'):
pos_or_neg = ['pos',
'neg']
data = []
for l
abel
in pos_or_neg:
files = os.listdir('../data/' + dir_url + '/' + seg + '/' + l
abel
+ '/')
for file in files:
with open('../data/' + dir_url + '/' + seg + '/' + l
b
+ '/'
with open('../data/' + dir_url + '/' + seg + '/' + l
abel
+ '/'
+ file, 'r', encoding='utf8') as rf:
review = rf.read().replace('\n','')
if l
b
== 'pos':
data
set
.append([review, 1])
elif l
b
== 'neg':
data
set
.append([review, 0])
return data
set
review = rf.read().replace('\n',
'')
if l
abel
== 'pos':
data.append([review, 1])
elif l
abel
== 'neg':
data.append([review, 0])
return data
if demo:
train_data
set
= readIMDB('aclImdb_tiny/', 'train')
test_data
set
= readIMDB('aclImdb_tiny/', 'test')
train_data = readIMDB('aclImdb_tiny/', 'train')
test_data = readIMDB('aclImdb_tiny/', 'test')
else:
train_data
set
= readIMDB('aclImdb/', 'train')
test_data
set
= readIMDB('aclImdb/', 'test')
train_data = readIMDB('aclImdb/', 'train')
test_data = readIMDB('aclImdb/', 'test')
# shuffle 数据集。
random.shuffle(train_dataset)
random.shuffle(test_dataset)
random.shuffle(train_data)
random.shuffle(test_data)
```
### 指定分词方式并且分词
...
...
@@ -91,15 +92,11 @@ def tokenizer(text):
```
{.python .input n=5}
train_tokenized = []
train_labels = []
for review, score in train_dataset:
for review, score in train_data:
train_tokenized.append(tokenizer(review))
train_labels.append(score)
test_tokenized = []
test_labels = []
for review, score in test_dataset:
for review, score in test_data:
test_tokenized.append(tokenizer(review))
test_labels.append(score)
```
### 创建词典
...
...
@@ -108,7 +105,7 @@ for review, score in test_dataset:
这里我们特别设置训练数据中没有的单词对应的符号'<unk
\>
',所有不存在在词典中的词,未来都将对应到这个符号。
```
{.python .input n=6}
token_counter = Counter()
token_counter =
collections.
Counter()
def count_token(train_tokenized):
for sample in train_tokenized:
for token in sample:
...
...
@@ -128,55 +125,53 @@ vocab = text.vocab.Vocabulary(token_counter, unknown_token='<unk>',
```
{.python .input n=7}
# 根据词典,将数据转换成特征向量。
def encode_samples(
x_raw
_samples, vocab):
x_encoded_sampl
es = []
for sample in
x_raw
_samples:
x_encoded_sampl
e = []
def encode_samples(
tokenized
_samples, vocab):
featur
es = []
for sample in
tokenized
_samples:
featur
e = []
for token in sample:
if token in vocab.token_to_idx:
x_encoded_sampl
e.append(vocab.token_to_idx[token])
featur
e.append(vocab.token_to_idx[token])
else:
x_encoded_sample.append(0)
x_encoded_samples.append(x_encoded_sample)
return x_encoded_samples
# 将特征向量补成定长。
def pad_samples(x_encoded_samples, maxlen=500, val=0):
x_samples = []
for sample in x_encoded_samples:
if len(sample) > maxlen:
new_sample = sample[:maxlen]
feature.append(0)
features.append(feature)
return features
def pad_samples(features, maxlen=500, padding=0):
padded_features = []
for feature in features:
if len(feature) > maxlen:
padded_feature = feature[:maxlen]
else:
num_padding = maxlen - len(sample)
new_sample = sample
for i in range(num_padding)
:
new_sample.append(val
)
x_samples.append(new_sampl
e)
return
x_sampl
es
padded_feature = feature
# 添加 PAD 符号使每个序列等长(长度为 maxlen )。
while len(padded_feature) < maxlen
:
padded_feature.append(padding
)
padded_features.append(padded_featur
e)
return
padded_featur
es
```
运行下面的代码将分好词的训练和测试数据转化成特征向量。
```
{.python .input n=8}
x_encoded_train
= encode_samples(train_tokenized, vocab)
x_encoded_test
= encode_samples(test_tokenized, vocab)
train_features
= encode_samples(train_tokenized, vocab)
test_features
= encode_samples(test_tokenized, vocab)
```
通过执行下面的代码将特征向量补成定长(我们使用500),然后将特征向量转化为指定context上的NDArray。
这里我们假定我们有至少一块gpu,context被设置成gpu。当然,也可以使用cpu,运行速度可能稍微慢一点点。
```
{.python .input n=9}
# 指定 context。
context = mx.gpu(0)
x_train = nd.array(pad_samples(x_encoded_train, 500, 0), ctx=context)
x_test = nd.array(pad_samples(x_encoded_test, 500, 0), ctx=context)
ctx = gb.try_gpu()
train_features = nd.array(pad_samples(train_features, 500, 0), ctx=ctx)
test_features = nd.array(pad_samples(test_features, 500, 0), ctx=ctx)
```
这里,我们将情感标签也转化成为了NDArray。
```
{.python .input n=10}
y_train = nd.array([score for text, score in train_dataset], ctx=context
)
y_test = nd.array([score for text, score in test_dataset], ctx=context
)
train_labels = nd.array([score for _, score in train_data], ctx=ctx
)
test_labels = nd.array([score for _, score in test_data], ctx=ctx
)
```
### 加载预训练的词向量
...
...
@@ -203,28 +198,28 @@ glove_embedding = text.embedding.create(
4.
`decoder`
: 最后,我们利用上一步所生成的特征,通过一个dense层做预测。
```
{.python .input n=12}
n
clas
s = 2
n
um_output
s = 2
lr = 0.1
num_epochs = 1
batch_size = 10
emsize = 100
em
bed_
size = 100
num_hiddens = 100
nlayers = 2
n
um_
layers = 2
bidirectional = True
```
```
{.python .input}
class SentimentNet(
gluo
n.Block):
def __init__(self, vocab, em
size, num_hiddens, nlayers, bidirectional
,
**kwargs):
class SentimentNet(
n
n.Block):
def __init__(self, vocab, em
bed_size, num_hiddens, num_layers
,
bidirectional,
**kwargs):
super(SentimentNet, self).__init__(**kwargs)
with self.name_scope():
self.embedding =
gluon.
nn.Embedding(
len(vocab), emsize, weight_initializer=init.Uniform(0.1))
self.encoder =
gluon.rnn.LSTM(num_hiddens, num_layers=nlayers,
self.embedding = nn.Embedding(
len(vocab), em
bed_
size, weight_initializer=init.Uniform(0.1))
self.encoder =
rnn.LSTM(num_hiddens, num_layers=num_layers,
bidirectional=bidirectional,
input_size=em
size)
self.decoder =
gluon.nn.Dense(nclas
s, flatten=False)
input_size=embed_
size)
self.decoder =
nn.Dense(num_output
s, flatten=False)
def forward(self, inputs, begin_state=None):
outputs = self.embedding(inputs)
outputs = self.encoder(outputs)
...
...
@@ -232,16 +227,14 @@ class SentimentNet(gluon.Block):
outputs = self.decoder(outputs)
return outputs
net = SentimentNet(vocab, em
size, num_hiddens, n
layers, bidirectional)
net.initialize(
mx.init.Xavier(), ctx=context
)
net = SentimentNet(vocab, em
bed_size, num_hiddens, num_
layers, bidirectional)
net.initialize(
init.Xavier(), ctx=ctx
)
# 设置 embedding 层的 weight 为词向量。
net.embedding.weight.set_data(
glove_embedding.idx_to_vec.as_in_context(context))
net.embedding.weight.set_data(glove_embedding.idx_to_vec.as_in_context(ctx))
# 对 embedding 层不进行优化。
net.embedding.collect_params().setattr('grad_req', 'null')
trainer = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': lr})
loss = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})
loss = gloss.SoftmaxCrossEntropyLoss()
```
## 训练模型
...
...
@@ -251,82 +244,46 @@ loss = gluon.loss.SoftmaxCrossEntropyLoss()
```
{.python .input n=13}
# 使用准确率作为评价指标。
def eval(x_samples, y_samples):
total_L = 0
ntotal = 0
accuracy = mx.metric.Accuracy()
for i in range(x_samples.shape[0] // batch_size):
data = x_samples[i * batch_size : (i+1) * batch_size]
target = y_samples[i * batch_size :(i+1) * batch_size]
data = data.as_in_context(context).T
target = target.as_in_context(context).T
output = net(data)
L = loss(output, target)
total_L += nd.sum(L).asscalar()
ntotal += L.size
predicts = nd.argmax(output, axis=1)
accuracy.update(preds=predicts, labels=target)
return total_L / ntotal, accuracy.get()[1]
def eval_model(features, labels):
l_sum = 0
l_n = 0
accuracy = metric.Accuracy()
for i in range(features.shape[0] // batch_size):
X = features[i*batch_size : (i+1)*batch_size].as_in_context(ctx).T
y = labels[i*batch_size :(i+1)*batch_size].as_in_context(ctx).T
output = net(X)
l = loss(output, y)
l_sum += l.sum().asscalar()
l_n += l.size
accuracy.update(preds=nd.argmax(output, axis=1), labels=y)
return l_sum / l_n, accuracy.get()[1]
```
运行下面的代码开始训练模型。
我们每800个batch,会输出一次当前的
`loss`
。
运行下面的代码开始训练模型。
```
{.python .input n=14}
start_train_time = time.time()
for epoch in range(num_epochs):
start_epoch_time = time.time()
total_L = 0
ntotal = 0
for i in range(x_train.shape[0] // batch_size):
data = x_train[i * batch_size : (i+1) * batch_size]
target = y_train[i * batch_size : (i+1) * batch_size]
data = data.as_in_context(context).T
target = target.as_in_context(context).T
for epoch in range(1, num_epochs + 1):
for i in range(train_features.shape[0] // batch_size):
X = train_features[i*batch_size : (i+1)*batch_size].as_in_context(
ctx).T
y = train_labels[i*batch_size : (i+1)*batch_size].as_in_context(
ctx).T
with autograd.record():
output = net(data)
L = loss(output, target)
L.backward()
l = loss(net(X), y)
l.backward()
trainer.step(batch_size)
total_L += nd.sum(L).asscalar()
ntotal += L.size
if i % 800 == 0 and i != 0:
print('[epoch %d] batch %d. loss %.6f' % (epoch, i,
total_L / ntotal))
total_L = 0
ntotal = 0
print('performing testing:')
train_loss, train_acc = eval(x_train, y_train)
test_loss, test_acc = eval(x_test, y_test)
print('[epoch %d] train loss %.6f, train accuracy %.2f'
% (epoch, train_loss, train_acc))
print('[epoch %d] test loss %.6f, test accuracy %.2f'
% (epoch, test_loss, test_acc))
print('[epoch %d] throughput %.2f samples/s'
% (epoch, (batch_size * len(x_train))
/ (time.time() - start_epoch_time)))
print('[epoch %d] total time %.2f s'
% (epoch, (time.time() - start_epoch_time)))
print('total training throughput %.2f samples/s'
% ((batch_size * len(x_train) * num_epochs)
/ (time.time() - start_train_time)))
print('total training time %.2f s' % ((time.time() - start_train_time)))
train_loss, train_acc = eval_model(train_features, train_labels)
test_loss, test_acc = eval_model(test_features, test_labels)
print('epoch %d, train loss %.6f, acc %.2f; test loss %.6f, acc %.2f'
% (epoch, train_loss, train_acc, test_loss, test_acc))
```
到这里,我们已经成功使用Gluon创建了一个情感分类模型。下面我们举了一个例子,来看看我们情感分类模型的效果。
```
{.python .input n=15}
review = ['this', 'movie', 'is', 'great']
print(review)
```
上面这个句子的情感是(1代表正面,0代表负面):
到这里,我们已经成功使用Gluon创建了一个情感分类模型。下面我们举了一个例子,来看看我们情感分类模型的效果(1代表正面,0代表负面)。如果希望在不同句子上得到较准确的分类,我们需要使用更大的数据集,并增大模型训练周期。
```
{.python .input}
review = ['this', 'movie', 'is', 'great']
nd.argmax(net(nd.reshape(
nd.array([vocab.token_to_idx[token] for token in review], ctx=c
ontext
),
nd.array([vocab.token_to_idx[token] for token in review], ctx=c
tx
),
shape=(-1, 1))), axis=1).asscalar()
```
...
...
chapter_recurrent-neural-networks/rnn-gluon.md
浏览文件 @
d02e0125
...
...
@@ -255,8 +255,9 @@ def train():
train_l_sum += l
if batch_i % eval_period == 0 and batch_i > 0:
cur_l = train_l_sum / eval_period
print('epoch %d, batch %d, loss %.2f, perplexity %.2f' % (
epoch, batch_i, cur_l.asscalar(), cur_l.exp().asscalar()))
print('epoch %d, batch %d, train loss %.2f, perplexity %.2f'
% (epoch, batch_i, cur_l.asscalar(),
cur_l.exp().asscalar()))
train_l_sum = nd.array([0], ctx=ctx)
val_l = eval_rnn(val_data)
print('epoch %d, time %.2fs, valid loss %.2f, perplexity %.2f'
...
...
chapter_recurrent-neural-networks/rnn-scratch.md
浏览文件 @
d02e0125
...
...
@@ -386,8 +386,8 @@ def train_and_predict_rnn(rnn, is_random_iter, num_epochs, num_steps,
outputs, state_h = rnn(
get_inputs(X, vocab_size), state_h, *params)
# 设 t_ib_j 为时间步 i 批量中的元素 j:
#
Y 形状:(batch_size * num_steps
)
#
Y
= [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ]。
#
y 形状:(batch_size * num_steps,
)
#
y
= [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ]。
y = Y.T.reshape((-1,))
# 拼接 outputs,形状:(batch_size * num_steps, vocab_size)。
outputs = nd.concat(*outputs, dim=0)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录