6.7_gru.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.7 门控循环单元（GRU）\n",
    "## 6.7.2 读取数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2.0 cpu\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"..\") \n",
    "import d2lzh_pytorch as d2l\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "(corpus_indices, char_to_idx, idx_to_char, vocab_size) = d2l.load_data_jay_lyrics()\n",
    "print(torch.__version__, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.7.3 从零开始实现\n",
    "### 6.7.3.1 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will use cpu\n"
     ]
    }
   ],
   "source": [
    "num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size\n",
    "print('will use', device)\n",
    "\n",
    "def get_params():\n",
    "    def _one(shape):\n",
    "        ts = torch.tensor(np.random.normal(0, 0.01, size=shape), device=device, dtype=torch.float32)\n",
    "        return torch.nn.Parameter(ts, requires_grad=True)\n",
    "    def _three():\n",
    "        return (_one((num_inputs, num_hiddens)),\n",
    "                _one((num_hiddens, num_hiddens)),\n",
    "                torch.nn.Parameter(torch.zeros(num_hiddens, device=device, dtype=torch.float32), requires_grad=True))\n",
    "    \n",
    "    W_xz, W_hz, b_z = _three()  # 更新门参数\n",
    "    W_xr, W_hr, b_r = _three()  # 重置门参数\n",
    "    W_xh, W_hh, b_h = _three()  # 候选隐藏状态参数\n",
    "    \n",
    "    # 输出层参数\n",
    "    W_hq = _one((num_hiddens, num_outputs))\n",
    "    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device, dtype=torch.float32), requires_grad=True)\n",
    "    return nn.ParameterList([W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.7.3.2 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_gru_state(batch_size, num_hiddens, device):\n",
    "    return (torch.zeros((batch_size, num_hiddens), device=device), )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gru(inputs, state, params):\n",
    "    W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q = params\n",
    "    H, = state\n",
    "    outputs = []\n",
    "    for X in inputs:\n",
    "        Z = torch.sigmoid(torch.matmul(X, W_xz) + torch.matmul(H, W_hz) + b_z)\n",
    "        R = torch.sigmoid(torch.matmul(X, W_xr) + torch.matmul(H, W_hr) + b_r)\n",
    "        H_tilda = torch.tanh(torch.matmul(X, W_xh) + torch.matmul(R * H, W_hh) + b_h)\n",
    "        H = Z * H + (1 - Z) * H_tilda\n",
    "        Y = torch.matmul(H, W_hq) + b_q\n",
    "        outputs.append(Y)\n",
    "    return outputs, (H,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.7.3.3 训练模型并创作歌词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2\n",
    "pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 40, perplexity 150.963116, time 1.11 sec\n",
      " - 分开 我想你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我\n",
      " - 不分开 我想你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我不你 我\n",
      "epoch 80, perplexity 31.683252, time 1.16 sec\n",
      " - 分开 我想要你的微笑 一定                                       \n",
      " - 不分开 不知不觉 我不要再想 我不要再想 我不 我不 我不 我不 我不 我不 我不 我不 我不 我不 我不\n",
      "epoch 120, perplexity 5.855305, time 1.49 sec\n",
      " - 分开我 想要你这样打我妈妈 难道你手不会痛吗 我想你这样打我妈妈 难道你手 你怎么在我想 说散 你说我久\n",
      " - 不分开  没有你在我有多烦熬多烦恼  没有你烦 我有多烦恼  没有你在我有多难熬多难多  没有你烦 我有多\n",
      "epoch 160, perplexity 1.815359, time 1.04 sec\n",
      " - 分开 我想要这样牵 对你依依不舍 连隔壁邻居都猜到我现在的感受 河边的风 在吹着头发飘动 牵着你的手 一\n",
      " - 不分开  是后过风 迷不知蒙 我给再这样活 我该好好生活 不知不觉 你已经离开我 不知不觉 我跟了这节奏 \n"
     ]
    }
   ],
   "source": [
    "d2l.train_and_predict_rnn(gru, get_params, init_gru_state, num_hiddens,\n",
    "                          vocab_size, device, corpus_indices, idx_to_char,\n",
    "                          char_to_idx, False, num_epochs, num_steps, lr,\n",
    "                          clipping_theta, batch_size, pred_period, pred_len,\n",
    "                          prefixes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.7.4 简洁实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 40, perplexity 1.018485, time 0.79 sec\n",
      " - 分开的快乐是你 想你想的都会笑 没有你在 我有多难熬  没有你在我有多难熬多烦恼  没有你烦 我有多烦恼\n",
      " - 不分开不 我不 我不要再想你 爱情来的太快就像龙卷风 离不开暴风圈来不及逃 我不能再想 我不能再想 我不 \n",
      "epoch 80, perplexity 1.028805, time 0.74 sec\n",
      " - 分开始想像 爸和妈当年的模样 说著一口吴侬软语的姑娘缓缓走过外滩 消失的 旧时光 一九四三 回头看 的片\n",
      " - 不分开不 我不 我不 我不要再想你 爱情来的太快就像龙卷风 离不开暴风圈来不及逃 我不能再想 我不能再想 \n",
      "epoch 120, perplexity 1.012296, time 0.73 sec\n",
      " - 分开的话像语言暴力 我已无能为力再提起 决定中断熟悉 然后在这里 不限日期 然后将过去 慢慢温习 让我爱\n",
      " - 不分开不 我不 我不能 爱情走的太快就像龙卷风 不能承受我已无处可躲 我不要再想 我不要再想 我不 我不 \n",
      "epoch 160, perplexity 1.184842, time 0.74 sec\n",
      " - 分开的快乐是你 想我想大声宣布 对你依依不舍 连隔壁邻居都猜到我现在的感受 河边的风 在吹着头发飘动 牵\n",
      " - 不分开 快使用双截棍 哼哼哈兮 如果我有轻功 飞檐走壁 为人耿直不屈 一身正气 他们儿子我习惯 从小就耳濡\n"
     ]
    }
   ],
   "source": [
    "lr = 1e-2\n",
    "gru_layer = nn.GRU(input_size=vocab_size, hidden_size=num_hiddens)\n",
    "model = d2l.RNNModel(gru_layer, vocab_size).to(device)\n",
    "d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,\n",
    "                                corpus_indices, idx_to_char, char_to_idx,\n",
    "                                num_epochs, num_steps, lr, clipping_theta,\n",
    "                                batch_size, pred_period, pred_len, prefixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}