From 617bc4e55a8464c25ccd01c77b3111dd177c9e78 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Thu, 13 Jul 2017 12:18:17 +0800 Subject: [PATCH] Refine README.md and refactor code structure for "machine translation with external memory" model. --- mt_with_external_memory/README.md | 190 +++++- mt_with_external_memory/data_utils.py | 15 + mt_with_external_memory/external_memory.py | 201 ++++++ .../image/lstm_c_state.png | Bin 0 -> 52288 bytes mt_with_external_memory/infer.py | 154 +++++ mt_with_external_memory/model.py | 206 ++++++ .../mt_with_external_memory.py | 598 ------------------ mt_with_external_memory/train.py | 157 +++++ 8 files changed, 891 insertions(+), 630 deletions(-) create mode 100644 mt_with_external_memory/data_utils.py create mode 100755 mt_with_external_memory/external_memory.py create mode 100644 mt_with_external_memory/image/lstm_c_state.png create mode 100644 mt_with_external_memory/infer.py create mode 100644 mt_with_external_memory/model.py delete mode 100644 mt_with_external_memory/mt_with_external_memory.py create mode 100644 mt_with_external_memory/train.py diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 583ab570..22c8effd 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -1,27 +1,29 @@ # 带外部记忆机制的神经机器翻译 -带**外部记忆**（External Memory）机制的神经机器翻译模型（Neural Machine Translation, NMT），是神经机器翻译模型的一个重要扩展。它利用可微分的外部记忆网络，来拓展神经翻译模型内部工作记忆（Working Memory）的容量或带宽，即引入一个高效的 “外部知识库”，辅助完成翻译等任务中信息的临时存取，有效改善模型表现。 +带**外部记忆**（External Memory）机制的神经机器翻译模型（Neural Machine Translation, NMT），是神经机器翻译模型的一个重要扩展。它引入可微分的记忆网络作为额外的记忆单元，拓展神经翻译模型内部工作记忆（Working Memory）的容量或带宽，辅助完成翻译等任务中信息的临时存取，改善模型表现。 -该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务，例如：机器阅读理解 / 问答、多轮对话、其他长文本生成等。同时，“记忆” 作为认知的重要部分之一，可用于强化其他多种机器学习模型的表现。 +类似模型不仅可应用于翻译任务，同时可广泛应用于其他需 “大容量动态记忆” 的任务，例如：机器阅读理解 / 问答、多轮对话、长文本生成等。同时，“记忆” 作为认知的重要部分之一，可用于强化其他多种机器学习模型的表现。 -本文所采用的外部记忆机制，主要指**神经图灵机** \[[1](#参考文献)\]，将于后文详细描述。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度学习的背景下，涌现出一系列有价值的工作，例如：记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。除神经图灵机外，其他均不在本文的讨论范围内。 +本文所采用的外部记忆机制，主要指**神经图灵机** \[[1](#参考文献)\] 方式（将于后文详细描述）。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度学习的背景下，涌现出一系列有价值的工作，例如记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。本文仅讨论和实现神经图灵机机制。 -本文的实现主要参考论文\[[2](#参考文献)\]。本文假设读者已充分阅读并理解PaddlePaddle Book中[机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章。 +本文的实现主要参考论文\[[2](#参考文献)\]， 并假设读者已充分阅读并理解 PaddlePaddle Book 中 [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation) 一章。 ## 模型概述 ### 记忆机制简介 -记忆（Memory)，是人类认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（不同于感知）成为可能。记忆，同样是机器学习模型需要拥有的关键能力之一。 +记忆（Memory)，是认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（如推理、规划，不同于静态感知）成为可能。灵活的记忆机制，是机器模仿人类智能所需要拥有的关键能力之一。 -可以说，任何机器学习模型，原生就拥有一定的记忆能力：无论它是参数模型（模型参数即记忆），还是非参模型（样本即记忆）；无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络连接权值即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的；在预测时，模型是静态一致的，不拥有额外的跨时间步的信息记忆能力。 +#### 静态记忆 + +任何机器学习模型，原生就拥有一定的静态记忆能力：无论它是参数模型（模型参数即记忆），还是非参模型（样本即记忆）；无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络连接权值即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的；在模型推断时，模型是静态一致的，不拥有额外的跨时间步的信息记忆能力。 #### 动态记忆 1 --- RNNs 中的隐状态向量 -当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个可持久的信息通路。带有隐状态向量 $h$（或 LSTM 中的细胞状态向量 $c$）的循环神经网络（Recurrent Neural Networks, RNNs） ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 +在处理序列认知问题（如自然语言处理、序列决策等）时，由于每个时间步对信息的处理需要依赖其他时间步的信息，我们往往需要在不同时间步上维持一个持久的信息通路。带有隐状态向量 $h$（或 LSTM 中的状态 $c$）的循环神经网络（Recurrent Neural Networks, RNNs） ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可往上持续叠加新的信息以更新记忆。在模型推断时，不同的样本具有完全不同的一组记忆信息（$h$ 或 $c$），具有 “动态” 性。 -我们注意到，LSTM 中的细胞状态向量 $c$ 的引入，或者 GRU 中状态向量 $h$ 的以门（Gate）控制的线性跨层结构（Leaky Unit）的引入，从优化的角度看有着不同的理解：即为了梯度计算中各时间步的一阶偏导矩阵（雅克比矩阵）的谱分布更接近单位阵，以减轻长程梯度衰减问题，降低优化难度。但这不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅，如图1（引自[此文](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)）所示的 LSTM 中的细胞状态向量 $c$ 可视为这样一个用于信息持久化的 “线性记忆通道”。 +尽管上述对 LSTM中细胞状态 $c$ 的直觉说法有着诸多不严谨之处：例如从优化的角度看， $c$ 的引入或者 GRU 中的线性 Leaky 结构的引入，是为了在梯度计算中使得单步梯度的雅克比矩阵的谱分布更接近单位阵，以减轻长程梯度衰减问题，降低优化难度。但这不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅，如图1（引自[此文](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)）所示的 LSTM 中的细胞状态向量 $c$ 可视为这样一个用于信息持久化的 “线性记忆通道”。

@@ -30,37 +32,59 @@ #### 动态记忆 2 --- Seq2Seq 中的注意力机制 -然而这样的一个向量化的 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着一定程度的信息丢失。 +然而上节所属的单个向量 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着较大的潜在信息丢失。 -于是，注意力机制（Attention Mechanism）\[[3](#参考文献)\] 被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量，而是依赖一个向量组，向量组中的每个向量为编码器的各字符（Tokens）级编码向量（状态向量），并通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式提权信息用于序列的不同位置的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容获取不同的读取强度，起到一种和源语言 “软对齐（Soft Alignment）” 的作用。 +\[[3](#参考文献)\] 提出了注意力机制（Attention Mechanism），以克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量的信息，而是依赖一个向量组的记忆信息：向量组中的每个向量为编码器的各字符（Token）的编码向量（例如 $h_t$）。通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式读取信息，用于序列的不同时间步的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（请参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容决定不同的读取强度，起到一种和源语句 “软对齐（Soft Alignment）” 的作用。 -这里的 “向量组” 蕴含着更多更精准的信息，它可以被认为是一个无界的外部记忆模块（Unbounded External Memory）。“无界” 指的是向量组的向量个数非固定，而是随着源语言的字符数的变化而变化，数量不受限。在源语言的编码完成时，该外部存储即被初始化为各字符的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于神经图灵机的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址（Location-based Addressing）。两种寻址方式不赘述，详见 \[[1](#参考文献)\]。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，有待进一步的探索。 +相比上节的单个状态向量，这里的 “向量组” 蕴含着更多更精准的信息，例如它可以被认为是一个无界的外部记忆模块（Unbounded External Memory），有效拓宽记忆信息带宽。“无界” 指的是向量组的向量个数非固定，而是随着源语句的字符数的变化而变化，数量不受限。在源语句的编码完成时，该外部存储即被初始化为各字符的状态向量，而在其后的整个解码过程中被读取使用。 #### 动态记忆 3 --- 神经图灵机 -图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。 +图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模拟可微分（即可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式的动态存储。神经图灵机正是要弥补这样的潜在缺陷。

图2. 图灵机结构漫画
-图灵机的存储机制，常被形象比喻成一个纸带（Tape），在这个纸带上有读头（Read Head）和 写头（Write Head）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （Contoller) 控制（见图2，引自[此处](http://www.worldofcomputing.net/theory/turing-machine.html)）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”，其中 $n$ 为记忆向量（又成记忆槽）的数量，$m$为记忆向量的长度，以前馈神经网络或循环神经网络来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（寻址方式不赘述，请参考论文\[[1](#参考文献)\]），并最终写入或读出信息，供其他网络使用。神经图灵机结构示意图，见图3，引自\[[1](#参考文献)\]。 +图灵机的存储机制，常被形象比喻成在一个纸带（Tape）的读写操作。读头（Read Head）和 写头（Write Head）负责在纸带上读出或者写入信息；纸袋的移动、读写头的读写动作和内容，则受控制器 （Contoller) 控制（见图2，引自[此处](http://www.worldofcomputing.net/theory/turing-machine.html)）；同时纸带的长度通常有限。 + +神经图灵机则以矩阵 $M \in \mathcal{R}^{n \times m}$ 模拟 “纸带”，其中 $n$ 为记忆向量（又成记忆槽）的数量，$m$ 为记忆向量的长度。以前馈神经网络或循环神经网络来模拟控制器，决定本次读写在不同的记忆槽上的读写强度分布，即寻址： + + - 基于内容的寻址（Content-based Addressing)：寻址强度依赖于记忆槽的内容和该次读写的实际内容； + - 基于位置的寻址(Location-based Addressing)：寻址强度依赖于上次寻址操作的寻址强度（例如偏移）； + - 混合寻址：混合上述寻址方式（例如线性插值）； + +（详情请参考论文\[[1](#参考文献)\]）。根据寻址情况，图灵机写入 $M$ 或从 $M$ 读出信息，供其他网络使用。神经图灵机结构示意图，见图3，引自\[[1](#参考文献)\]。

图3. 神经图灵机结构示意图
-和上述的注意力机制相比，神经图灵机有着诸多相同点和不同点。相同在于：均利用矩阵（或向量组）形式的存储，可微分的寻址方式。不同在于：神经图灵机有读有写（是真正意义上的存储器），并且其寻址不仅限于基于内容的寻址，同时结合基于位置的寻址（使得例如 “长序列复制” 等需要 “连续寻址” 的任务更容易），此外它是有界的（Bounded)；而注意机制仅仅有读操作，无写操作，并且仅基于内容寻址，此外它是无界的（Unbounded）。 +和上节的注意力机制相比，神经图灵机有着诸多相同点和不同点。相同点例如： + +- 均利用矩阵（或向量组）形式的外部存储。 +- 均利用可微分的寻址方式。 + +不同在于： + +- 神经图灵机有读有写，是真正意义上的存储器；而注意力机制在编码完成时即初始化存储内容（仅简单缓存，非可微分的写操作），在其后的解码过程中只读不写。 +- 神经图灵机不仅有基于内容的寻址，同时结合基于位置的寻址，使得例如 “序列复制” 等需 “连续寻址” 的任务更容易；而注意力机制仅考虑基于内容的寻址，以实现 Soft Aligment。 +- 神经图灵机利用有界（Bounded) 存储；而注意力机制利用无界（Unbounded）存储。 + +#### 三种记忆方式的混合，以强化神经机器翻译模型 -#### 三种记忆混合，强化神经机器翻译模型 +尽管在一般的序列到序列模型中，注意力机制已经是标配。然而，注意机制中的外部存储仅用于存储编码器信息。在解码器内部，信息通路仍依赖 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充解码器内部的单向量信息通路，成为自然而然的想法。 -尽管在一般的序列到序列模型中，注意力机制已经是标配。然而，注意机制的外部存储仅仅是用于存储源语言的字符级信息。在解码器内部，信息通路仍然是依赖于 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充解码器内部的单向量信息通路，成为自然而然的想法。 +于是，我们混合上述的三种动态记忆机制，即RNN 原有的状态向量、注意力机制被保留；同时，基于简化版的神经图灵机的有界外部记忆机制被引入以补充解码器单状态向量记忆。整体的模型实现参考论文\[[2](#参考文献)\]。少量的实现差异，详见[其他讨论](#其他讨论)一章。 + +这里额外需要理解的是，为什么不直接通过增加 $h$ 或 $c$的维度来扩大信息带宽？ + +- 一方面因为通过增加 $h$ 或 $c$的维度是以 $O(n^2)$ 的存储和计算复杂度为代价（状态-状态转移矩阵）；而基于神经图灵机的记忆扩展代价是 $O(n)$的，因其寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。 +- 基于状态单向量的记忆读写机制，仅有唯一的读写强度，即本质上是**全局**的；而神经图灵机的机制是**局部**的，即读写本质上仅在部分记忆槽（寻址强度的分布锐利，即真正大的强度仅分布于部分记忆槽）。局部的特性让记忆的存取更干净，干扰更小。 -当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储和计算复杂度为代价（状态-状态转移矩阵）。而基于神经图灵机的记忆扩展代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。另外值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的；而神经图灵机的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净，干扰更小。 -所以，在该示例的实现中，RNN 原有的状态向量和注意力机制被保留；同时，基于简化版的神经图灵机的有界外部记忆机制被引入以补充解码器单状态向量记忆。整体的模型实现参考论文\[[2](#参考文献)\]，但有少量差异，详见[其他讨论](#其他讨论)一章。 ### 模型网络结构 @@ -79,39 +103,141 @@ 1. $M_{t-1}^B$ 和 $M_t^B$ 为有界外部存储矩阵，前者为上一时间步存储矩阵的状态，后者为当前时间步的状态。$\textrm{read}^B$ 和 $\textrm{write}$ 为对应的读写头（包含其控制器）。$r_t$ 为对应的读出向量。 2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头，二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量。 -3. $y_{t-1}$ 为解码器上一步的输出字符并做词向量（Word Embedding)，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 +3. $y_{t-1}$ 为解码器上一步的输出字符并做词向量（Word Embedding）映射，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 4. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#参考文献)\] 基本一致（略有不一致之处为：用于 attention 的 decoder state 被改进，即叠加了一隐层并引入了 $y_{t-1}$）。 ## 算法实现 -算法实现的关键部分在辅助类`ExternalMemory` 和模型函数 `memory_enhanced_seq2seq`。 +算法实现于以下几个文件中： + +- `external_memory.py`: 主要实现简化版的 **神经图灵机** 于 `ExternalMemory` 类，对外提供初始化和读写函数。 +- `model.py`: 相关模型配置函数，包括双向 GPU 编码器（`bidirectional_gru_encoder`），带外部记忆强化的解码器（`memory_enhanced_decoder`），带外部记忆强化的序列到序列模型（`memory_enhanced_decoder`）。 +- `data_utils.py`: 相关数据处理辅助函数。 +- `train.py`: 模型训练。 +- `infer.py`: 部分示例样本的翻译（模型推断）。 ### `ExternalMemory` 类 `ExternalMemory` 类实现通用的简化版**神经图灵机**。相比完整版神经图灵机，该类仅实现了基于内容的寻址（Content Addressing, Interpolation），不包括基于位置的寻址（ Convolutional Shift, Sharpening)。读者可以自行将其补充成为一个完整的神经图灵机。 -类结构如下： +该类结构如下： ``` class ExternalMemory(object): - __init__(self, name, mem_slot_size, boot_layer, readonly, enable_projection) - __content_addressing__(self, key_vector) - __interpolation__(self, head_name, key_vector, addressing_weight) - __get_adressing_weight__(self, head_name, key_vector) - write(self, write_key) - read(self, read_key) + """External neural memory class. + + A simplified Neural Turing Machines (NTM) with only content-based + addressing (including content addressing and interpolation, but excluding + convolutional shift and sharpening). It serves as an external differential + memory bank, with differential write/read head controllers to store + and read information dynamically as needed. Simple feedforward networks are + used as the write/read head controllers. + + For more details, please refer to + `Neural Turing Machines `_. + """ + + def __init__(self, + name, + mem_slot_size, + boot_layer, + readonly=False, + enable_interpolation=True): + """ Initialization. + + :param name: Memory name. + :type name: basestring + :param mem_slot_size: Size of memory slot/vector. + :type mem_slot_size: int + :param boot_layer: Boot layer for initializing the external memory. The + sequence layer has sequence length indicating the number + of memory slots, and size as memory slot size. + :type boot_layer: LayerOutput + :param readonly: If true, the memory is read-only, and write function cannot + be called. Default is false. + :type readonly: bool + :param enable_interpolation: If set true, the read/write addressing weights + will be interpolated with the weights in the + last step, with the affine coefficients being + a learnable gate function. + :type enable_interpolation: bool + """ + + def _content_addressing(self, key_vector): + """Get write/read head's addressing weights via content-based addressing. + """ + pass + + def _interpolation(self, head_name, key_vector, addressing_weight): + """Interpolate between previous and current addressing weights. + """ + pass + + def _get_addressing_weight(self, head_name, key_vector): + """Get final addressing weights for read/write heads, including content + addressing and interpolation. + """ + pass + + def write(self, write_key): + """Write onto the external memory. + It cannot be called if "readonly" set True. + + :param write_key: Key vector for write heads to generate writing + content and addressing signals. + :type write_key: LayerOutput + pass + + def read(self, read_key): + """Read from the external memory. + + :param write_key: Key vector for read head to generate addressing + signals. + :type write_key: LayerOutput + :return: Content (vector) read from external memory. + :rtype: LayerOutput + """ + pass ``` -神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，注意这里的`is_seq`需设成`True`，该序列的长度表示记忆槽的数量，`size` 表示记忆槽（向量）的大小。同时依赖一个外部层作为初始化， 记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 +其中，私有方法包含： + +- `_content_addressing`: 通过基于内容的寻址，计算得到读写操作的寻址强度。 +- `_interpolation`: 通过插值寻址（当前寻址强度和上一时间步寻址强度的线性加权），更新当前寻址强度。 +- `_get_addressing_weight`: 调用上述两个寻址操作，获得对存储导员的读写操作的最终寻址强度。 + + +对外接口包含： + +- `__init__`：类实例初始化。 + - 输入参数 `name`: 外部记忆单元名，不同实例的相同命名将共享同一外部记忆单元。 + - 输入参数 `mem_slot_size`: 单个记忆槽（向量）的维度。 + - 输入参数 `boot_layer`: 用于内存槽初始化的层。需为序列类型，序列长度表明记忆槽的数量。 + - 输入参数 `readonly`: 是否打开只读模式（例如打开只读模式，该实例可用于注意力机制）。打开是，`write` 方法不可被调用。 + - 输入参数 `enable_interpolation`: 是否允许插值寻址（例如当用于注意力机制时，需要关闭插值寻址）。 +- `write`: 写操作。 + - 输入参数 `write_key`：某层的输出，其包含的信息用于写头的寻址和实际写入信息的生成。 +- `read`: 读操作。 + - 输入参数 `read_key`：某层的输出，其包含的信息用于读头的寻址。 + - 返回：读出的信息（可直接作为其他层的输入）。 -`ExternalMemory`类的寻址逻辑通过 `__content_addressing__` 和 `__interpolation__` 两个私有函数实现。读和写操作通过 `read` 和 `write` 两个函数实现。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 +部分重要的实现逻辑： -为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中，并且采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 +- 神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，并采用序列形式（`is_seq=True`），该序列的长度表示记忆槽的数量，序列的 `size` 表示记忆槽（向量）的大小。该序列依赖一个外部层作为初始化， 其记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 -`ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。便于用该类等价实现传统的注意力机制。 + ``` + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) + ``` +- `ExternalMemory`类的寻址逻辑通过 `_content_addressing` 和 `_interpolation` 两个私有方法实现。读和写操作通过 `read` 和 `write` 两个函数实现，包括上述的寻址操作。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 +- 为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中。控制器主要包括寻址操作和写操作时生成写入/擦除向量等，其中寻址操作通过上述的`_content_addressing` 和 `_interpolation` 两个私有方法实现，写操作时的写入/擦除向量的生成则在 `write` 方法中实现。上述均采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 +- `ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。主要目的是便于用该类等价实现传统的注意力机制。 -注意， `ExternalMemory` 只能和 `paddle.layer.recurrent_group`配合使用，具体在用户自定义的 `step` 函数中使用，它不可以单独存在。 +- `ExternalMemory` 只能和 `paddle.layer.recurrent_group`配合使用，具体在用户自定义的 `step` 函数中使用（示例请详细代码），它不可以单独存在。 ### `memory_enhanced_seq2seq` 及相关函数 diff --git a/mt_with_external_memory/data_utils.py b/mt_with_external_memory/data_utils.py new file mode 100644 index 00000000..6cc481fe --- /dev/null +++ b/mt_with_external_memory/data_utils.py @@ -0,0 +1,15 @@ +""" + Contains data utilities. +""" + + +def reader_append_wrapper(reader, append_tuple): + """ + Data reader wrapper for appending extra data to exisiting reader. + """ + + def new_reader(): + for ins in reader(): + yield ins + append_tuple + + return new_reader diff --git a/mt_with_external_memory/external_memory.py b/mt_with_external_memory/external_memory.py new file mode 100755 index 00000000..d3dbf32f --- /dev/null +++ b/mt_with_external_memory/external_memory.py @@ -0,0 +1,201 @@ +""" + External neural memory class. +""" +import paddle.v2 as paddle + + +class ExternalMemory(object): + """ + External neural memory class. + + A simplified Neural Turing Machines (NTM) with only content-based + addressing (including content addressing and interpolation, but excluding + convolutional shift and sharpening). It serves as an external differential + memory bank, with differential write/read head controllers to store + and read information dynamically. Simple feedforward networks are + used as the write/read head controllers. + + The ExternalMemory class could be utilized by many neural network structures + to easily expand their memory bandwidth and accomplish a long-term memory + handling. Besides, some existing mechanism can be realized directly with + the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an + unbounded external memory). + + Besides, the ExternalMemory class must be used together with + paddle.layer.recurrent_group (within its step function). It can never be + used in a standalone manner. + + For more details, please refer to + `Neural Turing Machines `_. + + :param name: Memory name. + :type name: basestring + :param mem_slot_size: Size of memory slot/vector. + :type mem_slot_size: int + :param boot_layer: Boot layer for initializing the external memory. The + sequence layer has sequence length indicating the number + of memory slots, and size as memory slot size. + :type boot_layer: LayerOutput + :param readonly: If true, the memory is read-only, and write function cannot + be called. Default is false. + :type readonly: bool + :param enable_interpolation: If set true, the read/write addressing weights + will be interpolated with the weights in the + last step, with the affine coefficients being + a learnable gate function. + :type enable_interpolation: bool + """ + + def __init__(self, + name, + mem_slot_size, + boot_layer, + readonly=False, + enable_interpolation=True): + self.name = name + self.mem_slot_size = mem_slot_size + self.readonly = readonly + self.enable_interpolation = enable_interpolation + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) + # prepare a constant (zero) intializer for addressing weights + self.zero_addressing_init = paddle.layer.slope_intercept( + input=paddle.layer.fc(input=boot_layer, size=1), + slope=0.0, + intercept=0.0) + # set memory to constant when readonly=True + if self.readonly: + self.updated_external_memory = paddle.layer.mixed( + name=self.name, + input=[ + paddle.layer.identity_projection(input=self.external_memory) + ], + size=self.mem_slot_size) + + def _content_addressing(self, key_vector): + """ + Get write/read head's addressing weights via content-based addressing. + """ + # content-based addressing: a=tanh(W*M + U*key) + key_projection = paddle.layer.fc( + input=key_vector, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + key_proj_expanded = paddle.layer.expand( + input=key_projection, expand_as=self.external_memory) + memory_projection = paddle.layer.fc( + input=self.external_memory, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + merged_projection = paddle.layer.addto( + input=[key_proj_expanded, memory_projection], + act=paddle.activation.Tanh()) + # softmax addressing weight: w=softmax(v^T a) + addressing_weight = paddle.layer.fc( + input=merged_projection, + size=1, + act=paddle.activation.SequenceSoftmax(), + bias_attr=False) + return addressing_weight + + def _interpolation(self, head_name, key_vector, addressing_weight): + """ + Interpolate between previous and current addressing weights. + """ + # prepare interpolation scalar gate: g=sigmoid(W*key) + gate = paddle.layer.fc( + input=key_vector, + size=1, + act=paddle.activation.Sigmoid(), + bias_attr=False) + # interpolation: w_t = g*w_t+(1-g)*w_{t-1} + last_addressing_weight = paddle.layer.memory( + name=self.name + "_addressing_weight_" + head_name, + size=1, + is_seq=True, + boot_layer=self.zero_addressing_init) + interpolated_weight = paddle.layer.interpolation( + name=self.name + "_addressing_weight_" + head_name, + input=[addressing_weight, addressing_weight], + weight=paddle.layer.expand(input=gate, expand_as=addressing_weight)) + return interpolated_weight + + def _get_addressing_weight(self, head_name, key_vector): + """ + Get final addressing weights for read/write heads, including content + addressing and interpolation. + """ + # current content-based addressing + addressing_weight = self._content_addressing(key_vector) + # interpolation with previous addresing weight + if self.enable_interpolation: + return self._interpolation(head_name, key_vector, addressing_weight) + else: + return addressing_weight + + def write(self, write_key): + """ + Write onto the external memory. + It cannot be called if "readonly" set True. + + :param write_key: Key vector for write heads to generate writing + content and addressing signals. + :type write_key: LayerOutput + """ + # check readonly + if self.readonly: + raise ValueError("ExternalMemory with readonly=True cannot write.") + # get addressing weight for write head + write_weight = self._get_addressing_weight("write_head", write_key) + # prepare add_vector and erase_vector + erase_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + add_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + erase_vector_expand = paddle.layer.expand( + input=erase_vector, expand_as=self.external_memory) + add_vector_expand = paddle.layer.expand( + input=add_vector, expand_as=self.external_memory) + # prepare scaled add part and erase part + scaled_erase_vector_expand = paddle.layer.scaling( + weight=write_weight, input=erase_vector_expand) + erase_memory_part = paddle.layer.mixed( + input=paddle.layer.dotmul_operator( + a=self.external_memory, + b=scaled_erase_vector_expand, + scale=-1.0)) + add_memory_part = paddle.layer.scaling( + weight=write_weight, input=add_vector_expand) + # update external memory + self.updated_external_memory = paddle.layer.addto( + input=[self.external_memory, add_memory_part, erase_memory_part], + name=self.name) + + def read(self, read_key): + """ + Read from the external memory. + + :param write_key: Key vector for read head to generate addressing + signals. + :type write_key: LayerOutput + :return: Content (vector) read from external memory. + :rtype: LayerOutput + """ + # get addressing weight for write head + read_weight = self._get_addressing_weight("read_head", read_key) + # read content from external memory + scaled = paddle.layer.scaling( + weight=read_weight, input=self.updated_external_memory) + return paddle.layer.pooling( + input=scaled, pooling_type=paddle.pooling.Sum()) diff --git a/mt_with_external_memory/image/lstm_c_state.png b/mt_with_external_memory/image/lstm_c_state.png new file mode 100644 index 0000000000000000000000000000000000000000..ce791573f359a2e3a346eb920b1edec6983cc184 GIT binary patch literal 52288 zcmd43XIK;47dMK>f`W)1K$?IeRjME$T|_{-h}2M}NtYU02sTivbm_e}>7A%Z?=?Vx zNG}0G4hotW$;imr z$;keCboC1GM8|1o2l#QxsaBM&k%)?cJQ7hHzDnjz-Wi^o0 zX!N6l9gjf_Mg=kK7Rxm-u!Y*l8TTK;&c|NTYD1$jxXSQ{F*S*~>Kxl#yu1K$|6c3q zHM#U0ruR1N`xvSdb3-_%@iYAY{gRAKQGvrg(k9&Z#b_IT4j}ixOR-a4z)B;{Fp}=S zCuC1Ha)C!4(E7jr_vpv)Q`6`FyCdUNxq0n>cbUsq*)RO>F7A#3$Nz4EV(x#wbbc4a zQN(LbPpSEi07!r=?jr(8_0K4VhVZ8}#?gOCKf25(@{W$T=C!x)AEUiO7|1~FT@M0X z9w;qdCFLBYr8RH}_6Ob(KDqVmAHry(qecgNXWv%|U$+jY{S^HrT!%Jza@R74qIQ1y z9^K?bE&pdaipymV#;X&*yOtr+jK|;<(hLp%?+jrxsliyW&|kw0Jt86r9KMl!uwtj@ zpFEQ?JIY+m7F1u}V|$~7_;vETmqiypCkVWK-Q`mBKcp+gHQ(PKQ}Nf4>z{EP?^u({AtF%u{*s>%hAo?sRnh!?Dos)wRK`3%|>s8{u6VTqP4p9ZGf@BuRjt{;LhcUnWgrnU$kCdJgdSblIxPPs@B5TuyUJT}ltC zbV?B9n!xZMHx>UI{%-h^zIdVoTAnvdFZkt1yasvvtKoI=nsG3%2|w*$Wp9ia5;a^U zD1kK;um7u7PsrVMN5~D;0Ev{{bhbNT#O1o&l z5s0a(P8X_8Y12-Uc2CjvIh-?s%=0dDs{z|62H-ZkiLT+_L(0x(ZC`8mPv3L&xUs}{ z^yV0E*%r(~PG)1-bVqz!O@}Eq?|P`G2noZN0BL=y zER(D&D;sd=7eEwS0i6Ht{QAEMZ#kkA^AUBU`dCXQW60Gjy058|GjL^H$9E`HDbI->*miLK1?i{M z=ljo;cQ#~u88c&yc%^5?tWoR-#kSsj8N(mn!S_`40qwrp3h2u6546cQZ+&qAjGt$u ziB`$X?N0RxF5va+&i`qkl68(a7y@~&p75ZrB*%>oQ4qH6SXu~4u#}s^GKRF;8y$f5 zc(zWTbz)cJUAP=rzgqGE6I{9d{vRj_VxMu>9K>S3wxR8ReP?7Wd2SG@W4*GUbWG*O zean00Oj$Xpa9ZG-Qw3(IW);<%KY68qc8%iwi@p^4&WQd5FaYB}#3TOIZVQ?$Uc>1# z|D$^)d?6v}rZN`%&y1wauX2>7NgCZ|p3J*jB?l93%(V}6_r*r+57YXUMRoF2C5Q`t zPWgK^ct9Y+TRkw(yh#+$Q41dSe{pN-nV^FQWn=BpJ?B1sn-;G1H^>%MY0G=UG0Rfr zU!+$+*V!+|)}vAkngj&O)bd0qG>0Ag8Yc@9u^Sr2xoZ-gh2{2cr4k4L?Sg(r{f|`1 zvDH-yK)%>?04+u=ZzkPHo_qS@J4eqE3!iPG^TW^mN@ZGd_2Mmu5bZV=^eBJt4u!^i zxuXd!D#D{+l(*zwGsish8i@hJPyRKh?_VaJa+u3Xs5eg2U$gM{F(v(Q=qYu}@j|6{ z;&a{jg=R)i;lYW%>M^~q8K$y@tVOaf6x31u@U zCmM~_ggdg;HG2a-wDIDz{P|hF@w%~hw&NScIPE2X<$`bgr*l-+IrYQpe?X@t8FGIo|6A zX+}66NW;Az$(SW`8mGAP;aBVO<%~qcuHP4MHDXB7*VX%O7(l`-=DYtX(GyVlbVNU( z>E&lPHWgR1_3OV<&Dd$_bF0)1U3KcM*gt4&DRzVlPDtfD`Psgj_BK*APBV=&YT;q7pef(40;z~`7g$y zO3SE|Trv_2Su3`_%UKJmNjCn%CdH=ztz~n!EKO&6a?+`b<{D>O@$qzA-5smGheak~ zR@Tdn>AVNQm+Mv-5bkP!KUEvuJZuI$gl=+Cys34#j_J~(;TwuWF#xGDS*Sxuk|ZNT zEoY0@zqlH~F`Y1G55Pg45p2Ae?O++vBP^0t%LRLs-n05B`?bb=Osh_G=a1#>BQZ|X zq1xBpN+Z(=pb`srZ_}l}VR36)#Tr~WfLnn`(a-?XUS$3k@_$H8^QOp38vEUvTm4BTYlxw@}fJelfSvFC1kniYnwiu&18zQh9yNNK<145 zQszTL6hbt2yh@l7{Tq2ExNCWVEeP-a7YLLt^@j2H#W)*1P>)lo)O!CEsTFMHSg>r; zrNWb*9R% zVwS~h9bE^kkO}?YcC30{YPy@WNq7BDt$J(iJAKw)V#EbA-TXmU&HC$~<14axM)&hS?b)}NTm)$mxpRP-*Cj44Uic?Kv?%OT z;^pQ6lna2F$)Qw@kcj!sRXQ^UWv)6pQFg$Z`k3fD)#m2Y->BCbM>MWnEY>8=;Qyc3 zfc)a2fS1oO9ZUv)oi3a(O){e$vN>Qfm6Wst&1EPwR5|O&Bo2AC1v{~cDjScSl(T)i z1JMSiKnSy50fYuz*eCsh)H29wexqI$$!0+bSb*vQswW{q-%(@x5}9Gzt)lY^j=#J<9OpcXtX!*b|&%WzX9aog|7YnxCQBazP=D8X8~tpJ=o|CicSp% zIWdU`l4AG0pf8v9SL>G+nv4=Ij%?;jNif$o;OA}tlN?g_F^tV3BS_=`kwwBKE>{(< zngHorF58mE2S@WDuL=&0dU}s-kXU=Lif%F30g;?Z;@}77`=jUYkYT?$p6Y13OWx0+ z(^L-tnT4!9oN<3@uVLY8Us3DDS3f=#IW#&m>6G_?sa2#t|9b~%_ZH7B|BVlR+_hE{ zrgIORJFqDV2pgN0LqyF9n66S$`i69-^hg0YSfw=W# z$XAj^WMXoSMJd_~#~{6Yu~plVACwX9n!ie}Lbz{rq&{1o-s`S@on_@K(+w=^wDj#3 z5Oxf!hqJF_$E7@Y)KnBu9ZNZ;WgQ;=@NZF;3uHgYhT|EdEopndMSkdhk)?OleH)^Y zVAE6OEScc=+C^p`m=f`YXNRM`LHCw%mG4a;hMQ78H}kj`a=ni?GJ90%lIJwCCa8Wr z>30CKBE;WT;TjANjG13Zy{x{kTE*WOS>j&tfCuA9sR}l<=vz<#D6+U}Wv>rxnpHTo z6i7_!=$Pg_vpNJWNp4F0MHZxRKX#CpLK^C=8rNkF$}7LnC?3au5OSF=(D~lRtIKst zJirvfZW^-F8gb`G$P|nS3X(3BLlLy>kb$1f1dryi403w_~O<`;reX;?Zx;#T3=JjS@LXCgWrG^(A?WSx@oK4yeF(9pQAxM*gDa9;$bmUqK7li>HF5x`KY zX`4Xx@_Eq_ea}Lp1gEZqrN{0piEnQK6!HRcal0cKS>1Cxheu@lf(@;5=&$e0cH+Iu zy2qox&Yq1{?5c2ZFaLe~y^M_t2nT}#6-W{6i;p_iWxf2N5;ppNot_2HqB;92m!%@_ zbj3T4mh@&{REaE|vbm z1q{dDB5~vgm%C-JQMw;rG+%CWD*H_2{h@=JG>-UEXJSR49>p)2ejj;I zbPCwiu%e@)s&+Z5=q4@g@zH>jjeRg?3G}(@{k)cnV+eJWm_nywUzXp|&NyP_s_5cJGUk zx=aQz$@NG7Ok!jGD#M7@wYzjz8nqn50YA8U7x*pq^O~=_sllhG3JNzp8J;uI6v@^0 z4+ZnuX}J4!R<;ZlyaWs@Q(LDWAloSx$g>OPHN@0~Y&cu=H4JzCC6Q7uvPd z|NatSId^4-bY*TO+IY>T(J5yo)V3vcE7>c@kakY?&(5nJRP)S#Ga9kHYLBQ}{L&y( z5t~Y?kKkY9tCQz?Lg_3GzbDkSrSpHggEu2R4<{D^FmT{aw_rq>st`tcW~ZTi@ykc2 z6cv(K-}P4UcPzOzrd_|ocS+LG2*unpe2L$cB(JD-8$f{&AH^D12%F(!4h6LSvz#7(96$bVd%T4U~E4Dx$D5PaZ?fwCl2~O1* z-h5pLoq-UWO?Ux%V663Ie;Zz>v3uu zsaif&I#30Eb^TuClBc0im81dM_E$o-*KdHr7smgv$SBt6b{~0C_R!4~P6^#45BwmL1UX?O2lM0WaE#;Ih-(s{go!#9quWt3z_f^}NQY1Lt8iK& zcF978IbzWMP zjL|XfpTg59b+!6=DRB*|oOIt((iE7KE&Isk?hrg;T-kiormD#_E;h2al9I4qZ zSkECx+p&23x)li0`(TdJ?guD%YO#MTxt3viJL$!m>G4EihCFa$z>vM31gOjhGY^+q z?xcZv@EwDMi48TB7{|5=tr&#Q>I%g!J@(r?3K@vh zwY7=W(iR|!~7&0?3@js&JP0bjjC1^PCnQtV<$qNtN}*|Yl{5XZ6$v33=N@;+R< zpAkyjDVOi76EOAOmce7;ut6vJz%?z!ZYtBjQ@Uxfk`&tVl*`Situ)#w&ih^N63|Q_ z9^iR*f%p~B>)|A)Zx66CR#fj++JOaL7_$gGWd3-1!PM|9LP@CO#w^YPeQIY01Uyaq zVjJzQ1i&X@}9ck$oJ4mA|322k!vMOFyOC>^GSO%Hu-dBz_C z82`lHNPn;8;phE}#|xEhwS)Nntghb)tsrhysRh2S_1`qwJaPULAiXiSsA>NzG1iC# zTeR31$6W6P7>59$nk`qh6ah#1#DMmf#>i3ad9e7Mx34Of-O^f)>6<9~2Od`zOyg1v z(ci0$!@l=W6}zcakQCjcI$b8a=B-;7#$mr1&RfhX*B`3O1~|V%cE5EXR?57s{AoZs zg1@AFWapf@bpBf58gG;+$Y(Du*CLxmPS-C&6KlM$EUm!JGwpM{dkw4VoAQ^u5oJ{D zb~}~Te^*v@=gR^1x~_DNoG&OfYo`4_eg!65|7^NtG{SKCqLcn|Reac-qadsjauk&l zC2vB0XLsT5!mHyqSFC8uAc6DFLuq=91?SV4{xeOO#&@HT=)4OPA%Lp`3Mn5H96#|$ zWMX>+n@-+8Q}=Mhp_bY_bBlPHQ_vf}qDkQ^O_9g5U~s&DElAG}*kESKEWrh!;)(XN zP;qMZ9+LkVeix}f#PD2UImRYzTN>)C?XL(`WWt80n@0G@Ok)FpCF<1@z(^~)&q(9u3x*Ye_sk=J-RlbA3j*} zj1L;lBo^8Pa3E7#c#GshfeR6yEym9INyqZXrD)8&hdFNf0;2Bli=w2zU0k+L(b_eb^ONPbuCrxLv9)MCVLL9W1b zxC1G$t;@{GGy1?@+l6cIjpr+)Plq(N3RrbU>DzcUt{7Ji1hN5x7}iNzv#!#}vy%Jk z3RF)w4Y#E$v_AJqy^J#~v8b`qh^8PMe#lbi;f=7etMImtcyN~dG+tT|Wmq1ad%4d% zjJQMa$7{3URH-mrJynK){s86}6PcdUUMPH*wGbKL-&VbV>e1!y^)hKpK-wvg&S@MQ z9@zGX$M3AC>Sc3Ul7bCVsW~X{CV-#WUJ7Rf`LiDUQY$rzEea1hH8Hd5j2Qi>x->B5 zW_5VduDziGC;Zx%G2qU?`=-6!tTd|=UkU#*@Vv~)cr|cyDPvFG=LhS4nvZn_B{v`% zGSBjM#^{UI_LE9s^e(iYwbG>&9=aY(=+x;0ke~2$fNl0{uBy(n8q$9)BSC_nl{0Pg zX;C>4qt*$K6z`BXXVpS!s~$7Y(tLJ?s&!ILa|kv-o#;kBEJzJ-51bstEj#zwQ`5Bz zkLN_mZKj{((flnWhbum61(nydTUt(6X2|sf;%gX8%+I^ zM{n*Ty{wzT-+cBSwfzb-6S(<>*zN7KUDs94weac^e%B44Ea<1>!JU^E^T?JyFWi@4 zdh^(3lcNJ~9hG8mb$`YKj(-hnBX%*#Iy;0cEV2>L$hBj@fgREc8X@@|4K0ut2OaF}A zhi#<1W_xj8**DGe8<=}zFE$@MWd zX~Bv3o9?Zq-qV~?`-MX_V1_Lzf9?*ep6)CX=#UKyTzpM6d1I?)Oe+f|wOxVq^@tYv zRL6R>z{ofka4^)O&Fc2mfv?l4hAAvsN8G_p)Uq1!&3L12ceao9uU+T=dI1tR2IlSZ zUk9@HejEUfzr63$+*n2c7J7xp7C@2Vl5m&iwbiPMw16y|a2-Ixb9-GW18ve!_FAzE zi}Z5P0K&Zgjt8gx>Q>sTTRG>X3CI0@{ZKPSZlYxmo07oMDhthWKP`3cWj6vz7W~0b zw0g6eLV)9G&f@uK31UZnxP=+ zUy;PMpESgq@z{6{$l)Kn*Yb$UN4ojXE@0MD39rrZZnrxT8ISbucilMm z9%RCoASh6x9h4Y?Y<7j!{s<&&B#IG_>f@(mZ$)wTJ%#T#o6KjG+ebs!e};6)L-(Vx zhxb(MVh6tu1VVlOXjA7+5{VEDvN>QPbmDlgAW+{t1+4tI?F?U}i6IAk+6@PiAk_z( z-h)7~RG~F@KQf6ZOHJSCk(3?MRkCTceM$1LWDHe{dkulUC}&1QUA&f^=ZFzf zUlH;_FT*_Ux~64VEt7{%udmxk>`DmRe$vTStDVD+?}57>p-{{2(W<+oEM_aLj$>fP zKmWnf#C1PF(`A(bLwO?Pv!6r^!yg|b>%lhz7UFWmtTvB7?Sz}{JO??Z*zQ!u26H;B zlVFD|xDbMwlSq|tbuo(}V0wWjl^#s3!x4qQ@9I$~fOGsJKb7BETR-Jvu5Ii?%&VEX zcYLy8OM`rCMXDS>gquK|ZG8fsx~Ri(Yp`@rkLlD0B@b$r2BjI0f(5KVfoA<-g6(Cp zO#1#5|97x9!QAaBsk3VGk_U2rC5hIiLku|;f!kYWc2P2&uY9cvM}Rnp?7F3aIYiUD zqV94}r?oMCW)63ouCT#KBfUJtcU=@{1 zDk$x!Ihw87gAh?uQ>7KxDEU>@Q5XNMQp5iG*WTylu=R(eWKwMldSLF6ak}QQofC_h zhSqUuroUO_EL?kIlu!=a33k#R0wPN?i?@%@cD2dLSx=(t8V?2|kjH)IzH1o-0aM5x zrgrX7HCe_h8<1S;DG{O`HgA;8tINGva-bEaC3|}4lw>ebhLlhDS!9a@5D8NbAd;fG zxXoQad!tA(!;9D8c}O#C0i_rvWmvnClhk^Zv}VaGef{K&xwC`*DCl8qKa_|wUpu8I zF14I2QtphdsBiCCaet%UX>~oUjAEnRX*)>2jBsyaxm#XFefnNNwHlLN6iT(Js7z6| z`YY715tM={^@&o~1fblRl4Qr6$IbqMXr~ju3hHf_*IF^=Q=FH%T*&w0V$!cD2MFl5*@K{@LGPToZjYtw z-vI@jdyN4DjAKxHSrkXIDnz19Sy9Sa#0xaT<=<=kDaBfJ1mWFy_w)o2A8!COA zZ>;xInyHEwY^W?|T)i~E^S12CxNA6X&Mc_`NB9N5QPp@%F7Ft3kDC)Cjr~UQr^+!c z$v)W5s;j-m%JS?a|0xjf{U9Y+YHt>@!Q#dVUi;(MLVX`v#s86S1dhe2K6QKvi1U`3 z<-T6KRtkynAAJ1$sAQ6dEl+B;UILGSR!GvDySts$2&HX9d?tSm-XdppZ9O}K;&ONa z64pO1Is1k;IgdadjkCD#7Er|0G?km31}K1MiPfRn1Cs2=D-|piQ7ppT1`ClEq(uI^ zo=*(m_dmXL-iR>MP7>F7Sn=rS9vVC6+#_-OY)D;2(@X_-c2ub{2RHLGEnnh^e$If@ zI-b_(r>d>ZUsm3;m@JijkDT>TIc`R_%5kf9GJJ6x2WYvd%zg{y>CeBae$n2xrS_YE$h}r33p@NY*33kFiSko%c6j)NeMfrTYi_?=;%^ zKfrCl%yxujfZXc|aW6nS62)^}9O_iV0$OSB#bxQOQ>(D}zFH`}AC-Lr&}bfBlJk7= zRMfWrb4<@O94c7PF^=~zz;_(v3X^O*a6!Whb#gTO1yRB#0gGRtk7FKr0QtnK!Lx*| z3u46KZx<%+09$@uT+W#P%qSoSa|f2>}EEDJT4X^5b9@XkEzuM$4nA43tu1(}B82DBiHE?QZ zxH{=n^J+B7EbOkhoK~le?ibYSS@)(uEgiw05JWDE2)B)`CG8*4|Aim zD{Rcs*?A09A@oYus+uWSAS~*OE7GCneN=a zp0QEAaf!hsU;1aq<-!FfG4EXED3+pVtKdEiMDhk7{-P;eaI~5Ko|k}sy%rxz-(G3X zVWOfmaSg{L>DQnf!I%Y%b?hBiO@3^XDuc9A{+sNH!p-%%j5(V*DEfoc5z73q?QQSF zoram7Q_K0nRHX?{02hAzgVi)#8lTHsk7u^*3({D%3r3r;qipM&Y^Dy}sWy8Kt^wYC zc3**pHr$jvTX=@|uf}6dZ~%PklyFhELs=HxuoO8GDZTS8F*gk6RwSf~gLt~Ky!}HM zIoZcxh}XL{8S7v$pg+lNd2K<8{NGn z<}S-B@J@B;T_Hp8 zAk3J>#l^%?$?DKd+|Eh07^X_}H91Q_rbPa0H9)rzB78I!H3KW%E|L~bh3J?+g zJA14|+|pnM8uAPm;8PbVw<9dCc11s4y1YFv?yuN-gPK+v;{1`kC;H^H6+16fE<(Ny zc{5Y}I&JWcle2TJnco7PdWyNn<)r|s7`N{lm>`fthSUA08NZl`LufIvr&M zJD;nF7SM7)ExY^OV|v!Hraf}(Fp+ylXiElCQOfBbHg=V`w@l;0@R3R1w1@>fGyi*Q zUrYL4U?46kmQR0jS_CAoI1=WI?M}su*)Lvlbvna4n@V;b{_*2U6AnL2F(;Twn7GAm z8i>`v&{K7C4G0(PG6CnVQOnYtTQ@+Wy z6su*N?S>o`Q9})-o$Av%!x^&nt*ao-50#HJHI3UG;?w;WF5@@$x{j7ThKg=pBH1jm zaoxWOMGqS@qEF2xxZ^(LSp!h_907e%7MZ46owZI~$=4+x9fJZUv z+V@!pQlpE(HsSU=Xl1|@Soo4N_qBxEoFH;s=-hI!Vyn>QqTZZcT z?*l$(BhU>Lw`r-^QAEt-pvCD{s-)D0XvT<~s9kSmvq4u$W0@wk?B2&Ve}uE&(SQU3 ziG);`dJbp{PI!xQ^|__S1YRMV(zhPn7`W0S8R?pDFt9npGy_@h<*Nuc4cO~Qlbf9! z*WU>D<=&X0iF#yJQn1X|8r`@vUO>ue{;7Pt3l6@5%1?p*?Z20|mP%|pZfgCr{Yn3Y zVM=>^e4_oNS#z?YJicf5po=ww%}7@Kg@iPfExOjf!(*t$VgSFIgq@ikYj?9jtCt~f zCk-tYG{EZbz_|C)5@jR?hKp`hxi5T zOgK)iBP5We=CfdP+B8UGkGLO;W0(g%9t0E|noCS#evPTUKOK6d!m$pACZBmY_?_ z$j@LKjvjFjSo+=GV|0EXWCW9i?ds*RF3F$ zi{kY3O!aQj{xx^7UWT2z;NjjrJ0hv7wvv*DwXihGlVCUBj_6CsKbljyISm84s=$Ix zI`Txqc1*6dOu6IQz*u7B<7|XzO`Y}6X7px1U&S~wJhU6Q7ZSKlw>*i~G z@nZrS{yc_c?Z`hp(VJ*LY-6$e)?j*D?WJ6|5gYNOt8YGViRIKg;EAWq3-&5#Dkk2w zF68TyO4Q^Mj8Lm3x!)ZV@-^m>*WxqTw_}=8sDq3|4gvG3r`)V=b?G>aI>q}7d(Kzm z!XGNz1uV22-*oS?A3Gd31s|zh!hgL<;KI60Jo-fljMk;2VQ!W!XWCk<0R}3*vMBxS6lk z6{g-JFm#i8bDTv{_u%8YG87=I0VFm-2CgR+KMB`vGeoylKht@}6p^(w(-&9>JKI3V z)w22LPZ%tN-a(IRR?7finN|5Ws=VT^EE6)u{gAIA@*32XIz>8#eRe{oGJz-so zr3Ew6w81TZME|Je#r#Xd73Mt|00pr(NV(gG*>xQ&cGU-b+PyJ(tC@m<@Qd;#4VWEV zTJ<2YicwcwV2o|sfrnpEoyQS@+lK*G`5**U|M__3WDt_?fYRCTmdPRFgN*X9Rm zW|*?AYeE}v3i#zF5N8?HDb&B6>VY;FZVOv2zk6Yhn8U`)8pfN_LjtVq%dP@1X&-i9 zBnwV1OS;vVHNnSxzVFXbfg{)@MN9|11k5=7i?N|7kUbn;Bh^lXyH^f;2P98uY=l}5 zvpssJ&4-Dy`?*y~sVE=q`U^Qg&-4e5O<^w3VP{-i};jWy*ARjM#&`M5SQMD+mWL>mGH!Tpx*o# zCK$spFl%Ku=S_@yrDr-VR9YCh-M@15<;%|^^s}1qU6}o4;-5O@Q^Qd_^gN35#Rj=PRtU@Z%6}0Cw z`}^!$08)ZWge$Rc8aR}jHqq~~{xxG6!t$$(O8pwUwD-P_`ItGIk4ZXeGb_b)GO*hA z2m}HU#?mgmNQHPLWR#F6&3aOFJP%E{)3s4AenHl4G}?TEBzhnId>GO@#oM2V!_~^t z&->)KG;E8jj*7`QYN*@+e-1#R;Nlm4JgG0c5{>Ghd6@Qgrc!F9#C$wgNB3ZYFqP^o zSvm~_kN>p#T?bCcJ7h8F#KWYE+Fv;!_%QSFiM7-*7*$M7ah3vFdiiE;-q?&kaT+`A z>hi)yPYEhPxn4&FZ5xz7wKff$Gy-|0iwz9Stc+yGctk*dUZ#(G^x4Gqa4MQ~ytOIX#paB?d0aW8q6f4|p!6v0l9X3cufGH|q>3KU}6Lsv>X7m^It*P}At$nL`B zf792u=%+^*7qJ1 zQyG%SS8;5-*VWRonf_7ag`Jl4s6pM3ejWqFd1(%gsz)ASKwHFXqgHuEz|otHtx-$r z*=4x5;SU|5+U3uMo?qcof!!k&_mm$4Co0$PQJ(=vypcdt>&Br~)Z8KgfNfK_c|~ia zq%siD80+#vTy{+GOUDR0p{f>VkW|^wKuUp=AUrS)QPHUELcb}Sz#S{K2I$EriTT1r z!a;FD8>j$7#fOd^6w9P##kQMss_N0vNgY*ZY_Cl^^;hu^NN;ekNVukN8kCKM4Yd_t zTnl&}smLZ0q|jSREcs|dIp7avtUewJW~V9Uh5bVQDO;rGMcptV4lN37y>6B`O|>1P ziorSrzouoxZ={DDO?}W_&lQdG(3U$mSXolh8ZbD`@u({#iKFfudJ!^XJta4g0yDh2 zrAxW7vEJU#3~tPqmJihlhO)~Z56g3?0%4u`$lc0s_B7m^U5O^yOF|7#Vh;BK;ehA7 zunQgZu61Z5B%yBJxTOJqBw%2YwU?s;Zj4a55 ztk&FdvkpZX&^4;eoOp$-MT!&dO&>b8{LCMi7xLCh!VH$NchTdGy}bpUXXgjyh^yPE zKl8M$;QS95i8k92f6LJA<{cF~yu36n6@@`>wcWOdClJhB@o=9d{~E&WR`387bhR!3 zEe?Z&X6rxrW9Bic+!@s+7>jmCCm-zz9fbHStQolDbFqYJ!a|x(4w_xo1Ov7u+LS{u zxqR^X!-J9(`FiH`=4v9Vlb;EBS2?0fr1Q6g1N?S*>jJzbXJ}@jb!rQl(GfMYM!BiJ z@yb}fWpxeP?#en2hwaf3hwoD#lh7bRx)y@C37`OnVduA1g3Bd9mrWcUOT4;FVf)M; ze6LfyUp!;>eq0NF>(Vl0nvo5BJF}5sTab}PMO}!f> z!-qgnymPS>eQ`T*&DKClJS@FhIxWTYOm6<^n(YX>onF5R-%QC z#k4-ho9Z&y9NHTBcs-LfEvKzWb?F#s4t%o|`BJFmMQHxUchRBo``U6wF&-CyaHJZO zRCLhhm2f=e;r+PS$z9wwF$6!y1#r!W)T<|awdG(?`j}V^UUREj@q!SMcJj3b$i{ZA zyTX6nc>e4?`(GkjFMeO|p(>JHHZw)`fXO`_8hjT#moV_W7^H(9q1k z>9YxM$*zIkUkXn0cLl@Uk2#^^qZ9smU@rr&Bh;zS;r8};k*glOXvcs4=ro|baWxs6 z4S!et6C(rJfuMEEUM6k0Ti!wK0pKAmM{rYpS*;IL3}z}e6fn&ha@Zp64ac?vX}V|8 z92+Cl(1v;3ssvlngDt&!WI6n7c4#0#(t^>aykw{q4JkG#>6#f*y&O$EfMGLyt_cYvur{ z;cB;n5V_rV2wO$Mwu*eQ3ZV~L6cw-^yHtUPRs`g+`XF(vr&2o}QT}&MtA1n);Fd6Q zn3lEkg)HZ^v@4)T+t?nq)6>!CX|vy4UW<{DJ?=VhAkpt{jbe6NKhh!mw+dYUy>(nqV#_n=u>>?nJ8)GLAU8Hd?p6Jq=U62Mk~Q(N zL+!WT9gZL_$n(tHq9v?^$mJTG9crUgpu6NzwZv>5PhSY$ z5Jz@<#REK7yUuyXp(;&;%p&l5I|EE2e2<zFKazIyd?w9%<4Ahrn*y z%(EWpnU}-yQ;tihj9qIpSs*Kj3_p()Is$gS-TO`5vMPo(C*q({^27o_&k1WG^8BjU zFA^{#IAq0OvnL9A7L9?zu{!bj%AvCD<0eyob2KYlY-Jniy$D9&-A+`k7=z{Td&C^!&o`*p*?=5!)4&Wj zbFn@;*9BIw*1D+|!_oKXD4z{Kq9}A<71OX;d=Zvmnz1~=0j%eq{pU~ zre^JA{qIB}>K<~rqAlge%DgnesoSkX?a&?nSr;ob;yZ-sPc#W%b9+JCl(z}f|QET7r*~nw-p)Hg6dP&a=m4^AWdT9RImMLZ|91eqf z;BTSZdG5#va!FJ<*9W}Cjm()2-J?9Fk>8`)2{SunC)Sjs%`lB2X2%(>su{Q~bQJ$s zJ{52P{`mZh*1@3K+S>j66qnFy8N>lcSmTQ?zk&A8fBs*9ZO5_HSLur!i#wQ=JI9rd zI3vMkxKud@(Qw-LFCfM-N`NeKKLPJ7eXorgO&eKTT|Z`~fs3|Z8&1P4K+oR6*JIgm zHFD7K&@u_w5{qA7Gg6jthD7@05XKHND!Kx?!0WLYr{TR-;~r+mBKRZ3y!U!)6JDb( z0Iw}iQ1n}jW#wys)Kzb{YbGrxe`1aHMC??~{iPkW1Ht>dhIQcTkpy-;p_yDl`1f8A2KRj<2&Tr~})b@5Grq})L;7OwpfmL?9IsJi$Pr!vWo#b3&Se5S| z@xTt!R4tPm>+5M^FGNl)NML{fJsP zJC_B_q0Cg&b6Eh=yUNd$lx70|20rZYO|Q??6BrbuK79?@x5}90ChoCCd7eT~#tROA zeJL(gk=?Syj3s>z!>M_#fA37|t9Y?o3+$_Nt$o|hz?^s%1i z`3)_FvEa+lG9?TnLplCM9zCX}f0uRQzHnauI zIm_RcU7V?Su6@=E9Gf)$BB}wj>;KMSJy?m!h2cV=-+Yx#7`psA2PF7DUpWY2s!a|& zs;!7eJ&wt(XgD^W!)DV*)I*8n@`WVt!(yKXJ$Xm{JZ3W@ne~JdBR?t^G6DqJZ0!HYHX%l6f8D;$)Z2n;MVszA9AiN3wU8qkwld|| zB}y{=pQ^8()xnMZj+`sH9@ht!wBdQRr#%qnJ`*LLFn=T{<`wOybl*OKCVvQI6NN=d?+m9U z3aZAG`|mHUobHS#UcOy4QjBs;KZ%dAm2T4-Lk2A0$8LQ;+Jnn&I4-BR-<}`~`T%6R z%o+IY4i+~23!;q#aqWi(et%aTkklO(QqKK@f>BS2&|{8|mjlC1LtKboh60pH?!GHS zekEe?@!J z{<3SES;>~sr?D~1I-8gBHyw-@@POrQy0)$AUp~DQ|A`5-kIX6Uyn@W(38=2^_@e# zlFSX=tc(iCH9I|91EBtcJkg_-py?{*>6}5>vODTDCwi%V_;L7a!ZF0`iXoRa-auQw zdg3cb>lT%(;jTj+?zf$sXt`71sD!KzrVS}e-@;DzqvcM$7zzSAk;8s6Zyubkk6{9*Y*e2nzcH%(`9b2 zb|97wMwQ2MKql_9W`&q~47`GS98_lrz}-6U*(TeB#y(4LhL%i_+#=tAh-73Lx_M)I zA2Or7=wx4;;x|7qG+8F(|AC~An;62O_EChk&#&x6PW!h3Hok&gl}C7?Q?p1RvY_IL z&^GiZWD0DNlPLLS5>aCT92irce>wF1wb^0l+J>nsPPRiQn+?0i25$~Lw!_Ts@Rhgf zo4ak-sx;kb7e4%DC+D%bP25*x8$|>XF`TMM?Tw;ZARwUU2-8$S+bzhD)oHot-PY8qppH}}V@Hv>SGQd~G;um(qfVmDEeq9%>!p)jT20YQ z$SbZbPRC_K1>jz{o4EcIo`n|Bx1Isk4-Pn6!eZpl+Er$>Coo4^R3bAcw1PBsb>h;X6w4n9RW$f7;xKH?c z>zS9t27$U^sPgz4nW0?1uMf*lJp=P$>ej7}Y07lW;&83(%y*#QEKdQ5wEz<>7IVOM z)Dz<-1#3v4;MFI9{eKAi@2IAl?hP2`RzXE6DhLP&A_!6hkzQ|9K)RFw2~C91i}YR; zkz1;?C|&6!Gy??bDAJ`HTIiuB(mM#gllc6;_5Ja#^=4&}ERvivGiPS+>$>*db2fMt z+>nlZ-Fnl1S`>>F|KOUEx#hQ~gX%Ij(_PaD);G$3TyE)S4PH^*>5m9{%e{gaX)j>KUsPAx{}560 zM)lX~v?)*mmmf|BE#;k^{>Ev#Bl@}JY-vqX@a{Z^*s?$Khz=aUN)@$azwX zghF5q7&L>16?R3G4eC=p-%c2CM79m{JouWJZ9Rr?k|8C2o<7Vt!o>v}B<$=;nig0D z^s#!y1(>B1z5eN*(~UdiJt|QQh`Q;%DxQ!l*#MBvM2sYXE<$Ox{&I~Um9$JgwWI!a zh5V4yi6uhk^l`&c7RTwv=e-Q3@Zc565PiD3S*nKL1;Of*!4?Sk3Qw8eC-Z<~fqA17 zzln1c%~vY~H6#_bWw)A+kDD$AA1qYw60ua9A^>7Oh(-&>g)9Pz>e)a4LrNYS{wG2R z=k9Gg{V(s=Z2oe)WDDQ4LF@iblrCx|OL6n<5h7%-Z-*u>Al)Ik%iy>ckLMVX{c{kv zn-+2qadiA=x2esd3ApTj4fE}X%@K~nJPV|2 zNsmhZOlV>^J!DA5Rqso7Re*PAQ%cghv1aVImB~okU)% zp=m~{B^nToXcbmxjpW2lV)S7H?iXJ@ZP5PH>;i6QjF*4I^4 zAEwIOpas8s-RSELAZaV~_AOF^ukCfE`GKItp)9PU#istzYITz>PVEaY{}=M4hH(24RkboBOqe=Yu<$h(`r-N){n4BP(-Q z$4fBr7v83gm8FVD8T-UC#f|FP?LSj_b!+t|K9hcG#P^c(QkZ*F+x(Mwb0)7{d0|4Z zMIeX#ChN_DcC&qmyQ%Gf#`#_`(4=ss!h1^dPT;$p}?HertV6m2$?ZGrwtA?}R|5GeTGSm~pD=z}b2G$`2kVxK3&YnI^@0(ZOJ8I?bSccjM)E;64lFGCAr zS9gA{Q%0FnmnZnMS&%*qWh5opQiKNW2E2^SHt45Iwmr;#f3`SDNf6pqq^OZ%qT zP&qQRsn0!rWBKIGEvw&mSl-+k5`#k#f{r^=36^?l7r;q0|L*!#2Li@Y+TKA8xK$$m zt0yzYk=ak1nAVQ`%W#Rr5}I%H6jtEKS`QYWjRL;u@3Z;xoQ0dDKUgy3Ua1X9ER15D z+83DtuBW&|u48}p-n^iCX=Gc_sGkl}0@3zUZYC^y9rJfjzEsU*q7HkCg?ashI)iuw zMRcJx?Gx^5>F23QKp;6{m(OeIFJh2w8LnZgms$8DMkIrWzPIOLAdp4bVG(27Qm44X zPK;Hzo0D`Ys9LawYPJF_+I&bZstk4h#yPS5o$#cIsiVwSR7<<~OojT8rY`gP@ky|7 zlEtQUevUdJF5?CJl9^gD${L&1ETH) zBL5i_9ig31)6eBIhHt*ankV8QNJeSeb|yoORGY{aV_k-IhiZo>?rf^PQof$r_21kw zFSooSesRwzkp@-o&X?|N2lhL2YCXua&vz*hZugo$&Xk+xNdk7XS$!@915Wym8xB=) zxh)yvQczZPspOjJm2w%CFjI=~J%bkCs(2P_giL7fkCJ2>)Kxt{ya zMlbp&!`8UrS4O1#T{aFS02(R45GHVE9J)8SSd{uSYIC2h=-M__qVEl?F|QWxTRQkQ zb`5OY({PHR7`*xB#i{fqj(sbz7O_b1M1!L%Q-6D!hdHA*y>Earf?s(8Z0F2nvbC04 zq7u4oi+aP>x&2T{(V)aUV;dQY9U$(Bw=QGNz#8hmWZ?L7v59ZyT&_K`*GA{Ce6>*Q z0w)oR40QEE1+?(#CD5Ax7%IftWbWFTAb>95kQ{krDMibjV<)0hp|=Tf9lr1a%S-|cJzV!n_`>y_7H5yApxEB3 z^=wBxw-#gDXYL}y5<+1Ujn)v``Gf|}r_9RyJmHqPVp>=yN&6ANK|&uyR49YdS_0FB zuy5naB3&eyXI>U^@TOV%XNKDzK^tD=JWE^d1YpIYzwK`+ahgXZ{$@sHCG5-(_l|!^GO7Z{eFkAK*2^!es)~czugLUH)3onb0d_%PL z%bX)(nA)pgb=4SdNq@IHmZSW;ZuZWF2c|C*5qT-4rZ6KmI%F_AZW!X|3<>aMI^?{qP7t+d^vmld4*hUyELzBhlg8`)p1pbf}8S&8AUcEar+DHF&331Sdt<;b~%(F%cX zv_U#Ib%diJSO~`tEg6&b9o#EfJ*$M606 zBu6@?6tC%XHo|dr@0dEr&gkdLIA**jUvqo8?2@ZwT5Ww}X|`Zh%>Xt$YSn5LRMt^Q zsr}9aqxPZ~6rGJiSk7NOp8bq^5?a9@dyQwa6T(NSR49l)`1YK^@4gGs*<`o~V+Hz@ z5gcN4L7GJpk>IaXST`bRb0l9mwu&5DU#GsMIuZc)@G$bN%|Z)$`dK~tdh=E;HxD#l zgGg*YlDALEhsCt{2DO6?2u}S#`7DJ9lK~kf-S*4OI?ycA(<>V?axC!!tV=M%ePdM7 zzQIKYppCZ{bmjUCt@sxulTIB{4Srzyc8+0=H%|ljF5CuEMzX8`iLcWcGW6%c)tpk`G@Y>PC(c8LuDwEbqDLhcCw$lREs>w{KjX<*24#nWKc3 zKG>Q?r263l$AhX2S6N}7xJyhwT_hH2)}C>+%=Cz9f&2^9_^R6_>_O=D(nOU{v#nL) zhnbR7Wmd8<$EvntP=#a3pjFib%ger00Afj9Al-u(hhDIM$e9%bp~3 z=|$$O@Jsk`)sh^I+Fy&xJIKx6w<|-FvlrCtCz2ZH5A)Ax-1~KWA^oFZH@DgpL-NaQ z8|F3x4|ebv^mUky=9?XxuBUbc|}RtH(tHy771Wi1N8Ev_aE|MTZwt z3(-Z47~y(3UAMr@Sxyg;=Ret)TGIBglw&E%-kEZ0`$B&#SC5|8@mGTi4xXY9&ohP$ zd~S#uXtV4>hrb@?#hNTd^sswyyb0pCx%nZv_aQ8A71lj16rYQf} zKby}m^`>ht?sKVPG^E7PS{4i#6ogRFnhSoPxV0d=0udH=v1o9?dwRc&4~Si^DZSlY zNws~O_e`U?)b9q#p0dUoCZA06Dmz_VP3WD}5DQ_&hSjjaNwPfwFG>GJJ+@ zLm;dimGpDPTW8fjD*LHK@$YTZ{bqVl6%p?JkCYHJGw!U>h!?H%MUo8_qz)+pm%O-8 zN;~QMP|tu2O@oW8Y0sH-FF$j>WnmpGqznfz#oG#YljKNYwaA3WN}fBB0an@JE0%E+ zVmOD8UoyQQGJ7}&0{q!c%qRj06AfPuP4nzOrA(x8k&${*JU_wbY17ouL#Pu1|I*H&1x$< z{&?yE|BBvdl8_-NPIkGB4Cg*=&8l_52iL-h>5z47>^RxPAlscj;x~iG{7m45PpZB`DsRWsrdj=`YV0Z)1z0$HM&08l6U>> zhZ^W#h@nx_OXrSEv}Uu`K5cUpzd_hL=0}tZH0#3sN@KKbCt|S}p{n6jQA3dz0LL~H zK5>!qwZg6K^ZBA_r(b-nWk+IO>;aq*>?)1A#p@P0TWR3P2xPz;eF48BK9xE-@nxYY zeHvica6^qoVi%_5tN;rM^sHji-%6C^VIFo2?8}eH6MCTPmTW6qZAVhCcLYd_qx**8 zf296*_wseSdkjQ@w^EU4!QIaSAp;ZUmu^8ZLay17b8^6`uiRg<%2V@SPM03>XyeU{ z1Mu6Oi_un{L4`UDPkG{J&L+c)MgPEaSj427BDvhufv2*qKylyveR`i85@@mOusa&J z&@k^;N@7UifofAMqq$w>;{;W7`E`q#Q!@8`#R%hZA)>N#>Evi`7sad zI6~3OeHsZ44<7A?D5Hhr%%g^Xbs4gJIcwDSs5!NU09KE;*i1Q3qU}IxATc=#iXY){ z69)_{5{@-{X-%)}&Ti7U)aldU!J(cHMT*2Swsv9=J?BUTV8%RTDHB}b(miT!p1W!f zc`Pqo6YW!*LD-~ZOBph>clrE?;t3^rIR5|JEtvN?+ml7{R^41a%_JIJ z+#P!)w^p9HPA0y?i(MLXyB%Y6=Re>??Nj+X^;*@k<)&{c+;}tZpBXrvxKuOfFyRDR z@QfVh2XE5(xGF_63-;4bfT_-F)exp6Kj2OZJTQUch-PA8$pxYg6Lwmv{J0vzQ6)OiC zp7nAAR1r~pAa7g+-jvg0oV#5{k>a^%!Q{M@1a62#F~4A>QYARfT<*K1webjhX0-RY z#6(w;KjeOdYch9ukrB{nzq2`0yDevA$iTl9I9={`yov^=U>mt<;5j~Qx-JQhJyZOD zd+J`YWq!jp5|5X`mn}d`#k7*|q<{`(GS!V|T!$BNb?^Phr;};`!dEYoxg8O-R8b*)gng$N zO|o)PJzo-2I502>I^->&6uk?lKKdlr2WikvgqvLYLh>kp!IQJ00eC|P`#q}ihmpn9@Hr{nr0?np26-gj)&3Tk7Vw+=&LjYw zz@^+GBTR~N1GL~so4;z~I!Qr&+h4**?Of`Uls|Uh*hoa0qqG1pbjK>jWz@hzxtDKKRey*J3(A%#XB+unniY-6?b!l~lU zvV%+J*i*FFP@9UXviXw)({{rg!$d0%MLx01k78h20LarnQHhVt{&|$F1@e<1$nJn9 zim0SJD<9EP(R7m%Z}rV;iIHIaF4eiq>qt1i1Lq2@AP6x5(Rd1^`T;uu$-XwS@FwVNFgFV;f^>6!c&0WUHv6WopB1RV{n2qjJI zndpvf(~O?XEy4Fob-)y}B&3$S(TwtTH>O$W<#y*&(=)57FLfvZ#^1jSH4eBoVABTb zVOF1Yjap^L`oDRY0nTk7RFPB9$3PV?*lkdSVcWlGsG7OZ74>k9#r_{Qs!vA^tJ_eT zNAB!KByZ>p&95mraeyNDBOm(d^7V+ex)pP?RvO(r>7_aPM2HzrNr#fr*;z8qdGN90 zlC$ZU0aTso9tNJec#!K@Fa!xG0ohe&JU&-Ev_Ia@4HkaN*6L76a8GZg*)>q=)0AO_s4 zgl&-^t-E5rC35b#*56sH$KRQ)nHGtbOv)QGbp;_WigE(k2q8=}7b0^WC(K_qna{h% z`!HzQq@z|OgD)EJbCARnW!{w{qEXIF6@L#@hbt&(Ycw%MxBTZ{#XoKKIz*aaKw5b_QcACcSmh>@-H!0&mvI|fRrwWm^+}m5_CM2E48z|uZ z4bKL;rLXO!U8%{+=Bun37+5?q44+d2int}c`i;tvVk4%2-yjFbrsR%V7uxAAYWRi3 z>TY%@x0=_&VC}2mFQJUiE8hNiapU|V)VMR`cVbGP1M?do$iZUXT&96Nc6IT*2ns$a zLhq8P-m}iUm5&L%7-yCiiD-m#5!U6NJM_cDtF92jp~CiYzS~c26flxC0cN`@TFHdY zb5|375_j7j*(gJg-_9Agey2hDP8~2SJOQfcPzg1x9Wy_ATpe6u$WNX91e%GOCek5u z-+QD_7lEZ_pTMuCmuG+H5pZROd2`*~2f`bbdnoOD-Adxm8B`o9Z@6n8D$s=4{XWCq zj;R`U%IunGi(Kl&2$uLrv@>QqB3iA;3e-8W7@%>ln@WgN=h#{}whUGV+%=X}{4hE3 zQoCWNn~efk?=A%eyWOn3qQdvPx7=DSTeq!o=Q$GJSQZWuof~!oBFt5x2-v!3+xwN> zV^J+)f4BIXPj*1pf7NP;5>n4xs&)5!WOT1YK*Qs(vFwsj71j=$Y#X+|TL*Fnwtq>@ z8?*gzn0#6`q*w(lQA$hYO zh^BK{8eQ2<()uw2U^_bh0t^t;2bWcNane~#ZVX_-&?Li)s!+oQuEX>Gu|OEdkAgUO zC&qdA_|s9PT4q;egf{)_+CY0Vw?K3SEULbL(O-3tbKb&Hf@!OsFp2@X(KD^q;nd#J zyK5~&ohe|JlxP31*{L@8I67408N;t~VJ*SJ{mB}v>WMoWF$E&jpdA^t)F>z_?(9P~ z!cHyjRhyb+y?kM@VW8jIj!+P)g~bm@PfwJW(LmK4pIGh^$prx}Ov8SZ1hOhJ_bZ-j zddW6asDdpj=iLhr{5=1;^Um?-9UUZZpo?1@q}uo8umWN2`l+I6Ala2dio9b_^VRM< zVNYNcb8;U%qT1#9s)FJA89k+4bIZrQgPO_;7mVdxF@9OAHddxJ5X7KLL5{oXTV0Y$ zRg3Rq#~D@_X07<&8A)c-^Q+;{7P)B@DuL624xj#M^_nPIdipA-g?&N!AGf6Ctx2Ls zS=Z$7^|5^$>SZ1E2E#=8%a3>;>4BzD;M^pDv3~Y8*IuG{>PC9MW`YtW!qTZKoJ|CT zGAfV1cS#o|9_lg+yuF4^^Ezyl++N_tY5T7^y|32P0HQPNq*DKfg&LWLM%b=yzp+CD zj>Wbd%ujiz6TpT=7Gz?9Hp<3>97aA^#|z$#d@d*4@8(of>M^sI*!O-LnP3K2mA|f9 z0m~85%fXITKlHkuGnV-#S7mnw(_zU_#Ol;@rUb%|gwQHppW_|?xxWMJ(~(OYxj}`r~$_OlNj>HXn9u z*}*EvBhX1pfaRt~DZWGQ+a-#?KGtgOWGYKFv0~V-og*Bq2t(MtU7f)OiOjRdY%#(z zdZrE}j;>Wm?LQcc-Tk=t$uHB?sN|sT3&~MFN2l?LUuYx^o!siMN`6DpOr3Z~{ zI9=T&{z$#a;E&w$2ewd#aEUrIM&(c0d_Z_4U|U)=rgz&)-iMuTy6}S5mm>>F;c{ax zEDUa{8xIG1`}jLUzL1UzNB-y5D&~o|981XJp(*_J#yOHbN`!;2xY$>J|@l5E&r!ko5L2k&^ zIQ}-8!{nCria@Sk@_fsy`)68pxw_G`C1_Llw#@Zo5>-Qu6?aR2J7G~aarP{@c%G*S zYqz|0RJ&z4Q$Mgum>VB&6y_RT_J&+`ExT1TX{UWsZ&qelHMLU6r_Q6s<GC3ry?Ae<>hBf`0j>w9)%L8^qQE4{h#Feg2A%Yo*(csP_F5HkCG zv7V&770iqc1dua<_mT!B5D?PZzbr##Nu04Feo)GVJ7+OTKWEmf8=IiEkE{Ljb{7aW zkeI(JOhggi2k>#v)tFGXXXf@A( zYiVSD;PcEGx!~ALVwV^VW2<4m{;oi~!?6sygv-56`h#t!^+K_@@-7Sk$C*!jwn#p- zyO)a}tu#>hB+;p?Yv++8Fv7~_n#1J$3ogd0cjNWz3M2#xXvAju+-Dr!yiD)2H`Y+6 z0HsiPs)P8A2NvzmoHMhf022bi>%Gg2$(JNsTk5S3ct9^=XD^?40lE1_W3+ZRP3sqy57Uu^w#-SU+AvI0=?Geb@@HFS=ox!xu|vaO0;^iR$An_n6*toWosWW~4tP$@v6 zhR4T$c~;BEj`y_1$E9={mii&GO32(*%JMgzX3by360cf0jK@Qymi!(vyegfBAr`&&X!>hH)VA&Sl@{A7eBn?Up& zSWzHo^sW-uJ|1#^iO>VLW)9=u1nUF3XfUhGK=V);C>gFURqFR5pPL5Vb8evcE*9fP zx$yD1YfYwxY7>V_1q_1V7eCy5k*1e$AG%{>7k8xu-*=}8^FUDXA|6_zTWsU8Upzd3 zKa>)R*J?4S8|Y7-t?#Jk!%!v;+TEKU$U^a=WO(;UcALy(p54{TWX=-*kF7^Zc73}? zJIeBgLzy!N_4QaJQmn8s)sXoyxRKPVxBf5##605dH{%AEtZR7Hr~f~yg}T=w;tiSW z;qUYC@;4~~gw(KgAG@;4t!9rX!mXopWsfumoSm)d?EL(O$1r)^ck_|sQQz#?km4pd z)RYqtL!s|UvGvkd6E&0eBzA*{Te`jIr|`R;IU*W$M`J$|E8`#>I5AdSiZ3)TiCt3C zzS*8hEcAX|^A5)%O7ln=TI#7`4SF=g_ZM{ZW1uzN)+FjKROTk};-twy)60X^6| z=T>^^W8)ux4gtTrsW zd2l9XCu!*sK$G0sRB)Qc{xAx{V~@0!5_YegHq7yB&daxj zA|eHGd(S~vzzrAU!5BNA5uP3 zd+RkZ6;5A@4fMNgzz;kZ;S0ERq97bv$}J1~f?L_Gu4;3}64Aw9KTiTDWu3AWxN*|s zV#G8u9Lvy{JGNZYV!RUhq-pLh^=;tlL@q#3Mn2AocZwuNBzan%RzTjb2b^Dfpho3k zrn9BY1`=UnukyHMbF0xcfoQHY%8Q|^7rzhHWx$!w-=={L^b#c(>|&OrpO6w__Q-d= zD}SwG`6}p%xV3IlWR9IFoRQA9kL88|;z?K}U6cF9Mc*GfVgm>-#BtDs-u{2LtU>w# ziPv<--;4W)oOa;TK+(Kw*6Jo|_daC_U9w`qN#FYUm0h-3*?Q+b<|gYKOS=-XOr3nA zGJJB`+v^iH)$c@C)R)$H$7yh=-%0Z4*fRezCCDRBCU73;BpT>42vCDGhrHpWlGt{} zcwP|`Ni{vnlV^eH_!k4;1+jjW@27x&{dM8uc6Pr_8ETIW$+azQ=t`2BQLjlh;#ns{ z>u_{Y%eX&6`I8(@MoHG8)gE@o19|-;a8H|t^kUr?SKKbRzji1FoXidGVbVHTZ;ar< zeYR%K*KB!bd3o`9mx#4yeM;N>$Q|#e4pKudzrTKuuA8*LnNI)YqK-5La*+At{s*A; z<*dZIv2~ZOl{VuvGMC2LdP9@Afs-x9`Z(L-FgJEn0T|@ItERvE>fhbFWvVwSJ#SXU z?$%>NHeZ5V1d!KynX6T|_@iRsISC<+Qeq+2rNb?w2)VK)$k7)sr>(rD4F+VHlGoiS zlegX3XZ+GDXCWv%?q<^rZ{#&f?Odc{5aqFXq^0tMzi#($$v{Gi6T6SGaxUBiDe0I% z^-`7oGLkc`gsKF}#2Gv1_8+D~$nh#K?qNK)cJg-+O}(osH-d##6bYoUs=pDQTG{h=cRuc)1{)Za3APL8m-iZLSf7$W`u_s5#-cfg z?*?ZX;}bv7)%b2cE??FLFXh z88-GRE;QdI@JoE3&=4^C@3^a2wLm@!N5p@ulcl8&)p0v=QmO0wx}-XPieKL?I>3vD z4av4tR9#@J+VhO-AHTMJpE_)keU&pTm@h;Xo`WC7tKai5R>Dj59ylf=L+t1$S9fem zzldIb1VjiRD^+?QNAin+6PV}3eFdd@x%SVVqR$8wwxoD5!XcE)q-Dqe2S zI?gQUVgUCsX?a>V#vEb}K)LMX*?af<=~~dZK-%Qy5uGr51GX|D~$bU94t2_eA zucH$$U7{yll-OMfOsQ2tbD*-7(pw=9`ouJPR5Pc9Jk>627_O&xT7`K`TR} zAL)#E#ERMO)U$?vK{R+dy;kC58LF;Xb@F6X-h5l^YEA!|js!l?4s%wIL@Q75?$7y& z&jg9ReGwd*^#XvIc^p)x*cg=dTcbUWK|@dR0)x)$_c0ET*oKp>Od`^- zC^{}nBo^%0%|GSTcg;=ophniKfixRM?r@w z5+r8j5jdnIoeX+pR{Kf?HM}A`xC`zh&&UyeSLl13QJS|dydQ_`Q?JVY$U$pEv3I;> zy3~&;;w>4PL^h)&9WI%5LqP=T6{kUJE`Np|IG!$pLsgo-S+GY=j+8ox?osE%yD)-K z43Fc%6o!f;Yxv(LsmQ7tH(#DLIp9MbO0peF(5uE?W=230inJ1U7?N)s5`*|Wzn4pt zI_~vy{^_7T=sBTMGeqpH$|!o0X%Q${Z7ggo>x z&|MPzn%=Yx?knOqHS^jJu^@U`-;$nR<3W{hUD&H0zY zR@+j1O@CByxi>wvx(PUfgd~*^hoppTZNE#O*Pn`I%9q)eCLQXn7N_NYYS=V!>CzGq zGyPs*bRWJq}7NPnlq{?ut0*2c@--q(k z)f7*w;_G(yxTQ73+!sEf18`Vg3`u4|Ck@xJn>kgB_nhU7#_$i>9yXA)`E;h=!a$>V z_BNUyFNjOM`*gG{yY&ASdXw^ViyrSsAYJJrQwLh`ka4}Ay2`rEp+m5n;oR&jtHTSu z!12gZxJVqQyTH%FOyj7>eftI_a1c&N5F7%Y%kX68BcA)QTk%vmq1)RDn`ZFnX2ep! z*6#+B)iqu%;Ph5v+nq-iYLEs&HKhSy%2h~}u)I(1DmUBthY_E--nqPUK6&P(u~dfp zVEUJLvQBTflNeByKf+fP!0lyG<{algKoO`RA_`i@;cJ%Y9MA4^#IFuIHX}#QMmK&3 zCMSqJkFXduN1ss+8BUySLAcAd6!+Q~VS(snfB&)-ys|EVtrxNJa;262Frj0{Zr5>UGM-zfg+JOs=J zHGRn^{JJH*{f*r=aHFnXT=8tgQcdlP&m}f>Zu9h_eSbIt_QxilB;?y4-G9LJf)Cjd z&5)dj@&}LJ-t@@|j=jEhx(}1mb+!Td1$70mq$ey&|11nTDTbzU&RWw)ekqelRyso) zaZTIj%-+Fq>WH-*)>ll|Oft@8Bzt6vIVT4nMx@B*(+HB1)t?kJ?<=6bwHzo`OM9zC zTt-TUlVXfw`e3--ib0yaNp>QIUT+YkbCrenb8IY~r>|I`_l3J=hH3jtj1gKx%Ynb^ z)qjp$f7rj$+Pb@M@Q7qo;_%l5fS;#2=pJ?*hV3406Jk28`H8=x7X663b&>cdKn_9j z^VnyLnVH^ywd%nuc6l37+n}`n_0;iN;A7ofCso`ghK38zZ5YzuaBZsbaVAr(RNY0f z{KS+wDLr0dVPcPfZdCnGPtl+q@v33M#=*ZiQJ=SM=rhQtw^~u zRP=knkwIDeyy~OHT--a%_CuBUq1nyZu0;W}bWJqo+oehc(!<8Qeu&GMco1mea1F!o*>5(PV-)vx_PEpZb02u z`p{#LwJoTQ^~xFZ%r|d3{wVJXF&z14Pv1YR=kkxHH*h6u235ko1@BM3t%o_=pV2O= z8O1rZNb$5qBoYP@ANCbP+*g@8PxD3|KG|*}al=YXj+rl#D@Ju2K;TSA!+8Qc1zowF z{qbAG$80{aOzrg_Chxtk($E=BpSd38$03#`M8{QzTU;}~a5u=`(U3`c_jppaB+sH{ zWb4S8o{8 z-72)d!3fFA7K9l3?EA5)66pNt7O%DXFI3b+TAzjVPmihByr+sEhVCAld7<|_gcKs~ zZ+m|);Lu(O>L0VqeYsD7qv+0tEtbS@XD5j%-i&DZ+LgrwdNt}rrhTl>Ga$pOa!~v5 z6Z-beL5Unl6sP-A$osW$j@}*6o=>nFv?+RMWF=2W^mZoR8Jn+Fq|gRmTzCX^tt zxL$8W8WooFsaMgwBjV$Lb#`r$SHFcB-*UsqN^3Z>7U4N`16pVFixfsFCS!k!^0P>! zd97yc;`Q*dRsQ4h;lh!1{%&*z`*PgelL*!GTT`eG4WPUt3)f!9fb$PKOGa|#WgME8 z>HcWPIA3}A6e85b_iO~`1=di^eZrooSHKv|gtFbOIL9YKMzQ&8QuP1j0yKQ6uI#-8 z=BrX!!ThzQ*ala`(haSGk&@5*Xz?w99v0B9(39`HFwneb-B?J27<2h>hjQH*)mCc! zQCKN@&Hqsh1ihEPCH?y2z@f39x!mE0)A9lrpX#vUYKLG!g@))Vm9a=zVre&eJLbOL zaL-So7hFQ@e7=XaDlcdaYmuNj#H%E3X-R#~ILutfZO6Wa{cz!TvDc94^f1w6toJ(^@}`w+ezGG$BS(dvSd@Th82tFK6|H_Gg~RpFxA z5bPtcC??UiykS=e)?=qH|9V&#axkLe0go;zL0v6j*u%@vs{|wryw6KD`Nx_;diT6M zkF4BzQcgnoUGki1MxeSDoS}H@{jd%&wsw&CF>mR5cIQ_e++!!@{DRA^W1V`tN0Y;! zrexrgnqWDvSm&=;Bk6O|rI6t&mBhDPOAIwnddItKpe3&sSL0cyJ0#5^`ly-_fiW4> zDfME}I((d0XpZFdvRY>kWxHr25fYmOvP!^D-Wfy^<&z z>eOu9B+yNJGxEB;rZ=dj=JNTI9XjLsszXfcC+iM_vmepc>fRD!)jO%rozc%vzS?ET zS6u6G9$)s&==NQ0D=(R%LG7+q($F^ZKCdL1?a}92_yyKK^P(zxJ8!o?t)`8MjX25Z zn@NhQs1Nxq&!i`M8EOp`mOAw0*8odQOB)e6cj%bjM;)Vjjh&5F;CcYgFx*)?>AoNj5!L+L>J$-T5k;2onK6T)=;XF^SM{M z&&$)jFyF<~O(Eg2>_m|Ms+xE%H#-~6^!N2@FGG7?d&bA!0W2Q>lDa7H_h#izpKZXG z9&-*4FF!9H^St>aH{)@brN!`g0f%+&*K%R2=w5})ZVODYXRlC*;8arWcvcN;x~p$f z(DVn{NRlxOMR{wz#m&bq#}m$JzyThPg?BwS9{^JpOZ3UAiZyRSjfoju*yeN>4oj6E zSSCVzpfMW^_hw0{WEkT#&V;V|c0Uf!quF0FuYb`6^ns0iI+=4^ zgzHWf)V}0e;fAz)Psm;>p%O&t7#zY(Yx{4NXH7nfKrd z_w8Q1SGu-MvBq)lg70w$>CQ%>*qKBi0G_nn`?JT-@6(}y0Ie#h5=$M{D4Jgz1}-ZRCt~Xw4Vc3vG$KUsetYoPqvmT zc%=w!9j}7JcYb5&U)c&1&l$=%UkH+V&>Xd)Mv?+X z^@PxL6Yfn8LXi`1Fy~ygo<@bmN6LUtD16EKFvTdWyM+!N$Er0naX977`Jdp{!wUPx zK&4*@3F&!4HR)g!-rAKbLso#2iUTG-^4pmP`5c5l3YK@(JI-#yvMVdEG*Of@qFI4iY zG_Qw1U@>v=Z5}R7+rtaJP-2L3H^0KdyNQw$#++xOtoQmQ3%>dcvdnUh^h=b)?0Ni> z&*OM~2sRt5x+x^;O!z|{FC+x4_R3U-wWQ)fAcbP^rAo!f$+tr)K{hdfeI(+udXb{PBA z5j<`FD@{igL7MeCe~sacminM^`+6)`qGD-_wNkA1PP~y^yiC&J^ae&TsriV{h7IQV z#cA@#eqG(1qMw9_Y09(}b?F}6OkBmJVK5|bAqnPuzH*D|nS=rsD>3FhEXU5zpD~15 zLo>;(nD?JmG6W&%JBmel4$Ab0i!3d2@vbQz0dkHsETn5=_&4Wt)X=ge^r&kj7TurG~M%;-|q1_Q^fCDC9{!QD#=J2d|<4OTWrcTY=W z@b%1QPmcAPP>#3&_^@2&2U$jRx(ll~W0I|`&3Hx?FX2#S^w*{)lM}xwM&SX7px5-? zG8^_4tG9gp)(xsi#!JVe)275soF4>4Fs3vWV+Ac|D!)6h|& zO#WE!Ek5cn1CZV+wd~ioP~b!ra9i8YA+G+{xO@7d>?8a%$||L;oLa)gNn>jAx3B^J zs)-(imV~?L2SED==J|Wc(>+XbukCirWIf=ALWT$@g~t1L<4;Tf@y$M;mU%`J19D{(la?=1ZP$7H}L&!Txf) zTGOORN`D{z-gl!=_+Br?&CwpQ=h@f45}M$c zI-d+FK^9}hloZZUaxi2{BX73H)4myt#9Z|~CbXgW9g`uq*Bf4Cal|(XABiU=9!3_S@gM8nd4oeW}N00mZOoOti2RYCgT&4a2LIqD=$M z16}Al{oM2sO=~|;WL`z)4$YaYI_@#;t@)B?6OhP(m9f{&&uO~QhVNYXgIna@HTB)d zk@BW<*;gnObi~HIcdZY+5SgAFRXo>mH}a;!Vq>k#Pt!Fbx3}nf`zN}?{88#Lbne6a zktKuN2~~Yr(A%#+%9R;lW9sQ5xlnxW>WRxoha3IYFgvj>$p#i*%GJmtX|OR502^di zHfl9wUnBCm()(uP_s@y(x)4buQbp^L*CbLbW~6n!S~$pv2p$*(?*b-0Xs~kzf|{gnxmBBAi0?jnqX{_xmV_M zg>SF?3Pjch`=&#ImeBFjPo*1QU+C#}--OvoH6A;wR1JAn>uOWsZh65k$UHhdS{y0R z=Xd4!rJ@9A?7=xQ2MR!zF1_puYA{DCHC1ffuc-godid+>hxq|Zqv1&dN9q0FT_r(2 z!S{pcAGQH?Mc2qhiN|!vHzQN)=4-=ALET}k(xz&I1|jPb$h)R%gF@6a>(k!iaRkPt zAAx_EbMozfif!+L21jNO*Cl>Uw46`Q-+SM5)`(aosxops67jL3q9~TOu4K`oXItHN zNndSS1gC6x>osu| z0oKWSOjN9u(?wE^2*oovc@^#b^wpEQz(7YU3eT{f-AYLEx+-n!gOIIr} zK3HAGhdjUCpJ$)j61JAJ`XHl1MtY67d z&4)A+f-E5^-3Um-E{aGjjf5=W2h!cmqDTuXAl|9&ra!=PNLG9M1V9E>@ z7HZB-#F(ty-gyq#x{GqG)OmKEe)V-f)K%e;!C1_VJKZ_;gaM@`FK zNW;7fDYgC|Bb?HFZ;L2hDsx=bO^OLQN!_d<#&CsfaT3?p6Ej2>-mB*(>cO#KDROEW z8@5TO4n@p74VetUd5&+m*Q+8-%7<0y9irEhp-H{S`HzphxEZ7Pi&dKFf$tG5%)>40 zn4L@^)LU`$A(68<08k10U#wn+$w8B3Lm7upc7_tKL!Zm_6)E5Lm`rur?!tBDS#0u5 z^S@UvAgcEhALRPbv=?%B7k@IQMUOf-IgobS(p53dW_FZ9hUmRD;uPU=G(rp&fXN}+ zK^M&U;5Y8lVZLAt`d)kVH>7P^o+If8IxXcno5RDva`_?2E${EbF1#Q)vm`IcDzMu# zwarI0J99NpW4n9JxZ?p2p>BsreqAcc zeJY6c^29hat7%8GLO9Hxs(8iXw$3`SRUzRcOEnaB%7;pqkARnxdN8jf&hR5GG;jiz zcK)i^k$v9pH4s#*z=>C0(IiJc|&Y-;;He(@S=TFU0myW}{imYT5YK7PHW zyrB-;SScT!AzB#@YMta$@W}HmdOg87{$Qfe+vh%%OOkhSoN6jBpz=Es)~-gfxu{L~ zK;1G{Dn1`d+uYCnDzT=fWSTvmDPu93vtzXRFm9@RbkZ#P_CM~IpE0{b(X>$1@tKp1 z2IipQ=$WL%`OJq8{leO|rxg}Jt=50v2dKhd;&WDCK462b6xi1}jf48y?e~)>}p$OI@k$8DSWbkv; zHC#lUUb>!9r;rg4CDLzcCL7)Plo%Vu9s;EX`gt&ASTeMrj8mf+$^OynQLQ!YaJN`C z^Ox&7hdg&`(2B!rh>1Bft_UU(jDF=mZbJseTX^xz(=KZmXM|H=i#7Y-6Eg zgZz9wh3n5!U@YJMD5$hxlpxD}KHKS1ZT#~0H%h~E=jAM1TSwU%z*p^=SNFiTR3JVV zz1*@tVD{|O1K#MQD6p?~m8GKL9(LB$%-Ab*Un|LwMfx*rs>q>OGA{{f5ya}HEMPUyRROKtxfGUTBbdN zKdMT5CVxH1(Cw$)AX^Aa3W+^*5ueJw~{HRzuy&Dpm{{0UgZAi;sLmbdA=n{uj|PRakUH zgtw>!a>8q)kt9jwh0@|Y$xwp>rtghYtNzUfKWBcac(tuGiZ%Jcqooo4$e%hm*r~Ek zfl)qNBD2fQfA$@O3VYM*n(i<@u+kjANTSvc==Qh-8Tm&`#U!?$xqrvmSZSgu@0p9X zLXhEpB4Qh=XZOzc&#Pi&1s#Qu=x<u5 ztNPyQlZfPq$W!*Nle(*G8?{WCUW)N7(f$0ecf?A*qGn;&0#qg)^Yn~zIoY9Zt{z5e zOs&q}_(T45nu?4C{8P*w3f}f+?|$v2k;}O%=Bl8K#hS**Di0{8sGV8IF$=PBqRmF^ zMl3SZYU&c)ES?!@#d=AXU_`J0U<<9V9IvyfTI$2zJqq2Z)|W{ z@OybfI^|middhW?SCB__X_j0N4KSA{&U)vDy;h7YwyWd&AUWa-A8<39bTsg z!fMak50c7H#@`Jkx64VCTj(iCyF|Yd+Blv+vMT>9!VD^jQ$6bfHUFsgGh$a6MXD_f z6VsDN?=HoWY0UjJAUNHM5%~?WZqd@-Lh4V_ef z>H;@fvz_fgQrxjyX~u^Jmg=RTusKC~HK`HK6(JOUPHHJ+&B%E9DDnVxrx%Ux86B-H zTTmYb(kr&d9+>?jLMkkmL5TU!eP z$szK>srs+o{O-CXtb=U`9eF;vUb;e;wYcDY=V^o{>S=-RBeuhgoLeIEK|l((CV~PV z{zQ}*t6_M6m=bw%NUaI|?c+9%ryhfsr;5+n@iPHQpH|;HIKw2#g4&iGW?pKQhp-wp zS6JIV9MX-%h)8k$v@s14D!P~R}84sZ-ui0 zgW0Tjl8)yvHG0QkfF;p~h+ax6p2RepT76QCfmhF+S zEH>PqLFfbKhgOCibBog7NWzbWPq-yfl)2+8pIGua325$l^EL2$l~@MSD=aRaMQM-* zIlg=A^xI#$+YiH7?aN@`>geWVQ|Nj}O)9`}wkhgc0kZHN&QruJte?|x^n|Gwfu7@} z-!HBZ3*5r`6brr=t#gA7nOIy32Hr8G+|zRU6ZzaB9^L+1t?Gb@LdXO{o9&?s6LiSNv=_4V|o ztQjBo{nsi58tUl;C3r^!O=hVxM2F^D=!(!GG_Kcb+KBztk8WaqPK)GtMs$)!GJ5w>C*?!hgDD;Ljidbd{){=! zU51X=*+woeuLo9E&VWdz(?xEAomSw)QUX>+Ip#&d@$4B-X|c@F?sh`!BMDo^ahtZm zpd*rLoT0n7ch-#GgoJQ`sQbSb0NcjD$~ji@P+Yf{a8YcK1`9y0y>(8Ml$2Vi!QaIk zpJ5g5w^Olf6&0rgLa}4fOZhqdi+gozAhOlCKY#v>C=fT!q=DGEpoNAbhG%WdHCQ*v zw`mSV&?|r&i4Y9R0*%>r4XCzsjPsDKcoN%-6THqA-rcmP2l@+lRHG&ulBZPIXI)@( z&Izub9zYSt^&b+u@9T2J<;$+sc5TK$ARN=&^REI)3WZ zJ+@*wn-{oNAQB0yI}%iz|CRQ0I;X(J)!o~8O?9n_|4 zc3Gm?A7exD^qRB%rL_UeE^%(+5#mDi|9tWWx1PAlI;K=%WyoQxq1~WHNmxYvR&hQ9 zdTnc9g?X657@zm~o5v{6?tC#^`^wc6D17s*+ZccMMSp!P`Yd5`TK$_>=Ajvv2|!3X zLX*>Gs&GfJ)Ap0W&S{mD5PILlVwpF^*C@iml0$G*LRjckFIVmJTG#k<(!#t~8$3XJ>_k3*bAt)&+^OK{qsLZswvLvfW0#o5HPUgh@ zN{LiAjA>g{U>%-a7;KH6pZjsuOK*)|F_PVikr>z@+ol)rQX+HFH>`Te%PT74Yb@oy z@3LZ5RJ4R`(zTNjqT@E$Y#LZar#R!DZOUEWcRs_8ayY{5B>63u&`k8UuReW24pQ`4 z^9#nVX!X^Pu#f)KzEqE}r87Bwd9PONbx`hR_M|44D!q9zXEMQdY|Bq=^t!0 zCS_?D3fPLzQ#2zgbAy>fu2kJsS}2;ibHZyt2vv1+~P-5%=Bw2ptB6zEZ%(YV#EDRs6tkfDX z+i0f9g{OI#h0iI5^u!ZGex1qjr}QEorKp~Cu-MHIv;<;?9x@m30`&hy3g z^S&>tRwsyanb**`vTs{(=e@0L?St>VDS}Koq*yfP5 zVhp@1KfLJIUXFFH&v}nJ_B(~Gr^{_oQeRS7%gS9kY59sm94OzDvKOSF1zQ5dAmVGvonYfi zPFC)9TgS>k_?INSB5*D4>sMz9Y&9lGyb%BA_$iM2xa>T?0i%|>e+G19 z6rnmx$+cSg`f;qx!CX5BIZszw%$y6XQl0UupBFveK z+x{&2mI^yt&Z*CTK?N5$&o=w%>CSgN@u!v3mo_=AxLH(SNaz1%e{4M{iy_$e7Toke zVa2Md2s3JyEL)eP%y^iw9E6a1%%DgsX5bwSSah?)1qptyXsQvCGKbHSbwWEf%mwS{ zo>9Lep|+ZPq%^-Iv%AqsFRIXp&iCl#v78P+&jbkrMGr#rQX_9Jlp9Qw!$ZMsjR$TK z*W8wf1&8wX1=~vfDZVn^9}#VmdnODBUAD4rroBtVx6LS(6#Dhyk`CPFV}!z#<7Cp;Gitfqy8xY(hUn$y;Ovi9s_ z6Ytbs>ZpG5IqhCbTj32LT^8%e? zJR2%!j)2_BA7;7h_<$2-^CDIe-?onUd62}cs-iF*K*RN@oaHYpjLSo;3_S0~moYk&Uu_eoa)F5a8Y=G%P#q_Xcy1`*)>`Jtc(LE(q z<|5tN?flsDr?}YDAbVcA(G~NhbT(pRI$Xm^!6_p7a2IHgk1w>x(jSqA7lMyllgFz; z)mEdYNNO@WWxdRWcew!1-D4F9!{e<6aey_UC-%>xa`|#Mr1UQ71x^h8ceKDulr79$ zRbSG|=|4Qc{mTUMP?usWf!?XYVv??n>LGUTD)p6U^?B2TjaeD9EIj9CeXI)<-RF#3 zs%Aa+n)uiy_#babPm;Ed91tnu9|$Tr8FC0z36^OKd4-8S20gNq#;`$@ANlZ_LLpzep5xo8V`s8#uwFH1~^ zPCe86Rs&y1zZuiudj51tP_j~ySw)kChlPDgX&-Out^OY9`}(6ey5|RI@%V#>0_4;F zw|40IlUO%HkSzK=^L<_FrJxCk?hlGeyG_xRKBBTeXrje3i>Pr&JJ&s)iLCj-_MmRb z_ySLRycaN-97?jc%7C?&<}r#6MeOk-GWgHzk95kjiBmDktsXt6UVP- zaWe^n8V6o0DrY97Y;4vs#==U<{mk#yXFBuKQ$l_iqt2(>RddqrJ>sVG%Vnxnmwzf8m4tOpN4dL_ znMu%XB09~};i))3Q5qNp{MnI3pR2+PQx5=}eL|5ClOVD;b9rTVVpQS}8XA-7c@HW2 zJTT`;RAvDOt|~;He3a)`t6o64>F28>E_)!El?q0g;VxgwT48>Ou?i}|D{K1bQxL*> zC6*di78`($LD!=?9*i4~o)MxT=rkmVrcu=^aQTNFI{ z&<(nFn%3lYzTE=fz(t+A4t)$w0Zu5v2z3Q;PpRaOTVvd-r{N8`8*;#;H^N6H8zv(& zT0CHP{j`gGkV!8`BM0flq^Gq0t0?!i{jswusBlC}4UmjIImE1G>%Ic73@=Eu4h@$h;H5;r>xhrylzYT70KG}T^Kf$NX4@{Bt zQCvr{1?^x0|04(FYTp5CY`F%R313mz>S_tdSXiF}VI6$zZM=dGX{Mx@_q4CXQN9hb zcF{MrPe01UAZoPVD=2Nx5m2NV=&#ufNXf#9VobkZ+ zNGf>qV%xyEPCS^iTiMXUZY+J{ca{K$^n&9pX{t!pC6WP@Ite!>a=oqvw=R7LK!}hO z>>I&N)+m+4Q?uJ8A ziBs$LSj76mHg=(Wox;nMZz&Sk5_T6laWCI$%^RErT#Ws8co=a%`X=T3;ePfI1Dqw~ zbTh9X{)PE*@PxbuP3P-!eothC6&20ia0L`OyyP+V^4>9v7PN2+c7_n^mfV}7hrObE zY)sZG+YkWIjia4_ZV(ehC6Y15S=MB5*QWH{7dDrQ(g{;K6@af12t^|U8^Af zUTNp+$vQ@gx5vwk=5-}Zui#$B#o>47)6(H*v#0|pQuzAOZ(nZ`P~p=5MTHTV4rFQO z)9^NY*Kjdu0fF3{E;S-rI1`*Y$a6a2D*IA_pKmu#TW}$mLzV9L{Z|)C}mj}^U?svYw#mT2S_tH zAJ;WV2*dl~2hU$ha`8V@*AF`5cIhhX4XwUYoC*IuFdF1q+4w2L=1jbXT4{(1T%}+R zA8wDuWc=hQmU-`Q%q!4cR`dAHO0HxB0%D8Df@aguSNEEv8BX7CV7F zYG%5K9Qr#9*~TcQE6%*)57&Wevx7SS8WQzgHN$0-tW`3ka;qe#V9RMZ-K_v{%feS% zF)8SGuZW19MM_*{!nucb8ev3$N7MWuJKd(icL0P}ns)N?PUPaSr+SOjT1Tbz z$D6sR9~(vkLs;RU>&D_WVjabW;-xlJxV;?(A0U^S#E#No2{`dc)iG{DVsg#j%nvmA z2hy{_-(w#xmA`e(uk|=(iuA1c&C1;mD;?G!Ma(y|UJI7F@gkowV0d7IiJ}M(?i$#x zuu|BTUvtHz`55=2eD)+Imd)(QxvgJMqCgjW)WB_z5U)AO&We?CMCJBWel8WL&5>U4 z=T8q}c(7AcD}3XtT6|uST@TAKYhiXSG7}}$me_r6JpI-$s}_ztguQ7st-+&H^p}h) zg&V7Ki%|7Fn$I}2esu+*kAFzHZ{TGc>=$3qSDN4$#KUfc-@;BSoM#T(;x^LuYLQjY zuJnFm*pb=0Azbb`eg?Vw6jxT_qF+}D=#u)&-50B#&qq?|BQ0Ml?wYSTb{u~p^VIMx!QuWl~w|e{zi zLmr<84O6(2s-W5R#@`21e$^E9B0N)5ZKqPTdZMhWgVRy@sH-fAYbY}v%+hh-eS8-0 z`mP-ux4)J`I?voc+jr?z&PE9W+t*k^)NB?8@*h6UB4&9IO=7dnKuQ!65(P>wONKRt z;}=`t*;~{gpz=c`za=&xW+z(Z1h0icO`7oVYhfCE+OGF@b5PM zOUUSRzqA2Noe?UH1%7d=`_G&1O&VpfMevJK^VlP9%@ox+?#&0$PqZEt93J=S3@V+d z&Z_+Sei;HGpL<1!P(mPFAoV_o^}0($%uiAL`pZdJn{Uy-FXS{HZ-18^AuEqyD;DjToSUlxL`!~YW7mhyFa*s zlau9YFDfhMeSkYp{+BMc84|0jM3o<#`Ri(JJGd|fZqn@6-(oQM6cb0dOz`wE-N6g; zLnIJm2_$7qnm`HseGv?<_x}foK=jq;iN`on6mY;m{Eo42}%)J1*gtQPwM~82lGatr&ItN(&9&=E0utz!>lNP1 z`7~o(ur?j>Kyi_%lcemtxBKW1&t~49c3T|sG|W!TMNY#{^|60m1-5da;OA#)-Gy~^ z;bEqQVb98XyT*=D&NLiK4@49OnNr&+nJ~%8u7&X=)x-yH*OHxaZcz<~wn_3awa2ku z`yNLJ)ARb(5+g;K)Kj*u^oHBBJ|C*q^zr{ffsFa8=lgz zS&pLPal)^m&e8(hB+{b$>c8u>4lAIZq=X;YfoQQ2p_G9??cLFl!}$#5x;o`CMa>#| zDFZ`khoXS5|&XpWFCzq;T+#qQQDS1>(R>*8q$k>?V=d^dK*c!^KqY%F7TfXOB?g_0+`yuIG|1Idvub#BnK#ZUr z1|#8dF5V$RLhXX_8)!VV(Me7=ZCxABk!breV-v8K1Df(4dd2T86I3h7o69>=pvV3XAHOQdhwkR7i)73ePSM%@>HKp>bX(rQ zSZi1DC5RHLuN@(x>LSiZceZw0rmXg;{SpE3{fQo?snbr+i&zUbash*Q z<8N8P@^m(+PIvYsa1?Kw=djIj0geSRQZRpTzTa1j@CYa#tHyx8crwTXrVkk^NF*za zr>5DvNw_|-%^qfdWPen?o8Mwg-n&Z&*iXUrEyCr2p#si$W8YNw`0C0$SIIsSYmu3&xdmIYy=KML_Tr#(NN=m^a6@zxT z-yZU;=DcZYuL}HcqMZ%g>uXN>!v!39G3G@202(EzA0KQLm6Q$YUd4UX7W9=o%R1U! z*2xrbzgT_AT>}0!CuzBrzxyJ*xIqX(vLBZS1Fzram1u=YbY5{JEFpWxQQH~#<8a>< zFFrtg=-PT!;zb9;xLYu1cZyN-HJL_7UTc{9UcGM=EjQ{12zZ*@{=5DVCt!8-_G#Y+ zf6Al_p`Pa|4(2aASu4g@cV<-s5YC8Du7IWL2HgFTfxTx~I#gg|2exxCY*wgOEkU?d zFd|2Ql}WqAovpEoe+{bz$IZE%dcYWMhuPLQZJV+LVdihxVhxrzqSr?0W>DM&_t}Q6 z{piq^W_Hj6XQJ(>arfb~Ln_*~J=+cwUREnCuPcwTi5W0@TB6Q&Ek0re{2T6^lHAVAn^DB&ReQ>?W$Zv$Gny|jPZAzd+sz>J`} zq=b00NRCZ62o#rG^m>_574DEMPFA8##_?I0%7Ng|RZ?7!Q4yL9pEB zae^}#GIhnSAflhAOYU%RajD)RQ2PSHo(`%59X!^>S&N#N$!Ak9Rq0Go!C4F$dG zYdtm}WFMcM7T!}|ras2W@rjT8{&WSRdlis*iTh@_REh#A%6s*5 z5Cb_D>v5$KP6g;HzyHv0Z}K?fm|jIPjUv}zCjEgR!6GNrz3MtW82FoJMqUaKs8)lg0Aq^ zZsNJQ6>--pkTy;I%6e->R7mB6uExb!|t>&Y-QXjt>{*y}sPw`GuVUA_*P zVFWZiqdod?dP8#ZY{Mpc)%EZSQHvg>$evE4e229*#NR{Uj?AyD^5s(mB0_(E@?dr- zVGDj!7y?NkWY1Oi4p~>jEA~$Zcm*(7;UXkpd1j@$26|$=T+uR3L5wUQZ)CY{b8#;e zVN5w*uvL}jcVElh3GpdBGL}7@$2VrMa--Yb9F_h`2^RJ9!|p%220V2H1{DhD#Py2BqYLHP4Q#L`x!3zr!E*&ntk|zw!ja- z{e)f`9s(v?q;DvATrLI|^OM9iojM<2`l7~Njfc8K+I=Mv6>RB6Yi#+v@vt>mZntbS zqTl2^=RVJR?~v4lhS0*8v0^O zjw`~8){K36dN4ZbOX5E}883O%f06 zx(#zbkd0tb)r!0B2vo(G^d~Hn?yl9-j+6?6m9xw|69A<7UCTDDnCsqXw+MK2*?Y9#~V@Jge>12mA zpv7F;0I2^y>Cz%`qas4~7*!8UI8e_rL7EE*oYFJW=xQt+b^vF&o-H+0T(1o{?S%Q0 zE(5atmzjM}*;#A3SzB_F_f%{9bmSh&%}N4>0^B>NKy0pYW|FE7{>(|blVDhHIBjgN zzIo8{I}5hlf7h=|qj<$Ogk{{XdIr333mUB7pQDSOdC&?~XUyH6%JcSV_jo zl7VLn$oBf~wBSpS?cVs>m9FFdexP~)rIwNai#Wv>zjS++F0N&j2ojV*j;MIb9#LIF zPZ1D~B*bb9{tbQH6(N67!l^|5VoZrF123Z%pbnsTUBEO>*OWh>Ai@9w-2)`pe2X-i zyI9N;aBT1l5ey&`vi?Pjc6)vx(}3fdphlGZ^CbQTqrl&32_(O9jK{3%b z&83az5dJ%aa1XU+XH1;u(}ZE--hS;1hg8EgKXnb#gzPX89zGOMC4sd62oyV@#R`sz zb41j!W(;h~2{>VLIYX^icS>GO8r#Q_kl(w9n62;~!+Y@>CXHXFNHSQq!rP@oIhDMSDqy#5;7^Zb;74q-Dh~~yM(zyQ* z2%&3|;6TcV>$enmh)Mc4dmUjHVr@T z$={h_S`H?96$f`cN2TE3o+VhiR!B^w_?FeISG(LOFVfrR`i3aaq_K0`hh`+H?MoM%E7#!b zcDBZr!a-K{k42nN9@axN^#Hx*CHdqqPu*(Ag=8LlE^$4*yo{tBW2y>CcW`i4k(p%j z=V>%*D(v%|TBQ^{g;Tx9u1e{z)Q+06e^>~sG2j6F;%&w|FAc5SV;<}a^|6dH`ra;t z+pFxX#53Ebm8kl?o<5Y?C-I*LoA|jw51L+h4(fP^$D-j51bCgdnzrVGK!ajNMokUb zMyb3hfnXK6FY_@Gi%<8S7)gEqOZtSe9dDUbP6JN#PJ2A3dV!`k3FkjqWuD_S5l@ml zUrEi%%g;fC8@Ku?;9{4Kj+hVJ4?d+`|8m%#hGrr0s1K&eZb6(#HJ!gAyH{H7$GYVN z6GRBx@v5z2_2ZpPzsN5{6~Ck6yx0*Tm3#Ypcl3OHESy@(2Nvbh+viZd=;)^#j!4BYhlXf6jJB_KXG(k=E-U z)@_`?eB#dO<-Bnb9p&W*|D!*0Za5b`WG1QSQ_Jd+XOas{M9{56BXj4eY+7ov@4%iP z#bS5Wn+S$QO4u#zr-|$>OJgTGTG~vX`cs^It%qYxB2)X$y1mpwj^pkEZF@tZ z)vDv}zzmq*Ou^xIdbiYH8PHq-L0elqZ*X#$pv#xl7cwnE&~Ll_cAp9^F4s4E6I67e z9|}mgz$C4d0Znfzv>l6qzKTDd4M5k6W#A5=%}4dPbGI}56IEA!AEN`Av;!fj)vB$@ ztt3+yo1Se?=a)pak)%(m+I@7UK85KzDwxxKI@}o`9D0chkZv<=lnMGcPuKiuV5GsO zFi%2#op!BZPP@xQC=mc?xrg`q_*GLsk4s&R&c;dO=FcKyug~+Sv9_jo++stRhR{aW z4*w-ME(1LO9f(O!rI%5~#;ER49rr+v@NLHW2DT>C&3XNE2e z5TgW)+%&NWV|o$NAd4N-v15MF4#sLsjC+lQhlYEo-9S{xGV${NJzx@o8c}T&EBL95 zQ}f+Dk;Qi-enQ@*p8XA;JB$u0g4lpg&}HQ)g)!b*mUzpcfxrp;Nj?zR^jcglo#g1L zi}tWvGF@7maVma%d1WBm#;i=>Y_Rd%Su@kJCUm;=ub3Z~1>YYr9<-%#Zu;()NEnG1 zj=eFmkKFW7eOB8a7~Z`e{Of-R%|g&M))sew!j1oLUc1GL;9z*|=BtZgK#N*5*6cmr zVw$7>kq8`hglA@w=-y5cBP0F^BmVBabIP`hYD?fBP8j}P-gYV3pHevZUFzz<8@4H% Pgcp?L)MSgEn|$~`&9t4_ literal 0 HcmV?d00001 diff --git a/mt_with_external_memory/infer.py b/mt_with_external_memory/infer.py new file mode 100644 index 00000000..d519ea8c --- /dev/null +++ b/mt_with_external_memory/infer.py @@ -0,0 +1,154 @@ +""" + Contains infering script for machine translation with external memory. +""" +import distutils.util +import argparse +import gzip +import paddle.v2 as paddle +from external_memory import ExternalMemory +from model import * +from data_utils import * + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--dict_size", + default=30000, + type=int, + help="Vocabulary size. (default: %(default)s)") +parser.add_argument( + "--word_vec_dim", + default=512, + type=int, + help="Word embedding size. (default: %(default)s)") +parser.add_argument( + "--hidden_size", + default=1024, + type=int, + help="Hidden cell number in RNN. (default: %(default)s)") +parser.add_argument( + "--memory_slot_num", + default=8, + type=int, + help="External memory slot number. (default: %(default)s)") +parser.add_argument( + "--beam_size", + default=3, + type=int, + help="Beam search width. (default: %(default)s)") +parser.add_argument( + "--use_gpu", + default=False, + type=distutils.util.strtobool, + help="Use gpu or not. (default: %(default)s)") +parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Trainer number. (default: %(default)s)") +parser.add_argument( + "--batch_size", + default=5, + type=int, + help="Batch size. (default: %(default)s)") +parser.add_argument( + "--infer_data_num", + default=3, + type=int, + help="Instance num to infer. (default: %(default)s)") +parser.add_argument( + "--model_filepath", + default="checkpoints/params.latest.tar.gz", + type=str, + help="Model filepath. (default: %(default)s)") +parser.add_argument( + "--memory_perturb_stddev", + default=0.1, + type=float, + help="Memory perturb stddev for memory initialization." + "(default: %(default)s)") +args = parser.parse_args() + + +def parse_beam_search_result(beam_result, dictionary): + """ + Beam search result parser. + """ + sentence_list = [] + sentence = [] + for word in beam_result[1]: + if word != -1: + sentence.append(word) + else: + sentence_list.append( + ' '.join([dictionary.get(word) for word in sentence[1:]])) + sentence = [] + beam_probs = beam_result[0] + beam_size = len(beam_probs[0]) + beam_sentences = [ + sentence_list[i:i + beam_size] + for i in range(0, len(sentence_list), beam_size) + ] + return beam_probs, beam_sentences + + +def infer(): + """ + For inferencing. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + beam_gen = memory_enhanced_seq2seq( + encoder_input=source_words, + decoder_input=None, + decoder_target=None, + hidden_size=args.hidden_size, + word_vec_dim=args.word_vec_dim, + dict_size=args.dict_size, + is_generating=True, + beam_size=args.beam_size) + + # load parameters + parameters = paddle.parameters.Parameters.from_tar( + gzip.open(args.model_filepath)) + + # prepare infer data + infer_data = [] + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [[ + random.gauss(0, memory_perturb_stddev) for i in xrange(args.hidden_size) + ] for j in xrange(args.memory_slot_num)] + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(dict_size), + append_tuple=(bounded_memory_perturbation, )) + for i, item in enumerate(test_append_reader()): + if i < args.infer_data_num: + infer_data.append((item[0], item[3], )) + + # run inference + beam_result = paddle.infer( + output_layer=beam_gen, + parameters=parameters, + input=infer_data, + field=['prob', 'id']) + + # parse beam result and print + source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) + beam_probs, beam_sentences = parse_beam_search_result(beam_result, + target_dict) + for i in xrange(args.infer_data_num): + print "\n***************************************************\n" + print "src:", ' '.join( + [source_dict.get(word) for word in infer_data[i][0]]), "\n" + for j in xrange(args.beam_size): + print "prob = %f : %s" % (beam_probs[i][j], beam_sentences[i][j]) + + +def main(): + paddle.init(use_gpu=False, trainer_count=1) + infer() + + +if __name__ == '__main__': + main() diff --git a/mt_with_external_memory/model.py b/mt_with_external_memory/model.py new file mode 100644 index 00000000..6ac00d1d --- /dev/null +++ b/mt_with_external_memory/model.py @@ -0,0 +1,206 @@ +""" + Contains model configuration for external-memory-enhanced seq2seq. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. vanilla attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories are exploited to enhance the vanilla + Seq2Seq neural machine translation. + + The implementation primarily follows the paper + `Memory-enhanced Decoder for Neural Machine Translation + `_, + with some minor differences (will be listed in README.md). + + For details about "external memory", please also refer to + `Neural Turing Machines `_. +""" +import paddle.v2 as paddle +from external_memory import ExternalMemory + + +def bidirectional_gru_encoder(input, size, word_vec_dim): + """ + Bidirectional GRU encoder. + """ + # token embedding + embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) + # token-level forward and backard encoding for attentions + forward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=False) + backward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=True) + forward_backward = paddle.layer.concat(input=[forward, backward]) + # sequence-level encoding + backward_first = paddle.layer.first_seq(input=backward) + return forward_backward, backward_first + + +def memory_enhanced_decoder(input, target, initial_state, source_context, size, + word_vec_dim, dict_size, is_generating, beam_size): + """ + GRU sequence decoder enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this enhanced RNN decoder. + + The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the + hidden state vector (or cell state in LSTM) carrying information through + a span of sequence time, which is a successful design enriching the model + with the capability to "remember" things in the long run. However, such a + vector state is somewhat limited to a very narrow memory bandwidth. External + memory introduced here could easily increase the memory capacity with linear + complexity cost (rather than quadratic for vector state). + + This enhanced decoder expands its "memory passage" through two + ExternalMemory objects: + - Bounded memory for handling long-term information exchange within decoder + itself. A direct expansion of traditional "vector" state. + - Unbounded memory for handling source language's token-wise information. + Exactly the attention mechanism over Seq2Seq. + + Notice that we take the attention mechanism as a particular form of external + memory, with read-only memory bank initialized with encoder states, and a + read head with content-based addressing (attention). From this view point, + we arrive at a better understanding of attention mechanism itself and other + external memory, and a concise and unified implementation for them. + + For more details about external memory, please refer to + `Neural Turing Machines `_. + + For more details about this memory-enhanced decoder, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. This implementation is highly + correlated to this paper, but with minor differences (e.g. put "write" + before "read" to bypass a potential bug in V2 APIs. See + (`issue `_). + """ + # prepare initial bounded and unbounded memory + bounded_memory_slot_init = paddle.layer.fc( + input=paddle.layer.pooling( + input=source_context, pooling_type=paddle.pooling.Avg()), + size=size, + act=paddle.activation.Sigmoid()) + bounded_memory_perturbation = paddle.layer.data( + name='bounded_memory_perturbation', + type=paddle.data_type.dense_vector_sequence(size)) + bounded_memory_init = paddle.layer.addto( + input=[ + paddle.layer.expand( + input=bounded_memory_slot_init, + expand_as=bounded_memory_perturbation), + bounded_memory_perturbation + ], + act=paddle.activation.Linear()) + unbounded_memory_init = source_context + + # prepare step function for reccurent group + def recurrent_decoder_step(cur_embedding): + # create hidden state, bounded and unbounded memory. + state = paddle.layer.memory( + name="gru_decoder", size=size, boot_layer=initial_state) + bounded_memory = ExternalMemory( + name="bounded_memory", + mem_slot_size=size, + boot_layer=bounded_memory_init, + readonly=False, + enable_interpolation=True) + unbounded_memory = ExternalMemory( + name="unbounded_memory", + mem_slot_size=size * 2, + boot_layer=unbounded_memory_init, + readonly=True, + enable_interpolation=False) + # write bounded memory + bounded_memory.write(state) + # read bounded memory + bounded_memory_read = bounded_memory.read(state) + # prepare key for unbounded memory + key_for_unbounded_memory = paddle.layer.fc( + input=[bounded_memory_read, cur_embedding], + size=size, + act=paddle.activation.Tanh(), + bias_attr=False) + # read unbounded memory (i.e. attention mechanism) + context = unbounded_memory.read(key_for_unbounded_memory) + # gated recurrent unit + gru_inputs = paddle.layer.fc( + input=[context, cur_embedding, bounded_memory_read], + size=size * 3, + act=paddle.activation.Linear(), + bias_attr=False) + gru_output = paddle.layer.gru_step( + name="gru_decoder", input=gru_inputs, output_mem=state, size=size) + # step output + return paddle.layer.fc( + input=[gru_output, context, cur_embedding], + size=dict_size, + act=paddle.activation.Softmax(), + bias_attr=True) + + if not is_generating: + target_embeddings = paddle.layer.embedding( + input=input, + size=word_vec_dim, + param_attr=paddle.attr.ParamAttr(name="_decoder_word_embedding")) + decoder_result = paddle.layer.recurrent_group( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings]) + cost = paddle.layer.classification_cost( + input=decoder_result, label=target) + return cost + else: + target_embeddings = paddle.layer.GeneratedInputV2( + size=dict_size, + embedding_name="_decoder_word_embedding", + embedding_size=word_vec_dim) + beam_gen = paddle.layer.beam_search( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings], + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=100) + return beam_gen + + +def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, + hidden_size, word_vec_dim, dict_size, is_generating, + beam_size): + """ + Seq2Seq Model enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this Seq2Seq model. + + Please refer to the function comments of memory_enhanced_decoder(...). + + For more details about external memory, please refer to + `Neural Turing Machines `_. + + For more details about this memory-enhanced Seq2Seq, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. + """ + # encoder + context_encodings, sequence_encoding = bidirectional_gru_encoder( + input=encoder_input, size=hidden_size, word_vec_dim=word_vec_dim) + # decoder + return memory_enhanced_decoder( + input=decoder_input, + target=decoder_target, + initial_state=sequence_encoding, + source_context=context_encodings, + size=hidden_size, + word_vec_dim=word_vec_dim, + dict_size=dict_size, + is_generating=is_generating, + beam_size=beam_size) diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py deleted file mode 100644 index 22f96a72..00000000 --- a/mt_with_external_memory/mt_with_external_memory.py +++ /dev/null @@ -1,598 +0,0 @@ -""" - This python script is an example model configuration for neural machine - translation with external memory, based on PaddlePaddle V2 APIs. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. vanilla attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories are exploited to enhance the vanilla - Seq2Seq neural machine translation. - - The implementation primarily follows the paper - `Memory-enhanced Decoder for Neural Machine Translation - `_, - with some minor differences (will be listed in README.md). - - For details about "external memory", please also refer to - `Neural Turing Machines `_. -""" - -import paddle.v2 as paddle -import sys -import gzip -import random - -dict_size = 30000 -word_vec_dim = 512 -hidden_size = 1024 -batch_size = 5 -memory_slot_num = 8 -beam_size = 3 -infer_data_num = 3 -memory_perturb_stddev = 0.1 - - -class ExternalMemory(object): - """ - External neural memory class. - - A simplified Neural Turing Machines (NTM) with only content-based - addressing (including content addressing and interpolation, but excluding - convolutional shift and sharpening). It serves as an external differential - memory bank, with differential write/read head controllers to store - and read information dynamically as needed. Simple feedforward networks are - used as the write/read head controllers. - - The ExternalMemory class could be utilized by many neural network structures - to easily expand their memory bandwidth and accomplish a long-term memory - handling. Besides, some existing mechanism can be realized directly with - the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an - unbounded external memory). - - Besides, the ExternalMemory class must be used together with - paddle.layer.recurrent_group (within its step function). It can never be - used in a standalone manner. - - For more details, please refer to - `Neural Turing Machines `_. - - :param name: Memory name. - :type name: basestring - :param mem_slot_size: Size of memory slot/vector. - :type mem_slot_size: int - :param boot_layer: Boot layer for initializing the external memory. The - sequence layer has sequence length indicating the number - of memory slots, and size as memory slot size. - :type boot_layer: LayerOutput - :param readonly: If true, the memory is read-only, and write function cannot - be called. Default is false. - :type readonly: bool - :param enable_interpolation: If set true, the read/write addressing weights - will be interpolated with the weights in the - last step, with the affine coefficients being - a learnable gate function. - :type enable_interpolation: bool - """ - - def __init__(self, - name, - mem_slot_size, - boot_layer, - readonly=False, - enable_interpolation=True): - self.name = name - self.mem_slot_size = mem_slot_size - self.readonly = readonly - self.enable_interpolation = enable_interpolation - self.external_memory = paddle.layer.memory( - name=self.name, - size=self.mem_slot_size, - is_seq=True, - boot_layer=boot_layer) - # prepare a constant (zero) intializer for addressing weights - self.zero_addressing_init = paddle.layer.slope_intercept( - input=paddle.layer.fc(input=boot_layer, size=1), - slope=0.0, - intercept=0.0) - # set memory to constant when readonly=True - if self.readonly: - self.updated_external_memory = paddle.layer.mixed( - name=self.name, - input=[ - paddle.layer.identity_projection(input=self.external_memory) - ], - size=self.mem_slot_size) - - def __content_addressing__(self, key_vector): - """ - Get write/read head's addressing weights via content-based addressing. - """ - # content-based addressing: a=tanh(W*M + U*key) - key_projection = paddle.layer.fc( - input=key_vector, - size=self.mem_slot_size, - act=paddle.activation.Linear(), - bias_attr=False) - key_proj_expanded = paddle.layer.expand( - input=key_projection, expand_as=self.external_memory) - memory_projection = paddle.layer.fc( - input=self.external_memory, - size=self.mem_slot_size, - act=paddle.activation.Linear(), - bias_attr=False) - merged_projection = paddle.layer.addto( - input=[key_proj_expanded, memory_projection], - act=paddle.activation.Tanh()) - # softmax addressing weight: w=softmax(v^T a) - addressing_weight = paddle.layer.fc( - input=merged_projection, - size=1, - act=paddle.activation.SequenceSoftmax(), - bias_attr=False) - return addressing_weight - - def __interpolation__(self, head_name, key_vector, addressing_weight): - """ - Interpolate between previous and current addressing weights. - """ - # prepare interpolation scalar gate: g=sigmoid(W*key) - gate = paddle.layer.fc( - input=key_vector, - size=1, - act=paddle.activation.Sigmoid(), - bias_attr=False) - # interpolation: w_t = g*w_t+(1-g)*w_{t-1} - last_addressing_weight = paddle.layer.memory( - name=self.name + "_addressing_weight_" + head_name, - size=1, - is_seq=True, - boot_layer=self.zero_addressing_init) - interpolated_weight = paddle.layer.interpolation( - name=self.name + "_addressing_weight_" + head_name, - input=[addressing_weight, addressing_weight], - weight=paddle.layer.expand(input=gate, expand_as=addressing_weight)) - return interpolated_weight - - def __get_addressing_weight__(self, head_name, key_vector): - """ - Get final addressing weights for read/write heads, including content - addressing and interpolation. - """ - # current content-based addressing - addressing_weight = self.__content_addressing__(key_vector) - # interpolation with previous addresing weight - if self.enable_interpolation: - return self.__interpolation__(head_name, key_vector, - addressing_weight) - else: - return addressing_weight - - def write(self, write_key): - """ - Write onto the external memory. - It cannot be called if "readonly" set True. - - :param write_key: Key vector for write heads to generate writing - content and addressing signals. - :type write_key: LayerOutput - """ - # check readonly - if self.readonly: - raise ValueError("ExternalMemory with readonly=True cannot write.") - # get addressing weight for write head - write_weight = self.__get_addressing_weight__("write_head", write_key) - # prepare add_vector and erase_vector - erase_vector = paddle.layer.fc( - input=write_key, - size=self.mem_slot_size, - act=paddle.activation.Sigmoid(), - bias_attr=False) - add_vector = paddle.layer.fc( - input=write_key, - size=self.mem_slot_size, - act=paddle.activation.Sigmoid(), - bias_attr=False) - erase_vector_expand = paddle.layer.expand( - input=erase_vector, expand_as=self.external_memory) - add_vector_expand = paddle.layer.expand( - input=add_vector, expand_as=self.external_memory) - # prepare scaled add part and erase part - scaled_erase_vector_expand = paddle.layer.scaling( - weight=write_weight, input=erase_vector_expand) - erase_memory_part = paddle.layer.mixed( - input=paddle.layer.dotmul_operator( - a=self.external_memory, - b=scaled_erase_vector_expand, - scale=-1.0)) - add_memory_part = paddle.layer.scaling( - weight=write_weight, input=add_vector_expand) - # update external memory - self.updated_external_memory = paddle.layer.addto( - input=[self.external_memory, add_memory_part, erase_memory_part], - name=self.name) - - def read(self, read_key): - """ - Read from the external memory. - - :param write_key: Key vector for read head to generate addressing - signals. - :type write_key: LayerOutput - :return: Content (vector) read from external memory. - :rtype: LayerOutput - """ - # get addressing weight for write head - read_weight = self.__get_addressing_weight__("read_head", read_key) - # read content from external memory - scaled = paddle.layer.scaling( - weight=read_weight, input=self.updated_external_memory) - return paddle.layer.pooling( - input=scaled, pooling_type=paddle.pooling.Sum()) - - -def bidirectional_gru_encoder(input, size, word_vec_dim): - """ - Bidirectional GRU encoder. - """ - # token embedding - embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) - # token-level forward and backard encoding for attentions - forward = paddle.networks.simple_gru( - input=embeddings, size=size, reverse=False) - backward = paddle.networks.simple_gru( - input=embeddings, size=size, reverse=True) - forward_backward = paddle.layer.concat(input=[forward, backward]) - # sequence-level encoding - backward_first = paddle.layer.first_seq(input=backward) - return forward_backward, backward_first - - -def memory_enhanced_decoder(input, target, initial_state, source_context, size, - word_vec_dim, dict_size, is_generating, beam_size): - """ - GRU sequence decoder enhanced with external memory. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories can be implemented with - ExternalMemory class, and are both exploited in this enhanced RNN decoder. - - The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the - hidden state vector (or cell state in LSTM) carrying information through - a span of sequence time, which is a successful design enriching the model - with the capability to "remember" things in the long run. However, such a - vector state is somewhat limited to a very narrow memory bandwidth. External - memory introduced here could easily increase the memory capacity with linear - complexity cost (rather than quadratic for vector state). - - This enhanced decoder expands its "memory passage" through two - ExternalMemory objects: - - Bounded memory for handling long-term information exchange within decoder - itself. A direct expansion of traditional "vector" state. - - Unbounded memory for handling source language's token-wise information. - Exactly the attention mechanism over Seq2Seq. - - Notice that we take the attention mechanism as a particular form of external - memory, with read-only memory bank initialized with encoder states, and a - read head with content-based addressing (attention). From this view point, - we arrive at a better understanding of attention mechanism itself and other - external memory, and a concise and unified implementation for them. - - For more details about external memory, please refer to - `Neural Turing Machines `_. - - For more details about this memory-enhanced decoder, please - refer to `Memory-enhanced Decoder for Neural Machine Translation - `_. This implementation is highly - correlated to this paper, but with minor differences (e.g. put "write" - before "read" to bypass a potential bug in V2 APIs. See - (`issue `_). - """ - # prepare initial bounded and unbounded memory - bounded_memory_slot_init = paddle.layer.fc( - input=paddle.layer.pooling( - input=source_context, pooling_type=paddle.pooling.Avg()), - size=size, - act=paddle.activation.Sigmoid()) - bounded_memory_perturbation = paddle.layer.data( - name='bounded_memory_perturbation', - type=paddle.data_type.dense_vector_sequence(size)) - bounded_memory_init = paddle.layer.addto( - input=[ - paddle.layer.expand( - input=bounded_memory_slot_init, - expand_as=bounded_memory_perturbation), - bounded_memory_perturbation - ], - act=paddle.activation.Linear()) - unbounded_memory_init = source_context - - # prepare step function for reccurent group - def recurrent_decoder_step(cur_embedding): - # create hidden state, bounded and unbounded memory. - state = paddle.layer.memory( - name="gru_decoder", size=size, boot_layer=initial_state) - bounded_memory = ExternalMemory( - name="bounded_memory", - mem_slot_size=size, - boot_layer=bounded_memory_init, - readonly=False, - enable_interpolation=True) - unbounded_memory = ExternalMemory( - name="unbounded_memory", - mem_slot_size=size * 2, - boot_layer=unbounded_memory_init, - readonly=True, - enable_interpolation=False) - # write bounded memory - bounded_memory.write(state) - # read bounded memory - bounded_memory_read = bounded_memory.read(state) - # prepare key for unbounded memory - key_for_unbounded_memory = paddle.layer.fc( - input=[bounded_memory_read, cur_embedding], - size=size, - act=paddle.activation.Tanh(), - bias_attr=False) - # read unbounded memory (i.e. attention mechanism) - context = unbounded_memory.read(key_for_unbounded_memory) - # gated recurrent unit - gru_inputs = paddle.layer.fc( - input=[context, cur_embedding, bounded_memory_read], - size=size * 3, - act=paddle.activation.Linear(), - bias_attr=False) - gru_output = paddle.layer.gru_step( - name="gru_decoder", input=gru_inputs, output_mem=state, size=size) - # step output - return paddle.layer.fc( - input=[gru_output, context, cur_embedding], - size=dict_size, - act=paddle.activation.Softmax(), - bias_attr=True) - - if not is_generating: - target_embeddings = paddle.layer.embedding( - input=input, - size=word_vec_dim, - param_attr=paddle.attr.ParamAttr(name="_decoder_word_embedding")) - decoder_result = paddle.layer.recurrent_group( - name="decoder_group", - step=recurrent_decoder_step, - input=[target_embeddings]) - cost = paddle.layer.classification_cost( - input=decoder_result, label=target) - return cost - else: - target_embeddings = paddle.layer.GeneratedInputV2( - size=dict_size, - embedding_name="_decoder_word_embedding", - embedding_size=word_vec_dim) - beam_gen = paddle.layer.beam_search( - name="decoder_group", - step=recurrent_decoder_step, - input=[target_embeddings], - bos_id=0, - eos_id=1, - beam_size=beam_size, - max_length=100) - return beam_gen - - -def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, - hidden_size, word_vec_dim, dict_size, is_generating, - beam_size): - """ - Seq2Seq Model enhanced with external memory. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories can be implemented with - ExternalMemory class, and are both exploited in this Seq2Seq model. - - Please refer to the function comments of memory_enhanced_decoder(...). - - For more details about external memory, please refer to - `Neural Turing Machines `_. - - For more details about this memory-enhanced Seq2Seq, please - refer to `Memory-enhanced Decoder for Neural Machine Translation - `_. - """ - # encoder - context_encodings, sequence_encoding = bidirectional_gru_encoder( - input=encoder_input, size=hidden_size, word_vec_dim=word_vec_dim) - # decoder - return memory_enhanced_decoder( - input=decoder_input, - target=decoder_target, - initial_state=sequence_encoding, - source_context=context_encodings, - size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=is_generating, - beam_size=beam_size) - - -def parse_beam_search_result(beam_result, dictionary): - """ - Beam search result parser. - """ - sentence_list = [] - sentence = [] - for word in beam_result[1]: - if word != -1: - sentence.append(word) - else: - sentence_list.append( - ' '.join([dictionary.get(word) for word in sentence[1:]])) - sentence = [] - beam_probs = beam_result[0] - beam_size = len(beam_probs[0]) - beam_sentences = [ - sentence_list[i:i + beam_size] - for i in range(0, len(sentence_list), beam_size) - ] - return beam_probs, beam_sentences - - -def reader_append_wrapper(reader, append_tuple): - """ - Data reader wrapper for appending extra data to exisiting reader. - """ - - def new_reader(): - for ins in reader(): - yield ins + append_tuple - - return new_reader - - -def train(num_passes): - """ - For training. - """ - # create network config - source_words = paddle.layer.data( - name="source_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - target_words = paddle.layer.data( - name="target_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - target_next_words = paddle.layer.data( - name='target_next_words', - type=paddle.data_type.integer_value_sequence(dict_size)) - cost = memory_enhanced_seq2seq( - encoder_input=source_words, - decoder_input=target_words, - decoder_target=target_next_words, - hidden_size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=False, - beam_size=beam_size) - - # create parameters and optimizer - parameters = paddle.parameters.create(cost) - optimizer = paddle.optimizer.Adam( - learning_rate=5e-5, - gradient_clipping_threshold=5, - regularization=paddle.optimizer.L2Regularization(rate=8e-4)) - trainer = paddle.trainer.SGD( - cost=cost, parameters=parameters, update_equation=optimizer) - - # create data readers - feeding = { - "source_words": 0, - "target_words": 1, - "target_next_words": 2, - "bounded_memory_perturbation": 3 - } - random.seed(0) # for keeping consitancy for multiple runs - bounded_memory_perturbation = [ - [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] - for j in xrange(memory_slot_num) - ] - train_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.train(dict_size), - append_tuple=(bounded_memory_perturbation, )) - train_batch_reader = paddle.batch( - reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), - batch_size=batch_size) - test_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=(bounded_memory_perturbation, )) - test_batch_reader = paddle.batch( - reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), - batch_size=batch_size) - - # create event handler - def event_handler(event): - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 10 == 0: - print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( - event.pass_id, event.batch_id, event.cost, event.metrics) - else: - sys.stdout.write('.') - sys.stdout.flush() - if isinstance(event, paddle.event.EndPass): - result = trainer.test(reader=test_batch_reader, feeding=feeding) - print "Pass: %d, TestCost: %f, %s" % (event.pass_id, event.cost, - result.metrics) - with gzip.open("params.tar.gz", 'w') as f: - parameters.to_tar(f) - - # run train - trainer.train( - reader=train_batch_reader, - event_handler=event_handler, - num_passes=num_passes, - feeding=feeding) - - -def infer(): - """ - For inferencing. - """ - # create network config - source_words = paddle.layer.data( - name="source_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - beam_gen = memory_enhanced_seq2seq( - encoder_input=source_words, - decoder_input=None, - decoder_target=None, - hidden_size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=True, - beam_size=beam_size) - - # load parameters - parameters = paddle.parameters.Parameters.from_tar( - gzip.open("params.tar.gz")) - - # prepare infer data - infer_data = [] - random.seed(0) # for keeping consitancy for multiple runs - bounded_memory_perturbation = [ - [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] - for j in xrange(memory_slot_num) - ] - test_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=(bounded_memory_perturbation, )) - for i, item in enumerate(test_append_reader()): - if i < infer_data_num: - infer_data.append((item[0], item[3], )) - - # run inference - beam_result = paddle.infer( - output_layer=beam_gen, - parameters=parameters, - input=infer_data, - field=['prob', 'id']) - - # parse beam result and print - source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) - beam_probs, beam_sentences = parse_beam_search_result(beam_result, - target_dict) - for i in xrange(infer_data_num): - print "\n***************************************************\n" - print "src:", ' '.join( - [source_dict.get(word) for word in infer_data[i][0]]), "\n" - for j in xrange(beam_size): - print "prob = %f : %s" % (beam_probs[i][j], beam_sentences[i][j]) - - -def main(): - paddle.init(use_gpu=False, trainer_count=1) - train(num_passes=1) - infer() - - -if __name__ == '__main__': - main() diff --git a/mt_with_external_memory/train.py b/mt_with_external_memory/train.py new file mode 100644 index 00000000..91fc2750 --- /dev/null +++ b/mt_with_external_memory/train.py @@ -0,0 +1,157 @@ +""" + Contains training script for machine translation with external memory. +""" +import argparse +import sys +import gzip +import distutils.util +import random +import paddle.v2 as paddle +from external_memory import ExternalMemory +from model import * +from data_utils import * + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--dict_size", + default=30000, + type=int, + help="Vocabulary size. (default: %(default)s)") +parser.add_argument( + "--word_vec_dim", + default=512, + type=int, + help="Word embedding size. (default: %(default)s)") +parser.add_argument( + "--hidden_size", + default=1024, + type=int, + help="Hidden cell number in RNN. (default: %(default)s)") +parser.add_argument( + "--memory_slot_num", + default=8, + type=int, + help="External memory slot number. (default: %(default)s)") +parser.add_argument( + "--use_gpu", + default=False, + type=distutils.util.strtobool, + help="Use gpu or not. (default: %(default)s)") +parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Trainer number. (default: %(default)s)") +parser.add_argument( + "--num_passes", + default=100, + type=int, + help="Training epochs. (default: %(default)s)") +parser.add_argument( + "--batch_size", + default=5, + type=int, + help="Batch size. (default: %(default)s)") +parser.add_argument( + "--memory_perturb_stddev", + default=0.1, + type=float, + help="Memory perturb stddev for memory initialization." + "(default: %(default)s)") +args = parser.parse_args() + + +def train(): + """ + For training. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + target_words = paddle.layer.data( + name="target_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + target_next_words = paddle.layer.data( + name='target_next_words', + type=paddle.data_type.integer_value_sequence(args.dict_size)) + cost = memory_enhanced_seq2seq( + encoder_input=source_words, + decoder_input=target_words, + decoder_target=target_next_words, + hidden_size=args.hidden_size, + word_vec_dim=args.word_vec_dim, + dict_size=args.dict_size, + is_generating=False, + beam_size=None) + + # create parameters and optimizer + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Adam( + learning_rate=5e-5, + gradient_clipping_threshold=5, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + trainer = paddle.trainer.SGD( + cost=cost, parameters=parameters, update_equation=optimizer) + + # create data readers + feeding = { + "source_words": 0, + "target_words": 1, + "target_next_words": 2, + "bounded_memory_perturbation": 3 + } + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [[ + random.gauss(0, args.memory_perturb_stddev) + for i in xrange(args.hidden_size) + ] for j in xrange(args.memory_slot_num)] + train_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.train(args.dict_size), + append_tuple=(bounded_memory_perturbation, )) + train_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), + batch_size=args.batch_size) + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(args.dict_size), + append_tuple=(bounded_memory_perturbation, )) + test_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), + batch_size=args.batch_size) + + # create event handler + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 10 == 0: + print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( + event.pass_id, event.batch_id, event.cost, event.metrics) + with gzip.open("checkpoints/params.latest.tar.gz", 'w') as f: + parameters.to_tar(f) + else: + sys.stdout.write('.') + sys.stdout.flush() + if isinstance(event, paddle.event.EndPass): + result = trainer.test(reader=test_batch_reader, feeding=feeding) + print "Pass: %d, TestCost: %f, %s" % (event.pass_id, event.cost, + result.metrics) + with gzip.open("checkpoints/params.pass-%d.tar.gz" % event.pass_id, + 'w') as f: + parameters.to_tar(f) + + # run train + if not os.path.exists('checkpoints'): + os.mkdir('checkpoints') + trainer.train( + reader=train_batch_reader, + event_handler=event_handler, + num_passes=args.num_passes, + feeding=feeding) + + +def main(): + paddle.init(use_gpu=args.use_gpu, trainer_count=args.trainer_count) + train() + + +if __name__ == '__main__': + main() -- GitLab