From ce8b8d63944f9ba5a8696990143cc5415c2fef98 Mon Sep 17 00:00:00 2001 From: guosheng Date: Thu, 6 Dec 2018 13:46:59 +0800 Subject: [PATCH] Add fluid_transformer.md --- .../transformer/fluid_transformer.md | 777 ++++++++++++++++++ .../transformer/images/attention_formula.png | Bin 0 -> 24299 bytes 2 files changed, 777 insertions(+) create mode 100644 fluid/PaddleNLP/neural_machine_translation/transformer/fluid_transformer.md create mode 100644 fluid/PaddleNLP/neural_machine_translation/transformer/images/attention_formula.png diff --git a/fluid/PaddleNLP/neural_machine_translation/transformer/fluid_transformer.md b/fluid/PaddleNLP/neural_machine_translation/transformer/fluid_transformer.md new file mode 100644 index 00000000..1cd8389a --- /dev/null +++ b/fluid/PaddleNLP/neural_machine_translation/transformer/fluid_transformer.md @@ -0,0 +1,777 @@ +## Transformer 概述 + +### 背景简介 + +Transformer 是论文 [Attention Is All You Need](https://arxiv.org/abs/1706.03762) 中提出的用以完成机器翻译（machine translation, MT）等序列到序列（sequence to sequence, Seq2Seq）学习任务的一种全新网络结构，其完全使用注意力（Attention）机制来实现序列到序列的建模。 + +相较于此前 Seq2Seq 模型中广泛使用的循环神经网络（Recurrent Neural Network, RNN），使用（Self）Attention 进行输入序列到输出序列的变换主要具有以下优势： + +- 计算复杂度低 + - 特征维度为 d 、长度为 n 的序列，在 RNN 中计算复杂度为 `O(n * d * d)` （n 个时间步，每个时间步计算 d 维的矩阵向量乘法），在 Self-Attention 中计算复杂度为 `O(n * n * d)` （n 个时间步两两计算 d 维的向量点积或其他相关度函数），n 通常要小于 d 。 +- 计算并行度高 + - RNN 中当前时间步的计算要依赖前一个时间步的计算结果；Self-Attention 中各时间步的计算只依赖输入不依赖之前时间步输出，各时间步可以完全并行。 +- 容易学习长程依赖（long-range dependencies） + - RNN 中相距为 n 的两个位置间的关联需要 n 步才能建立；Self-Attention 中任何两个位置都直接相连；路径越短信号传播越容易。 + +Transformer 模型在训练时间大幅减少的同时取得了 WMT'14 英德翻译任务 BLEU 值的新高。此外，Transformer 或其部件在其他模型和任务中也取得了良好的效果。 + +### Transformer 模型概览 + +Transformer 使用了 Seq2Seq 模型中典型的编码器-解码器（Encoder-Decoder）的框架结构，整体网络结构如图1所示。 + +

+
+图 1. Transformer 网络结构图 +

+ +Encoder 由若干相同的 layer 堆叠组成，每个 layer 主要由 Multi-Head Attention 和 Position-wise Feed-Forward Network 这两个 sub-layer 构成。 +- Multi-Head Attention 在这里用于实现 Self-Attention，相比于简单的 Attention 机制，其将输入进行多路线性变换后分别计算 Attention 的结果，并将所有结果拼接后再次进行线性变换作为输出。其中 Attention 使用的是 Dot-Product，并在点积后进行了 scale 的处理以避免因点积结果过大进入 softmax 的饱和区域。 +- Position-wise Feed-Forward Network采用的是两次线性变换中间加以 ReLU 激活的结构。 + +此外，每个 sub-layer 后还施以 Residual Connection 和 Layer Normalization 来促进梯度传播和模型收敛。 + +Decoder 具有和 Encoder 类似的结构，只是相比于组成 Encoder 的 layer ，在组成 Decoder 的 layer 中还多了一个 Multi-Head Attention 的 sub-layer 来实现对 Encoder 输出的 Attention，这个 Encoder-Decoder Attention 在其他 Seq2Seq 模型中也是存在的。 + + +## Fluid Transformer 实现 + +代码： https://github.com/PaddlePaddle/models/tree/develop/fluid/PaddleNLP/neural_machine_translation/transformer + +```text +. +├── config.py # 训练、预测以及模型参数配置 +├── infer.py # 预测脚本 +├── model.py # 模型定义 +├── optim.py # learning rate scheduling 计算程序 +├── reader.py # 数据读取接口 +├── train.py # 训练脚本 +└── gen_data.sh # BPE 数据生成脚本 +``` + +### Fluid Transformer 训练网络 +模型定义代码 `model.py` + +```text +. +├── transformer + ├── make_all_inputs + ├── wrap_encoder + ├── prepare_encoder + └── word_embedding + position_encoding + └── encoder + ├── stack of encoder_layer + ├── multi_head_attention + └── positionwise_feed_forward + └── pre_process_layer + ├── wrap_decoder + ├── prepare_decoder + └── word_embedding + position_encoding + └── decoder + ├── stack of decoder_layer + ├── multi_head_attention + ├── multi_head_attention + └── positionwise_feed_forward + └── pre_process_layer + └── loss +``` + +- `make_all_inputs` 数据输入的定义 + + APIs：fluid.layers.data + + 相关 Q&A：如何处理变长数据 + + ```python + def make_all_inputs(input_fields): + """ + Define the input data layers for the transformer model. + """ + inputs = [] + for input_field in input_fields: + input_var = layers.data( + name=input_field, + shape=input_descs[input_field][0], + dtype=input_descs[input_field][1], + lod_level=input_descs[input_field][2] + if len(input_descs[input_field]) == 3 else 0, + append_batch_size=False) + inputs.append(input_var) + return inputs + ``` + + ```python + # The shapes and sizes are placeholders in compile-time(when building network). + batch_size = -1 + seq_len = ModelHyperParams.max_length + input_descs = { + "src_word": [(batch_size, seq_len, 1), "int64"], + "src_pos": [(batch_size, seq_len, 1), "int64"], + "src_slf_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, + seq_len), "float32"], + "trg_word": [(batch_size, seq_len, 1), "int64", 2], + "trg_pos": [(batch_size, seq_len, 1), "int64"], + "trg_slf_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, + seq_len), "float32"], + "trg_src_attn_bias": [(batch_size, ModelHyperParams.n_head, seq_len, + seq_len), "float32"], + "lbl_word": [(batch_size * seq_len, 1), "int64"], + "lbl_weight": [(batch_size * seq_len, 1), "float32"], + "init_score": [(batch_size, 1), "float32", 2] + } + ``` + +- `warp_encoder`/`warp_decoder` encoder/decoder 的 wraper + + - prepare_encoder/prepare_decoder 产生 encoder/decoder 的输入 + + APIs：fluid.layers.embedding、fluid.layers.scale、fluid.layers.elementwise_add、fluid.layers.dropout + + 相关 Q&A：如何进行权值共享、如何导入外部计算的参数值 + + ```python + def prepare_encoder_decoder(): + """Add word embeddings and position encodings""" + src_word_emb = layers.embedding( + src_word, + size=[src_vocab_size, src_emb_dim], + padding_idx=ModelHyperParams.bos_idx, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0., src_emb_dim**-0.5))) + + src_word_emb = layers.scale(x=src_word_emb, scale=src_emb_dim**0.5) + src_pos_enc = layers.embedding( + src_pos, + size=[src_max_len, src_emb_dim], + param_attr=fluid.ParamAttr( + name=pos_enc_param_name, trainable=False)) + src_pos_enc.stop_gradient = True + enc_input = src_word_emb + src_pos_enc + return layers.dropout( + enc_input, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False) if dropout_rate else enc_input + ``` + - encoder/decoder + - encoder/decoder layer + - multi_head_attention + +

+
+ 图 2. Multi-Head Attention +

+ +

+
+

+ + APIs：fluid.layers.fc、fluid.layers.reshape、fluid.layers.transpose、fluid.layers.matmul、fluid.layers.elementwise_add、fluid.layers.softmax、fluid.layers.dropout + + ```python + def __compute_qkv(queries, keys, values, n_head, d_key, d_value): + """ + Add linear projection to queries, keys, and values. + """ + q = layers.fc(input=queries, + size=d_key * n_head, + bias_attr=False, + num_flatten_dims=2) + k = layers.fc(input=keys, + size=d_key * n_head, + bias_attr=False, + num_flatten_dims=2) + v = layers.fc(input=values, + size=d_value * n_head, + bias_attr=False, + num_flatten_dims=2) + return q, k, v + + def __split_heads(x, n_head): + """ + Input a tensor with shape [bs, max_sequence_length, n_head * hidden_dim], + then output a tensor with shape [bs, n_head, max_sequence_length, hidden_dim]. + """ + hidden_size = x.shape[-1] + reshaped = layers.reshape( + x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True) + return layers.transpose(x=reshaped, perm=[0, 2, 1, 3]) + + def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate): + """ + Scaled Dot-Product Attention + """ + scaled_q = layers.scale(x=q, scale=d_key**-0.5) + product = layers.matmul(x=scaled_q, y=k, transpose_y=True) + if attn_bias: + product += attn_bias + weights = layers.softmax(product) + if dropout_rate: + weights = layers.dropout( + weights, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False) + out = layers.matmul(weights, v) + return out + + def __combine_heads(x): + """ + reverse to __split_heads. + """ + trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) + return layers.reshape( + x=trans_x, + shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], + inplace=True) + + def multi_head_attention(): + """Multi-head Attention""" + q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value) + + q = __split_heads(q, n_head) + k = __split_heads(k, n_head) + v = __split_heads(v, n_head) + + ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, + dropout_rate) + + out = __combine_heads(ctx_multiheads) + + # Project back to the model size. + proj_out = layers.fc(input=out, + size=d_model, + bias_attr=False, + num_flatten_dims=2) + ``` + + - positionwise_feed_forward + + APIs：fluid.layers.fc + + ```python + def positionwise_feed_forward(x, d_inner_hid, d_hid, dropout_rate): + """ + Position-wise Feed-Forward Networks. + """ + hidden = layers.fc(input=x, + size=d_inner_hid, + num_flatten_dims=2, + act="relu") + if dropout_rate: + hidden = layers.dropout( + hidden, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False) + out = layers.fc(input=hidden, size=d_hid, num_flatten_dims=2) + return out + ``` + + - pre_post_process_layer 对 sub-layer 的输入/输出进行预/后处理 + + APIs：fluid.layers.layer_norm、fluid.layers.dropout、fluid.layers.elementwise_add + + ```python + def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): + """ + Add residual connection, layer normalization and droput to the out tensor + optionally according to the value of process_cmd. + This will be used before or after multi-head attention and position-wise + feed-forward networks. + """ + for cmd in process_cmd: + if cmd == "a": # add residual connection + out = out + prev_out if prev_out else out + elif cmd == "n": # add layer normalization + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.initializer.Constant(1.), + bias_attr=fluid.initializer.Constant(0.)) + elif cmd == "d": # add dropout + if dropout_rate: + out = layers.dropout( + out, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False) + return out + ``` + +- loss 计算 + + APIs：fluid.layers.label_smooth、fluid.layers.one_hot、fluid.layers.softmax_with_cross_entropy、fluid.layers.elementwise_mul、fluid.layers.reduce_sum、fluid.layers.elementwise_div + + ```python + if label_smooth_eps: + label = layers.label_smooth( + label=layers.one_hot( + input=label, depth=trg_vocab_size), + epsilon=label_smooth_eps) + + cost = layers.softmax_with_cross_entropy( + logits=predict, + label=label, + soft_label=True if label_smooth_eps else False) + weighted_cost = cost * weights # to mask out the loss from paddings + sum_cost = layers.reduce_sum(weighted_cost) + token_num = layers.reduce_sum(weights) + token_num.stop_gradient = True + avg_cost = sum_cost / token_num + return sum_cost, avg_cost + ``` + +### Fluid Transformer 解码 + +#### Preliminaries + +- while_op 如何工作 + - 使用一个 scalar 的 tensor variable 的输入用以判别循环结束；一个 BlockDesc 的 attribute 作为循环体 + ```python + while_op = layers.While(cond) + with while_op.block(): + pass + + parent_block.append_op( + type='while', + inputs={ + 'X': [ + parent_block._var_recursive(x_name) + for x_name in x_name_list + ], + 'Condition': [self.cond_var] + }, + outputs={'Out': out_vars, + 'StepScopes': [step_scope]}, + attrs={'sub_block': while_block, + "is_test": self.is_test}) + ``` + - 执行 block 内的 program 直到作为判别条件的 variable 值变为 false。 + ```c++ + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + bool is_test = Attr("is_test"); + auto ctx = executor.Prepare(*program, block->ID()); + while (cond.data()[0]) { + auto ¤t_scope = scope.NewScope(); + step_scopes->push_back(¤t_scope); + executor.RunPreparedContext(ctx.get(), ¤t_scope, false, true, true); + if (is_test) { + scope.DeleteScope(¤t_scope); + } + } + ``` + 每次运行循环体都在当前 scope 内创建一个新的子 scope，循环体内的使用这个子 scope，保证不同时间步（子 scope）内的 variable 具有相同的 name 但相互隔离正确运行，子 scope 中的 variable 在上层 scope 中不可见，不同时间步的交互需要借助于上一级 scope 内的 variable。 + +- LoDTensorArray + + ```c++ + using LoDTensorArray = std::vector; + ``` + + 通常用于将 while_op 内每一步中的 tensor variable 保存下来以供在外部的 scope 访问，相关 Operator： + + - fluid.layers.array_read 从 LoDTensorArray 读出一个 LoDTensor + - fluid.layers.array_write 往 LoDTensorArray 写入一个 LoDTensor + + +#### Transformer 解码 + +- APIs： + - fluid.layers.beam_search + 接受上一时间步 shape 为 `(batch_size * beam_size, 1)` 的 pre_ids、pre_scores 以及当前时间步 shape 为 `(batch_size * beam_size, topK)` 的 ids、scores 作为输入，在 beam 间取 topK，包含了对 end beam 和 end sentence 的处理，将 end beam（eos）下一词预测的概率密度全分配到 eos token 上，对 end sentence（达到 beam width 个 end beam 的 sentence） 进行 prune（batch reduction），输出的 selected_id 的 lod 中保存了 pre_ids 中每一个在 select 后对应 selected_id 中的哪些。更详细的说明可以参考[这里](https://github.com/guoshengCS/models/blob/beam_search_op_review/fluid/neural_machine_translation/transformer/beam_search_op.md)。 + - fluid.layers.beam_search_decode + 接受分别保存了每一步 beam_search 返回的 selcted_ids 和 selcted_scores 的两个 LoDTensorArray 作为输入，根据其中 lod 回溯路径进行解码，使用 LoDTensor 保存结果。更详细的说明可以参考[这里](https://github.com/guoshengCS/models/blob/beam_search_op_review/fluid/neural_machine_translation/transformer/beam_search_op.md)。 + - fluid.layers.sequence_expand 使用 lod 对输入的 Tensor 进行 expand；由于 beam_search_op 输出的 selected_id 的 lod 中保存了 pre_ids 中每一个在 select 后对应 selected_id 中的哪些（expand 了多少次），因而可以将 sequence_expand 作为 gather 使用，从上一时间步状态更新当前时间步的状态。 + +- BeamSearchDecoder 构建： + + ```python + def fast_decode(): + ########################################################### + # all inputs required by beam search decoder + ########################################################### + enc_output = wrap_encoder() + start_tokens, init_scores, trg_src_attn_bias = make_all_inputs( + fast_decoder_data_input_fields) + ########################################################### + + def beam_search(): + ########################################################### + # definition of while_op + ########################################################### + max_len = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=max_out_len) + step_idx = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=0) + cond = layers.less_than(x=step_idx, y=max_len) + while_op = layers.While(cond) + ########################################################### + + ########################################################### + # definition of beam search states and cell states + ########################################################### + ids = layers.array_write( + layers.reshape(start_tokens, (-1, 1)), step_idx) + scores = layers.array_write(init_scores, step_idx) + + caches = [{ + "k": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0), + "v": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0) + } for i in range(n_layer)] + ########################################################### + + with while_op.block(): + ########################################################### + # update inputs and states required for the current step + ########################################################### + pre_ids = layers.array_read(array=ids, i=step_idx) + pre_ids = layers.reshape(pre_ids, (-1, 1, 1)) + pre_scores = layers.array_read(array=scores, i=step_idx) + pre_src_attn_bias = layers.sequence_expand( + x=trg_src_attn_bias, y=pre_scores) + pre_enc_output = layers.sequence_expand(x=enc_output, y=pre_scores) + pre_caches = [{ + "k": layers.sequence_expand( + x=cache["k"], y=pre_scores), + "v": layers.sequence_expand( + x=cache["v"], y=pre_scores), + } for cache in caches] + pre_pos = layers.elementwise_mul( + x=layers.fill_constant_batch_size_like( + input=pre_enc_output, + value=1, + shape=[-1, 1, 1], + dtype=pre_ids.dtype), + y=step_idx, + axis=0) + ########################################################### + + ########################################################### + # cell calculations + ########################################################### + logits = wrap_decoder( + dec_inputs=(pre_ids, pre_pos, None, pre_src_attn_bias), + enc_output=pre_enc_output, + caches=pre_caches) + ########################################################### + + ########################################################### + # compute accumulated scores and search + ########################################################### + topk_scores, topk_indices = layers.topk( + input=layers.softmax(logits), k=beam_size) + accu_scores = layers.elementwise_add( + x=layers.log(topk_scores), + y=layers.reshape( + pre_scores, shape=[-1]), + axis=0) + topk_indices = layers.lod_reset(topk_indices, pre_ids) + selected_ids, selected_scores = layers.beam_search( + pre_ids=pre_ids, + pre_scores=pre_scores, + ids=topk_indices, + scores=accu_scores, + beam_size=beam_size, + end_id=eos_idx) + ########################################################### + + ########################################################### + # save states + ########################################################### + layers.increment(x=step_idx, value=1.0, in_place=True) + layers.array_write(selected_ids, i=step_idx, array=ids) + layers.array_write(selected_scores, i=step_idx, array=scores) + layers.assign(pre_src_attn_bias, trg_src_attn_bias) + layers.assign(pre_enc_output, enc_output) + for i in range(n_layer): + layers.assign(pre_caches[i]["k"], caches[i]["k"]) + layers.assign(pre_caches[i]["v"], caches[i]["v"]) + ########################################################### + + ########################################################### + # update condition variable + ########################################################### + length_cond = layers.less_than(x=step_idx, y=max_len) + finish_cond = layers.logical_not(layers.is_empty(x=selected_ids)) + layers.logical_and(x=length_cond, y=finish_cond, out=cond) + ########################################################### + + ########################################################### + # decode according to selected ids and scores + ########################################################### + finished_ids, finished_scores = layers.beam_search_decode( + ids, scores, beam_size=beam_size, end_id=eos_idx) + ########################################################### + return finished_ids, finished_scores + + finished_ids, finished_scores = beam_search() + return finished_ids, finished_scores + ``` + +- 在 cell 内使用 cache + + - self-attention 进行 cache + + ```python + def multi_head_attention(): + """Add word embeddings and position encodings""" + q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value) + + ########################################################### + # use cache and concat time steps + ########################################################### + if cache is not None: + k = cache["k"] = layers.concat( + [layers.reshape( + cache["k"], shape=[0, 0, d_key * n_head]), k], + axis=1) + v = cache["v"] = layers.concat( + [layers.reshape( + cache["v"], shape=[0, 0, d_value * n_head]), v], + axis=1) + ########################################################### + + q = __split_heads(q, n_head) + k = __split_heads(k, n_head) + v = __split_heads(v, n_head) + + ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, + dropout_rate) + + out = __combine_heads(ctx_multiheads) + + # Project back to the model size. + proj_out = layers.fc(input=out, + size=d_model, + bias_attr=False, + num_flatten_dims=2) + ``` + + - 同时支持 encoder-decoder attention 进行 cache + + https://github.com/PaddlePaddle/models/pull/1476 + + encoder output 在每一个时间步都相同，对以 encoder output 作为输入的运算结果做 cache 需要将这些运算定义在上层 block 而非在 while block 内每一个时间步都执行，需要能够在指定的 block 中添加 OP 而现有的 API 只能在当前的 block 内添加，因而需要在上层 block 和 while block 之间进行切换。 + + ```python + def wrap_layer_with_block(layer, block_idx): + """ + Make layer define support indicating block, by which we can add layers + to other blocks within current block. This will make it easy to define + cache among while loop. + """ + + class BlockGuard(object): + """ + BlockGuard class. + + BlockGuard class is used to switch to the given block in a program by + using the Python `with` keyword. + """ + + def __init__(self, block_idx=None, main_program=None): + self.main_program = fluid.default_main_program( + ) if main_program is None else main_program + self.old_block_idx = self.main_program.current_block().idx + self.new_block_idx = block_idx + + def __enter__(self): + self.main_program.current_block_idx = self.new_block_idx + + def __exit__(self, exc_type, exc_val, exc_tb): + self.main_program.current_block_idx = self.old_block_idx + if exc_type is not None: + return False # re-raise exception + return True + + def layer_wrapper(*args, **kwargs): + with BlockGuard(block_idx): + return layer(*args, **kwargs) + + return layer_wrapper + ``` + + ```python + def __split_heads_qkv(queries, keys, values, n_head, d_key, d_value): + reshaped_q = layers.reshape( + x=queries, shape=[0, 0, n_head, d_key], inplace=True) + q = layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) + # For encoder-decoder attention in inference, insert the ops and vars + # into global block to use as cache among beam search. + reshape_layer = wrap_layer_with_block( + layers.reshape, + fluid.default_main_program().current_block() + .parent_idx) if cache is not None and static_kv else layers.reshape + transpose_layer = wrap_layer_with_block( + layers.transpose, + fluid.default_main_program().current_block(). + parent_idx) if cache is not None and static_kv else layers.transpose + reshaped_k = reshape_layer( + x=keys, shape=[0, 0, n_head, d_key], inplace=True) + k = transpose_layer(x=reshaped_k, perm=[0, 2, 1, 3]) + reshaped_v = reshape_layer( + x=values, shape=[0, 0, n_head, d_value], inplace=True) + v = transpose_layer(x=reshaped_v, perm=[0, 2, 1, 3]) + + if cache is not None: # only for faster inference + if static_kv: # For encoder-decoder attention in inference + cache_k, cache_v = cache["static_k"], cache["static_v"] + # To init the static_k and static_v in cache. + # Maybe we can use condition_op(if_else) to do these at the first + # step in while loop to replace these, however it might be less + # efficient. + static_cache_init = wrap_layer_with_block( + layers.assign, + fluid.default_main_program().current_block().parent_idx) + static_cache_init(k, cache_k) + static_cache_init(v, cache_v) + else: # For decoder self-attention in inference + cache_k, cache_v = cache["k"], cache["v"] + # gather cell states corresponding to selected parent + select_k = layers.gather(cache_k, index=gather_idx) + select_v = layers.gather(cache_v, index=gather_idx) + if not static_kv: + # For self attention in inference, use cache and concat time steps. + select_k = layers.concat([select_k, k], axis=2) + select_v = layers.concat([select_v, v], axis=2) + # update cell states(caches) cached in global block + layers.assign(select_k, cache_k) + layers.assign(select_v, cache_v) + return q, select_k, select_v + return q, k, v + ``` + +- 从输出的 LoDTensor 解析获取翻译结果 + + ```python + seq_ids, seq_scores = exe.run(infer_program, + feed=data_input, + fetch_list=[out_ids, out_scores], + return_numpy=False) + ######################################################################## + # How to parse the results: + # Suppose the lod of seq_ids is: + # [[0, 3, 6], [0, 12, 24, 40, 54, 67, 82]] + # then from lod[0]: + # there are 2 source sentences, beam width is 3. + # from lod[1]: + # the first source sentence has 3 hyps; the lengths are 12, 12, 16 + # the second source sentence has 3 hyps; the lengths are 14, 13, 15 + ######################################################################## + hyps = [[] for i in range(len(data))] + scores = [[] for i in range(len(data))] + for i in range(len(seq_ids.lod()[0]) - 1): # for each source sentence + start = seq_ids.lod()[0][i] + end = seq_ids.lod()[0][i + 1] + for j in range(end - start): # for each candidate + sub_start = seq_ids.lod()[1][start + j] + sub_end = seq_ids.lod()[1][start + j + 1] + hyps[i].append(" ".join([ + trg_idx2word[idx] + for idx in post_process_seq( + np.array(seq_ids)[sub_start:sub_end]) + ])) + scores[i].append(np.array(seq_scores)[sub_end - 1]) + ``` + +## Fluid Transformer 中的 Q&A + +### 如何处理变长数据 + +Transformer 等 NLP 模型的数据输入中多存在 batch size 和 sequence length 两个大小可变的维度，这种变长数据如何定义和处理 + +- Paddle Fluid 中在网络定义时（compile-time）设置和传递的数据大小都可以看作 placeholder，可以进行任意设置，保证能够通过 compile-time 的检查即可；在执行时 runtime 用到的数据大小会从实际输入数据重新获取。Transformer 定义网络时使用了类似如下的输入数据定义， 实际运行时可以接受任何 batch size 和 sequence length 的输入数据。 + + ```python + batch_size = -1 + max_length = 256 + src_word = layers.data( + name="src_word", + shape=[batch_size, max_length, 1], + dtype="int64", + append_batch_size=False) + ``` + +- 这种具有多个大小可变的维度的数据需要 reshape 时，由于 reshape_op 中 shape 这个参数会作为写入网络配置被运行时使用，为保证运行时使用的是实际大小，可以使用类似 Transformer 中如下方式进行 reshape，能够保证 batch size 和 sequence length 的正确大小。 + + ```python + reshaped_k = layers.reshape( + x=keys, shape=[0, 0, n_head, d_key]) + ``` + +### 如何进行权值共享 + +Transformer 中源语言和目标语言共享词表和 embedding，权值共享也是一种常见的使用场景，如何实现权值共享 + +- 通过设置 ParamAttr 中的 name 来实现权值共享，相同的 name 指定使用相同的权重参数，在 Transformer 中使用类似下面的代码实现 source 和 target 共享 embedding。 + + ```python + src_word_emb = layers.embedding( + src_word, + size=[vocab_size, emb_dim], + padding_idx=pad_idx, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0., emb_dim**-0.5))) + trg_word_emb = layers.embedding( + trg_word, + size=[vocab_size, emb_dim], + padding_idx=pad_idx, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0., emb_dim**-0.5))) + ``` +- 如果在权值共享的同时对权重参数有一些额外的操作，如 Transformer 中还对输出层 fc 的权重与 embedding 进行了权值共享，这时需要对 embedding 进行额外的转置，可以使用参数名获取参数对应的 variable 带入额外的操作，对应代码实现如下： + + ```python + predict = layers.matmul( + x=dec_output, + y=fluid.default_main_program().global_block().var( + word_emb_param_name), + transpose_y=True) + ``` + +### 如何导入外部计算的参数值 + +Transformer 中的 Position Encoding 可以在外部使用 python 代码方便的计算出来，对参数进行特殊的初始化也是一种常见的使用场景，如何实现导入外部计算的参数值的功能 + +- 可以将参数看作一般的 variable（只是在多个 iteration 之间不会被清空），和数据输入同等对待，在运行第一个 iteration 时和其他输入数据一起 feed 进去即可。 +注意在使用 ParallelExecutor 多卡运行时由于每张卡对应有自己的 scope（存放各自用到的 variable），输入数据需要为每张卡准备一份；对于单卡运行的程序还有另外一种设置的方法（多卡时由于有多个 scope 和 place，这些没有在 python 端暴露，无法使用），Transformer 单卡 Position Encoding 可以使用如下的代码导入，这也方便实现预测使用比训练更大的长度。 + + ```python + for pos_enc_param_name in pos_enc_param_names: + + pos_enc_param = fluid.global_scope().find_var( + + pos_enc_param_name).get_tensor() + + pos_enc_param.set( + + position_encoding_init(ModelHyperParams.max_length + 1, + + ModelHyperParams.d_model), place) + ``` + +- 一些时候不希望对这种从外部设置的参数值进行训练和更新，如 Transformer 中的 Position Encoding，这可以通过设置 ParamAttr 中的 trainable 属性或者 variable 的 stop_gradient 属性来实现 + + ```python + src_pos_enc = layers.embedding( + src_pos, + size=[src_max_len, src_emb_dim], + param_attr=fluid.ParamAttr( + name=pos_enc_param_name, trainable=False)) + + src_pos_enc.stop_gradient = True + ``` + +- 除参数值以外，其他一些方便在外部计算的值也可以使用类似的方法作为数据输入 feed 进去，如根据特殊的 learning rate scheduling 产生的每一步的学习率。 diff --git a/fluid/PaddleNLP/neural_machine_translation/transformer/images/attention_formula.png b/fluid/PaddleNLP/neural_machine_translation/transformer/images/attention_formula.png new file mode 100644 index 0000000000000000000000000000000000000000..249857f524b4137bafc2d4d1b779ed62d1437b6d GIT binary patch literal 24299 zcmeFXQ*>qFvNjy2W7~FCY;-!dZQFLzvDHC`9d~Toww)E*=AV7e*?XV;UH_M7jCZXu zuWHs?H49HY6`>#}jsS}b3jzXyASofD1Ofui`1L&f4eIM3*vGC90s@b2DJ-lYDJ)E^ z;AC%RX=4fkA`y|S4y~a)f|;$Y^bLjq8I&fMQ^+fMlXtd9Oad$zm6+u3@{B0t($C0m z;gOY*%Zp&cS~YZ00UZL!ikg~Oa2g7pNnov=4;{C@&X+l?hnde)BQ3WRAb%_-M#sj) zl0k-nSl{x%ebXW&M`UKv0zlxYL0t_djo@zE^YeN^e$_l~nLcsUhdT8s;h*XsewI-? zH~-)OkwEV+!6)H`-pL2qJ~`tP0tE@?_{*858&5SDgj@zoECdqXj9n~U(~4azbxj^2 zYf}&C*8__E3_OW9*MK371vw zic0!uN-NoA)-1nsq@NHaH4)5a0%Wj1e$o$prexa8EaSd7D6m2Z&Fo|vZakRmhek4E z?4Z5Phf9R35_gX8N}LTm`Grzb%b5vwW4`pJgF=B&Z&$zlAX)sGMlu?Cn_{S)@_h&H zeCC^;kV83qF|1R*>GjMHKR1yy`pHOmyDRLrMIIpH`=(|$?`&%8r8 z&ubK9y6@{G4Q&jO62S25UiDB*bnnFu6)sg&QsQ&tU5xO;2bndmsAYcbCt$IFPx zI3!M@D4aqf42|)*bOilrpriZthenzWtJ{3?Ogkw&R}@*i%TqE4uo)^?-6S-ySr67! zLGWa>pwA>J>ByxT7ajD3RLXzh#r|a4hI<1+3*VN2Y#~OwT7^!B+6ElUfEa*+Nd^UW{bU=l$?YeD;|t<~Uh3y};N*iB z=-93v$Z||&7A)FXE*0_5BVrE6{?3MX*b`b_2Sf(h?3xMb<-{_hI_M=D18Vd2vi_g~ z!T>RIGaQimgNhTWS;{oqje3yAe-q=o^jUz`o+))rA##S66F3m96WoNYJ3>{$&^vmk zMQ~ogI6DWdUgAU!w)mFq$DKxK`sc*SR2fum6B{S+d`vyXYQJL-JhtIFZu8!WOWQ!Y zL9_*D4OALXRa18N%Nb8-;JGP95Agy^3eX1CqEZ~jemZ&>b1Wh(zAxtZYh+Mp>+1jv zLEYs7pNy|2CNk)@3(e7)I)HZLrck#lYDsC0iyV_6;g4>|i`OY6D|IM?3-+yVtRT(5 zewe;+5{!Wa>`sCFLGqu&X%9T_#0*Fxuu1r%P>NT{fr_@!^VrfCiY&m zA%m_-%&u4`YTdO(pamt8WgvJ*E~3yxDMf*n2AUd+8i~e!e}DgeUw5x~^mY$xOnsCn zv5S6->@%VC2Yd00Qn9k={KEXkJnOvod=ne)MB0Ik1+T4ldp+%e}5&99V8f*_N zeuyaS0&EPdE^HP0K6(s#2ZlK{&Ijnv5^K;dP9{-4JZR8?7(AXQe? zF_p)%#xlgxlD{iu17#D6Woo=7ixb`0gV6?J9L5N{OuHPra}eh+n`q5wYI0mmxz@ej z_Yo(0Cpz~^_dzF3^MaO4>~HL~Y>Sp`mVC4Q>=W$8R+Ee63+;cnnG{{~) zHkW=hj5p*p99W)DTg~ClF3up%PtI+Y`^tHL56j&xOe(1pI{YQq#i+X^oLbx_^jLpE zB*R|B$(o@#!Z(FKKQ{|Ei*{_gWW9tp!#NW-2UryR`#Bt7v~3e_R^9Vw!C+cpiD})T z(=Z6=OU6xRo)4cdE0!9)iI;&*g$>}eWAQW0vqQB}GMP2Fnf_&VVwrBQW$rZ>ZE;|e zV%j)R-iOcb&NRhT#nELKsKupqMc=65B1bCY^t(Sth4VO;VqtczcDlBwb|}&i!%Wp? z!Q|em7Pz-Ja!RS6{Z|l`(QGwsId}DD9x#D*ADyJ>xOlmse>}g~Ae4zikFzD*$o9&npW>OK(u2~& zuhP|<>X>L}?RfO;x~X|UdBVT-ygWT=>wb(TI>0(oKUun{+95jB-~Z(LE=bJ(hs1zs zn#4hG`#13q1&is{KfFDwJ!ji5+das|@sK2k!X>%PxvIJHz2kilh73Rp4~eIjt1uKX zl(+CA31l4n7J(bZUy>?*^Y+RS%K1fV^1hh1X#9dNO7Q$WG z$(dRh$x2%LZJwZ>xRiC7-o@;%fK0ngArj0QJONpHl{QDog7Hp%LUTu5&iKZ}>-Uhr zJ?i}Ayn2v6rOW;3srY>caC_x>=2>xje*pNsRoCs^^AIxJJ0DUJ1&_&~J>SITaGg!@ zi$YV{575ZS2=HrSGqfCIIX8_;z5hUeQWlQ*5%Fhl(y3`*TYa0(gKkxQwPL;f#+s)1 z&g!?Trs&F{>Y1v#f8%q@&Ed>~W!c&iS7nK=w9c5mw|CZIEC2rJzUY3(Y2Om3oq-+o z#+BV}d5k*k8C|1Yo(-z)NhN%l&8AJW=lQ{uDvoMXsc$LLqQydaxo1~MU18Z`*+aM5 zd4;LApzW@0TN}TFw{e!E?f~LWY%jr-+u&vUUDPe$zKzX2`TV6eqpQtZ{x19WUhe+h zyWyeI@;jag!uxB{<>~TeQ`d9Ae&9JY4)hpepm3hB|6BUa>3Lsnn06c&5x2wqk;Me_ zOU;dT6O1%n;KkYC-4?ITPF+5**rTqU$xG#0 ziTIFAT+{VzAe(1U0Vxnuhy)-C=mqY82;$k-Jcz*0;KE1N2rjVdHMoH00g&JxRw)L2 zF=S<+w$EMcJ=ou@ji>ksrin>4+$dOkreLJ}eESAa;s%f_Q6?UqT2q(1F*%6`LJ;$t zLU(~0WZ<)3MhmuXPF@Z68}TOuxw;Kj8t!Y=7fgV0kkE7n0l}pB#}8CeiTnx#(g0=y?Z%hl+Z~<>x*h45vkRFU|Ue3q%Y zPwT0%><;q>nFEHA$w~LyU`_Ay`oE@n0S!?IM`|3TR0jVkz!!v z-~Z$IfwAdmphyHj{ON*8Dt>1r~aIy$GwUyY@G zi|psIYajyV-e03>?lze5q<-c$)jxOtTSQ%U_Z7~#wpyYPyNv~2nYf}IRFa7jt5ksS zfEl_VE7kwXe(^9ag8F3v)x2~w%c~)QxhT(vYNfmAPSU6ggB64WJK5Q2aaB8a$w0-G9Kp z@`6fh=6BAJs^HmJ2Yy54qSscaW^+NkZreWBAa5_f^u;=C68ZD{yqHbL%o|3@@88uT zZ6AUsTkv>k`_$V$E;j;+fI(2LQuCq}gU6R;)ODps?XLNX3G1tz9MH;J3IDU47*K)< zAbG7#?T8HYEvvN>-X2WMy;T4$URc9+xI&xN&yw0*36MTcefiV*~VoO18ihI zy`?wU>)%~UcRlJALiRxHKA!R^_lQ>D#^+*?^r%9W*{}8c=A-20RHv%`?SWLb7wycv zAu5{ufv^X(%0DwkqN>Um7h4MP3YU?T#-Eb?ZzO}ZM^;n z=tze-L}ueby@>2<=jGqEGo&m;?P*tUDFyQ;+nVA5p=0HAEU+^Fw4?zh*ybjbkHbBt zsX&y+ocQT5TJ>KJH5rwq_BBDsFxsnJU%~D_Cwzv*RY-@3VWDLsHS`+l)R+(w+h1NB zXgu3rCJAlhF4bc4bK<;lMh(ZN8a5aTVIq>2JbTyPDd>?tApYJrfWI%n@UI9r!vnRa zdMKww==ufsY~9uw!v1o*LAAtK2J+Zk4s^&{+86Xn(x+VEe3I%vG19)uhM;_P&nfBt znSA5GNmsfS>L4T2E8>pDkrT4NSBiSFF;613gx!k3H(G1ggsA4$-C* zlL>X&=37ed^uS;JEeGCF&P@o<9zaH!UOBEA+H0-J#g*uU{m3t&U)(sqXYWwHl=dzE zjrLEmATZy+-aTy22@zn-cYTxYtSo*~`P5Pa<9h-m$L(?W{LwQO&u@@i%WO`1E)bqa z4md2Idi1Rhn=~w%WJbrwLAg}M#|x!5{$*`=@js7uXy5D#v42ej%5M<*GcCH%pzwF? zGiE(A=wkSmn@XOC?b z+D$BHi!ugLhFG>13l`L#AUb8QS$qz26xIiv$4QM@J6r(wwWuY8){yFYi^80cc7Uqv*82 z4VACQ*vZxXnJ3pw)?1|I@#ydqt8_r)$@q==#lPyMb9lGC{!^}+b`;zqn>wm8>Yd(? zq!(Fzoo32Qg3hD>pjOL+n(Tn z63zmw7Q46Udz`UXcL2L+!G1r3q>Z+q$8{!$pEH76GW{(c!&lvx!C#AQUkCFkU9E&s zuiNE4LXeZ4(brY)DzAQEmbHoS z)(F_;&D?7De`jK2(^lNz2sU0@3O~LI%>Q#Iq`_`UFj$fz6oVu$VL`@9ew6SSE1pCLRWulpk+wF}JnU}cCEz;nomT3+r#I-0=OcK3zDCo@9 zT*G~6JSN;+fV?E}+!y>aNvED8`)4L41|aDT2aCVsoR+5KC zd_`j*n(sZd0=n_+d+U^sWQP7NneRJVRw~ZD|ML z%2hJa$8GtmaH}q~?w^BIKIkLmEg0Ng13T^f7ff~KK6s_eP~DXJA*5tgx8O#ieX;9D#^!*5tVl3FJ2!N5>| zy^xyv=-;3}u!N%e;R2c+hwt`{F5TJQ9^6i|6LX*4Kd`J7rJ~~AK`WzXx5$)hqwK=_ zc&GM`+Be$w&7C06|f#(_3W-$nHoy(BiqbXDw;3&TD zf2nmNoJ@_PAR7qCj8=BAr2|6h<{ZB=;YuFCR1uo^esUX>zJsPJVfR-74Uqecr-YZ6 zAjbH5vr_IL7dewD1Rs~T%hz<>0d^(`x`cu#nrI2)xD&D3o z3sT!Nsx&nZheDffyGd89O_3}b#XKKx;AG@p6S8^8783CjY~vlPy2*m#firlR>rHI3 zBSj?|lRHL$uj==fQONyZ@L)s;ME9!qrSiI;_@SSQewu7sYxVg&<{!tYgJ|FmfsMFu zAI{y4?R0a4W2N-L=g&rDJa+=9$_~mBQ~E<5o)aR{_#o>_j7yHTMMJ}56nLjh&u{Sl zk_J}a3Rdj^jAFLx1mz?p>5s-OPX-Imr%P{TK^Pt8nn#g0uMeU-2V&f6RXS1eS%%IF zh88<%i^GaWN4!4ShIfyT#?@U|Sy+=h!<(-I+l=?uSnW5Tt21kBaUYv@cSvPn#U*{u zO!TwI7wrMXw4G(0+wPb8ua5YZ&BU|E1KSqmYn!CgWGKw>0znI7wKDH#Jt+Hgu#>!2(+jMclxx?@1784J;5~w66O1ior-j+M(R(Q{ z5fqk2I=DospOgg=5#2dFz<7P)@n*K%&cC~Ay`lva#k*Y`nqWMZgLkl_gz<`aw3;o# zHB@i(!P$P#!ku8s5`H~2FW9NLx{cqt*JulKk3jC2SpsVWN@`XqRw`1qJFF7QIl@%d zX3b4X4*%NVSTu?AkT)@`Gls32fAt5r*my}|p-=^kGqLI|_=y_PGL~Ho9L4kV?5G32 zMg`(lmgf;`9E~l)u5_7t)cWsRQkL|zR6yp%(gaCCJ0>LcGCf$RTj{!UwsF>O0}WRC zNj`3V#T6OfCn!9HdcOL>7I59&A%*zyN|~Y}oHI#x27xv&M zMRbWs=7fwtMo5?iy0s#5o!b(0sC-58HVrf1dld246=M&kDWWh;-6LWq)xxVh)I+aE zk*posHF59}!rBP+x6GvhTMBw5A1_`toZc8B{>r^!{{>6ft)pli{|A^jRi0L>7?m4y z&Tf591m&M%8L8LVF)C(X?;%3UseGW2_}>wbhVt8JhgYy0ITD+btrBeno4C1r8|mIU zreN~;_?A+&hBiJArhcA!wYpd_x_4UJ%E(vaS=+&0q<+wnp-s7FOXM{r7!`R_k*AxT z`M0AHj_Puu4Eje=#|ky~*`gx8<=uNJE2bO4>jX(nLVnM-J=tio+#y`c1+}9^%N@=;)_<4hSEt6$9;f&Ajo*-s2^LD- zrWhMu?AI7aav8m?3)ws;m^&-Y;+c}p8`FoD72@VsXUP|qmZa}vVw8-Hljc_!DG(yE z(cPs1>|@~JN(I$OUJIL({?)m2W+){gzPDhDnAE~90|RH8@Q5c(^F|86I!R%T;< zS4ltBLSazJcSV}BQ%Y8tM|s-5T-I|1Hd(z~Y&D*iitd+`zPPP=ZA~9==(T`0to3z^ zRF#6I-Xu@HbRIiWt)vF9@Pdy?g~qr~Qr_OYQkar!Vv0Aj`KiXdS5zX5J4;}c3{wJb zS>H+qk65NHg;VyI>?Bp)+%<|Var>wdK}j zsaA*pNb*X2?@VfEawZn@vMqMZ20fsoovWfAGRx_a|BF$lTt(tV)HM8n0# z{XJoFcs!Ux|Ngr(JJPS^abB;($&kj)i|7LP%p>h z&o|CIUWxec_YBc#f?(W6`1{ECEAA`ks;ry=qQnR(8EgihjG;N2fwt-QOk&mj6`rmDJZr z$+;B#AVxcPqkSHAoIiD9KLhk6o_P-IRYQbOv`yp7%Dc0c^>1q-DEwZ2a;F_-!jJ zEGT^?2zV#>dL=nA@r5FUTthX^fOGe#Xj1uW|40O4e5tHz0iDw5;g#146 zZ1RUS;!1QH&Im)GHD}$vx(Di5bjbTZV3yNG>k+ERv)F%X+KlPN5`68^{I%+N8rc}m zIT+w_s`Br$ii09Cbw<;ZHKcKqf1L{y#iO_5BQ!<< z@LK9~vhh=q#LKN&7MORFTTQl(6>;daZvY_Nl3nNXt16_lxrXF-j9>w%K#_Kj=Vy~e zLD(Vq<}cg)&Yqc^n*qQ*i&`t_v_nBk7eB{1w=yAe^kL|5ZK<^|JOfNN<<2xQv}gdt zF1AZe)s!zAl*}=E6B9a%4jMD3H$)vLm)=0>Ip7UKCu&XWGG@DY2aYy8N$Zq~L61}m z^l0QsBMJVRCLil7&_wjE`I{#d>6W^_e%LW#Q%Ns zfDzca`OvX}P@Vc$7yg5edg??Fv<~qVhaJ$u)g`KhkS@R=D%htOte^?e2{(Mj32-X z7YteE&VLM4(k!1t*RCCas5UhA0n7CW#)pXAiB?m}<2s2Q@dOKVyYFQHpoq^q!g8l{ zB^jcMZzmVfqXx`uf9vNoeNj3Q2U*4BfyVD?e_WXAm@Kj7p7L4iCRWm_w74gWA#9%X z6Gd&^n3EGNwuT3TxAlCc{qU3M@6v#T`87FXe(>3RgYxB~tls@&A7-Zc{u+c!q}b=r z>4Vom;oON15ute>xGpXyKxC5F$n}oRfSQe}??rY_2XTawET^A~S~k#6=Vh08^X)Vk zJC-Hztn5bNDjSTPV{{ZcX9@c9iL2Uy*=d2j_i*578QbUMv4WmnuT_JWnN15Fn8f|= zL+XE-hNKzgoUaGwT9?A_OL+IF)4P3@(K9l=qS4AiRHjBR``yj8^!GuYl=t0|M-jJ! z-HMZSqVDV0G^wH#G0G$}=23PQFL#s|rr2uU7qx~3nJwa(A?Ps9Xs%WFW3z0ui12Ll zPo1O|l!%l-gbuUeA|l)Hp!l}Oak3!TK*Swb%+B#A0XEc;-7^pD{G$nVO@aSywVY2g zH#q=mW;$Ho+s#N8);TwEaxoso2b%!D|I{90RYp^!WuhHObW`)y&)y@r=NEc|HKA*x zuBEYo+sZXsiJa|CCCN0@L^L>R!eJU1+S#o?%$?#4JT*(mSC6AkB@A$nsB7i8qX+qW z5cvDuRy~#RRkFOE50CkC-lXCde1Q@-+;be{S@VZa z%B|LM)GYmv8CE|@-N*H!_PYbjO`1iW)qGsut?0vaFU5Axn^b}+1AyM?CqcHL3fQ%w zx-*|LP~?-hVQ}n8f3U`HOKAB9JjSl|njHs99%>EmYI%vU2N-&Vcy!`mswzvp)gsjJ zBgRJ+o|UH=dh25!ui57qHJ!K5aiDf`nTk^KDU?_7Rs_-wAGXT{a<(BdD^qvb+um^= zoZt7oolCg}S~crOiJZgu&PvIc%<>k0DQ+2{;5B98KBd5~b6)p=|)5Y|_f!~va zwb}dGd_pjcGUhXy(KE)|)ki1b_rBy0oLCaCHmmX=#0rik?)e->v+z9a0;ap~J-Pd) zIw$RVY`&x<#=C^nu}op2LQMpe8t@RfYE`FeE^F}`+f|n7|*fpi^20l92fY~Np zYJVL60_9ckA{8bUHY)i*kbY#X@<3F>T~^T8Nn+_^^uyd9RpMb3Q?lt8^QK-kbd*Ct zxNtS%Rpw5^!epVkNu~5*7N%dSV?)iU&^QHEqT!rvmDM0+Ck}P(7oI|FJrt)uUD=Y& zB89c!CWap~5}OKU33H&V8P(u`^!sw!6JF|Im=;@G+gL_#NY$7jn1AVE-iephbsq&M z<3eglG_|+t9bzk~gSxH7F8>UdxDEADs;h~F)trr%X~Mee0%%8Y%^b<4O`*UbFIxus zDw&%n(rHesUm#m(L1l{8z9D6*VtZ=&)kR}%wj518uwKh$WkdN=g}-3bMPI7ur*r{+ zB2!Wy-snofBDouDTxDClRfC$PiyO?V1t(`lw0G|aC4s{a?Jm)0UOezBDJLZ|0@rkJ z`INbxXeOJ)NoOn4XjhSXc9cg$mrBD80$2W-)VX`bXi~>2aw8FnUt7U!jH`pkPf->L zsELf|)Hd)#oZr?;g$KQD`Y4=jW1p%?voUtjb`xI;mnj9rhW&C6C5WTjXAFh`6C4~9 zja!OqrPzTNgO$ie`%SX9p#3xj3sjKxuZl&v~{ zf!UyS=>7%Elx#Yb1%ummdl#7eQRMA-R^1MPfT*L_F^ zEzEI`JG|sA6+oyinfn_oOvyW^zr9bM;i;b*MAdD-!dfNN&+Z6s1Am43fv)r~Ni1O9 zYlMz@s8`&4M_ao+#w9b2Zl`=d4>3Ag z_*tzkE;Y{puY3>pGxOtDJD)tQeB9u7%-!w+u}`z>FLYJ0`UJmZ`z$c*a@k*CM2Skg zBM|Q21)EQQ*E68?Tctxn1l=Fo6(?$AC8_$iu?dRpS?KCK&2xilF}3rPaa=eM*hWl^ulHc`Q4}3d__Ty&`*<5+paC_fIZmtG~AjusvU|cTzImjpcBUT zbdJq@yRLzEo3e#-MH~y4lGSIAsrG8>iewUSwrM$)#*=n9ncy|PvO2p&Z$=e>h?ivJ znqkRNdIN-#Eic){9F9NK_$t^mtX-$-K)deW`%AHlfOE5sh-$j(7Nt5lnppo+$mmE+ zj(f6W1YFWuDW=Y1g3mU#cK7Ixms7_<1HgbeQcJQAViW>oSaMywF}^)p*m|J;Bj@8_ zCcMwNY*Q$>`v+{Hm(eElLQ{3;}&EGQDR zNm8p|*3EW$dHqFhpAH8|EZx7}tDqez(o||rdPbg;F}X!mK#Fz_=e0X>DtGF;eHF+D z!>h-@mzni><`JpHF^5~-nj(IA(+o5vmW3^owPm(dE=>___6NJsIV=u$NGX+XfKOS$ zw95)XYXI9DeAgV!4&P~3>4eVyvu2W@$KqM#kA@SJTJv)%2emr?1SIoO_t!5!mB9c6 zTDd>?Q=Qr>R1?gG9 zbEa;Sm1>}Vb=pwqICLQwZDVWTnJ9qWrSO*GPORY(?&@MXDjHwzthJ3SVx-&XV>g;0 z|IoTx-&4I|+uxUXP*{a5*W*uXdE``XBGpWp_L=gqsEmy+dg8Xza6)&LheKz<5#G11 zYvBkx6;98WF7FZ=(s;Bc@)j9q5ipu8L_NOZm?fPcU5c^lU60~wO)X|e8{QiyxGS1_ z?TB#2FOG2#?z!K<(qwaFrS?R&d=Unr? zHAoW5B0IMA6vfM)RA~IIaF;>vMEFsw57Fmgc!g_B$lUNbvJ1;n=V#h#1$R;7#Ec0$ z0@ZPYqn){BX01lBX=%}MOfIrdg%aK<>fITZ%?7_=x^0P7tu1Z`v9B&FO{`(g@-qGltA)TIblI?Gy8vsWs;@v{Tru z*0LE8m6}J7wO_jB_+Cry1yZkA{5T$VJGJn)c0K%*(ijE4KTc8Cn+R&6GwJ+zH4b_t4{Dj+D-U+?v0iUuPoZEMXH$8_hwuV_j(f5W zIH<_SUEV1d1~nC%jSPDe8#*6C?Uu{irTd&lkGsNR6Z&xEaiyMS@meFC57TL>R=*V=K%5~pJK71)%vm~GeA z8!X(4l)p;gPly!R>DA}GrrXz{zo)K>Av&zKrOzaY@z@g}YJ3E)LRjC_?ynXGy+VtL z$R`~jP(;R~H2*iPXe!?0>Xk#4-=8zzxAvY_`bdlX*3)L6&8Bw%eh17nGnP#Vnh0q` zgU;hr*Z!gx`#Z@eJ`8g~B}b7swsR49$x=Qi+kZtAQ$J`oFEF+S^>SX=>!I}-bCjHJ zGiA5>M+uIME;{W|X~dzWlBf-frU}}54kQ}Bye;{e0TedNN2}4o9xoLTr0cu&%!X3D zF9yXI_sE(feYxJ)nz(_{#Y4$DcvN00m=t=RDjed$ESkex)dX>N3f^PhXLmp)dm_Kj zA!=LecOprnudVv?pAi{G{D8!qh*tHiSyJs@>gduI7Ceejim__nO7Gg8~G zh2m>6z_JUmy1H6zwNT17D_eBVcEd`H0KtmSe!XZO3Mote+LM!`7OJ#>ze;qS$0-S$ z^?F^mxiQBkV~ncl0+6T9Cjd*uQ}9v_zAwa^3eMhay7F$&l8p9^!zY+qCFz{C)+1kl z!5cdFrK{y6iMTjvwtD(6x>Y#*UHBqB1t_&@lYh_@4#@kWT#X*y%3sG*`$KEB`y#?U zrSCV^YWNqiXB>ttolOB=8I)Dh_VAD#$8awX*?th-=>w9Q`~p-D3Omf~Uxb~_k!5** zDeCz_1#rE8Q#EXWRrsyvKurByfLhycIPkzA1I>)}p7GLv%FpNJ_341n6NzBKup#&C z9r-n;x_e`x8P=M_%8 z=jxpQp_yKoL8KnrQshY6&R?73zT(eAu<#fkV?s^M)a#14s@{(ShxnwaPs|K=tPp z1fmg^>`!QXd0u~_E~wc2_VY)JDk3Nne=8*1YST}y8RIfI#ookES>aU+2a>K2=>@#Q zWUv&_vnCd1kGiJdu_y3qe{NqwK45Jed9HVfUXV8CHu$ufPW}RxDNapOJ!iNH<8DPm z)mZM3wIR~rt&72IR1=j6KItD>V~IKc9mx!`l|xT0xw=BwQF->WQMGrYx5w0}!E)Dv ztC{~CzX2Wynj2fomV)hdKP@|@u8jH&Y5nX$Zd^gQI2jE#)8j%+|cxJ^~*6gf&7f%eg+V zpM0co^<`~37u=rJ=RW8JXuC(JR>H=*45hTX)!JmVA!$gCTf2vS*>%+ul4UUn)&j1K zHHn*jqGRl0QDv0~y987HdQ)$GE4?a)Ht7A5EL(*sf{)>;67Kp>kbU zmdO+L>$!+XfLd|0s*kQ(Z0HhGVggl5;!ZtJ-kmh%qfSkEnJNZw*0h2ozZLyB^Qp|O z;*V&QksNAdboy`SdpL!DUfXqer>dTc^wF2E?SbOb7ZLf)&kz>5ynL+eAC?YsX@Y43 zA*DE{j%R%#;n_K^DeYXd(2NWWA13{?!xMks{9X&Li|D>jxBIs?P=Wkq$hl3~TZrg+Qx@hHI!8 zhzg-_*mh#^x)XJeIHQKPrYjV+4yEJi!vK;5;np(iSC{Kg2Tfo+9IS?D{}%ntEwzP! z=D<3Bcqn4cn40qhD{G1zdBUVRvC=i{iM8*^R^9;KdQ^TDeGki8Y(k_^eZv}0;#z~7{&t!!(R>Rmw z3g%}JZLXyQEchOHZfd@CpCopUr$*T`J-u4Q+{#q zx)Hv(=*3krvFtdxTsOT(N2AjaEJWQh_ z**A75v=NETA=BQju(78nuA&=z8ykGI`c`SZxNOLR36eLtGhoHH8@YNbT0sCn!%|CA5_=UxUnJK;vGLBc=UVfY!tfk5ww5` zPV#3_X>IkqNh1M((ma4Rh>K7xa ziEMx89Z%T47UrxXf0UZ;GvCNHvs&EQlY5f7F=%<=)%eB-gI5b`)zRRUvRu=+FCXA0 z6kEGLa`0V5UyxQxJx@ooaOmf1EQ*c%m36WBSzvsJ2?eZuD@;gPsKc=lGU zC@&*pd3`pWJ3{AntyLxd>Ld8k9jj&UU-=?20CyPuUMJoBK+)t(gn!n7q35PFRE&wF}BVqv4AB{$qdoOlv^*6eYa@!pBQvo~-< z54wR>N2xE+BCB2i7{^*2pE`f_&CmmfwBk-@Ox}S}ks2gLlqwkg_k}9Ph*VDLryknN z7SbpcHv-Jipc0*u!Kt1D`}$rtzAKEc_N{vq`sQoN_<>R@i>ea>C$8dzjykqhJCV+> z20HJ+!+YFZw)7<=@e#O?YW0Jv`z<{BUA=#4vpC~w+m8>Qqiqk?O6y6)z0XT)y%$95Bw zgB{V$|GVEL4j;qS{)u(8>5UP)(SQjoAxAXa^HIjrixC=nciP}&>zsJy+!1$=D{hJQ zAj(F9qnK^#^~!${6;3L=82MUxn!A`IN#U&OywW_Ac36C!Kca|A55lwa$azww7wvje zwbhSqGfEd!2>rxE_Lc~ajUj?{eGTzB7_@hO4zg&j;+K<8eL6z7@ur{j{}EV3lh(>% zzB9T{4oAGfp_ac~;3?kt4h+lEn-8ydUA;V2wf*B`pNW ztKTdr6+5!Y`dsa^uIS*IN5?;T(|H*?a+pZY?=GK6_BYdUL)M@tlnlylVkRb{E zxcS9ZYE_=|#O=c8Iv(1>qDDqb0~5Yjv)$PMiTtvB+0rUE$vn-u0+Qn6zvW3Rq}l{4n&$SRQ_Oo)hZu2_#wED~tXF)8E)$=tK% z7Z-Msh$46~-oc#izqORIzx3?HpJ)Z^IBAL;aWAS?&GhIkkcvVi3_`Dxr!tJgJy=Or znG^*@bzzHXq5lp6Eu{V3pJ}Y?D&^zwgz>j`kZ%m61sz#!7Zr^$JrZU+b_30etJE{s2Bz zt}>M^3`H8M(1KQjEC%P2OOm#E9yUr>73k4YDL!x06nCEaA!6VHoM`zS9;iJj+V zr3yJ#;;02piiUbf(-<4~S(_%|2Ajq=H-C*-JrzKAtl|212TY=Fyu%`2ofXRF*HbsG)Q-M!x`^$l;`{p&%5i@-fLfLuixI+ z^;zHF&gn%#c0n=U_6&@KHRrM2|2C~?KCq!A?%-LD&-g0K0Iro7o^5lgH85M@9})8M zb)&6VUky?H{7ZuZq;|;}MemzbtLPGH)I#m1A%kH_0gL{W)O-poD%u6??&b_GRQzNW z;=*Ob#e!;sXh--Lpn$`?74n7b%Jb0C!gD=(+>}oEs(khjhc`s89p^`X?$FS9(QY|A06YNdUm zjEroF{6k4xItt43u@^Kk|j@gLf%q13D zbd|QPng`_YH{SU)RVV9_)ZTo2PuOd7QovOhbSS5S+Aivd@vC^(9-T{NY*cV_F;md| zD8Tn+X>G~(xfUz;snDpe`h>#;J1e7Gdr``d_TrZCcjJzZO}dmlAaOFdqkFCkhieb3 z=mnO}^+N@^>@fonV#X3dTXA}EWFqct6j%2q`)9GiGm^C2bnNh#MG!Dlb)tk`h)UlV z*W(ij8zKy1TK;?&8S7IrUV-7|ENh|61nS*YUwA_Kf!hu!$DirlcV zbXU%MzUkyVF8;USk9`0zi%R;_@#px%dxJ9$ev8Xk2JmtT^NAkz@fryY^uTVpHkf#c8081LHe$lsGQ-#SAAL7frs~H-E(NyUCCduE?kln0L7Y^2WUT1z z`V&xKtv6U7$QCuZtT37AQ*P)pZs#%}Bs7QG$B~xf1eCUvvq=Hv>qfZI4@8)XJg+0p zu=(tb;<dWlqk>26m?otnFc*A@S9F&`{Cv`O5+F74=lXb8=g%fT!)DuYyyQHct=h z+d4@~sW?-wF>0W!yTXs^2u=@A#eNDPUP;IcCWK*fVPW5x#XC33~g!fB%E18~7!8Ft5dnCROTDzquZQfZmJx)(T zGFWBLaha7Rf&u351B)9a>UtB-HA}TCGW}1I%QTL0PteulM@~u7v(U-8Ge4uX`ePflv6~yM_oF%y_H3JqI&JQD5V6vk}L$-gc3 z!9+^=I>W%?$j=$`dyA6gkX@!6%R{^_LMgo?D~H||m&{HE&=yus!8F3X?ISMg0L-Rg zrnr!JSHUJ!4vKKYGlY`fxkKdq+Y2DbcyOa&(aIWUQbmfie}g-e6UwjED{CabQ zaWoM1*n(S)r*w^X%%%J0NNfA`mz#aLtdI|s_eEs>!3ZW!4087|2s;cnjdw8)w`NZz z$+Y4->Mi}{PSf2AdL4?QEg!L46ElK)w+xZTyYyFz^@NLu^r=XMAg%CU=7nly=$M4_?Q?EXM9w4zcD%F|TOplyKXF)j>pwV9cRG zhyDncyODjfnPAc{Mpn>nt!zpgZO)J{z zLQv#DD?-@#`E<(Q&%0q3T?cv#bfo-KHF3vgavP#YyaB3_lSln=D$7x6u)U&0<>F}3 zeScD>;~Y_)X-}X`n7m2O{TFhzBE&U#^OG0&*jyYJLZaU!Ye5o1L{~N0VCrdSRvk-@ zHK!U1cjQ(tXaJW{4h0MbB|Qc(D2s2N8x5}>$~|vb@;CQy)x(F4z-WjewC-isFYHYy z4!jScgMGw$?8O3R#qBGwI4UF3v}9-B7rCRe6uvEuTm-YWE?L&@y-;2iXgUJ}L3q>0 z%1I;_Lg;4hfVbHcB|a@h*Ebve!?bLKhxP6A?J97v4QrR>NOJ{usW5jc<^Lf&(QH9= z%1Z=PPhi&H$4f>_fk%`miK#j5pqjBNr0^)K#y6m8Mm@Av!vJB-q(5#~ zMH~sy?iq0Ii0qM43BLZe#be#f3~?4T$Ht~%0L5WRn^pS@9`pCa(QoYSXYoz@1#7?u zg8NFX2FE#%c?pz&#muL2zD~;i54DvnNI&ZTq3ISOWy=MdsZ@IgE(46D;p1Ys$b$UUyrpAtX@*%`NAwhB(wZd9r z|7w6)lERx2A>Lpb3PWza$atSC@&FHRBiDzk@Z=r_`kQjwDC`PSi&s#Ky>d*_abMmM z&D1o7mg4L*weMS^`)H{AZYmy8Q0LRKYM#e6O#wk0lUwT4uNtvxlVu1e?7;r}zaEQP zw{ct_;EbB3P0@CRu6|Dvs2ER|Y8!LE_n_zHU>-pcFNy@WckQv~K#iOpF+0@NQoFf@^w6?EOfMCH5EO zN@dBG)`RX@{NpO}xyTawTm5s1U2NCpXAT?#E!^xYgw+OLx-wt8%~AZpI8_r9&V zIl~fVh4den1^aM2{XB{=d&9WWk2SQ^KSkd}3$NDx^)um?$(r2~r2!dWg?ATE#nlbY zQ?;FGk}$KUhGIX2n~Z<6(EH|u@t~4MIv0n0>pB}!Cf@5|7C(@))+ zAG@EfrStRT^11z4bKBklU0V~cuIX9(RWunNE=@aPhA1|+u{7vTDC~QPMrOn8$+Cms%EY10W4HWk=`!aI2DG6760@}49b=- zX@mqUUM76(2`4N^(@@r4yAij8;mS7!bCXP#)Hyv>9Pz`o3ehAS4E;pw$3Q*^?YHk# z0ugem1lXime!J4$scen7SRPo~E ze!>Ar6Ta7s&usY-d^?l8Nev*veSWs%A>l+1;p)5Eb74MUt@_@sDso57WIwEQPn5y7 z!c!<9kiFT6Qs2#Rm~XMeH<(&hN!fa$ArT(X5$Videt4ZtwiK!z)^Km~Vln60 z%YboQ)yv?wto#Q+7RPkHk830l)B?240uoet-8h#qjk7x+=NTpn@6WaNDYIe%l+PQ< ztPr?G;-t8>TTh`#Gdzz>bzX45w=?$Ygso@>fRr)^P+3kv{iT^4K)xgOsW{4_k3^7! z@elu(h=)~&zKQv6;hWlz4gTa0^v?D6)WJgtOs7>*W?4Cd#w;{-PAJf8YC14X)$(|+ z_wsZWmN?B$zU{BT{rM|%^BMazXPUNab>XT|n=%d{4=~`QK9rtv`^s=7pR6dlq zCMOpQ!(-jwokOF9PxZflFk3ycse>S1o{lu9OQ?s0iYC)>zg)!bw-eJnwu>p&uhd(> z2X8pwQ$5+gW*=>IU> ze53liDl;Z>61G7NS3?rzp$(I7FhY;{FNy_i>E3gT0fo-9;VC=_pvR@^_v&MFg6}X4 zuCq&0V3D_H(iPu;gL>xxp#&Nt#3Dfa#{^)&V&)-hJmnz%Rzb7*!-(>L6}6Jl)}T;P zqx3Mbvo3mV0#?!JMm;y$IqGfMR}p9Ih-RZh{B_lCitX!cuDag7Gj>8XuY5*vl#Ft) z!!4Z%0tL0n+~8Jv=PRxF`!@RVRJ8tb7e+Fbbi?b<|IzzF2`#^`%l7=>1;61vOemCH8+pad$tibsU8v_b zxe#{@fMIwtz8Zvcrbpu-UT_+B~H0ZsO!>Yj|{iniAY%%rEJkv)q)c2@V z`JTEEXrH`Mr7wv^;`OPL*|acue>=U}XoK`NXH^5>uA5JdZzwK@)(bP#u4kxcV400QnMhm+yN#)i;(Y|)dyh*DgKR2fLJ#y=xm z1reeHVPtucFZm}B5dZzhzL!BI68~sz3vbGXdnYs$lXXm@Ci4C5W-lY*qXu#M@G zaJ{13Z4Gl*7WNA6&5Q@OoR&1@Q---)&=c4nws<`8KsPbEKVBuLo-Uj_V;`Nj5Pn%j zbdkcIw{iYx$_~JW6|L>Wv_56w^s!oPW>4UO#;V|t``_dAl}r$0g<8qwKQx_gcVl7} z7KheH$kz2e86P_f*1j#H1+g8+vGDk87cnRa`_mOQYAY}Z7?gYb<8DioUUcN zsO^K~Y;O5XQwF7%XJVa%B`+V_8E}O2CDp0~@?%EPrMK^!9$M#Z@s3k)8WbZ$UA+qT z0`;_4+JKbjxZF%LRhi>Ta<@?s&F=Q3vbnSTRYTAQ0QZy3+_%X5__|GN;uLMtrNf0e zFIH!HW^3p2^aXL}ds;^s7rjHLWG5Uyd2r%N$7atjX*bIpn|_!$!W^q;R7LiAHw9^V z^4RdpQqLG+2d*`Svi)XPv2NXJ;PPm`P^v!8o%UpsO(XR;NgEE79RHsb*H>%`<@O0L zi<{rL*xjAqfs6}z8HKIZldm`BWVdTMp87(g-!vKv_-H?*z3J5Uysjk~Q|D+;z>cax zINSH-?h(y{M*{>qEQ&_&tgHUbyb{2C?l`pJbIg^~T6g*&eJ`mM)HIS*-x^(Hks~Qc zS19)6qf}|tSDxR?;yAB$1|7>TgjGImeK2?@4iFDrL(47Lk5E(MZ{kXcI=d?~gm^rR zvOZ%Q2duhBMj=&Nui3j3xp7)9D(@&2UD3hv;PE7hB6%~7_=bE9f#4+;nm3VXp1WFS z{wMs8ayLn1xgfn9Ey_eKj638Gm*b^FQz_``uMFSDnm78z&}()@?%-D81bm=@Ohgpg ziu^FJ5?4t2FfNZyojn_G_g^OsaZCx!>fgd<`wTK(yPXjwZY5M_nuu3$?1+RvewLzr zj*ZKa%~nhHs(fH)cy~;6&#t+sR`;U+Y-S2=VrJox@Vxx*fd#_*SFunN0_#q?4Kd$I zXQPVWoj5*wR>Q$o>$F4~Hr)L-WUNSz7BvWa%`7qn@Np|lEH$&Q=Yj1qETA>Wocb9t z!=K~tNzfWFy3&RTGbenk&Uxn51Z~6`2{2=-=w`sL1QyMwD3bfi)e*CHUhhNbLSxY zm^t3J*C(|9o5?*)w3AAR2AUW7Pd7Ld7xcaDg;A4M)Soo`>n_nNj&_5iP>d0}C9~1) zIDuWiv+sqWDe`CTf7b=Ni`ExQW|{T>99SHakqK=#vWWfS=&u3B?}sMb-@2Uf_{-ft zy8N&0P9hrbCyOHk4c{{EzsKe#KzqXdH|Rgx=f7|IzuV(?MWPGw^2P%LMQ$s%dW3eT PyrUqaB3&eD^7elKWhPFp literal 0 HcmV?d00001 -- GitLab