From af85268f718ec91c62b1f27fd4640a22c7f6b044 Mon Sep 17 00:00:00 2001 From: dyonghan Date: Mon, 18 May 2020 09:45:17 +0800 Subject: [PATCH] !1 Initial version Initial version --- .gitignore | 136 +++++ README.en.md | 35 +- README.md | 38 +- experiment_1/1-LeNet5_MNIST.ipynb | 517 ++++++++++++++++ experiment_1/main.py | 108 ++++ experiment_2/2-Save_And_Load_Model.ipynb | 566 ++++++++++++++++++ experiment_2/main.py | 142 +++++ experiment_3/3-Computer_Vision.md | 344 +++++++++++ experiment_3/dataset.py | 86 +++ experiment_3/resnet50_train.py | 172 ++++++ experiment_4/4-Natural_Language_Processing.md | 374 ++++++++++++ experiment_4/CRF.py | 177 ++++++ experiment_4/cluener_evaluation.py | 73 +++ experiment_4/evaluation.py | 161 +++++ experiment_4/evaluation_config.py | 53 ++ experiment_4/finetune.py | 152 +++++ experiment_4/finetune_config.py | 124 ++++ experiment_4/pretrain.py | 167 ++++++ experiment_4/sample_process.py | 100 ++++ experiment_4/tokenization.py | 388 ++++++++++++ experiment_4/utils.py | 263 ++++++++ project_1/1-Model_Optimization.ipynb | 565 +++++++++++++++++ 22 files changed, 4674 insertions(+), 67 deletions(-) create mode 100644 .gitignore create mode 100644 experiment_1/1-LeNet5_MNIST.ipynb create mode 100644 experiment_1/main.py create mode 100644 experiment_2/2-Save_And_Load_Model.ipynb create mode 100644 experiment_2/main.py create mode 100644 experiment_3/3-Computer_Vision.md create mode 100644 experiment_3/dataset.py create mode 100644 experiment_3/resnet50_train.py create mode 100644 experiment_4/4-Natural_Language_Processing.md create mode 100644 experiment_4/CRF.py create mode 100644 experiment_4/cluener_evaluation.py create mode 100644 experiment_4/evaluation.py create mode 100644 experiment_4/evaluation_config.py create mode 100644 experiment_4/finetune.py create mode 100644 experiment_4/finetune_config.py create mode 100644 experiment_4/pretrain.py create mode 100644 experiment_4/sample_process.py create mode 100644 experiment_4/tokenization.py create mode 100644 experiment_4/utils.py create mode 100644 project_1/1-Model_Optimization.ipynb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09417a1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,136 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# system file +.DS_Store +.swap + +# IDE +.idea/ diff --git a/README.en.md b/README.en.md index cef451a..054d389 100644 --- a/README.en.md +++ b/README.en.md @@ -1,36 +1,7 @@ # course -#### Description -{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} +The experimental guidance based on the MindSpore open source deep learning framework. It is only used for teaching or training purposes. -#### Software Architecture -Software architecture description +Part of the content comes from the open source community, internet or third party. If something violates your rights, please leave a message via issue or submit a pull request. -#### Installation - -1. xxxx -2. xxxx -3. xxxx - -#### Instructions - -1. xxxx -2. xxxx -3. xxxx - -#### Contribution - -1. Fork the repository -2. Create Feat_xxx branch -3. Commit your code -4. Create Pull Request - - -#### Gitee Feature - -1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md -2. Gitee blog [blog.gitee.com](https://blog.gitee.com) -3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) -4. The most valuable open source project [GVP](https://gitee.com/gvp) -5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) -6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +Please go to [MindSpore Open Source Community] (https://www.mindspore.cn/) for more videos and documentation tutorials. diff --git a/README.md b/README.md index 4d9ba31..698e3a7 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,7 @@ # course -#### 介绍 -{**以下是码云平台说明，您可以替换此简介** -码云是 OSCHINA 推出的基于 Git 的代码托管平台（同时支持 SVN）。专为开发者提供稳定、高效、安全的云端软件开发协作平台 -无论是个人、团队、或是企业，都能够用码云实现代码托管、项目管理、协作开发。企业项目请看 [https://gitee.com/enterprises](https://gitee.com/enterprises)} +基于MindSpore开源深度学习框架的实验指导，仅用于教学或培训目的。 -#### 软件架构 -软件架构说明 +部分内容来源于开源社区、网络或第三方。如果有内容侵犯了您的权力，请通过issue留言，或者提交pull request。 - -#### 安装教程 - -1. xxxx -2. xxxx -3. xxxx - -#### 使用说明 - -1. xxxx -2. xxxx -3. xxxx - -#### 参与贡献 - -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request - - -#### 码云特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. 码云官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目 -5. 码云官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. 码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +请前往[MindSpore开源社区](https://www.mindspore.cn/)获取更多视频和文档教程。 diff --git a/experiment_1/1-LeNet5_MNIST.ipynb b/experiment_1/1-LeNet5_MNIST.ipynb new file mode 100644 index 0000000..57f357e --- /dev/null +++ b/experiment_1/1-LeNet5_MNIST.ipynb @@ -0,0 +1,517 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

基于LeNet5的手写数字识别

\n", + "\n", + "[TOC]\n", + "\n", + "## 实验介绍\n", + "\n", + "LeNet5 + MINST被誉为深度学习领域的“Hello world”。本实验主要介绍使用MindSpore在MNIST数据集上开发和训练一个LeNet5模型，并验证模型精度。\n", + "\n", + "## 实验目的\n", + "\n", + "- 了解如何使用MindSpore进行简单卷积神经网络的开发。\n", + "- 了解如何使用MindSpore进行简单图片分类任务的训练。\n", + "- 了解如何使用MindSpore进行简单图片分类任务的验证。\n", + "\n", + "## 预备知识\n", + "\n", + "- 熟练使用Python，了解Shell及Linux操作系统基本知识。\n", + "- 具备一定的深度学习理论知识，如卷积神经网络、损失函数、优化器，训练策略等。\n", + "- 了解华为云的基本使用方法，包括[OBS（对象存储）](https://www.huaweicloud.com/product/obs.html)、[ModelArts（AI开发平台）](https://www.huaweicloud.com/product/modelarts.html)、[Notebook（开发工具）](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0033.html)、[训练作业](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0046.html)等功能。华为云官网：https://www.huaweicloud.com\n", + "- 了解并熟悉MindSpore AI计算框架，MindSpore官网：https://www.mindspore.cn\n", + "\n", + "## 实验环境\n", + "\n", + "- MindSpore 0.2.0（MindSpore版本会定期更新，本指导也会定期刷新，与版本配套）；\n", + "- 华为云ModelArts：ModelArts是华为云提供的面向开发者的一站式AI开发平台，集成了昇腾AI处理器资源池，用户可以在该平台下体验MindSpore。ModelArts官网：https://www.huaweicloud.com/product/modelarts.html\n", + "\n", + "## 实验准备\n", + "\n", + "### 创建OBS桶\n", + "\n", + "本实验需要使用华为云OBS存储实验脚本和数据集，可以参考[快速通过OBS控制台上传下载文件](https://support.huaweicloud.com/qs-obs/obs_qs_0001.html)了解使用OBS创建桶、上传文件、下载文件的使用方法。\n", + "\n", + "> **提示：**华为云新用户使用OBS时通常需要创建和配置“访问密钥”，可以在使用OBS时根据提示完成创建和配置。也可以参考[获取访问密钥并完成ModelArts全局配置](https://support.huaweicloud.com/prepare-modelarts/modelarts_08_0002.html)获取并配置访问密钥。\n", + "\n", + "创建OBS桶的参考配置如下：\n", + "\n", + "- 区域：华北-北京四\n", + "- 数据冗余存储策略：单AZ存储\n", + "- 桶名称：如ms-course\n", + "- 存储类别：标准存储\n", + "- 桶策略：公共读\n", + "- 归档数据直读：关闭\n", + "- 企业项目、标签等配置：免\n", + "\n", + "### 数据集准备\n", + "\n", + "MNIST是一个手写数字数据集，训练集包含60000张手写数字，测试集包含10000张手写数字，共10类。MNIST数据集的官网：[THE MNIST DATABASE](http://yann.lecun.com/exdb/mnist/)。\n", + "\n", + "从MNIST官网下载如下4个文件到本地并解压：\n", + "\n", + "```\n", + "train-images-idx3-ubyte.gz: training set images (9912422 bytes)\n", + "train-labels-idx1-ubyte.gz: training set labels (28881 bytes)\n", + "t10k-images-idx3-ubyte.gz: test set images (1648877 bytes)\n", + "t10k-labels-idx1-ubyte.gz: test set labels (4542 bytes)\n", + "```\n", + "\n", + "### 脚本准备\n", + "\n", + "从[课程gitee仓库](https://gitee.com/mindspore/course)上下载本实验相关脚本。\n", + "\n", + "### 上传文件\n", + "\n", + "将脚本和数据集上传到OBS桶中，组织为如下形式：\n", + "\n", + "```\n", + "experiment_1\n", + "├── MNIST\n", + "│   ├── test\n", + "│   │   ├── t10k-images-idx3-ubyte\n", + "│   │   └── t10k-labels-idx1-ubyte\n", + "│   └── train\n", + "│   ├── train-images-idx3-ubyte\n", + "│   └── train-labels-idx1-ubyte\n", + "└── 脚本等文件\n", + "```\n", + "\n", + "## 实验步骤（方案一）\n", + "\n", + "### 创建Notebook\n", + "\n", + "可以参考[创建并打开Notebook](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0034.html)来创建并打开本实验的Notebook脚本。\n", + "\n", + "创建Notebook的参考配置：\n", + "\n", + "- 计费模式：按需计费\n", + "- 名称：experiment_1\n", + "- 工作环境：Python3\n", + "- 资源池：公共资源\n", + "- 类型：Ascend\n", + "- 规格：单卡1*Ascend 910\n", + "- 存储位置：对象存储服务（OBS）->选择上述新建的OBS桶中的experiment_1文件夹\n", + "- 自动停止等配置：默认\n", + "\n", + "> **注意：**\n", + "> - 打开Notebook前，在Jupyter Notebook文件列表页面，勾选目录里的所有文件/文件夹（实验脚本和数据集），并点击列表上方的“Sync OBS”按钮，使OBS桶中的所有文件同时同步到Notebook工作环境中，这样Notebook中的代码才能访问数据集。参考[使用Sync OBS功能](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0038.html)。\n", + "> - 打开Notebook后，选择MindSpore环境作为Kernel。\n", + "\n", + "> **提示：**上述数据集和脚本的准备工作也可以在Notebook环境中完成，在Jupyter Notebook文件列表页面，点击右上角的\"New\"->\"Terminal\"，进入Notebook环境所在终端，进入`work`目录，可以使用常用的linux shell命令，如`wget, gzip, tar, mkdir, mv`等，完成数据集和脚本的下载和准备。\n", + "\n", + "> **提示：**请从上至下阅读提示并执行代码框进行体验。代码框执行过程中左侧呈现[\\*]，代码框执行完毕后左侧呈现如[1]，[2]等。请等上一个代码框执行完毕后再执行下一个代码框。\n", + "\n", + "导入MindSpore模块和辅助模块：" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "# os.environ['DEVICE_ID'] = '0'\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "import mindspore as ms\n", + "import mindspore.context as context\n", + "import mindspore.dataset.transforms.c_transforms as C\n", + "import mindspore.dataset.transforms.vision.c_transforms as CV\n", + "\n", + "from mindspore.dataset.transforms.vision import Inter\n", + "from mindspore import nn, Tensor\n", + "from mindspore.train import Model\n", + "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor\n", + "from mindspore.train.serialization import load_checkpoint, load_param_into_net\n", + "\n", + "context.set_context(mode=context.GRAPH_MODE, device_target='Ascend')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 数据处理\n", + "\n", + "在使用数据集训练网络前，首先需要对数据进行预处理，如下：" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DATA_DIR_TRAIN = \"MNIST/train\" # 训练集信息\n", + "DATA_DIR_TEST = \"MNIST/test\" # 测试集信息\n", + "\n", + "def create_dataset(training=True, num_epoch=1, batch_size=32, resize=(32, 32),\n", + " rescale=1/(255*0.3081), shift=-0.1307/0.3081, buffer_size=64):\n", + " ds = ms.dataset.MnistDataset(DATA_DIR_TRAIN if training else DATA_DIR_TEST)\n", + " \n", + " # define map operations\n", + " resize_op = CV.Resize(resize)\n", + " rescale_op = CV.Rescale(rescale, shift)\n", + " hwc2chw_op = CV.HWC2CHW()\n", + " \n", + " # apply map operations on images\n", + " ds = ds.map(input_columns=\"image\", operations=[resize_op, rescale_op, hwc2chw_op])\n", + " ds = ds.map(input_columns=\"label\", operations=C.TypeCast(ms.int32))\n", + " \n", + " ds = ds.shuffle(buffer_size=buffer_size)\n", + " ds = ds.batch(batch_size, drop_remainder=True)\n", + " ds = ds.repeat(num_epoch)\n", + " \n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "对其中几张图片进行可视化，可以看到图片中的手写数字，图片的大小为32x32。" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAD7CAYAAAAVQzPHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAcm0lEQVR4nO3deZRV1Zk28OepQWaBYrIQAkZBIKyICjjE1U3aEDHdaU268QuiTRxCVqKt+aJGErOiMdqxTaL9pfOZDh0ZooKxo+0QtQnNEhLRBis4oSggDhArTIIWU0FVvf3HPexzCupW3enc4ezntxar3nvGXfCy795n2JtmBhGRpKsqdQFERIpBlZ2IeEGVnYh4QZWdiHhBlZ2IeEGVnYh4QZVdBkguIHlbqcshUmg+5XZFVnYk3yG5lWSvyLIrSS4vYbEKiuRnSK4huZfkZpIXlbpMEr+k5zbJO4N8/ojkuyRvKta5K7KyC9QAuLbUhcgWyeoMthkHYBGAmwD0BTABwB9jLpqUj8TmNoB7AYwxs2MBnA3gYpJfjLdkKZVc2f0IwPUk+x25guRIkkayJrJsOckrg/jLJFeSvJvkbpKbSJ4dLN9MchvJWUccdiDJpSSbSK4gOSJy7DHBug9IvhlthQXdhJ+TfIrkXgCfzuB3+y6AX5jZ02bWYmY7zeytLP9+pHIlNrfN7E0z2xtZ1AbgpIz/ZvJQyZVdA4DlAK7Pcf8zALwCYABSragHAUxC6i/+EgA/I9k7sv1MAD8AMBDASwAeAICgu7E0OMZgADMA3EPyE5F9LwZwO4A+AJ4leTHJVzop25nBsV8l2UjyfpJ1Of6eUnmSnNsgOYfkHgBbAPQKjh+7Sq7sAOB7AP6R5KAc9n3bzOabWSuAXwMYDuBWM2s2s98BOIj23zhPmtnvzawZqe7lWSSHA/gbAO8Ex2oxszUAHgbw95F9HzOzlWbWZmYHzGyRmX2yk7INA3ApgL8DMApADwD/msPvKJUrqbkNM7sDqcrxNAD3Afgwh98xaxVd2ZnZWgC/BTAnh923RuL9wfGOXBb99tscOe8eAB8AGApgBIAzgi7DbpK7kfqmPK6jfTO0H8B8M1sfnOufAHwuy2NIBUtwbh8+j5nZi0FZvp/LMbJV0/UmZe9mAGsA/CSy7PA1gZ4APgri6D9QLoYfDoIuQB2A95H6x15hZlM72TfboWVeyWEfSZ4k5vaRagCcmOcxMlLRLTsAMLONSDXVr4ks2w7gTwAuIVlN8nLk/xf6OZLnkDwGqesbq8xsM1LfvqNJXkqyNvgzieTYPM41H8BlJD9OsieAG4PziEeSltskq0h+lWR/pkwGcBWAZXmWPyMVX9kFbkXqQmfUVwDcAGAngE8AeC7PcyxC6pv2AwCnI9Wch5k1AfgsgC8h9W34ZwD/DKBbugORnEnytXTrzWwegF8BWAXgXQDNiCS8eCVRuQ3gCwDeAtAE4H6krkUX5Xo0NXiniPggKS07EZFOqbITES+oshMRL+RV2ZGcFrxCspFkLs8DiZQl5Xby5HyDgqmXftcDmIrUax8vAJhhZq8XrngixafcTqZ8HiqeDGCjmW0CAJIPArgAQNqEOIbdrPtRd9GlFJqwa4eZ5fIqkg+U2xXqAPbioDWzo3X5VHbHo/2rIluQegG5HZKzAcwGgO7oiTN4bh6nlEL5b/vNu6UuQxlTbleoVZb++eR8rtl1VHse1Sc2s7lmNtHMJtamfxZRpJwotxMon8puCyLv1CE1Usf7+RVHpCwotxMon8ruBQCjSJ4QvFP3JQCPF6ZYIiWl3E6gnK/ZmVkLyasBLAFQDWCemXX2TpxIRVBuJ1NeQzyZ2VMAnipQWUTKhnI7efQGhYh4QZWdiHghCSMVl43G6852cdO4g1ntywPhLHQnzwmfXW1rasq/YCKilp2I+EGVnYh4Qd3YAqo7L3zu9JXxj2a177qD+1x83a0XhivUjfVaVc+eLn7v2gkubiuTFzYGvdTi4h6Pri5hSbqmlp2IeEGVnYh4Qd3YPO37YjgYxuQBL5SwJJIU1YPCkbca/88oFy/52p0uHlbTG+Vg6rrPu/jg/okuPmZJQymK0ym17ETEC6rsRMQLquxExAu6ZpdGVZ8+Lm457aS02116+xMunt03uyHPtrXudfFdWz8brmhp6WBr8cWhccNc/OJ37omsKY/rdFFLx4b5f8nNU1y89cCpLmZLOO5p1aq1LrYi57ladiLiBVV2IuIFdWMj2C18LH3flLEuXvGLubGcb+6u01383hl7I2v2Hr2xSJm7f+Ty8MPiMF5/KMznb5x/WbjNzt0ubNv9YbtjWXNzoYunlp2I+EGVnYh4Qd3YiN3TwztIi2//cWRN+d0FE6kUJ9b0cPG/PD3fxa0Wzlh52Xe/2W6fvvf/T8HLoZadiHhBlZ2IeMH7buy2q8Oh1L99zQMuPqG2cF3XU1bPcPHgu7u7uHrvochWayECADVrNrr4L78628ULfnaXizPJz3R5l6np/7bExdk+MB9VzbBNNbq2V4fbtNayw+WF1GXLjuQ8kttIro0sqyO5lOSG4Gf/eIspUnjKbb9k0o1dAGDaEcvmAFhmZqMALAs+i1SaBVBue6PLbqyZ/Z7kyCMWXwBgShAvBLAcwI0FLFfRHBgYxhf1/jD9hnn4aGfYdD9uRTjOl3W0sRRNueZ2dEa5Hr972cUzbrrexZl0+4as3+9irnyxw22q+/V18Z8WDG23bkrPDZFPHXc/03loT3jcH/50pouf+FbpxuTL9QbFEDNrBIDg5+B0G5KcTbKBZMMhFP6paJECU24nVOx3Y81srplNNLOJtSiTWUJECkC5XVlyvRu7lWS9mTWSrAewrZCFitsHl53l4jOnvRrLOaasDWcIG/6EnvCpIGWV29F3RPN50NY+Fc5MtuHy8L99VbdWFzec/v/b7dO/Oruua9T6A/UuPm7eSy4+r8+3XBydIW3kml3t9m/L+czp5fq/8HEAs4J4FoDHClMckZJTbidUJo+eLAbwPICTSW4heQWAOwBMJbkBwNTgs0hFUW77JZO7sTPSrDq3wGWJ1e5/CLuu478aPsA7/2N/iP3c2yeEf83HHhuWo9+vno/93JJeUnI7ndYpp7l4y9fDB9jfPmdBmj16plmemQeaBrj4vsc/7eKR+8I8H/bD5zrcN45u65F0MUlEvKDKTkS84M27sT0uaXRxMbquy8c/Gn4YH4bXNYZdi9V7wgm2ez6yKvYySTIdPC+cnHrP0FoXN52/x8VvnHNf7OVYs2eEiwevKUbHNDtq2YmIF1TZiYgXvOnGlouf1K9x8W237HPxHx7JfggeEQCw63a4+IXo5ZMii+b23Nv/7OLfROZE1ryxIiIxU2UnIl5IdDe2ekCdi7vXHOpky9LoVhWWqXrQcBe37gi7JTANBCWVJzqy8ZRF4Tu30Xlj29ZvcnExurRq2YmIF1TZiYgXEt2NPf6pcHicu49/KrKmPO58XtP/DRePem6ri+eeHb4/27p9e1HLJFJo6eaNvfriq1zMlS8hbmrZiYgXVNmJiBdU2YmIFxJ9zW5Ej50u7l3V9XW6S96Z4uLXfzXWxWu+9/OCluuwbgxf2j6/Zzgs9YZnwlvyy74cXr+zBk2kLUfr8c0wt0+5LRyi7+XJi0tRnKOkmyTbasIZ0uKfIlstOxHxhCo7EfFCIrqx6Sb6vajvLyNbdT1T0pY9/Vx83MMbXTyp+Wtp97nh24vC8+UxyXa0S3tD3Vsu/l2vv3CxvpmkI21rw0eYhvwonEVs0uj0eZutv7g6HG8x+sJ/JdH/HxHxgio7EfFCIrqx6BbOtvvghHtdHL3zk63omwt189O/xfDDXjNdfPPAcHl08u1iDAMvArR/E6FuZeGO+8cZI8MPSe3GkhxO8hmS60i+RvLaYHkdyaUkNwQ/+8dfXJHCUW77JZNubAuA68xsLIAzAVxFchyAOQCWmdkoAMuCzyKVRLntkUwmyW4E0BjETSTXATgewAUApgSbLQSwHMCNsZSySP56aNj1nHfLeSUsiRRDueR21Slj231+5wvl15C8fOiSUhchb1ndoCA5EsCpAFYBGBIky+GkGZxmn9kkG0g2HEJzR5uIlJxyO/kyruxI9gbwMIBvmNlHme5nZnPNbKKZTaxFt653ECky5bYfMrobS7IWqWR4wMweCRZvJVlvZo0k6wFsi6uQxRJ9mPeG2feUsCRSLOWQ2ztO69fu8zrlXiwyuRtLAPcCWGdmd0VWPQ5gVhDPAvBY4YsnEh/ltl8yadl9CsClAF4lefghnu8AuAPAQySvAPAegOnxFFEkNsptj2RyN/ZZpB+B5dzCFkekeEqZ2zXDh7m4aWQxBjgSvS4mIl5QZSciXkjGu7Ft4UTSbx6KPhIV3kQbXhPW65mMWlwMzRZOkr3pUMeTeLNFk2Qn0aYrPubiN76SvLuv5ZjbatmJiBdU2YmIFxLRjW3dscPF0QmmURXe5WpbHI4E/F9jnixKubry011jXPzMuSd2uE3VznCSHXVopVKUY26rZSciXlBlJyJeSEQ3FhY2gqMjDEc1t4wsUmGOdsrqcC7PwXeHd4Kr94Z3qWyr5oT1ycfvfc/FY/j1dusq9e5sdN7lHdeED02XS26rZSciXlBlJyJeSEY3NgP8STgbzqShhZtPMxND1u8Py7HyRRfr7qq/WjZvcfGJC9v/NxyDsFtb7l3aqes+7+KWO4e4+JiGhlIUp1Nq2YmIF1TZiYgXVNmJiBe8uWZ3zJLwGkJdCcshcqSWTe+0+3ziL1tcPNa+jnI26KWwrD2WrC5hSbqmlp2IeEGVnYh4wZturEiliD6W8rFbtnSypWRDLTsR8YIqOxHxQibzxnYnuZrkyyRfI/n9YHkdyaUkNwQ/+8dfXJHCUW77JZOWXTOAvzKzUwBMADCN5JkA5gBYZmajACwLPotUEuW2R7qs7CxlT/CxNvhjAC4AsDBYvhDAhbGUUCQmym2/ZHTNjmR1MGP6NgBLzWwVgCFm1ggAwc/BnR1DpBwpt/2RUWVnZq1mNgHAMACTSY7P9AQkZ5NsINlwCM25llMkFsptf2R1N9bMdgNYDmAagK0k6wEg+LktzT5zzWyimU2sRbc8iysSD+V28mVyN3YQyX5B3APAZwC8AeBxALOCzWYBeCyuQorEQbntl0zeoKgHsJBkNVKV40Nm9luSzwN4iOQVAN4DMD3GcorEQbntkS4rOzN7BcCpHSzfCeDcOAolUgzKbb/QrHiDg5PcDuDdop1QOjPCzAaVuhBJodwuG2nzuqiVnYhIqejdWBHxgio7EfGCKrsMkFxA8rZSl0Ok0HzK7Yqs7Ei+Q3IryV6RZVeSXF7CYhUMyTtJbib5Ecl3Sd5U6jJJcSQ9twGA5GdIriG5N8jzi4px3oqs7AI1AK4tdSGyFTzT1ZV7AYwxs2MBnA3gYpJfjLdkUkYSm9skxwFYBOAmAH2RGm3mjzEXDUBlV3Y/AnD94Sfgo0iOJGkkayLLlpO8Moi/THIlybtJ7ia5ieTZwfLNJLeRnHXEYQcGY5s1kVxBckTk2GOCdR+QfDP6TRV0E35O8imSewF8uqtfzMzeNLO9kUVtAE7K+G9GKl1icxvAdwH8wsyeNrMWM9tpZm9l+feTk0qu7BqQepfx+hz3PwPAKwAGIPVN8yCASUhVKpcA+BnJ3pHtZwL4AYCBAF4C8AAABN2NpcExBgOYAeAekp+I7HsxgNsB9AHwLMmLSb7SWeFIziG5B8AWAL2C44sfkpzbZwbHfpVkI8n7SRZldtNKruwA4HsA/pFkLg/Hvm1m882sFcCvAQwHcKuZNZvZ7wAcRPvW1JNm9nsza0aqCX4WyeEA/gbAO8GxWsxsDYCHAfx9ZN/HzGylmbWZ2QEzW2Rmn+yscGZ2B1IJdBqA+wB8mMPvKJUrqbk9DMClAP4OwCgAPQD8aw6/Y9YqurIzs7UAfovcRpLdGon3B8c7cln0229z5Lx7AHwAYCiAEQDOCLoMu0nuRuqb8riO9s1GMLjki0FZvp/LMaQyJTi39wOYb2brg3P9E4DPZXmMnCRhKsWbAawB8JPIssPXu3oC+CiIo/9AuRh+OAi6AHUA3kfqH3uFmU3tZN98X1OpAXBinseQypPE3H4lh30KoqJbdgBgZhuRaqpfE1m2HcCfAFzC1Ei0lyP/yuJzJM8heQxS1zdWmdlmpL59R5O8lGRt8GcSybG5nIRkFcmvkuzPlMkArkJqLgTxSNJyOzAfwGUkP06yJ4Abg/PEruIru8CtSF3Ej/oKgBsA7ATwCQDP5XmORUh9034A4HSkmvMwsyYAnwXwJaS+Df8M4J+B9KM5kpxJ8rVOzvUFAG8BaAJwP1LXNIpyXUPKTqJy28zmAfgVgFVIDZzQjEhlHicNBCAiXkhKy05EpFOq7ETEC3lVdiSnBU9VbySpiYQlMZTbyZPzNTum3oNbD2AqUk/5vwBghpm9XrjiiRSfcjuZ8nnObjKAjWa2CQBIPojUTOppE+IYdrPuR91YklJowq4dGpY9LeV2hTqAvThozexoXT6V3fFo//T0FqTeyUurO3rhDGoek3Lw3/YbzZeQnnK7Qq2y9I+j5lPZdVR7HtUnJjkbwGwA6I6eeZxOpGiU2wmUzw2KLYi8ZoLUC77vH7mRZk2XCqTcTqB8KrsXAIwieULwmsmXkJpJXaTSKbcTKOdurJm1kLwawBIA1QDmmVlnr0CJVATldjLlNeqJmT0F4KkClUWkbCQttw+eN9HFdt2OjPbp8c3uLm5b+0bBy1RseoNCRLygyk5EvJCEwTtFpAt7hta6+IXxj2a0z9QBl7k4Ca2iJPwOIiJdUmUnIl5IdDd229Vnu/jAwPjPN/I/d7m47eV18Z9QRDKmlp2IeEGVnYh4IRHdWHYL30vcPf1UF3/7mgdcfFHv+OeYPmHwbBcPfOEsF/dfv9/FXPlS7OUQkaOpZSciXlBlJyJeUGUnIl5IxDW7qn59XTz/trtcPPaY4g6o+PaFc8MPF4bhKatnuHjoh2NcnISXq6V81Qwf5uKmkR2OVO4VtexExAuq7ETEC4noxpa7lycvdvGUu8L+bbfPlqI0kmRVffq4eP1V4cjyG/7hnlIUp6yoZSciXlBlJyJeUDdWJEHevGOci//nb38cWaMJvNWyExEvqLITES8kohvbtvMDF1878+sutpqOH6Tc9n8PuDh6p1Sk0ln3VhcPrlbXNarLlh3JeSS3kVwbWVZHcinJDcHP/vEWU6TwlNt+yaQbuwDAtCOWzQGwzMxGAVgWfBapNAug3PZGl91YM/s9yZFHLL4AwJQgXghgOYAbC1iurFhLi4uj48WlextwSMsEF08a/bUuj9/SKzzSE9+6s926YTW9MyyllJtKyO24bWnZ4+LP3/mtduvqX9/g4lZUvlxvUAwxs0YACH4OLlyRREpKuZ1Qsd+gIDkbwGwA6I7ijkIiEifldmXJtbLbSrLezBpJ1gPYlm5DM5sLYC4AHMs6y/F8BRXt6tat7Hr76iHhl3vT9XpaJ+EqOrez1dQW5nP9f2xst651+/ZiFydWuf7PfRzArCCeBeCxwhRHpOSU2wmVyaMniwE8D+BkkltIXgHgDgBTSW4AMDX4LFJRlNt+yeRu7Iw0q84tcFnKSnSU1+hQOQOqK7K3Ih3wNbd9pQtQIuIFVXYi4oVEvBubLftU+FDxrtE9OtwmOkFJ+1Fe9b6h+OfgeRNdvGdobVb7Vh8KL/30+48XXWzNzfkXLAtq2YmIF1TZiYgXvOzGbrg8/LXfPv/nRT33sN67Xbxj4ngXW8PajjYX6VLV+HAu4mMH7O1y+22t4TZ3bY3M+hR5xxwAGMnP428OHzi+f+TyrMr39qHw/dsvf/hNF/dcvs7FbU1NWR0zF2rZiYgXVNmJiBe87MaWUrQLcNu8sPvxh092L0FpJAn23xUZeXv8o11uP3fX6S5+78x9Lq4e2H4wg3MXPO/iG+reyrl8J9SGw6Ct+MVcF0+dcZmLq1a8iLipZSciXlBlJyJeUDdWxGPVAwe6ePZzz7dbd37PXZFP2T1IXI7UshMRL6iyExEvqLITES/omp2Iz6rCAS9Orm0/An03dj2vximrwyEBm18Op9h94yv3dLR5O9P/bYmL77vp8+3W9XxkVZf7Z0stOxHxgio7EfGCl93YUfPCF54nPdv1JNmdueHbi1x8Ue8P8zqWSDFc1PePLn528YkuHl6TWdtnzLOXuvhj/6/axbtGZzdlwey+77v43/u2P3ccE1OqZSciXlBlJyJe8LIbm+0k2VV9+rj4zTvGtVs3snZH5FPlP2UuyTe6Npxa4L/GPBlZk34wimjXddg9YZ5z5ZrIgc8qSPniksm8scNJPkNyHcnXSF4bLK8juZTkhuBn/66OJVJOlNt+yaQb2wLgOjMbC+BMAFeRHAdgDoBlZjYKwLLgs0glUW57JJNJshsBNAZxE8l1AI4HcAGAKcFmCwEsB3BjLKUsMfYMZyB78q/vbrdu7DFx3DeSYlBuZ67P0+GYdAf7tbp4+y1nu9jGZTe0evSB5CHr9+dRusxkdYOC5EgApwJYBWBIkCyHk2ZwoQsnUizK7eTLuLIj2RvAwwC+YWYfZbHfbJINJBsOobjzRIpkQrnth4zuxpKsRSoZHjCzR4LFW0nWm1kjyXoA2zra18zmApgLAMeyLrunDjPEbt1cvHv6qS5urWVHm2etpVd4nD5VbXkda/n+8Pvl3tXnuHg0GvI6ruSm3HM7E++vqXfxAyMGuHhmn50FO8eOSWHX9aSTG128buwTOR9z8N3h3V+uLINh2UkSwL0A1pnZXZFVjwOYFcSzADxW+OKJxEe57ZdMWnafAnApgFdJHn5A7TsA7gDwEMkrALwHYHo8RRSJjXLbI5ncjX0WQLr+4LmFLU7mog/67psy1sWLb/+xi6OzGhVOfse85a2/dfHoK9V1LaVyze1snTAnHE79u4O+4OKZ5/+yYOd4+8K5XW+URrMdcvFPd4Uz6lXvDZcX4xqAXhcTES+oshMRL1Tsu7Etp53k4ujEu/l2M+OwqzWciHjXvvAB5eNKURhJNB4Ih1xadzDMu+hTBMNq4v8/Eu26Pr0vfNvumU9/3MW2fW3s5YhSy05EvKDKTkS8ULHd2EoyccVVLj756k0ubu1oY5E8nDzndRdfd+uFLm6cHl72efE7XU+Gk6/oXddo17V1x46ONi8KtexExAuq7ETEC+rGFkFbc3iHrHW3JuWR+LQ1RYZZisT1vw4f25366mWxl6PdA8NFvuuajlp2IuIFVXYi4oWK7cbWvr7FxZNu6nju11LO6RqdoCQ6T61IKbRu3+7iqhXbO9myMEo23lUn1LITES+oshMRL6iyExEvVOw1u+g1iLr5HV+D+GGvmS6+eWDsRWpn2PIDLm43kbCIlIRadiLiBVV2IuKFiu3GZmLwz54rdRFEpEyoZSciXlBlJyJeUGUnIl7IZJLs7iRXk3yZ5Gskvx8sryO5lOSG4Gf/ro4lUk6U237JpGXXDOCvzOwUABMATCN5JoA5AJaZ2SgAy4LPIpVEue2RLis7S9kTfKwN/hiACwAsDJYvBHBhB7uLlC3ltl8yumZHsprkSwC2AVhqZqsADDGzRgAIfg6Or5gi8VBu+yOjys7MWs1sAoBhACaTHJ/pCUjOJtlAsuEQmnMtp0gslNv+yOpurJntBrAcwDQAW0nWA0Dwc1uafeaa2UQzm1iLbnkWVyQeyu3ky+Ru7CCS/YK4B4DPAHgDwOMAZgWbzQLwWFyFFImDctsvmbwuVg9gIclqpCrHh8zstySfB/AQySsAvAdgeozlFImDctsjNCveAMoktwN4t2gnlM6MMLNBpS5EUii3y0bavC5qZSciUip6XUxEvKDKTkS8oMpORLygyk5EvKDKTkS8oMpORLygyk5EvKDKTkS8oMpORLzwv9NPrlrn6D7QAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ds = create_dataset(training=False)\n", + "data = ds.create_dict_iterator().get_next()\n", + "images = data['image']\n", + "labels = data['label']\n", + "\n", + "for i in range(1, 5):\n", + " plt.subplot(2, 2, i)\n", + " plt.imshow(np.squeeze(images[i]))\n", + " plt.title('Number: %s' % labels[i])\n", + " plt.xticks([])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 定义模型\n", + "\n", + "定义LeNet5模型，模型结构如下图所示。\n", + "\n", + "\n", + "[1] 图片来源于http://deeplearning.net" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "class LeNet(nn.Cell):\n", + " def __init__(self):\n", + " super(LeNet, self).__init__()\n", + " self.relu = nn.ReLU()\n", + " self.conv1 = nn.Conv2d(1, 6, 5, stride=1, pad_mode='valid')\n", + " self.conv2 = nn.Conv2d(6, 16, 5, stride=1, pad_mode='valid')\n", + " self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n", + " self.flatten = nn.Flatten()\n", + " self.fc1 = nn.Dense(400, 120)\n", + " self.fc2 = nn.Dense(120, 84)\n", + " self.fc3 = nn.Dense(84, 10)\n", + " \n", + " def construct(self, input_x):\n", + " output = self.conv1(input_x)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.conv2(output)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.flatten(output)\n", + " output = self.fc1(output)\n", + " output = self.fc2(output)\n", + " output = self.fc3(output)\n", + " \n", + " return output" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 推理（训练前）" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "使用matplotlib定义一个将推理结果可视化的辅助函数，如下：" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_images(pred_fn, ds, net):\n", + " for i in range(1, 5):\n", + " pred, image, label = pred_fn(ds, net)\n", + " plt.subplot(2, 2, i)\n", + " plt.imshow(np.squeeze(image))\n", + " color = 'blue' if pred == label else 'red'\n", + " plt.title(\"prediction: {}, truth: {}\".format(pred, label), color=color)\n", + " plt.xticks([])\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "使用随机初始化的LeNet模型对手写数字进行识别，可以看到识别结果是随机的（大多数情况下是错误的）。\n", + "\n", + "> **提示：**MindSpore提供的基础数据类型为Tensor，Tensor支持numpy、list、tuple作为输入，并将其转换为Tensor类型。" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD7CAYAAAAMyN1hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZgc1XX38e/RaKRBEkYMCNDKsEggB9sCyyC2RC+YzYDxEjAEE6EACokJkAA2ONjgBAIP8fba2BjZgBTAYCwwm3EACxQjkCUUDDYggQALIbQvI7Qzy8kfVaqqGXVP1Ux3T3fP/D7P08+c6rpVdbvnzJ17azV3R0RE8utT7gqIiFQ6NZQiIinUUIqIpFBDKSKSQg2liEgKNZQiIikqs6E0m4bZDWF8LGZvdHE9P8HsG8WsWtUwa8DMMetb7qpIgnK7cGXI7cpsKJPcn8P9oNRyZudjNrvdshfj/u+lqlpi2/0x+x5myzBbj9mPMavNuOxEzJYWoQ6LMft0wevZeb39MFtYlDpKW9WR24dg9iRmazDr3EnXlZrbwd/rTzBbidk6zB7DbHhHi5S+oewdPZqrgfHAIcAY4DDg2qKtvbzf4VXAqjJuv3L1jtxuAh4ALijJ2svzHV4GHAl8HBgGNAI/7HAJd+/8CxY7XOPwusN6h7sc6sJ5Ex2WOnzNYYXD3eH7pzm87NDo8ILDxxPrO9ThJYeNDr9wuN/hhjbri8uOdHjIYbXDWodbHcY6bHNocdjk0BiWnRatJ5i+yOEth3UOjzoMS8xzh4sdFoWf6UcOlvH7mO9wZmL6bxzey7DcQIetDq1hvTc5DHO43mGGwz0OHzhcmOOzxN8L3B2uY2u4jq86NISfaZLDEoc1Dv/ayd/zfg4LHE5p8zvoyS/ldr7v5UAH70T5ys1tuM3hlsT0qQ5vdLRMIT3Kc4GTgAMIelHJHtQ+QD2wLzAFs8OAO4G/B/YAbgceDbvA/YCHgbvDZX4JfDHnFs1qgMeBd4EGYDhwP+4LgIuBObgPwn1wjmWPA24CzgKGhuu4v12p04BPAZ8Iy50ULjsKs0bMRuX5Lix8JadHYLZbnvIB983AKcCysN6DcF8Wzj0DmAEMBu5NWc95wBLg9HAdtyTmHgMcBBwPfBOzseFnOgazxg7XG/yX/TqwNaVcT6PcLlRl5/YdwNGYDcNsAMHv+zcdVaOQhvJW3N/DfR1wI3BOYl4rcB3u23HfClwE3I77XNxbcJ8ObAcmhK9a4Pu4N+E+A3gxzzYPJ+gqX4X7Zty34T47T9n2zgXuxP0l3LcD1wBHYtaQKHMz7o24LwGeBcYB4L4E98Hh+7n8BrgMsyGY7QNcGr4/IGPdcpmD+8O4t4bfYVd9C/etuL8CvELwhwLus3P+0e1g9nmgL+6/KmDb1Uq5XVrlzW14k6DxfR/4ABgL/FtHGyqkoXwvEb9L8EveYTXu2xLT+wJXhP+5GsPWfmS4zDDgfdy93fpyGQm8i3tzF+o7rM163TcBawn+c++wIhFvAQZlXPeNwB+Al4EXCHoRTRS2b++99CKZdP4zmQ0EbgH+qUh1qDbK7dIqX24HbgPqCEYAA4GHKGGPcmQiHgUsS0x7u7LvATeG/7l2vAbgfh+wHBiOWXLomm8Y8B4wKs8O4PbbbG8ZQVIHgsZgD4L/KoUJ/qtdgvtw3PcnSNL/xb0ly9IZ399M2x7qPhnX0xWjCYZ/z2G2giCRhmK2ol0vpadSbhdHJeY2BD3PabivC3vgPwQOx2zPfAsU0lB+BbMRmNUT7Mf6RQdlfwpcjNkRmBlmAzE7FbNdgTlAM3ApZn0x+wLBMCSXeQTJd3O4jjrMjg7nrSTYL9gvz7I/ByZjNg6z/sB/AHNxX9yZD52T2fBwf4dhNgH4BnBdYv40zKblWXolsEfq/sygt/oZzOrD4f3lOdazf5fqv7NXCRqLceHrwnD94yheb6CSKbd3CD5THdAvnK4Lt7FjfrXlNgS7P/4Ws93C0/j+kWBf6pp8CxTSUP4ceAp4J3zdkLek+3yCfTm3AuuBt4Dzw3kfAl8Ip9cDXyLoweRaTwtwOnAgwT6GpWF5gGeA14AVmO38gd1nEjRgDxIk5AHA2Zk+abDDe1MHO7wPIBhybwamA1fj/lRi/kjg+TyfaSFwH/BOOHQblrNccEDgFWAxwffe/o/3JuDacB1XZvhMx2K2KU+dmnFfEb1gHdAaTmfpJVc75XZsX4KDea+F01uB5Eny1ZXbgSuBbcAiYDXwGeDzHa6y7e6TjMwWAxfi/tvOL9zLBL2AV4CP495U7upICuV2dr0ot3vDCbPlFfQqxpa7GiJF14tyu/IvYRQRKbOuDb1FRHqRgnqUZnaymb1hZm+Z2dXFqpRIuSm3JanLPUoLLrl6EziB4Ajdi8A57v568aon0v2U29JeIQdzDgfecvd3AMzsfoJrOPMmUz/r73UMLGCTUiwbWb/G3YeUux4VSrldpbaxmQ99u6WX7JxCGsrhtD35eClwRPtCZjYFmAJQxwCOsOML2KQUy299Rr5L6US5XbXm+sySrLeQfZS5Wu2dxvHuPtXdx7v7+Fr651hEpOIot6WNQhrKpbS9JnYEba+JFalWym1po5CG8kVgtJntZ8EZ+mcDjxanWiJlpdyWNrq8j9Ldm83sEuBJoAa4091fS1lMpOIpt6W9gi5hdPcngCeKVBeRiqHcliRdwigikkINpYhICjWUIiIp1FCKiKRQQykikkI37hXpZfqOHBHF71xQ2OO8h7wcPzRyl4fnFbSuSqYepYhICjWUIiIpNPQW6aH6fCJ+nM2awwZH8caG+J4fCy/6cUHbOGHB6VG8bvcjo7huQ2sUD3hobkHbqATqUYqIpFBDKSKSQg2liEiKHr2P0sYfEsUtA2tzlqldtiEus+idom27ZvT+Udw0bLco7rt2axS3vrqwaNsTgbZ5t+AfB0Xxn0+/rSTbe3rsY/HEjXF4w5qDo/i5h+pKsu3upB6liEgKNZQiIil69ND7mDvnR/G1e+Ye5h7wzOQoPuiSeIjc0rghV/HMFlxdH8V/PuVnUXzywlOjuObMuEzL2nUFbU8E2ufdT6N4U+u2KH6vuZVC7FMTx7vXDChoXdVCPUoRkRRqKEVEUvTooXcW8//qR1E8cdoFUbzP5wobeuczY8yDUfzPT8TPgV6y01OjRYrnr9/8YhT3OaepoHW98Z3hUfz2cXcVtK5qoR6liEgKNZQiIil69NB79t+Nj+Iv/2CfKL6nYVYUJ4/a7T4gPhm8VAb1iU++3XeXtVG8hOo/KVe6z7rHx0TxGaP+GMX/vmvyxPL4IottzXHcf+X7BW27dXth97CsRqk9SjO708xWmdmriffqzexpM1sU/ty9tNUUKT7ltmSVZeg9DTi53XtXAzPdfTQwM5wWqTbTUG5LBqlDb3f/nZk1tHv7DGBiGE8HZgFfK2K9isLnRx0Flm5qKF9FpCJVQ27XDI4vgnh/2rAo/uXH4osYxtQOTCyR+54GhfrzzfG9Jm845oGSbKOSdfVgzt7uvhwg/LlXvoJmNsXM5pvZ/Ca2d3FzIt1GuS07KflRb3ef6u7j3X18Lf1LvTmRbqPc7j26etR7pZkNdfflZjYUWFXMShXL4hvi4cI39y39cGH5FUdF8fnjZ5V8e1ISZc/t5K3S3rguHnrP/2R8ccTuNQNJM3nJsVG89Z6hUdyfxZnqkfz7ue7z8d/PubuuzVW8jRN3/VMU3/mjS6L4oKtfj+LWjRsz1aMSdLVH+SgwKYwnAY8UpzoiZafclp1kOT3oPmAOcJCZLTWzC4CbgRPMbBFwQjgtUlWU25JVlqPe5+SZdXye9yvGeZ99NoqzDBcKVX/Ssii+bsjrHZSUSlCpuZ28I37ba6nTb2mWHG7/7vd/EcX99o+fvPjB9UeRxQ/Pjm/TduKAzl0ffnj/+Oj77z/73Sg+euOVUdx3W1ynhl+tb7N86ysLOrW9UtMljCIiKdRQioik6NHXeneHLV+I7492+B4vlrEmIjBnSUMU1w7bHMULzrq7aNv4z3UHRPGvl30sij+5x5Io/s7Ql6J4r8QR+kV/m/shZ5969x/aTNe/UnA1i0o9ShGRFGooRURS9Pqh97zt8dG8d5fsGcVjEiflWt/4a2o9In5WOMB5N8bPNZ6y2zJEymnhMcUbYk/dEF9bvqrpI1H84NTjonivW1+I4nmJ3VA3XL8livv3if/GLt09fshffyvNdemloB6liEgKNZQiIil6/dD7S8/ER9vGXBg/Bzw53O4zJr729gc/j6+3hfa3uBIpnDV7FL/ZFB+5PqDvLlFcY13v42z3eCj8TlP+E8lnXHhiXKfnX47ivXghV3EGPDQ3ip97KL5jf83e8R3RT5sbXwM+tl889G4eGJ98DtBnQHxyfeuWLZSbepQiIinUUIqIpOj1Q+98kke3k8Pt5PBHpBT6zI3vzH/5KZOj+Pu/ia/7LmSXzw/WHxzFzx5/QN5yfdbG9fC8pYrjsa/e0mb6pF2/GsUjbso91O9O6lGKiKRQQykikkINpYhIil65j3K/xy6K4rHfju9TuTFxZUHyihudAiTdyZub44m1jVHY4pajdDZfXjwxitdcOiLe1spXc5TufiP6Dmoz3VphjyBSj1JEJIUaShGRFL1y6F23PL4iYO2Re0fxMZfFVxZkvcHFwT/9xyg+9bTfR3Hyfnwi5XDCgtOjuPmWOM/7zZ+fq7h0QD1KEZEUaihFRFL0yqH3hJP/lPP9fMPlVS3xjQmOvvfKNvPG/Cy+/f3/Hhlf/I+G3lJmby2Oh9tjnuze4XbN6PhGMm9cFz9Vcp+abq1G0WR5rvdIM3vWzBaY2Wtmdln4fr2ZPW1mi8Kfu5e+uiLFo9yWrLIMvZuBK9x9LDAB+IqZfRS4Gpjp7qOBmeG0SDVRbksmqUNvd18OLA/jjWa2ABgOnAFMDItNB2YBXytJLYvsrlHPpZZJPiLi7Cf+JYoPurHtCbrNGzcmphoKrZp0o56Y25OXHBvFe84u36MWmobFw+23j7srMWfAzoVpe4QeYMjLzTnLlUunDuaYWQNwKDAX2DtMtB0Jt1eeZaaY2Xwzm9/E9sJqK1Iiym3pSOaG0swGAQ8Cl7v7B1mXc/ep7j7e3cfXUmHXJYmg3JZ0mY56m1ktQSLd6+4PhW+vNLOh7r7czIYCq0pVye4ya2v8f2Pys4lHRHwlPhG9tVtrJKXW03L79//9sSgedVf33sex7/4NUfz2cZ27b2vjPSPaTNc/PKcYVSqaLEe9DbgDWODu303MehSYFMaTgEeKXz2R0lFuS1ZZepRHA+cBfzKzHU8Y+jpwM/CAmV0ALAHOLE0VRUpGuS2ZZDnqPRvId3+n44tbneJ6ZMnHO1X+jnnHRHHyiYxZvbtkzyieNzo+an54/+p50HtvUs25XU7Jk8nbHt2Oh9sLL/px6nqmbhgWxXUbKnunli5hFBFJoYZSRCRFj77Wu/60N6P4Oeo6KBkYQ2HXwyaH61+6Iz5q/udTflbQekWyaq6Ln5dYs3fO0z8hcQf1lrXr8q6rZo/6eKJv3FQkr91uezJ5bi0eD6vfbt4axTMuPDGKBzw/l0qmHqWISAo1lCIiKXr00Fukt3n+3G9H8dqzcx/Q/+7KE6J4yRE5iwAw/In4ssx/2fvxKG57q7Tc124nJYfbl58yOYr7vBnfN8GpbOpRioikUEMpIpJCQ2+RHmSvmoGJOHeZG4c9FcVT//jJvOuasvvzOdebRfJk8uTR7TbD7ebKupVaR9SjFBFJoYZSRCSFht4iFay1cUMUT742vtN+S218RPsvL4lP1s7yPPnkMPraPRd2UDL3cPvg2edF8a6/GZSzTPLa7eTJ5JV+dDsf9ShFRFKooRQRSaGGUkQkhfZRilQw3x5fHbPbPb/PWWZ2y5FRPHb/CSWv04hZ26K4ZlZlPbKhVNSjFBFJoYZSRCSFht4iVW7wf8XD38FlrEdPph6liEgKNZQiIilSh95mVgf8Dugflp/h7teZWT3wC6ABWAyc5e7rS1fV6rLr6/2ieOLIz+Usk3xqY6GPoZDOU25LVll6lNuB49z9E8A44GQzmwBcDcx099HAzHBapJootyWT1IbSA5vCydrw5cAZwPTw/elA7m6TSIVSbktWmY56m1kN8L/AgcCP3H2ume3t7ssB3H25meV55FvvNPQ7L8QT38ldZgyLu6Uukp9yW7LIdDDH3VvcfRwwAjjczA7JugEzm2Jm881sfhPb0xcQ6UbKbcmiU0e93b0RmAWcDKw0s6EA4c9VeZaZ6u7j3X18Lf0LrK5IaSi3pSOpDaWZDTGzwWG8C/BpYCHwKDApLDYJeKRUlRQpBeW2ZJVlH+VQYHq4L6cP8IC7P25mc4AHzOwCYAlwZgnrKVIKym3JJLWhdPc/AofmeH8tcHwpKiXSHZTbkpW5d9/N2c1sNfBut21QOrKvuw8pdyV6CuV2xShJXndrQykiUo10rbeISAo1lCIiKSqzoTSbhtkNYXwsZm90cT0/wewbxaxa1TBrwMwx0z1HK4lyu3BlyO3KbCiT3J/D/aDUcmbnYza73bIX4/7vpapaYtuHYPYkZmsw69xOX7OJmC0tQh0WY/bpgtez83r7YbawKHWUtqoht9vW45lONVCVnNtmh2H2O8w2YbYSs8s6Kl76hrJ39GiagAeAC0qy9vJ+h1eR58qUXq935HbA7FxK8USEcnyHZnsC/w3cDuxBcJ3/Ux0u4+6df8Fih2scXndY73CXQ104b6LDUoevOaxwuDt8/zSHlx0aHV5w+HhifYc6vOSw0eEXDvc73NBmfXHZkQ4POax2WOtwq8NYh20OLQ6bHBrDstOi9QTTFzm85bDO4VGHYYl57nCxw6LwM/3Iw7MCOvG9HOjBTWmylh/osNWhNaz3JodhDtc7zHC4x+EDhwtzfJb4e4G7w3VsDdfxVYeG8DNNcljisMbhXzv5efZzWOBwSpvfQU9+KbdzfSe7ObzpMCFcV9+qzm34j+h3l/FVSI/yXOAk4ABgDHBtYt4+QD2wLzAFs8OAO4G/D1vw24FHMeuPWT/gYeDucJlfAl/MucXgCorHCc5XawCGA/fjvgC4GJiD+yDcd350iNlxwE3AWQRXZLwL3N+u1GnAp4BPhOVOCpcdhVkjZqMyfTNZuW8GTgGWhfUehPuycO4ZwAyCx6Dcm7Ke8wiuIDk9XMctibnHAAcRnED9TczGAmB2DGaNKTX8IfB1YGunPlf1U2639R/AbcCKDsq0Vdm5PQFYh9kLmK3C7LG0v+1CGspbcX8P93XAjcA5iXmtwHW4b8d9K3ARcDvuc3FvwX06wU1TJ4SvWuD7uDfhPgN4Mc82DweGAVfhvhn3bbjPzlO2vXOBO3F/CfftwDXAkZg1JMrcjHsj7kuAZwlu5gruS3AfHL7fXebg/jDureF32FXfwn0r7q8ArxD8oYD77Jx/dDuYfR7oi/uvCth2tVJu72A2Hjia4J9msZQ3t4M7RU0CLgNGAX8G7utoQ4U0lO8l4ncJfsk7rMZ9W2J6X+CK8D9XY9jajwyXGQa8j7u3W18uI4F3cW/uQn2HtVlvcMPWtQT/uXdI/sfcAgzqwnaK5b30Ipl0/jOZDQRuAf6pSHWoNsptALM+wI+By7pYr3zKl9uBrcCvcH8x/F1+CzgKs93yLVBIQzkyEY8CliWmvV3Z94Abw/9cO14DcL8PWA4Mx8zarS+X94BReXYAt99me8sIkjoQNAZ7AO+nLFdq+erd/v3NwIDE9D4Z19MVowmGf89htgJ4CBiK2Yp2vZSeSrkd+AgwHvhFmAc7esNLMTs2w/KVmNsAf2y3zh2x5SgLFNZQfgWzEQQPYvo6wcOY8vkpcDFmR2BmmA3E7FTMdgXmAM3ApZj1xewLBMOQXOYRJN/N4TrqMDs6nLcSGBHuF8rl58BkzMZh1p9gv8tc3Bd35kPnFHymOqBfOF0XbmPH/GmYTcuz9Epgj47+m4VeBj6DWT1m+wCX51jP/l2ofS6vEjQW48LXheH6x1G83kAlU24HNhD0VnfkwWfC9z8JzAWqMbcB7gI+H35ftcA3gNkE9yTNqZCG8ucEh9TfCV835C3pPp9gX86twHrgLeD8cN6HwBfC6fXAlwh6MLnW0wKcTnA4fwmwNCwP8AzwGrACszU5lp1J8IU8SJCQBwBnZ/qkwQ7vTR3s8N2XoDv/Wji9FUieSDwSeD7PZ1pIsH/knXDoNixnueCAwCsETwV8ip3/eG8Crg3XcWXHH4gdJztvyjnPvRn3FdEL1gGt4XRL6rqrn3I7WK+3y4PV4ZyV4WeDasvtoF7PEPwD/DXBqW8HAn/T4Srb7j7JyGwxcCHuv+38wr1M0At4Bfg47k3lro6kUG5n14tyu/ecMFsuwX/eseWuhkjR9aLcrvxLGEVEyqxrDaV7A+6/NbOTzewNM3vLzPSQeKl+ym3Jocs37g2fM/ImcALBjucXgXPc/fXiVU+k+ym3pb1C9lEeDrzl7u8AmNn9BJcm5U2mftbf6xhYwCalWDayfo3rURD5KLer1DY286Fvz3s+ZFcV0lAOp+05dUuBIzpaoI6BHGF6ZlMl+K3P0PNd8lNuV6m5PrMk6y2koczVau80jjezKcAUgLo2J9+LVCzltrRRyFHvpbS91GsEbS/1AsDdp7r7eHcfX0v/9rNFKpFyW9oopKF8ERhtZvtZcOLp2cCjxamWSFkpt6WNLg+93b3ZzC4BngRqgDvd/bWUxUQqnnJb2ivoyhx3fwJ4okh1EakYym1J0pU5IiIp1FCKiKTQTTFEJNXyK46K4o0f/bCDkjuzbTVRfNDV8Tn7rRs3Fl6xbqIepYhICjWUIiIpNPQWkVT1J8Xn2//xkIc7teyCD7dE8RX/9rl4hobeIiI9hxpKEZEUGnqXUc2Q+C5n6046IIoH//IPUezbt3drnUR22PKF+IZJh+/xYgclez71KEVEUqihFBFJoYZSRCSF9lF2s+R+yRVfPDCKH7zmP6P4/A3/EsUDZi2I4mq6kkEqV59dd43i5sMOzFvuvBsfi+Ipu+10O84OrWrZHMXfXXliPKO5uVPrqRTqUYqIpFBDKSKSQkPvbrb6tHio89I3b0vMGRRF/3P71Cg+4ZzJUdznf/6ASFdY//hRFVsmjo3iZK4V09T1n4ziJUdsTszZvHPhKqAepYhICjWUIiIpNPQW6QUazzw0iu+78duJOYN2Liw7UY9SRCSFGkoRkRS9fui96pL4FveN4+Jb3I+5cH5JtnHNpffmLPPnpk1RfP4liRPOX0qccF60GklvkC/v9qst3nD7E/POieK9vlcXxTWbmxKlXi3a9soltUdpZnea2SozezXxXr2ZPW1mi8Kfu5e2miLFp9yWrLIMvacBJ7d772pgpruPBmaG0yLVZhrKbckgdejt7r8zs4Z2b58BTAzj6cAs4GtFrFdJJZ8od97kJ6P418s+VpLtbdszjs8atCF3GY//Zw2ctziKW3R9d8n0xNxOypJ3hfpg7cAo3ud/4t1VXpKtlU9XD+bs7e7LAcKfexWvSiJlpdyWnZT8YI6ZTQGmANQxoNSbE+k2yu3eo6sN5UozG+ruy81sKLAqX0F3nwpMBfiI1VdEjzz5APer6t+O4mG1jVH8bzecFcUN187p9DbWTT4yiiec/KecZZ7aUhvF/3zHlVE8auPLnd6eFE1V53aWvCvUxFfjJymOfKx3nGHY1U/5KDApjCcBjxSnOiJlp9yWnWQ5Peg+YA5wkJktNbMLgJuBE8xsEXBCOC1SVZTbklWWo97n5Jl1fJHrUlKNfxsPSSb+RelPgF1zTHzC7V2jnstZZt6W+MmLI256IYp1Ynn36Im5fcjfx7mdL++KafW4uAn5yEfiegz+r87vrqpkvWMHg4hIAdRQioik6DXXeu/y5eVRnG9I8tN3j43irhzp/vCk8VF8YMPKnGVmbY3/N90x75goHkPxri2X3iVLbhfTrEMejicOicMrlh8WxfM2HRHFAx6aW/I6lZp6lCIiKdRQioik6NFDbxsfjwtGDFqaWn7EoPiE8zWJZX1+/qPkyW0Mv+6tKL6nYVbO8te//dkoLuat3ETK7TtDX4riG67fEsXPPVSXq3hVUY9SRCSFGkoRkRQ9euh9zJ3x0PbaPRemlk8Ol/9zWnwy+LPHH5CjdOD/TYuPjievGxcppZo96qO4rm9TByXLo3+fuE41Q0ZGccuaNXEhr4jL4zNRj1JEJIUaShGRFD166F2IS3ePh+qnzc1/u6r9a2sTU7V5y4kU0/Antkfx94Y/kZhTGUeYk38/o1+IL76YelR8PXjL6tXdWqdCqEcpIpJCDaWISAo1lCIiKbSPMo/+Fu9vHNuvsH2PyYfED7s2PiVC952Urtp3l7VRPKhP+n7JLy+eGMWv/9fYKH7pm7cVtV47JP9+ThmwPooXPftOFM88P95f2dHVb5VAPUoRkRRqKEVEUvTKoffBs8+L4l1/MyhnmY0NFsULL/pxQdsY9f9rorj1VT1hUbKrGbxbFL8/bVgUn7XbzxKlBqauZ+mmwVG8z4PxzVs+tf0f8i5z1TU/j7c3aEPqNvJJDsOTV689NfAvo7jSe2yVXj8RkbJTQykikqJHD70fnHpcFN+7ZxyPmLUtimtm5X7kw+C/OjSeuKjz27bXd43j51/ooKRIB/r3j8L7x90RxWNq04fb+SSviKm/K//VMTcNPDeKr9szfn/CyfGVat3x6IlKkOW53iPN7FkzW2Bmr5nZZeH79Wb2tJktCn/uXvrqihSPcluyyjL0bgaucPexwATgK2b2UeBqYKa7jwZmhtMi1US5LZmkDr3dfTmwPIw3mtkCYDhwBjAxLDYdmAV8rSS17KK9bu3eIe8JC06P4iEvN3frtqXzqjm3O+vUYfFw+c7rTypjTapTpw7mmFkDcCgwF9g7TLQdCbdXsSsn0l2U29KRzA2lmQ0CHgQud/cPOrHcFDObb2bzm9ievoBIN1NuS5pMR73NrJYgke5194fCt1ea2VB3X25mQ4FVuZZ196nAVICPWH313Pu9CxrvGRHF9Q/nPpoulaW35HbyRO+rpnT+AoreLstRbwPuABa4+3cTsx4FJg+TGr4AAAN6SURBVIXxJOCR4ldPpHSU25JVlh7l0cB5wJ/MbMf1d18HbgYeMLMLgCXAmaWpokjJKLclkyxHvWcDlmf28cWtTnnVDBkSxavHZrul/tQN8fW3dRt047Rq0ptyWwqjSxhFRFKooRQRSdGjr/XOos+AAVG8/Eujo/gPX4+PDLZ4PKR+u3lrm+VnXHhiFA94fm4pqii9WWt8MP2NpuTpnPGB+JF94/5Olrudd4ft3hTF7zQ15SxjzRV9okAb6lGKiKRQQykikqLXD72XXDYuip/8h1sSc+I7nyeH25efMrnN8n3ejB+KVD0DCakWLWvWRPHUo+KHcdEnPljfel98B/H/PvjX3VKvND9Yf3AUP3v8ATnL9FlbPX876lGKiKRQQykikqLXD71b4xtIM6Jv7geNtXjinOS1jW3mebNupyYl5PGgNHln8qTtzQ3dVJmdJZ9Zv9f34iPuNZvjI92+srKf2Z2FepQiIinUUIqIpOiVQ+/lVxwVxX/310+WsSYihbPvxE/++tSw/M/pLoW934zPCLHn/xDFlX4Uu7PUoxQRSaGGUkQkhRpKEZEUvXIf5caPfhjFyVvkJz21Jb7a4Z/vuDKKR218OVdxkbLp9+T8KK4vYz16MvUoRURSqKEUEUnRK4fee86Oh9X7cWHOMv2Wx2UabnohivWwB5HeRz1KEZEUaihFRFL0yqF3/V1zEnEZKyIiVSG1R2lmdWY2z8xeMbPXzOxb4fv1Zva0mS0Kf+5e+uqKFI9yW7LKMvTeDhzn7p8AxgEnm9kE4GpgpruPBmaG0yLVRLktmaQ2lB7YFE7Whi8HzgCmh+9PBz5XkhqKlIhyW7LKdDDHzGrM7GWCZ2Q+7e5zgb3dfTlA+HOvjtYhUomU25JFpobS3VvcfRwwAjjczA7JugEzm2Jm881sfhPbu1pPkZJQbksWnTo9yN0bgVnAycBKMxsKEP5clWeZqe4+3t3H19I/VxGRslNuS0eyHPUeYmaDw3gX4NPAQuBRYFJYbBLwSKkqKVIKym3JKst5lEOB6WZWQ9CwPuDuj5vZHOABM7sAWAKcWcJ6ipSCclsyMffuu2m7ma0G3u22DUpH9nX3IeWuRE+h3K4YJcnrbm0oRUSqka71FhFJoYZSRCSFGkoRkRRqKEVEUqihFBFJoYZSRCSFGkoRkRRqKEVEUqihFBFJ8X+Fuw9uqyBQJwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def infer(ds, model):\n", + " data = ds.get_next()\n", + " images = data['image']\n", + " labels = data['label']\n", + " output = model.predict(Tensor(data['image']))\n", + " pred = np.argmax(output.asnumpy(), axis=1)\n", + " return pred[0], images[0], labels[0]\n", + "\n", + "ds = create_dataset(training=False, batch_size=1).create_dict_iterator()\n", + "net = LeNet()\n", + "model = Model(net)\n", + "\n", + "plot_images(infer, ds, model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 训练\n", + "\n", + "使用MNIST数据集对上述定义的LeNet模型进行训练。训练策略如下表所示，可以调整训练策略并查看训练效果，要求验证精度大于95%。\n", + "\n", + "| batch size | number of epochs | learning rate | optimizer |\n", + "| -- | -- | -- | -- |\n", + "| 32 | 3 | 0.01 | Momentum 0.9 |" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 1 step: 1875 ,loss is 2.3086565\n", + "epoch: 2 step: 1875 ,loss is 0.22017351\n", + "epoch: 3 step: 1875 ,loss is 0.025683485\n", + "Metrics: {'acc': 0.9742588141025641, 'loss': 0.08628832848253062}\n" + ] + } + ], + "source": [ + "os.system('rm -f *.ckpt *.ir *.meta') # 清理旧的运行文件\n", + "LOOP_SINK = context.get_context('enable_loop_sink')\n", + "\n", + "def test_train(lr=0.01, momentum=0.9, num_epoch=3, ckpt_name=\"a_lenet\"):\n", + " ds_train = create_dataset(num_epoch=num_epoch)\n", + " ds_eval = create_dataset(training=False)\n", + " steps_per_epoch = ds_train.get_dataset_size()\n", + " \n", + " net = LeNet()\n", + " loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')\n", + " opt = nn.Momentum(net.trainable_params(), lr, momentum)\n", + " \n", + " ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5)\n", + " ckpt_cb = ModelCheckpoint(prefix=ckpt_name, config=ckpt_cfg)\n", + " loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch)\n", + " \n", + " model = Model(net, loss, opt, metrics={'acc', 'loss'})\n", + " model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb], dataset_sink_mode=True)\n", + " metrics = model.eval(ds_eval)\n", + " print('Metrics:', metrics)\n", + "\n", + "test_train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 推理（训练后）\n", + "\n", + "使用训练后的LeNet模型对手写数字进行识别，可以看到识别结果基本上是正确的。" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD7CAYAAAAMyN1hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dfZhVZb3/8feXYRgEFBgedHgSJED6eRIN0dJ+keZRKy899mhWqChZerSTmVb2UztSZpnnMrPEUvBZQ49ix1IjPYoSOJkUCooiAoI8KBgoEDPz/f2xF2utPew9a83svWfvzXxe17Wv+e697nWve+35zj3rXo/m7oiISH7dyt0AEZFKp45SRCSBOkoRkQTqKEVEEqijFBFJoI5SRCRBRXaUZsw048og/ogZL3Wwnl+Z8f3itq46mDHSDDeje7nbIhHlduHKkdsV2VHGufOUO+OSyplxuhnzWs17jjv/WbrWhcuuM+NaM9aYscmMG8yoTTnvZDNWF6ENK8z4eKH1xOrrZ8YsM9YHr8uLVbdkVENut2rHn9rTQVVqbsfq7WHG0jRtLHlH2UW2aC4BJgIHAWOBQ4FLi1V5mb7Da4FewEhgEvBlM84oQzsqVhfJbQDMOA2Kv75l/g4vAtanKunu7X6BrwD/DviL4JvAbwHvGUybDL4a/GLwN8FvCz7/FPjz4JvBnwH/QKy+Q8CfA98Cfg/43eBXxuuLlR0Ofj/4BvC3wK8HHw++HbwZfCv45qDszF31BO/PBn8F/G3wOeBDYtMc/BzwZcE6/QLcUn4fjeCfjb3/IviqFPP1Bt8G3hK0eyv4EPDLwWeD3w7+D/CzcqxL+L2A3xbUsS2o49vgI4N1mgK+Enwj+Pfa8TveCH5Y7P13wZ/qSL5U00u5nfM76Qv+MvgRQV3dqzm3g3pHgS8BPyH+O8j3KmSL8jTgOGA0ma2o+BbUfkA9sD8wzYxDgZuBrwIDgBuBOcGQtQfwAHBbMM9vgU/nWqAZNcDvgNfJbOkMBe52ZwlwDjDfnT7u9Msx79HAj4DPAQ1BHXe3KvYp4DDg4KDcccG8I8zYbMaIPN+FBa/4+2Fm9M1THgB33gVOANYE7e7jzppg8knAbKAfcEdCPV8GVgInBnVcHZt8FDAOOAb4f2aMD9bpKDM2t1VvjnU6KKH8nkK5ne2HwC+BN9sok6UKcvvnwHeBbWnWp5CO8np3VrnzNjAdODU2rQW4zJ0d7mwDzgZudGeBO83uzAJ2AEcEr1rgv9zZ6c5s4Nk8y5wEDAEucuddd7a7Z++7acNpwM3uPOfODuA7wIfMGBkrc5U7m91ZCTwOTABwZ6U7/YLPc/k9cIEZg8zYDzg/+LxXyrblMt+dB9xpCb7DjrrCnW3uLAIWkflDwZ15uf7oYv4AXGLG3ma8DziTwtanmii3A2ZMBI4k07EUS1lz24x/A7q7899pF1RIR7kqFr9O5pe8ywZ3tsfe7w9cGPzn2hz09sODeYYAb7gTvzvH63mWORx43Z2mDrR3SLxed7YCb5H5z71L/D/me0CflHVPB/4KPA88Q2YrYidp93/ktiq5SCodXafzyfy3XQY8CNwFhe+YrxLKbcCMbsANwAUdbFc+ZcttM3oDVwP/3p4FFdJRDo/FIyDcrAayEgMyX8z04D/Xrlcvd+4C1gJDzbKGefmGAauAEXl2ACfdBmkNmaQGwi9sAPBGwnyJgv9q57kz1J0DyCTpX9xpTjN7ys/fJXuLbr+U9XSIO2+7c5o7+7nzf8jkysJiLqOCKbcz9iFzkPIeM94k2hpebcZHUsxfibk9hsyujaeCdbofaDDjzVZb4FkK6SjPNWOYGfVkxvr3tFH2JuAcMw43w8zobcYnzdgbmA80Aeeb0d2MU8gMQ3JZSCb5rgrq6GnGkcG0dWT2C/bIM++dwBlmTDCjjsx+lwXurGjPSudixlAzhgTrdgTwfeCy2PSZZszMM/s6YEDS/kwyW6ufMKM+GN5/I0c9B3RsDXZnxmgzBphRY8YJwDTInP/XBSi3M94hs7U6IXh9Ivj8g8ACqMrcXkzmH+GudTorqH8CbWzpFtJR3gk8CiwPXnn/iNxpJLMv53pgE/AKcHow7Z/AKcH7TcDnyfTyueppBk4E3kdmB+/qoDzAn4AXgDfN2Jhj3rlkOrD7yCTkaOALaVY02OG9tY0d3qPJDLnfBWYBl7jzaGz6cODpPOu0lMywdnkwdBuSqxyZAwKLgBVkvvfWf7w/Ai4N6vhWinX6iBlb2yjyQeDvwJag7tPceSGp3j2EcjtTr7vz5q4XsCGYtC5YN6iy3HanqdU6vQ20BO/zjgAtc6i8fcxYAZzlzh/bPXMXE2wFLAI+4M7OcrdH2qbcTq8r5XaXOWG2XIL/vOPL3Q6RYutKuV3xlzCKiJRbh4beIiJdSUFblGZ2vJm9ZGavmNklxWqUSLkptyWuw1uUZlYDvAwcS+YI3bPAqe7+YvGaJ9L5lNvSWiEHcyYBr7j7cgAzu5vMNZx5k6mH1XlPehewSCmWLWza6O6Dyt2OCqXcrlLbeZd/+g5LLtk+hXSUQ8k+QXM1cHjrQmY2jczJyvSkF4fbMQUsUorljz4736V0otyuWgt8bknqLWQfZa5ee7dxvLvPcPeJ7j6xlroCFifSaZTbkqWQjnI12dfEDiP7mliRaqXcliyFdJTPAmPMbJSZ9SBzydSc4jRLpKyU25Klw/so3b3JzM4DHgFqgJvdvatcCyx7MOW2tFbQJYzu/jDwcJHaIlIxlNsSp0sYRUQSqKMUEUmgjlJEJIE6ShGRBOooRUQSqKMUEUmgjlJEJIE6ShGRBHpmjogURfcDRobx+o82JJYf9LtXwrh5w4Y2SpaftihFRBKooxQRSaCOUkQkQZfcR1kz5oAw3jmkb8mXV7vmnTBuXra85MsTKYf4fslnp/8ysfyxy88I427/q32UIiJVTR2liEiCLjP0rukXDbFfuiyKXz36lpIve/SfoiHGuPOiZTdvfidXcZGSsu7Rn323AfV5y7W89XYYe1NTzjLdevUK46beRX/4YcXQFqWISAJ1lCIiCbrM0PuNmUPCuPGDv4hN6bV74SJr/Gi0vMkzp4bxfidr6C2dr+Xwg8L42jtuyFvugtO+Hsb29PM5y6y8YEIYP/K1q2NT+nS8gRVIW5QiIgnUUYqIJOgyQ+/+vbZFcU3ph9tZy44tL94OkXLw7tHR6fE98v8txMvlO57dUhfFw7rvWcPtuMQtSjO72czWm9ni2Gf1ZvaYmS0LfvYvbTNFik+5LWmlGXrPBI5v9dklwFx3HwPMDd6LVJuZKLclhcSht7s/aWYjW318EjA5iGcBTwAXF7FdRfHyryeG8S2jb27XvPdujU4M/8mPvphqnv973oIwvqbhuZxlLh89J4zP+PWZYTz2rMZ2tU8KV8253V7/PC76Wxh62SttlEy29sIPh/GZn3kksfzqpq1hfOLV3w7jhheXhXFzQS0qvY4ezNnX3dcCBD8H5ytoZtPMrNHMGneyo4OLE+k0ym3ZTcmPerv7DHef6O4Ta6lLnkGkSii3u46OHvVeZ2YN7r7WzBqA9cVsVLFMnTQvjCfv1ZJY/idvjw7jO391XBgPvuWZVMv7y6kjozd5ht7xdsTb9xQ9Uy1DSq4qcjuNbSdPCuO6f18bxrePfKLddcWH25/+SjT/RfWvJs67pSXaHmv4bfU8/iGuo1uUc4ApQTwFeLA4zREpO+W27CbN6UF3AfOBcWa22symAlcBx5rZMuDY4L1IVVFuS1ppjnqfmmfSMUVuS1lcseH9YXzfrZPDuOH6dMNtqV57Ym7Hh9vdvh7tNXhs/EMF1Vt/3JowvmzQi4nlF+7YGcZfePg/wnjce8nzViJdwigikkAdpYhIgi5zrXc+Mxujo3ljr9FwW6pP/GTy+NHtNMPtrS3bw/gzL386a1r3t6L7EuykfR7d8i9hPObc6EKM5HNPKpO2KEVEEqijFBFJ0OWH3iLVzi/cGMbtPbq9qikaDHc7NXuA3TI86h6G9dmcWNfLO98N4wdXfiCM63m5XW2qRNqiFBFJoI5SRCRBlx96d6uLbvAUf/Z31jO3Lbq/c83Aga0qiKb17N7eY4MiHVMTex53qfLuqJujW/9dOnBpYvnzX/1cGA+aGhuqDxoUhs0bo90EuBfWwE6kLUoRkQTqKEVEEnT5oXeaZ27Hh9vTnpmfNf+42uh62uHd4/93dNs0KZ2hD0c3Cr526MOxKeXLu9lj7wvjVQuio+kv7YzufTzjwx8K465wmzURkS5DHaWISAJ1lCIiCbr8Psr+NdED4H97yK/D+N6/fTCM67otD+MTem3Kmr/O8j9AXqRQ8dOA4vslpw95NIz7dOvd4foPqK0N44/NzX6sw5S+f4u9S15Gn27R/tHxPeJTYk/TiJ1OV020RSkikkAdpYhIgj166P3I5R8N403fi4bI1+R5QuLY2mh4kf9KhNo8n6dz4dpDw3jh9MPCuBcLchWXLiB+P0nIvslF/Kqb+GlAhQy34+osyufdn6jY8WXcuzW6yu1nPzgnjPtt/muH6ywnbVGKiCRQRykikmCPHnr3uj8azv7lnJHRhDxD787wl7dGhHG8fdJ1bR2SvTvn2YMeyFOy8q72OmPlR8L4z3+IHv/QM3bvi8G3R49YqZ7bYGRL81zv4Wb2uJktMbMXzOyC4PN6M3vMzJYFP/uXvrkixaPclrTSDL2bgAvdfTxwBHCumb0fuASY6+5jgLnBe5FqotyWVBKH3u6+FlgbxFvMbAkwFDgJmBwUmwU8AVxcklYWwduPDAnjUavOKlq99xz9yzCeVFfYEXHpXHtKbh+75MQwfmXFvonl9xkQPbJh0aS72r28+HB78Y0HhfGIW/bcp5i262COmY0EDgEWAPsGibYr4QbnmWeamTWaWeNOduQqIlJ2ym1pS+qO0sz6APcB33D3f6Sdz91nuPtEd59YS11H2ihSUsptSZLqqLeZ1ZJJpDvc/f7g43Vm1uDua82sgawLOitPwzXRsKChiPU++rfoSN+kuuTb5UtlqYTc7rMm+1EOkxef3K757ZrofqljH2nMWab7ASPD+NUp0W4oJqVbRvxCiRdviIbb9bfOz1V8j5PmqLcBvwGWuPvPYpPmAFOCeArwYPGbJ1I6ym1JK80W5ZHAl4G/m9nzwWffBa4C7jWzqcBK4LOlaaJIySi3JZU0R73nAfnujXRMcZsj0nkqJbd7tB4uP9LeGlbk/LT78GFhHB9uLz37hvYugCevPzyMu8pwO06XMIqIJFBHKSKSYI++1lukK1s+NbqvQEeG26/t3BrGNTur9Srt4tAWpYhIAnWUIiIJNPQWkZxOP++bYdzv0ejO5F1xEK4tShGRBOooRUQSaOgtIjn1+Ed0Dbrv6Np3R9IWpYhIAnWUIiIJ1FGKiCRQRykikkAdpYhIAnWUIiIJdHqQSBe2vjl6IuORd3wra9rY5avCuKnTWlSZtEUpIpJAHaWISAINvUX2UIOejwbMo35/Vs4ytr0mjMdNX5w1rWnLltI0rAppi1JEJIE6ShGRBIlDbzPrCTwJ1AXlZ7v7ZWZWD9wDjCTzGLjPufum0jW1Mv1m4VFh/McRByaWX/NcQxiPyvP0POkce3pu7/XAwjAe+0By+ZYStqXapdmi3AEc7e4HAxOA483sCOASYK67jwHmBu9FqolyW1JJ7Cg9Y9dThmqDlwMnAbOCz2cBJ5ekhSIlotyWtFId9TazGuAvwPuAX7j7AjPb193XArj7WjMbXMJ2VqyxZzUmF4rRcLuyKLcljVQHc9y92d0nAMOASWZ2UNoFmNk0M2s0s8addO2bf0rlUW5LGu066u3um4EngOOBdWbWABD8XJ9nnhnuPtHdJ9ZSV2BzRUpDuS1tSewozWyQmfUL4r2AjwNLgTnAlKDYFODBUjVSpBSU25JWmn2UDcCsYF9ON+Bed/+dmc0H7jWzqcBK4LMlbKdIKSi3JZXEjtLd/wYckuPzt4BjStEokc6g3Ja0zL3zHmduZhuA1zttgdKW/d19ULkbsadQbleMkuR1p3aUIiLVSNd6i4gkUEcpIpKgIjtKM2aacWUQf8SMlzpYz6/M+H5xW1cdzBhphpvpnqOVRLlduHLkdkV2lHHuPOXOuKRyZpxuxrxW857jzn+WrnXhsqeY8Rcz/mHGajOuTvtLNGOyGauL0IYVZny80Hpi9V1kxmIztpjxmhkXFatuyaiG3A6W/x9mvGnGO2bcbJbu7PoKzm0z48dmvBW8rjbD2pqn5B1lF9mi6QV8AxgIHE7m1JJvtTlHO5TpOzTgK0B/MlernGfGF8rQjorVFXLbjOPI3D3pGDK3nTsAuKKI9ZfjO5xG5kYnBwMfAD4FfLXNOdy93S/wFeDfAX8RfBP4LeA9g2mTwVeDXwz+JvhtweefAn8efDP4M+AfiNV3CPhz4FvA7wG/G/zKeH2xssPB7wffAP4W+PXg48G3gzeDbwXfHJSduaue4P3Z4K+Avw0+B3xIbJqDnwO+LFinX4BbB7+fb4I/lKJcb/Bt4C1Bu7eCDwG/HHw2+O3g/wA/K8e6hN8L+G1BHduCOr4NPjJYpyngK8E3gn+vI+sTLOM68J93dP5qeSm3d/s+7gT/Yez9MeBvVnNuB7+jabH3U8H/3OY8BSTT4uAXWw/+dKtffhP4j8HrwPcCPxR8Pfjh4DXBCq4IpvcAfx38P8BrwT8DvjNXMgXzLgK/NvhF9AQ/Kph2Ovi8Vu2cGavn6OALPTRY7s/Bn2yVTL8D7wc+IkjW44NpI4I/ghEpv58HwK9KWTbrjyX47PLgOzgZvFvwHeZNptjv5OOx97uS6aZg/oPBd4CPD6YfteuPLkUbDfyv4Od0tAOqlpdye7fvYxH452PvBwb1DajW3AZ/B/zw2PuJ4FvaWpdCht7Xu7PKnbeB6cCpsWktwGXu7HBnG3A2cKM7C9xpdmcWmZumHhG8aoH/cmenO7OBZ/MscxIwBLjInXfd2e6eve+mDacBN7vznDs7gO8AHzJjZKzMVe5sdmcl8DiZm7nizkp3+gWft8mMM4CJwE9Ttiuf+e484E5L8B121BXubHNnEbCIzHADd+a50y9lHZeT2U1zSwHtqCbK7Ugf4J3Y+13x3inblku5czvXOvVpaz9lIR3lqlj8Oplf8i4b3Nkee78/cKEZm3e9gOHBPEOAN9yJn/me7wqH4cDr7h16HvuQeL3ubAXeAobGyrwZi98j84WmZsbJwFXACe5s7EAb41YlF0ml0HU6j8y+yk8Gf4RdgXI7shXYJ/Z+V1zIIxrLndu51mlrq99TlkI6yuGxeASwJva+9QJXAdOD/1y7Xr3cuQtYCwxt1ZuPyLPMVcCIPDuAky4xWkMmqQEwozcwAHgjYb5UzDgeuAk40Z2/t2PWfO1u/fm7ZA4a7bJfyno6zIwzCXbkuxd+9LKKKLcjLxBsqQUOBta581aKeSs1t3Ot0wttzVBIR3muGcPMqAe+S+ZhTPncBJxjxuHBofneZnzSjL2B+UATcL4Z3c04hcwwJJeFZJLvqqCOnmYcGUxbBwwzo0eeee8EzjBjQnB6ww+BBe6F33LcjKOBO4BPu7Mwx/SZZszMM/s6YIAZfRMW8zzwCTPqzdiPzFH21vUc0L6W52fGaWS+o2PdWV6sequEcjtyKzDVjPeb0R+4FKJcrsbcJrNO3zRjqBlDgAsh7zoAhXWUdwKPAsuD15X5CrrTSGZfzvXAJuAV4PRg2j+BU4L3m4DPA/fnqacZOJHMbftXAquD8gB/IvNf4U2z3Ye97swFvg/cRyYhR0O6013MGGHGVrO8WwPfB/oCDwfltprx+9j04cDTedZpKXAXsDwYug3JVQ64jcx+mBVkvvfWf7w/Ai4N6kg8NSk42XlrG0WuJLNV8mxsnX6VVO8eQrkd1f0H4Goy+zVfD16XxYpUY27fCDwE/B1YDPxP8Fn+OjNHfdrHjBXAWe78sd0zdzHBVsAi4APu7Cx3e6Rtyu30ulJu7/EnzJZbsFUxvtztECm2rpTbFX8Jo4hIuXWoo3RnpDt/NLPjzewlM3vFzPSQeKl6ym3JpcM37g2eM/IycCyZHc/PAqe6+4vFa55I51NuS2uF7KOcBLzi7ssBzOxu4CQgbzL1sDrvSe8CFinFsoVNG12PgshHuV2ltvMu//Qdbd4JqCMK6SiHkn2G/Woyd87Jqye9Odz0zKZK8Eefree75KfcrlILfG5J6i2ko8zVa+82jjezaWRua0TPrJPvRSqWcluyFHLUezXZl3oNI/tSLwDcfYa7T3T3ibXp7vcpUm7KbclSSEf5LDDGzEaZWQ8yVwLMKU6zRMpKuS1ZOjz0dvcmMzsPeASoAW529zYvLBepBsptaa2gK3Pc/WHg4SK1RaRiKLclTlfmiIgkUEcpIpJAHaWISAJ1lCIiCdRRiogkUEcpIpJAHaWISAJ1lCIiCfQoCJEq9N4p0c2Mtvct3/ZO/5e3hbE9/XzZ2lFq2qIUEUmgjlJEJIE6ShGRBF1mH2W3gw4M46YBe5WxJZGad6NHIXvj4jK2RCqVdY/+RFsOPyiMvzz9oTCe1ne3W2V2moMXnhrGQ96J/sZaFi8tR3NKRluUIiIJ1FGKiCTYM4beFj3ipGbgwOjzbtHnLddtCePHDry7U5qV5Cdvjw7jx48ZnbNMy1tvh7E3NZW8TVJZug2oD+Nr77ghjMf3qIxn9CyadFcYH3/dJ8O45tTBUaGW6HFDzRs3Rp938FHZ5aAtShGRBOooRUQS7BFD7/hwe9oz88N4XO36MB7ePf4/oWdnNCvR+f2jI4OfWvD3nGUuOO3rYbwnX/kg1W/22PvCeNWCljB+aWc0DJ/x4Q+FcfOGDZ3TsCLQFqWISAJ1lCIiCap26G0To5NvPzYzGm6f0GtTGNdZZRwZzKfOasN4fI/anGU+8+tHw/i2752YNa3X/QtK0zCRVg68KdoFVHdw9DcWP+rdp1u0S2t8j2jeA2qj8sseXx7Gc0+PhuGVfsFF4halmd1sZuvNbHHss3oze8zMlgU/+5e2mSLFp9yWtNIMvWcCx7f67BJgrruPAeYG70WqzUyU25JC4tDb3Z80s5GtPj4JmBzEs4AngIuL2K5Ezb2joepF9a/GpuQewqZx4dpDw/jJ6w9vo2RxbBkZnRC/9OwbcpaJX8d7U6v7Dlb2joXKV6m5XYn2XhGdHN7/0bowPmzs18J442HNYfzayTPCOL6LKf63uui64WH8xhUTs5bX45HGAltcXB09mLOvu68FCH4OTigvUi2U27Kbkh/MMbNpwDSAntoGkj2Icrvr6GhHuc7MGtx9rZk1AOvzFXT3GcAMgH2svlMv7hz10Nlh3HNt8pB8n+VR8+pvnd9GyeIYPHxYGI+pi4YwT5/206hMTe+St0OylD23a8YcEMYvXdY3jPerKU79j74X/S2cO/usMI7nHeTPvfiFD/VPR58PfG58GI+qif72Xjvxppz13D7yiTA+9tt7Z03bttekMN7rgYU55+9MHR16zwGmBPEU4MHiNEek7JTbsps0pwfdBcwHxpnZajObClwFHGtmy4Bjg/ciVUW5LWmlOep9ap5JxxS5Le1Su+adMB71+7Nylhn/07fCuHnZ8pxlpOuq1NzeOSQabr969C2xKR3fD3rHlgFh/IPffi6M3/fj6ETvI/b+ZtY83jM6ij0m9rTFfFoWLQnj8T+Ndh+M7n1GGDd+9Bdh3L8mWp/Hxkd3bAcYPyE6wX3EA4mLLjldwigikkAdpYhIgqq91js+lB47Nfewujnnp+XV/YCRYfzqlCFhvOwr8RPOdaRbChcfbl/x39Fwe9Sl0RkdLbHyY84t3r0D4n+f4y6MTkV9M7aI/m0cxff3R08k8CMnhHG5bjWoLUoRkQTqKEVEElTt0LuadI+dWB4fbue7vjtuxjtR+Z7vtLRRUiTbpfP+LYzHXlL6CyiKaelRt4XxqC3RWS1jn85VuvS0RSkikkAdpYhIAg29i6imX3SiMHXRrahePje6nVT20e1Is0fD6lebopN7Z5/1r2Hc62nd0Vzatr753TC27UW6OFy0RSkikkQdpYhIAg29i+iNmdER6rsn/CaMB9TE78CV+2Ty+HD7GydE18Z2ezm6FrdT71EnVenIO74VxuOmR7mj8yUKoy1KEZEE6ihFRBKooxQRSaB9lCnUDKgP46EP78iatv9e0T0vP9f312E8tjb5xhbxq27ipwFl7ZdsampfY6XqvHdK9hM/vzz9oTwlczvwpujejWN/vTKMm7ZsyVVcOkBblCIiCdRRiogk0NA7j24HHRjGLddFQ5hrhz6cVa5Pt56xd8nD7QvXHhrGC6cfFsbxq250GlDXsr1v9vbKtL5r2jX/3iuijGlatboobZJs2qIUEUmgjlJEJIGG3jHxW86vuSg6ur3owP+JlYoPtfObvPjkMF73THR0e5/l0TCp3/3VdY9AkbRqxkRPYXzpsuhmMftV6X060jzXe7iZPW5mS8zsBTO7IPi83sweM7Nlwc/+pW+uSPEotyWtNEPvJuBCdx8PHAGca2bvBy4B5rr7GGBu8F6kmii3JZXEobe7rwXWBvEWM1sCDAVOAiYHxWYBTwAXl6SVRbbt5ElhvGFC9BXEn/y2dNJd7a732CUnhnHLDdGT50Y88Ey765LSq+bcPnjhqWG878vb2ijZebodPD6Ml3y9Txi/dvRNsVK9UtV1xsqPhPHAebUFt61Q7TqYY2YjgUOABcC+QaLtSrjB+ecUqWzKbWlL6o7SzPoA9wHfcPd/tGO+aWbWaGaNO9mRPINIJ1NuS5JUR73NrJZMIt3h7vcHH68zswZ3X2tmDcD6XPO6+wxgBsA+Vl8R51KvOjG6O99rJyQ/CbEtX1oxOYybrt43jPd6ZGFB9UrnqNbc3rEoOr5UuzY6Qb2z7wwQH24v/ereYfzaiTMKqvfPf/iXMB5xS/l3XaU56m3Ab4Al7v6z2KQ5wJQgngI8WPzmiZSOclvSSrNFeSTwZeDvZvZ88Nl3gauAe81sKrAS+GxpmihSMsptSSXNUe95gOWZfExxm1OZdvjOML5u04FZ0zaePyyMezQ2dlqbpHDVnNtLz452GeqhLkQAAAQ5SURBVB224mthXL98RUmWF7/3QdOAvcL45S9GXUghw+17t/bNet9zY4erKgldwigikkAdpYhIgi5zrXdNv2jTvltdc2L5+HD79+9FRxgf/9gBWeV8w2JEClGzM/uA+Ws7t4bxqNo+rYvvpql3tPegW6/ohO6W994rrF2xO/vHbzX42IF3F1TvLvH1/NkPzsmaNvj28h/pjtMWpYhIAnWUIiIJuszQ+42Z0a3OGj/4i9iU3Neexo9ux4fbzRsr7HCcVL1+v/1r1vvT3/lmGP/vjclHkh/69tVhfNze3w7jYT8qbPgaf5Be9p39091qMMnp50Xr2e/R7O+gIq5MidEWpYhIAnWUIiIJuszQu3+v6FZU/WuSb/U0pe/fwnjH3Py3eZp35sQw9kYdAZf28x3ZN9To9cSSMP7oV6eF8czro6ss40fDh3WP4munRrc0W3jq6ILaNa3/02Hcp1vyg/PyiR/djg+34+vZsqOybyqiLUoRkQTqKEVEEnSZoXd7Da6JhhqXDlyat9yXrtsvjFdvHZlY75rnGsJ41CV6uJjsrmVLdHJ374Urwni7J2/X/GuvnbE4f96m0/Hhdvza7fjJ5PGj25U+3I7TFqWISAJ1lCIiCdRRiogk6DL7KLfdHu0bPOPr0RPebhnxVEH13j7yiXaVv2P/AWH8g6bPhfHIS7W/UnbnW6JTaz5z47fCuKUuKnPmZx4J44vqX+2Udu0Sf1pi/PEN8ftJxm9wUWlX3KSlLUoRkQTqKEVEEnSZoXe/W6Oh7eKaD4XxGV+NyhQ6DD92yYlhfNTA3EOgmY0fDuMeBS1NuoL4PSXz3eTiti3HhfEN7/9nydsUN3BedNVaJTwtsVS0RSkikkAdpYhIgi4z9I6rvyUahr/YHA3DJ39pQK7iqdk1A8P4voOG5ywz9po9d3gi5dEQy6mGNspJxyVuUZpZTzNbaGaLzOwFM7si+LzezB4zs2XBz/5JdYlUEuW2pJVm6L0DONrdDwYmAMeb2RHAJcBcdx8DzA3ei1QT5bakkjj0dncHdp31Whu8HDgJmBx8Pgt4Ari46C0ssfjRcG4ttLYVYdTwSP5SUhn29NyW4kl1MMfMaszseWA98Ji7LwD2dfe1AMHPwaVrpkhpKLcljVQdpbs3u/sEYBgwycwOSrsAM5tmZo1m1riT6rmtknQNym1Jo12nB7n7ZjLDkOOBdWbWABD8XJ9nnhnuPtHdJ9ZSl6uISNkpt6UtaY56DzKzfkG8F/BxYCkwB5gSFJsCPFiqRoqUgnJb0kpzHmUDMMvMash0rPe6++/MbD5wr5lNBVYCny1hO0VKQbktqVjmwF8nLcxsA/B6py1Q2rK/uw8qdyP2FMrtilGSvO7UjlJEpBrpWm8RkQTqKEVEEqijFBFJoI5SRCSBOkoRkQTqKEVEEqijFBFJoI5SRCSBOkoRkQT/H2y1o7k2YlTdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "CKPT = 'a_lenet-3_1875.ckpt'\n", + "\n", + "ds = create_dataset(training=False, batch_size=1).create_dict_iterator()\n", + "net = LeNet()\n", + "param_dict = load_checkpoint(CKPT)\n", + "load_param_into_net(net, param_dict)\n", + "model = Model(net)\n", + "\n", + "plot_images(infer, ds, model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 实验步骤（方案二）\n", + "\n", + "### 代码梳理\n", + "\n", + "创建训练作业时，运行参数会通过脚本传参的方式输入给脚本代码，脚本必须解析传参才能在代码中使用相应参数。如data_url和train_url，分别对应数据存储路径(OBS路径)和训练输出路径(OBS路径)。脚本对传参进行解析后赋值到`args`变量里，在后续代码里可以使用。\n", + "\n", + "```python\n", + "import argparse\n", + "parser = argparse.ArgumentParser()\n", + "parser.add_argument('--data_url', required=True, default=None, help='Location of data.')\n", + "parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.')\n", + "parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.')\n", + "args, unknown = parser.parse_known_args()\n", + "```\n", + "\n", + "MindSpore暂时没有提供直接访问OBS数据的接口，需要通过MoXing提供的API与OBS交互。将OBS中存储的数据拷贝至执行容器：\n", + "\n", + "```python\n", + "import moxing as mox\n", + "mox.file.copy_parallel(src_url=args.data_url, dst_url='MNIST/')\n", + "```\n", + "\n", + "如需将训练输出（如模型Checkpoint）从执行容器拷贝至OBS，请参考：\n", + "\n", + "```python\n", + "import moxing as mox\n", + "mox.file.copy_parallel(src_url='output', dst_url='s3://OBS/PATH')\n", + "```\n", + "\n", + "其他代码分析请参考方案一。\n", + "\n", + "### 创建训练作业\n", + "\n", + "可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。\n", + "\n", + "创建训练作业的参考配置：\n", + "\n", + "- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore\n", + "- 代码目录：选择上述新建的OBS桶中的experiment_1目录\n", + "- 启动文件：选择上述新建的OBS桶中的experiment_1目录下的`main.py`\n", + "- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_1目录下的MNIST目录\n", + "- 训练输出位置：选择上述新建的OBS桶中的experiment_1目录并在其中创建output目录\n", + "- 作业日志路径：同训练输出位置\n", + "- 规格：Ascend:1*Ascend 910\n", + "- 其他均为默认\n", + "\n", + "启动并查看训练过程：\n", + "\n", + "1. 点击提交以开始训练；\n", + "2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理；\n", + "3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看；\n", + "4. 在训练日志中可以看到`epoch: 3 step: 1875 ,loss is 0.025683485`等字段，即训练过程的loss值；\n", + "5. 在训练日志中可以看到`Metrics: {'acc': 0.9742588141025641, 'loss': 0.08628832848253062}`字段，即训练完成后的验证精度。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 实验小结\n", + "\n", + "本实验展示了如何使用MindSpore进行手写数字识别，以及开发、训练和使用LeNet模型。通过对LeNet模型做几代的训练，然后使用训练后的LeNet模型对手写数字进行识别，识别结果基本上是正确的。即LeNet学习到了如何进行手写数字识别。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/experiment_1/main.py b/experiment_1/main.py new file mode 100644 index 0000000..5970660 --- /dev/null +++ b/experiment_1/main.py @@ -0,0 +1,108 @@ +# LeNet5 mnist + +import os +# os.environ['DEVICE_ID'] = '0' +# Log level includes 3(ERROR), 2(WARNING), 1(INFO), 0(DEBUG). +os.environ['GLOG_v'] = '1' + +import matplotlib.pyplot as plt +import numpy as np + +import mindspore as ms +import mindspore.context as context +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as CV + +from mindspore.dataset.transforms.vision import Inter +from mindspore import nn, Tensor +from mindspore.train import Model +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +context.set_context(mode=context.GRAPH_MODE, device_target='Ascend') + +DATA_DIR_TRAIN = "MNIST/train" # 训练集信息 +DATA_DIR_TEST = "MNIST/test" # 测试集信息 + + +def create_dataset(training=True, num_epoch=1, batch_size=32, resize=(32, 32), + rescale=1/(255*0.3081), shift=-0.1307/0.3081, buffer_size=64): + ds = ms.dataset.MnistDataset(DATA_DIR_TRAIN if training else DATA_DIR_TEST) + + # define map operations + resize_op = CV.Resize(resize) + rescale_op = CV.Rescale(rescale, shift) + hwc2chw_op = CV.HWC2CHW() + + # apply map operations on images + ds = ds.map(input_columns="image", operations=[resize_op, rescale_op, hwc2chw_op]) + ds = ds.map(input_columns="label", operations=C.TypeCast(ms.int32)) + + ds = ds.shuffle(buffer_size=buffer_size) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(num_epoch) + + return ds + + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = nn.ReLU() + self.conv1 = nn.Conv2d(1, 6, 5, stride=1, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, stride=1, pad_mode='valid') + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.fc1 = nn.Dense(400, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.flatten(output) + output = self.fc1(output) + output = self.fc2(output) + output = self.fc3(output) + + return output + + +LOOP_SINK = context.get_context('enable_loop_sink') + +def test_train(lr=0.01, momentum=0.9, num_epoch=3, ckpt_name="a_lenet"): + ds_train = create_dataset(num_epoch=num_epoch) + ds_eval = create_dataset(training=False) + steps_per_epoch = ds_train.get_dataset_size() + + net = LeNet() + loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + opt = nn.Momentum(net.trainable_params(), lr, momentum) + + ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5) + ckpt_cb = ModelCheckpoint(prefix=ckpt_name, config=ckpt_cfg) + loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch) + + model = Model(net, loss, opt, metrics={'acc', 'loss'}) + model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb], dataset_sink_mode=True) + metrics = model.eval(ds_eval) + print('Metrics:', metrics) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') + args, unknown = parser.parse_known_args() + + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, dst_url='MNIST/') + + os.system('rm -f *.ckpt *.ir *.meta') # 清理旧的运行文件 + + test_train() diff --git a/experiment_2/2-Save_And_Load_Model.ipynb b/experiment_2/2-Save_And_Load_Model.ipynb new file mode 100644 index 0000000..7cb3a14 --- /dev/null +++ b/experiment_2/2-Save_And_Load_Model.ipynb @@ -0,0 +1,566 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

训练时模型的保存和加载

\n", + "\n", + "[TOC]\n", + "\n", + "## 实验介绍\n", + "\n", + "本实验主要介绍使用MindSpore实现训练时模型的保存和加载。训练过程中保存模型以及训练中断后基于断点继续训练是一项非常常用的功能。建议先阅读MindSpore官网教程中关于模型参数保存和加载的内容。\n", + "\n", + "## 实验目的\n", + "\n", + "- 了解如何使用MindSpore实现训练时模型的保存。\n", + "- 了解如何使用MindSpore加载保存的模型文件并继续训练。\n", + "- 了解如何MindSpore的Callback功能。\n", + "\n", + "## 预备知识\n", + "\n", + "- 熟练使用Python，了解Shell及Linux操作系统基本知识。\n", + "- 具备一定的深度学习理论知识，如卷积神经网络、损失函数、优化器，训练策略、Checkpoint等。\n", + "- 了解华为云的基本使用方法，包括[OBS（对象存储）](https://www.huaweicloud.com/product/obs.html)、[ModelArts（AI开发平台）](https://www.huaweicloud.com/product/modelarts.html)、[Notebook（开发工具）](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0033.html)、[训练作业](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0046.html)等功能。华为云官网：https://www.huaweicloud.com\n", + "- 了解并熟悉MindSpore AI计算框架，MindSpore官网：https://www.mindspore.cn/\n", + "\n", + "## 实验环境\n", + "\n", + "- MindSpore 0.2.0（MindSpore版本会定期更新，本指导也会定期刷新，与版本配套）；\n", + "- 华为云ModelArts：ModelArts是华为云提供的面向开发者的一站式AI开发平台，集成了昇腾AI处理器资源池，用户可以在该平台下体验MindSpore。ModelArts官网：https://www.huaweicloud.com/product/modelarts.html\n", + "\n", + "## 实验准备\n", + "\n", + "### 创建OBS桶\n", + "\n", + "本实验需要使用华为云OBS存储实验脚本和数据集，可以参考[快速通过OBS控制台上传下载文件](https://support.huaweicloud.com/qs-obs/obs_qs_0001.html)了解使用OBS创建桶、上传文件、下载文件的使用方法。\n", + "\n", + "> **提示：**华为云新用户使用OBS时通常需要创建和配置“访问密钥”，可以在使用OBS时根据提示完成创建和配置。也可以参考[获取访问密钥并完成ModelArts全局配置](https://support.huaweicloud.com/prepare-modelarts/modelarts_08_0002.html)获取并配置访问密钥。\n", + "\n", + "创建OBS桶的参考配置如下：\n", + "\n", + "- 区域：华北-北京四\n", + "- 数据冗余存储策略：单AZ存储\n", + "- 桶名称：如ms-course\n", + "- 存储类别：标准存储\n", + "- 桶策略：公共读\n", + "- 归档数据直读：关闭\n", + "- 企业项目、标签等配置：免\n", + "\n", + "### 数据集准备\n", + "\n", + "MNIST是一个手写数字数据集，训练集包含60000张手写数字，测试集包含10000张手写数字，共10类。MNIST数据集的官网：[THE MNIST DATABASE](http://yann.lecun.com/exdb/mnist/)。\n", + "\n", + "从MNIST官网下载如下4个文件到本地并解压：\n", + "\n", + "```\n", + "train-images-idx3-ubyte.gz: training set images (9912422 bytes)\n", + "train-labels-idx1-ubyte.gz: training set labels (28881 bytes)\n", + "t10k-images-idx3-ubyte.gz: test set images (1648877 bytes)\n", + "t10k-labels-idx1-ubyte.gz: test set labels (4542 bytes)\n", + "```\n", + "\n", + "### 脚本准备\n", + "\n", + "从[课程gitee仓库](https://gitee.com/mindspore/course)上下载本实验相关脚本。\n", + "\n", + "### 上传文件\n", + "\n", + "将脚本和数据集上传到OBS桶中，组织为如下形式：\n", + "\n", + "```\n", + "experiment_1\n", + "├── MNIST\n", + "│   ├── test\n", + "│   │   ├── t10k-images-idx3-ubyte\n", + "│   │   └── t10k-labels-idx1-ubyte\n", + "│   └── train\n", + "│   ├── train-images-idx3-ubyte\n", + "│   └── train-labels-idx1-ubyte\n", + "└── 脚本等文件\n", + "```\n", + "\n", + "## 实验步骤（方案一）\n", + "\n", + "### 创建Notebook\n", + "\n", + "可以参考[创建并打开Notebook](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0034.html)来创建并打开本实验的Notebook脚本。\n", + "\n", + "创建Notebook的参考配置：\n", + "\n", + "- 计费模式：按需计费\n", + "- 名称：experiment_2\n", + "- 工作环境：Python3\n", + "- 资源池：公共资源\n", + "- 类型：Ascend\n", + "- 规格：单卡1*Ascend 910\n", + "- 存储位置：对象存储服务（OBS）->选择上述新建的OBS桶中的experiment_2文件夹\n", + "- 自动停止等配置：默认\n", + "\n", + "> **注意：**\n", + "> - 打开Notebook前，在Jupyter Notebook文件列表页面，勾选目录里的所有文件/文件夹（实验脚本和数据集），并点击列表上方的“Sync OBS”按钮，使OBS桶中的所有文件同时同步到Notebook工作环境中，这样Notebook中的代码才能访问数据集。参考[使用Sync OBS功能](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0038.html)。\n", + "> - 打开Notebook后，选择MindSpore环境作为Kernel。\n", + "\n", + "> **提示：**上述数据集和脚本的准备工作也可以在Notebook环境中完成，在Jupyter Notebook文件列表页面，点击右上角的\"New\"->\"Terminal\"，进入Notebook环境所在终端，进入`work`目录，可以使用常用的linux shell命令，如`wget, gzip, tar, mkdir, mv`等，完成数据集和脚本的下载和准备。\n", + "\n", + "> **提示：**请从上至下阅读提示并执行代码框进行体验。代码框执行过程中左侧呈现[\\*]，代码框执行完毕后左侧呈现如[1]，[2]等。请等上一个代码框执行完毕后再执行下一个代码框。\n", + "\n", + "导入MindSpore模块和辅助模块：" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "# os.environ['DEVICE_ID'] = '0'\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "import mindspore as ms\n", + "import mindspore.context as context\n", + "import mindspore.dataset.transforms.c_transforms as C\n", + "import mindspore.dataset.transforms.vision.c_transforms as CV\n", + "\n", + "from mindspore.dataset.transforms.vision import Inter\n", + "from mindspore import nn, Tensor\n", + "from mindspore.train import Model\n", + "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor\n", + "from mindspore.train.serialization import load_checkpoint, load_param_into_net\n", + "\n", + "import logging; logging.getLogger('matplotlib.font_manager').disabled = True\n", + "\n", + "context.set_context(mode=context.GRAPH_MODE, device_target='Ascend')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 数据处理\n", + "\n", + "在使用数据集训练网络前，首先需要对数据进行预处理，如下：" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "DATA_DIR_TRAIN = \"MNIST/train\" # 训练集信息\n", + "DATA_DIR_TEST = \"MNIST/test\" # 测试集信息\n", + "\n", + "def create_dataset(training=True, num_epoch=1, batch_size=32, resize=(32, 32),\n", + " rescale=1/(255*0.3081), shift=-0.1307/0.3081, buffer_size=64):\n", + " ds = ms.dataset.MnistDataset(DATA_DIR_TRAIN if training else DATA_DIR_TEST)\n", + " \n", + " # define map operations\n", + " resize_op = CV.Resize(resize)\n", + " rescale_op = CV.Rescale(rescale, shift)\n", + " hwc2chw_op = CV.HWC2CHW()\n", + " \n", + " # apply map operations on images\n", + " ds = ds.map(input_columns=\"image\", operations=[resize_op, rescale_op, hwc2chw_op])\n", + " ds = ds.map(input_columns=\"label\", operations=C.TypeCast(ms.int32))\n", + " \n", + " ds = ds.shuffle(buffer_size=buffer_size)\n", + " ds = ds.batch(batch_size, drop_remainder=True)\n", + " ds = ds.repeat(num_epoch)\n", + " \n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 定义模型\n", + "\n", + "定义LeNet5模型，模型结构如下图所示。\n", + "\n", + "\n", + "[1] 图片来源于http://deeplearning.net" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class LeNet(nn.Cell):\n", + " def __init__(self):\n", + " super(LeNet, self).__init__()\n", + " self.relu = nn.ReLU()\n", + " self.conv1 = nn.Conv2d(1, 6, 5, stride=1, pad_mode='valid')\n", + " self.conv2 = nn.Conv2d(6, 16, 5, stride=1, pad_mode='valid')\n", + " self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n", + " self.flatten = nn.Flatten()\n", + " self.fc1 = nn.Dense(400, 120)\n", + " self.fc2 = nn.Dense(120, 84)\n", + " self.fc3 = nn.Dense(84, 10)\n", + " \n", + " def construct(self, input_x):\n", + " output = self.conv1(input_x)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.conv2(output)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.flatten(output)\n", + " output = self.fc1(output)\n", + " output = self.fc2(output)\n", + " output = self.fc3(output)\n", + " \n", + " return output" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 保存模型Checkpoint\n", + "\n", + "使用MNIST数据集对上述定义的LeNet5模型进行单机单卡训练，包含：\n", + "\n", + "- 在MNIST数据集上训练模型。\n", + "- 通过`ModelCheckpoint`保存Checkpoint。\n", + "- 通过`LossMonitor`输出训练过程中的Loss。\n", + "\n", + "Callback是模型训练/测试过程中的一种调试工具，可用在训练/测试过程中执行特定的任务。MindSpore框架提供的Callback：\n", + "\n", + "- `ModelCheckpoint`：保存网络模型和参数，默认会保存最后一次训练的参数。\n", + "- `SummaryStep`：对Tensor值进行监控。此功能会在MindData平台训练脚本中使用。\n", + "- `LossMonitor`：监控loss值，当loss值为Nan或Inf时停止训练。此功能会在MindData平台训练脚本中使用。\n", + "\n", + "`ModelCheckpoint`用于保存模型和参数，如每个epoch结束时，都保存一次checkpoint。\n", + "\n", + "1. 首先需要初始化一个`CheckpointConfig`类对象，用以声明保存策略。调用方法如：\n", + " \n", + " ```py\n", + " CheckpointConfig(save_checkpoint_steps=1, keep_checkpoint_max=5)\n", + " ```\n", + " \n", + " 参数说明：\n", + " \n", + " - `save_checkpoint_steps`：每多少step保存一个checkpoint文件，单位为step；\n", + " - `keep_checkpoint_max`：最多保留checkpoint文件的数量（按最新的文件）。\n", + "\n", + "2. 创建`ModelCheckpoint`对象。调用方法如：\n", + " \n", + " ```py\n", + " ModelCheckpoint(prefix=DEFAULT_CHECKPOINT_PREFIX_NAME, config=None)\n", + " ```\n", + " \n", + " 参数说明：\n", + " \n", + " - `prefix`：保存的文件前缀名，如'ck_lenet'。\n", + " - `config`：配置策略信息，传入上文创建的CheckpointConfig对象。\n", + "\n", + "> `ModelCheckpoint`会生成和保存模型（.pkl）和Chekpoint（.ckpt）文件。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 1 step: 1875 ,loss is 2.3151364\n", + "epoch: 2 step: 1875 ,loss is 0.3097728\n", + "Metrics: {'acc': 0.9417067307692307, 'loss': 0.18866610953894755}\n", + "b_lenet-1_1875.ckpt\n", + "b_lenet-2_1875.ckpt\n" + ] + } + ], + "source": [ + "os.system('rm -f *.ckpt *.ir *.meta') # 清理旧的运行文件\n", + "LOOP_SINK = context.get_context('enable_loop_sink')\n", + "\n", + "def test_train(lr=0.01, momentum=0.9, num_epoch=2, check_point_name=\"b_lenet\"):\n", + " ds_train = create_dataset(num_epoch=num_epoch)\n", + " ds_eval = create_dataset(training=False)\n", + " steps_per_epoch = ds_train.get_dataset_size()\n", + " \n", + " net = LeNet()\n", + " loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')\n", + " opt = nn.Momentum(net.trainable_params(), lr, momentum)\n", + " \n", + " ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5)\n", + " ckpt_cb = ModelCheckpoint(prefix=check_point_name, config=ckpt_cfg)\n", + " loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch)\n", + " \n", + " model = Model(net, loss, opt, metrics={'acc', 'loss'})\n", + " model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb], dataset_sink_mode=True)\n", + " metrics = model.eval(ds_eval)\n", + " print('Metrics:', metrics)\n", + "\n", + "test_train()\n", + "print('\\n'.join(sorted([x for x in os.listdir('.') if x.startswith('b_lenet')])))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 加载Checkpoint继续训练\n", + "\n", + "模型训练过程偶尔会中断，可以通过加载Checkpoint文件继续训练。\n", + "\n", + "1. 读取Checkpoint文件，调用方法如：\n", + " \n", + " ```py\n", + " load_checkpoint(ckpoint_file_name)\n", + " ```\n", + " \n", + " 参数说明：\n", + " \n", + " - `ckpoint_file_name`：checkpoint文件名，如'ck_lenet-7_1875.ckpt'。\n", + " - 返回值：返回一个字典。key为参数name，value为parameter类型的实例。\n", + "\n", + "2. 加载参数后继续训练，调用方法如：\n", + " \n", + " ```py\n", + " load_param_into_net(net, param_dict)\n", + " ```\n", + " \n", + " 参数说明：\n", + " \n", + " - `net`：初始不带优化器和损失函数的网络，如：`Resnet()`。\n", + " - `param_dict`：加载checkpoint文件后生成的字典。\n", + "\n", + "> 使用load_checkpoint接口加载数据时，需要把数据传入给原始网络，而不能传递给带有优化器和损失函数的训练网络。" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 1 step: 1875 ,loss is 0.1638589\n", + "epoch: 2 step: 1875 ,loss is 0.060048036\n", + "Metrics: {'acc': 0.9742588141025641, 'loss': 0.07910804035148034}\n", + "b_lenet_1-1_1875.ckpt\n", + "b_lenet_1-2_1875.ckpt\n" + ] + } + ], + "source": [ + "CKPT = 'b_lenet-2_1875.ckpt'\n", + "\n", + "def resume_train(lr=0.001, momentum=0.9, num_epoch=2, ckpt_name=\"b_lenet\"):\n", + " ds_train = create_dataset(num_epoch=num_epoch)\n", + " ds_eval = create_dataset(training=False)\n", + " steps_per_epoch = ds_train.get_dataset_size()\n", + " \n", + " net = LeNet()\n", + " loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')\n", + " opt = nn.Momentum(net.trainable_params(), lr, momentum)\n", + " \n", + " param_dict = load_checkpoint(CKPT)\n", + " load_param_into_net(net, param_dict)\n", + " load_param_into_net(opt, param_dict)\n", + " \n", + " ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5)\n", + " ckpt_cb = ModelCheckpoint(prefix=ckpt_name, config=ckpt_cfg)\n", + " loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch)\n", + " \n", + " model = Model(net, loss, opt, metrics={'acc', 'loss'})\n", + " model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb])\n", + " \n", + " metrics = model.eval(ds_eval)\n", + " print('Metrics:', metrics)\n", + "\n", + "resume_train()\n", + "print('\\n'.join(sorted([x for x in os.listdir('.') if x.startswith('b_lenet')])))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 推理\n", + "\n", + "加载Checkpoint，并执行验证。读取模型和Checkpoint文件，调用方法如：\n", + " \n", + " ```py\n", + " load(model_file_name, ckpoint_file_name)\n", + " ```\n", + " \n", + " 参数说明：\n", + " \n", + " - `model_file_name`：模型文件名，如'ck_lenet-model.pkl'。\n", + " - `ckpoint_file_name`：checkpoint文件名，如'ck_lenet-7_1875.ckpt'。\n", + " \n", + "使用matplotlib定义一个将推理结果可视化的辅助函数，如下：" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_images(pred_fn, ds, net):\n", + " for i in range(1, 5):\n", + " pred, image, label = pred_fn(ds, net)\n", + " plt.subplot(2, 2, i)\n", + " plt.imshow(np.squeeze(image))\n", + " color = 'blue' if pred == label else 'red'\n", + " plt.title(\"prediction: {}, truth: {}\".format(pred, label), color=color)\n", + " plt.xticks([])\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "使用训练后的LeNet模型对手写数字进行识别，可以看到识别结果基本上是正确的。" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD7CAYAAAAMyN1hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAcv0lEQVR4nO3de5RU5Znv8e9D03TLJUIraCMgXgDJMlEZguRoEuJl1IkecpxlJo7jQpfaIdE1OmO8xMl9NPHkmMvMMpmIE4TxbtBRYuIkSjQRZVCWgSSKCqMIKHJROgEEpJvn/FGbXbuaqn5316Wrqvv3WatXP7v27d3VTz+1330rc3dERKSwAdVugIhIrVOhFBEJUKEUEQlQoRQRCVChFBEJUKEUEQmoyUJpxjwzbozij5nxSpHL+bEZXylv6+qDGePNcDMGVrstkqXcLl01crsmC2WSO0+7Myk0nRkXmbG4y7yz3fnnyrUuXvexZvzSjC1m9OjCVDNmmLG+DG1YY8ZppS4nsbxPmvGkGX8yY025litZyu3Uy6l6ble8UPaTPZo9wAPAJZVYeJXewx3AXOCaKqy7Lii3S1c3ue3uPf4BXwP+JfCXwLeC3wHeHI2bAb4e/Drwt8HvjF4/G3w5eDv4s+AfTizvBPAXwLeB3w9+H/iNyeUlph0L/hD4ZvB3wG8Fnwy+C7wTfDt4ezTtvH3LiYYvA18N/i74QvDRiXEOPht8VbRNPwS3Hr4vR4N7D6YfAr4TfG/U7u3go8G/Dr4A/C7wP4Nfmmdb4vcF/M5oGTujZVwLPj7aplnga8G3gP9TEX/r08DXFJMn9fij3FZu5/spZY/yAuAM4ChgIvDlxLhDgRbgcKDNjClkKvjngIOA24CFZjSZMQh4GLgzmuenwF/nW6EZDcCjwBvAeOAw4D53VgKzgSXuDHVneJ55TwG+DXwGaI2WcV+Xyc4GPgIcF013RjTvODPazRiX9s1Jw50dwFnAW1G7h7rzVjR6JrAAGA7cHVjOhcBa4JxoGd9JjD4ZmAScCnzVjMnRNp1sRns5t6cPUW6XqK/ldimF8lZ31rnzLnATcH5i3F7ga+7sdmcncBlwmztL3el0Zz6wG5ge/TQCP3BnjzsLgOcLrHMaMBq4xp0d7uxyzz12040LgLnuvODObuBLwEfNGJ+Y5mZ32t1ZCzwJHA/gzlp3hkev95Yl7jzszt7oPSzWN9zZ6c4KYAWZfxTcWZzvn04A5Xal1V1ul1Io1yXiN8j8kffZ7M6uxPDhwNXRJ1d7VO3HRvOMBt50zzlQ/EaBdY4F3nCno4j2jk4u153twDtkPrn3eTsRvwcMLWI95bIuPEkqtbRN9UK5XVl1l9ulFMqxiXgcxLvVwH5nx9YBN0WfXPt+BrtzL7ABOMwM67K8fNYB4wocAA6dkXuLTFIDYMYQMl2lNwPzVVqhdnd9fQcwODF8aMrlSM8pt8ujz+R2KYXycjPGmNEC3ADc3820twOzzTjRDDNjiBmfMmMYsAToAP7ejIFmnEumG5LPc2SS7+ZoGc1mnBSN2wiMiY4L5XMPcLEZx5vRBHwLWOpe+qUv0TY1Q2bdUbuaEuPnmTGvwOwbgYPMODCwmuXAX5nRYsahwFV5lnNkURuQhxkDom1qhMz2dfPe9jXK7YhyO6OUQnkP8CvgtejnxkITurOMzLGcW4GtwGrgomjc+8C50fBW4G+AhwospxM4BziazAHe9dH0AL8GXgTeNmNLnnkXAV8BHiSTkEcBn02zodEB7+3dHPA+HNgZrZ8oTl5IPBZ4psA2vQzcC7wWdd1G55uOzAmBFcAaMu9713/ebwNfjpbxxcAm7bvYeXs3k3w82o5fkNkL2hmttz9QbmcptwHLnCbvmegizUvdeaLHM/cz0SfVCuDD7uypdnuke8rt9PpTbveHC2arKtqrmFztdoiUW3/K7Zq/hVFEpNqK6nqLiPQnJe1RmtmZZvaKma02s+vL1SiRalNuS1LRe5Rm1gC8CpxO5gzd88D57v5S+Zon0vuU29JVKSdzpgGr3f01ADO7j8w9nAWTaZA1eTNDSlillMs2tm5x95HVbkeNUm7XqV3s4H3fbeEpe6aUQnkYubcirQdO7DqRmbUBbQDNDOZEO7WEVUq5POELCt1KJ8rturXUF1VkuaUco8xXtffrx7v7HHef6u5TG7MX9IvUMuW25CilUK4n957YMeTeEytSr5TbkqOUQvk8MMHMjjCzQWRumVpYnmaJVJVyW3IUfYzS3TvM7Argl0ADMNfdXwzMJlLzlNvSVUm3MLr7L8jcWC7Spyi3JUm3MIqIBKhQiogEqFCKiASoUIqIBKhQiogE6MG9KQwYnP3eo7VXHp8zbm+KGzJGLs9+sd4BDz9XtnaJSO/QHqWISIAKpYhIgLreKdiw7PeqL/jcLTnjJg8a3HXy/Rzx2KVxPPHh8rVLJK2BR46P402faE01z8hHV8dx5+bN5W5SXdEepYhIgAqliEiACqWISICOUYr0A8njks/f9G+p5pnS9Pk4PvTB7Ov98Xil9ihFRAJUKEVEAtT1FpG8Xvhqtov+kd3ZbnjLHep6i4hIFyqUIiIBKpQiIgEqlCIiASqUIiIBOust0g8kH3CRvJA8eWZbCgvuUZrZXDPbZGZ/TLzWYmaPm9mq6PeIyjZTpPyU25JWmq73PODMLq9dDyxy9wnAomhYpN7MQ7ktKQS73u7+WzMb3+XlmcCMKJ4PPAVcV8Z2iVRcf8rt5P3ZB60c0+P5P37F0jhe3PnROB7+H0tKa1idKPZkziHuvgEg+j2q0IRm1mZmy8xs2R52F7k6kV6j3Jb9VPyst7vPcfep7j61kRTfxCVSJ5Tb/UexZ703mlmru28ws1ZgUzkbJVJFyu08vtv6QhxPPnJ6HA+vRmOqoNg9yoXArCieBTxSnuaIVJ1yW/aT5vKge4ElwCQzW29mlwA3A6eb2Srg9GhYpK4otyWtNGe9zy8w6tQyt0WkVym3JS3dwigiEqBCKSISoHu9C2gYOTKON5x3dBwPG7C3Gs0RkSrSHqWISIAKpYhIgLreBez5YPZ+2N/d8KPEmKG93xgRqSrtUYqIBKhQiogEqOudYE3ZBxu8/4HGkpa1qXNHdrm7GkpalohUl/YoRUQCVChFRALU9U5oP++EOL73plsSY3p+pvuku78Yx5Nuir+SBV2uLlJ/tEcpIhKgQikiEqBCKSISoGOUCZ2NFsdHNJZ2B87AXdll7d22raRliUh1aY9SRCRAhVJEJECFUkQkQIVSRCRAhVJEJECFUkQkIM33eo81syfNbKWZvWhmV0avt5jZ42a2Kvo9ovLNFSkf5baklWaPsgO42t0nA9OBy83sg8D1wCJ3nwAsioZF6olyW1IJXnDu7huADVG8zcxWAocBM4EZ0WTzgaeA6yrSygrqnDEljredtb2KLZHe1tdzW8qnR8cozWw8cAKwFDgkSrR9CTeqwDxtZrbMzJbtYXdprRWpEOW2dCd1oTSzocCDwFXu/ue087n7HHef6u5TG2kKzyDSy5TbEpLqXm8zaySTSHe7+0PRyxvNrNXdN5hZK7CpUo2spDdnNMfxyyfPrWJLpBr6cm5L+aQ5623AT4CV7v69xKiFwKwongU8Uv7miVSOclvSSrNHeRJwIfAHM1sevXYDcDPwgJldAqwFzqtME0UqRrktqaQ5670YsAKjTy1vc3pHw4Qj43hX654qtkSqqS/mtlSG7swREQlQoRQRCeiXTzhfeX1LHL9+1u1lW+76juwF6wN0WZ3UKOvwOH51z444PmrgATnTNVj+/aiO5uz8A4YNi+O+/CR/7VGKiASoUIqIBPTLrnelnPOda+N43Nzlcby3Go0RKWDA0j/G8VVnXRzHP3jsjpzpJjYOyTv/MxfcEsfTh/1jHE+4fGm5mlhztEcpIhKgQikiEqCud4mmfPPzcdz64Ko47nzvvWo0RyTIOzqyA++0x2GnF7r2PteohmyX3Js7y9auWqY9ShGRABVKEZEAdb1TSF5InjyzDV2625s391qbRKT3aI9SRCRAhVJEJECFUkQkoF8eoxz7s+znw+R1XwhOn3zARfKOG9BlQCL9gfYoRUQCVChFRAL6Zdf7gIefi+NxD/dsXj3gQvoSf29nHH/q5/+QOy7FXTcHL24se5tqkfYoRUQCVChFRAKCXW8zawZ+CzRF0y9w96+ZWQtwPzAeWAN8xt23Vq6pIuWl3M79+oa+/DzJUqXZo9wNnOLuxwHHA2ea2XTgemCRu08AFkXDIvVEuS2pBAulZ+y72bkx+nFgJjA/en0+8OmKtFCkQpTbklaqY5Rm1mBmy4FNwOPuvhQ4xN03AES/R1WumSKVodyWNFIVSnfvdPfjgTHANDM7Nu0KzKzNzJaZ2bI96DtcpbYotyWNHp31dvd24CngTGCjmbUCRL83FZhnjrtPdfepjTSV2FyRylBuS3eChdLMRprZ8Cg+ADgNeBlYCMyKJpsFPFKpRopUgnJb0kpzZ04rMN/MGsgU1gfc/VEzWwI8YGaXAGuB8yrYTpFKUG5LKsFC6e6/B07I8/o7wKmVaJRIb1BuS1rm7r23MrPNwBu9tkLpzuHuPrLajegrlNs1oyJ53auFUkSkHulebxGRABVKEZGAmiyUZswz48Yo/pgZrxS5nB+b8ZXytq4+mDHeDDfrn88crVXK7dJVI7drslAmufO0O5NC05lxkRmLu8w7251/rlzr4nUfa8YvzdhiRo8O+poxw4z1ZWjDGjNOK3U5ieV90ownzfiTGWvKtVzJqofcjtZ/pBmPmrEtyvHvpJyvJnM7WuYUM35rxnYzNppxZXfTV7xQ9pM9mj3AA8AllVh4ld7DHcBc4JoqrLsu9IfcNmMQ8Djwa+BQMrd63lXG5ff6e2jGwcB/AbcBBwFHA7/qdiZ37/EP+BrwL4G/BL4V/A7w5mjcDPD14NeBvw1+Z/T62eDLwdvBnwX/cGJ5J4C/AL4N/H7w+8BvTC4vMe1Y8IfAN4O/A34r+GTwXeCd4NvB26Np5+1bTjR8Gfhq8HfBF4KPToxz8Nngq6Jt+iG49fB9OTp6Jk3a6YeA7wTfG7V7O/ho8K+DLwC/C/zP4Jfm2Zb4fQG/M1rGzmgZ14KPj7ZpFvha8C3g/1TE3/o08DXF5Ek9/ii393s/2sCfLuJ9rNncBv/Wvr9d2p9S9igvAM4AjgImAl9OjDsUaAEOB9rMmEJm7+RzZCr4bcBCM5qiT6yHgTujeX4K/HW+FZrRADxK5nq18cBhwH3urARmA0vcGerO8DzzngJ8G/gMmTsy3gDu6zLZ2cBHgOOi6c6I5h1nRrsZ49K+OWm4swM4C3gravdQd96KRs8EFgDDgbsDy7mQzB0k50TLSHaNTgYmkbmA+qtmTI626WQz2su5PX2IcjtrOrDGjMeibvdTZnyowLSxGs/t6cC7ZjxrxiYzfhb63y6lUN7qzjp33gVuAs5PjNsLfM2d3e7sBC4DbnNnqTud7swn89DU6dFPI/ADd/a4swB4vsA6pwGjgWvc2eHOLvfcYzfduACY684L7uwGvgR81IzxiWludqfdnbXAk2Qe5oo7a90ZHr3eW5a487A7e6P3sFjfcGenOyuAFWT+UXBncb5/OgGU20ljgM8C/xq17+fAI9GHQLGqndtjyNzDfyUwDngduLe7FZVSKNcl4jfIvIn7bHZnV2L4cODq6JOrPar2Y6N5RgNvuuecBCl0h8NY4A13Oopo7+jkct3ZDrxD5pN7n7cT8XvA0CLWUy7rwpOkUkvbVC+U21k7gcXuPObO+8AtZPacJxfRzn2qnds7gf905/nob/kN4H+ZcWChGUoplGMT8TiId6uB/c78rgNuij659v0MdudeYANwmBnWZXn5rAPGFTgAHDrb/BaZpAbAjCFk/uBvBuartELt7vr6DmBwYvjQlMuRnlNuZ/0+xfoLqdXc7rpN+2LLMy1QWqG83IwxZrQAN5D5MqZCbgdmm3GiGWbGEDM+ZcYwYAnQAfy9GQPNOJdMNySf58gk383RMprNOCkatxEY002X4B7gYjOON6MJ+Baw1L30S1+ibWqGzLqjdjUlxs8zY16B2TcCB3X3aRZZDvyVGS1mHApclWc5Rxa1AXmYMSDapkbIbF+J3a16otzOuguYbsZp0XHUq4AtwEqoz9wG7gD+T/R+NQJfIbPXXPC4ZimF8h4yp9Rfi35uLDShO8vIHMu5FdgKrAYuisa9D5wbDW8F/gZ4qMByOoFzyJzOXwusj6aHzOULLwJvm7Elz7yLyLwhD5JJyKPIHHsJig54b+/mgO/hZHbnX4yGd0LOhcRjgWcKbNPLZI6PvBZ13Ubnm47MCYEVZL4V8Ffs/8/7beDL0TK+GNikfRc7b+9mko9H2/ELMntBOwldQtF3KLezy34F+Dvgx9E2zAT+d7RtUIe57c6vyXwA/pzMQ5mPBv6222VmTpf3jGUuQL7UnSd6PHM/E+0FrAA+7M6eardHuqfcTq8/5Xafv2C22qJP3lIOfIvUpP6U2zV/C6OISLUVVSjdGe/OE2Z2ppm9YmarzUxfEi91T7kt+RT94N7oe0ZeBU4nc+D5eeB8d3+pfM0T6X3KbemqlGOU04DV7v4agJndR+aMWMFkGmRN3syQElYp5bKNrVtcXwVRiHK7Tu1iB+/77oLXQxarlEJ5GLlX2K8HTuxuhmaGcKLpO5tqwRO+QN/vUphyu04t9UUVWW4phTJf1d6vH29mbUAbQHPOxfciNUu5LTlKOeu9ntxbvcaQe6sXAO4+x92nuvvUxuzNKiK1TLktOUoplM8DE8zsCDMbROZOgIXlaZZIVSm3JUfRXW937zCzK4BfAg3AXHd/MTCbSM1TbktXJd2Z4+6/IHMvsEifotyWJN2ZIyISoEIpIhKgQikiEqBCKSISoEIpIhKg51GK9FGdM6bE8ZszmvNOM2B3Nh73L8tzxu19772KtKseaY9SRCRAhVJEJEBd7xK9d272oTK7Dsz/uTPi1ex3vNszy/NOI9ITafJu21nZ79d6+eS5eadZ35Gd5pxt1+aMa71/VRx3bt5cVDv7Cu1RiogEqFCKiASoUIqIBOgYZQo2MPs27T3x2JxxF970szhuO3C/RxYCcMRjl8bxxLxfFS/SM2d8/Tdx/OWDXy56OWMGDo3j393wo5xxp//h4jge8BsdoxQRkW6oUIqIBKjrncKAg1ri+Pt353ZPJg/Sd6VIBVn263saDj44jpsGvFaN1vRb2qMUEQlQoRQRCVDXW6SGJbvbbc8uieOzBm9NTNXYiy3qn7RHKSISoEIpIhKgrrdIjbGp2ZsaPjkvf3e7ycLd7eOeOz+OR30///MouzPwhdVxvLfHc/ctwT1KM5trZpvM7I+J11rM7HEzWxX9HlHZZoqUn3Jb0krT9Z4HnNnlteuBRe4+AVgUDYvUm3kotyWFYNfb3X9rZuO7vDwTmBHF84GngOvK2K6qG3DsMXG891+3xfHYgTqs21fUam53Dsl2q69p+Z/EmHB3+5jFF8bxuH9piGN75nc9bkd/724nFftff4i7bwCIfo8qX5NEqkq5Lfup+MkcM2sD2gCa0e1+0ncot/uPYgvlRjNrdfcNZtYKbCo0obvPAeYAfMBavMj19bqOgw6I48ePuS8xpudnD6Wu1HVu20vDsvEzz1axJX1LsV3vhcCsKJ4FPFKe5ohUnXJb9pPm8qB7gSXAJDNbb2aXADcDp5vZKuD0aFikrii3Ja00Z73PLzDq1DK3paY0vvWnOD7q19knPS/7xA9zphvRkP/Y1MVrPxbHBy/Wvbi1qFZye8Bxk3OGX/3bnh0RO33lOXE8cnlHWdqUVueMKXH85oziD0uN/8+tOcN7V6wselmVoGtdREQCVChFRAJ0r3cBnauyT5CedHX2Urq3l+ZON6KBvP77vz4Ux+Pu0NlHKWzLlOE5w6+f8289mr/9rjFx3PLwkm6mLL9kd3tl24+6mbJ7R4xqyxk+5rbs4Yha6IZrj1JEJECFUkQkQF3vAgYMy164u2Pa+DhuNt0BK/1bw4Qj43hX656yLPP1T8/JGZ686QtxPG5FWVZREu1RiogEqFCKiASo611Ax5Sj4/g3tyW7BUN7vzEiVdYw/MA4fuVr2fj1U26vyPo6mrO3zicPg+3dti3f5BWnPUoRkQAVShGRAHW9RSTozXmj43jZXySfd1CZ53A+c8EtcTx92D/G8YTLl+abvOK0RykiEqBCKSISoEIpIhKgY5RldMzt2bsJjvrJ2jju3ScEipTHu49OjOOffujf43hEw5AeLef1Pdvj+KIrsscbL/jOo3HcduBbOfOMSqzDmzt7tL5K0B6liEiACqWISIC63mU0bE32boKOdeur2BKpJ6N+syFnOHkI5+XLws94/PgV2UtmFnd+NI6H/0dpz6acOe73cTyxsWfd7Qe2Z+/e+b/f/XwcN8zeEsczBq9KzNGz5fc27VGKiASoUIqIBKjrneAnHR/Hm/5hV6p5jvjZZXF8zAvtcaynVkpaHa+tyRk+6t+z10lMaMp2W5N3qyTPCn+39YU4vvgL2def+sTUOB720qA4bv1uuq8meXDOKXHcNDv73MlrWv4nOO+ru1qzbb0z+0DJtS3Z/7E1k7JfgTGxsTzPtayUNN/rPdbMnjSzlWb2opldGb3eYmaPm9mq6PeIyjdXpHyU25JWmq53B3C1u08GpgOXm9kHgeuBRe4+AVgUDYvUE+W2pBLserv7BmBDFG8zs5XAYcBMYEY02XzgKeC6irSyl2ydeEAcr5g2L9U8R9+T7SbVwrfFSXq1mtvJKyYmfP/9OH7nsxbHowp8++cd457ODiTi/3fiUXE8d9gZZWhl96YNznbP7772lG6mrA89OpljZuOBE4ClwCFRou1LuFGF5xSpbcpt6U7qQmlmQ4EHgavc/c89mK/NzJaZ2bI97C6mjSIVpdyWkFRnvc2skUwi3e3uD0UvbzSzVnffYGatwKZ887r7HGAOwAesxfNNI1ItNZ/bu7MF+LPLL4njn56Qvfc6zcXgyTPV17SFL2Iv1V8Ozp7FXtkL66u0NGe9DfgJsNLdv5cYtRCYFcWzgEfK3zyRylFuS1pp9ihPAi4E/mBmy6PXbgBuBh4ws0uAtcB5lWmiSMUotyWVNGe9FwNWYPSp5W2OSO+ph9zubP9THB/66Wx8waMXx3Hynuy/HPaHOJ7W1Fjh1pVP8t5wyL1gfdCG6m+HbmEUEQlQoRQRCdC93iJ1qOXsV+P4aZrjeO4Pr4jjn3/q+73aplJ875uzc4YPvOu/43g8pT0urhy0RykiEqBCKSISoK63SB8y6fqX4vjqb366ii3pmeHtv8sZrrU7U7RHKSISoEIpIhKgrrdIH7J327bsQDKWkmiPUkQkQIVSRCRAhVJEJECFUkQkQIVSRCRAhVJEJECFUkQkQIVSRCRAhVJEJEB35iSMeHVnHB/x2KWp5pn81rtx3Fn2FolILdAepYhIgAqliEiAut4J9szyOJ74TLp51N0W6fuCe5Rm1mxmz5nZCjN70cy+Eb3eYmaPm9mq6PeIyjdXpHyU25JWmq73buAUdz8OOB4408ymA9cDi9x9ArAoGhapJ8ptSSVYKD1jezTYGP04MBOYH70+H6if586LoNyW9FKdzDGzBjNbDmwCHnf3pcAh7r4BIPo9qnLNFKkM5bakkapQununux8PjAGmmdmxaVdgZm1mtszMlu1hd7HtFKkI5bak0aPLg9y9HXgKOBPYaGatANHvTQXmmePuU919aiNNJTZXpDKU29KdNGe9R5rZ8Cg+ADgNeBlYCMyKJpsFPFKpRopUgnJb0kpzHWUrMN/MGsgU1gfc/VEzWwI8YGaXAGuB8yrYTpFKUG5LKubee181bmabgTd6bYXSncPdfWS1G9FXKLdrRkXyulcLpYhIPdK93iIiASqUIiIBKpQiIgEqlCIiASqUIiIBKpQiIgEqlCIiASqUIiIBKpQiIgH/HyKwOLKinO/gAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "CKPT = 'b_lenet_1-2_1875.ckpt'\n", + "\n", + "def infer(ds, model):\n", + " data = ds.get_next()\n", + " images = data['image']\n", + " labels = data['label']\n", + " output = model.predict(Tensor(data['image']))\n", + " pred = np.argmax(output.asnumpy(), axis=1)\n", + " return pred[0], images[0], labels[0]\n", + "\n", + "ds = create_dataset(training=False, batch_size=1).create_dict_iterator()\n", + "net = LeNet()\n", + "param_dict = load_checkpoint(CKPT)\n", + "load_param_into_net(net, param_dict)\n", + "model = Model(net)\n", + "plot_images(infer, ds, model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 实验步骤（方案二）\n", + "\n", + "### 代码梳理\n", + "\n", + "创建训练作业时，运行参数会通过脚本传参的方式输入给脚本代码，脚本必须解析传参才能在代码中使用相应参数。如data_url和train_url，分别对应数据存储路径(OBS路径)和训练输出路径(OBS路径)。脚本对传参进行解析后赋值到`args`变量里，在后续代码里可以使用。\n", + "\n", + "```python\n", + "import argparse\n", + "parser = argparse.ArgumentParser()\n", + "parser.add_argument('--data_url', required=True, default=None, help='Location of data.')\n", + "parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.')\n", + "parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.')\n", + "args, unknown = parser.parse_known_args()\n", + "```\n", + "\n", + "MindSpore暂时没有提供直接访问OBS数据的接口，需要通过MoXing提供的API与OBS交互。将OBS中存储的数据拷贝至执行容器：\n", + "\n", + "```python\n", + "import moxing as mox\n", + "mox.file.copy_parallel(src_url=args.data_url, dst_url='MNIST/')\n", + "```\n", + "\n", + "如需将训练输出（如模型Checkpoint）从执行容器拷贝至OBS，请参考：\n", + "\n", + "```python\n", + "import moxing as mox\n", + "mox.file.copy_parallel(src_url='output', dst_url='s3://OBS/PATH')\n", + "```\n", + "\n", + "其他代码分析请参考方案一。\n", + "\n", + "### 创建训练作业\n", + "\n", + "可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。\n", + "\n", + "创建训练作业的参考配置：\n", + "\n", + "- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore\n", + "- 代码目录：选择上述新建的OBS桶中的experiment_2目录\n", + "- 启动文件：选择上述新建的OBS桶中的experiment_2目录下的`main.py`\n", + "- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_1文件夹下的MNIST目录\n", + "- 训练输出位置：选择上述新建的OBS桶中的experiment_1目录并在其中创建output目录\n", + "- 作业日志路径：同训练输出位置\n", + "- 规格：Ascend:1*Ascend 910\n", + "- 其他均为默认\n", + "\n", + "启动并查看训练过程：\n", + "\n", + "1. 点击提交以开始训练；\n", + "2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理；\n", + "3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看；\n", + "4. 在训练日志中可以看到`epoch: 3 step: 1875 ,loss is 0.025683485`等字段，即训练过程的loss值；\n", + "5. 在训练日志中可以看到`Metrics: {'acc': 0.9742588141025641, 'loss': 0.08628832848253062}`等字段，即训练完成后的验证精度；\n", + "6. 在训练日志里可以看到`b_lenet_1-2_1875.ckpt`等字段，即训练过程保存的Checkpoint。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 实验小结\n", + "\n", + "本实验展示了MindSpore的Checkpoint、断点继续训练等高级特性：\n", + "1. 使用MindSpore的ModelCheckpoint接口每个epoch保存一次Checkpoint，训练2个epoch并终止。\n", + "2. 使用MindSpore的load_checkpoint和load_param_into_net接口加载上一步保存的Checkpoint继续训练2个epoch。\n", + "3. 观察训练过程中Loss的变化情况，加载Checkpoint继续训练后loss进一步下降。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/experiment_2/main.py b/experiment_2/main.py new file mode 100644 index 0000000..9e5c98b --- /dev/null +++ b/experiment_2/main.py @@ -0,0 +1,142 @@ +# Save and load model + +import os +# os.environ['DEVICE_ID'] = '0' +# Log level includes 3(ERROR), 2(WARNING), 1(INFO), 0(DEBUG). +os.environ['GLOG_v'] = '2' + +import matplotlib.pyplot as plt +import numpy as np + +import mindspore as ms +import mindspore.context as context +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as CV + +from mindspore.dataset.transforms.vision import Inter +from mindspore import nn, Tensor +from mindspore.train import Model +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +import logging; logging.getLogger('matplotlib.font_manager').disabled = True + +context.set_context(mode=context.GRAPH_MODE, device_target='Ascend') + +DATA_DIR_TRAIN = "MNIST/train" # 训练集信息 +DATA_DIR_TEST = "MNIST/test" # 测试集信息 + + +def create_dataset(training=True, num_epoch=1, batch_size=32, resize=(32, 32), + rescale=1/(255*0.3081), shift=-0.1307/0.3081, buffer_size=64): + ds = ms.dataset.MnistDataset(DATA_DIR_TRAIN if training else DATA_DIR_TEST) + + # define map operations + resize_op = CV.Resize(resize) + rescale_op = CV.Rescale(rescale, shift) + hwc2chw_op = CV.HWC2CHW() + + # apply map operations on images + ds = ds.map(input_columns="image", operations=[resize_op, rescale_op, hwc2chw_op]) + ds = ds.map(input_columns="label", operations=C.TypeCast(ms.int32)) + + ds = ds.shuffle(buffer_size=buffer_size) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(num_epoch) + + return ds + + +class LeNet(nn.Cell): + def __init__(self): + super(LeNet, self).__init__() + self.relu = nn.ReLU() + self.conv1 = nn.Conv2d(1, 6, 5, stride=1, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, stride=1, pad_mode='valid') + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.fc1 = nn.Dense(400, 120) + self.fc2 = nn.Dense(120, 84) + self.fc3 = nn.Dense(84, 10) + + def construct(self, input_x): + output = self.conv1(input_x) + output = self.relu(output) + output = self.pool(output) + output = self.conv2(output) + output = self.relu(output) + output = self.pool(output) + output = self.flatten(output) + output = self.fc1(output) + output = self.fc2(output) + output = self.fc3(output) + + return output + + +LOOP_SINK = context.get_context('enable_loop_sink') + +def test_train(lr=0.01, momentum=0.9, num_epoch=2, check_point_name="b_lenet"): + ds_train = create_dataset(num_epoch=num_epoch) + ds_eval = create_dataset(training=False) + steps_per_epoch = ds_train.get_dataset_size() + + net = LeNet() + loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + opt = nn.Momentum(net.trainable_params(), lr, momentum) + + ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5) + ckpt_cb = ModelCheckpoint(prefix=check_point_name, config=ckpt_cfg) + loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch) + + model = Model(net, loss, opt, metrics={'acc', 'loss'}) + model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb], dataset_sink_mode=True) + metrics = model.eval(ds_eval) + print('Metrics:', metrics) + + +CKPT = 'b_lenet-2_1875.ckpt' + +def resume_train(lr=0.001, momentum=0.9, num_epoch=2, ckpt_name="b_lenet"): + ds_train = create_dataset(num_epoch=num_epoch) + ds_eval = create_dataset(training=False) + steps_per_epoch = ds_train.get_dataset_size() + + net = LeNet() + loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + opt = nn.Momentum(net.trainable_params(), lr, momentum) + + param_dict = load_checkpoint(CKPT) + load_param_into_net(net, param_dict) + load_param_into_net(opt, param_dict) + + ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5) + ckpt_cb = ModelCheckpoint(prefix=ckpt_name, config=ckpt_cfg) + loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch) + + model = Model(net, loss, opt, metrics={'acc', 'loss'}) + model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb]) + + metrics = model.eval(ds_eval) + print('Metrics:', metrics) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') + args, unknown = parser.parse_known_args() + + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, dst_url='MNIST/') + + os.system('rm -f *.ckpt *.ir *.meta') # 清理旧的运行文件 + + test_train() + print('\n'.join(sorted([x for x in os.listdir('.') if x.startswith('b_lenet')]))) + + resume_train() + print('\n'.join(sorted([x for x in os.listdir('.') if x.startswith('b_lenet')]))) + \ No newline at end of file diff --git a/experiment_3/3-Computer_Vision.md b/experiment_3/3-Computer_Vision.md new file mode 100644 index 0000000..0030994 --- /dev/null +++ b/experiment_3/3-Computer_Vision.md @@ -0,0 +1,344 @@ +

计算机视觉应用

+ +[TOC] + +## 实验介绍 + +本实验主要介绍使用MindSpore在CIFAR10数据集上训练ResNet50。本实验建议使用MindSpore model_zoo中提供的ResNet50。 + +## 实验目的 + +- 了解如何使用MindSpore加载常用的CIFAR-10图片分类数据集。 +- 了解MindSpore的model_zoo模块，以及如何使用model_zoo中的模型。 +- 了解ResNet50这类大模型的基本结构和编程方法。 + +## 预备知识 + +- 熟练使用Python，了解Shell及Linux操作系统基本知识。 +- 具备一定的深度学习理论知识，如卷积神经网络、损失函数、优化器，训练策略、Checkpoint等。 +- 了解华为云的基本使用方法，包括[OBS（对象存储）](https://www.huaweicloud.com/product/obs.html)、[ModelArts（AI开发平台）](https://www.huaweicloud.com/product/modelarts.html)、[训练作业](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0046.html)等功能。华为云官网：https://www.huaweicloud.com +- 了解并熟悉MindSpore AI计算框架，MindSpore官网：https://www.mindspore.cn/ + +## 实验环境 + +- MindSpore 0.2.0（MindSpore版本会定期更新，本指导也会定期刷新，与版本配套）； +- 华为云ModelArts：ModelArts是华为云提供的面向开发者的一站式AI开发平台，集成了昇腾AI处理器资源池，用户可以在该平台下体验MindSpore。ModelArts官网：https://www.huaweicloud.com/product/modelarts.html + +## 实验准备 + +### 创建OBS桶 + +本实验需要使用华为云OBS存储脚本和数据集，可以参考[快速通过OBS控制台上传下载文件](https://support.huaweicloud.com/qs-obs/obs_qs_0001.html)了解使用OBS创建桶、上传文件、下载文件的使用方法。 + +> **提示：**华为云新用户使用OBS时通常需要创建和配置“访问密钥”，可以在使用OBS时根据提示完成创建和配置。也可以参考[获取访问密钥并完成ModelArts全局配置](https://support.huaweicloud.com/prepare-modelarts/modelarts_08_0002.html)获取并配置访问密钥。 + +创建OBS桶的参考配置如下： + +- 区域：华北-北京四 +- 数据冗余存储策略：单AZ存储 +- 桶名称：如ms-course +- 存储类别：标准存储 +- 桶策略：公共读 +- 归档数据直读：关闭 +- 企业项目、标签等配置：免 + +### 数据集准备 + +CIFAR-10是一个图片分类数据集，包含60000张32x32的彩色物体图片，训练集50000张，测试集10000张，共10类，每类6000张。CIFAR-10数据集的官网：[THE MNIST DATABASE](http://www.cs.toronto.edu/~kriz/cifar.html)。 + +从CIFAR-10官网下载“CIFAR-10 binary version (suitable for C programs)”到本地并解压。 + +### 脚本准备 + +从[MindSpore tutorial仓库](https://gitee.com/mindspore/docs/tree/r0.2/tutorials/tutorial_code/sample_for_cloud/)里下载相关脚本。 + +### 上传文件 + +将脚本和数据集上传到OBS桶中，组织为如下形式： + +``` +experiment_3 +├── 脚本等文件 +└── cifar10 + ├── batches.meta.txt + ├── test + │   └── test_batch.bin + └── train + ├── data_batch_1.bin + ├── data_batch_2.bin + ├── data_batch_3.bin + ├── data_batch_4.bin + └── data_batch_5.bin +``` + +## 实验步骤 + +参考MindSpore官网[计算机视觉应用](https://www.mindspore.cn/tutorial/zh-CN/0.1.0-alpha/advanced_use/computer_vision_application.html)教程，使用MindSpore在CIFAR10数据集上训练ResNet50，并进行验证。建议： + +- 使用单卡训练即可； +- 理解并熟悉教程中涉及的源码； +- 使用MindSpore model_zoo中提供的ResNet50。 + +### 代码梳理 + +- resnet50_train.py：主脚本，包含性能测试`PerformanceCallback`、动态学习率`get_lr`、执行函数`resnet50_train`等函数； +- dataset.py：数据处理脚本。 + +`PerformanceCallback`继承MindSpore Callback类，并统计每个训练step的时延： + +```python +class PerformanceCallback(Callback): + """ + Training performance callback. + + Args: + batch_size (int): Batch number for one step. + """ + def __init__(self, batch_size): + super(PerformanceCallback, self).__init__() + self.batch_size = batch_size + self.last_step = 0 + self.epoch_begin_time = 0 + + def step_begin(self, run_context): + self.epoch_begin_time = time.time() + + def step_end(self, run_context): + params = run_context.original_args() + cost_time = time.time() - self.epoch_begin_time + train_steps = params.cur_step_num -self.last_step + print(f'epoch {params.cur_epoch_num} cost time = {cost_time}, train step num: {train_steps}, ' + f'one step time: {1000*cost_time/train_steps} ms, ' + f'train samples per second of cluster: {device_num*train_steps*self.batch_size/cost_time:.1f}\n') + self.last_step = run_context.original_args().cur_step_num +``` + +`get_lr`生成学习率数组，其中每个元素对应每个step的学习率，这里学习率下降采用二次曲线的形式： + +```python +def get_lr(global_step, + total_epochs, + steps_per_epoch, + lr_init=0.01, + lr_max=0.1, + warmup_epochs=5): + """ + Generate learning rate array. + + Args: + global_step (int): Initial step of training. + total_epochs (int): Total epoch of training. + steps_per_epoch (float): Steps of one epoch. + lr_init (float): Initial learning rate. Default: 0.01. + lr_max (float): Maximum learning rate. Default: 0.1. + warmup_epochs (int): The number of warming up epochs. Default: 5. + + Returns: + np.array, learning rate array. + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(int(total_steps)): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = ( 1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps)) ) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate +``` + +MindSpore支持直接读取cifar10数据集： + +```python +if device_num == 1 or not do_train: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle) +else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle, + num_shards=device_num, shard_id=device_id) +``` + +导入并使用model_zoo里的resnet50模型： + +```python +from mindspore.model_zoo.resnet import resnet50 +# create model +net = resnet50(class_num = class_num) +``` + +使用数据增强，如随机裁剪、随机水平反转： + +```python +# define map operations +random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) +random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1)) +``` + +`model_zoo.resnet`中resnet50定义如下： + +```python +def resnet50(class_num=10): + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) +``` + +ResNet类定义如下： + +```python +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ +``` + +ResNet的不同版本均由5个阶段（stage）组成，其中ResNet50结构为Convx1 -> ResidualBlockx3 -> ResidualBlockx4 -> ResidualBlockx6 -> ResidualBlockx5 -> Pooling+FC。 + +`ResidualBlock`为残差模块，相比传统卷积多了一个short-cut支路，用于将浅层的信息直接传递到深层，使得网络可以很深，而不会出现训练时梯度消失/爆炸的问题： + +```python +class ResidualBlock(nn.Cell): + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + # 如果in + self.down_sample = False + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + # output为残差支路，identity为short-cut支路 + out = self.add(out, identity) + out = self.relu(out) + + return out +``` + +创建训练作业时，运行参数会通过脚本传参的方式输入给脚本代码，脚本必须解析传参才能在代码中使用相应参数。如data_url和train_url，分别对应数据存储路径(OBS路径)和训练输出路径(OBS路径)。脚本对传参进行解析后赋值到`args`变量里，在后续代码里可以使用。 + +```python +import argparse +parser = argparse.ArgumentParser() +parser.add_argument('--data_url', required=True, default=None, help='Location of data.') +parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') +parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') +args, unknown = parser.parse_known_args() +``` + +MindSpore暂时没有提供直接访问OBS数据的接口，需要通过MoXing提供的API与OBS交互。将OBS中存储的数据拷贝至执行容器： + +```python +import moxing as mox +mox.file.copy_parallel(src_url=args.data_url, dst_url='cifar10/') +``` + +如需将训练输出（如模型Checkpoint）从执行容器拷贝至OBS，请参考： + +```python +import moxing as mox +mox.file.copy_parallel(src_url='output', dst_url='s3://OBS/PATH') +``` + +### 创建训练作业 + +可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。 + +创建训练作业的参考配置： + +- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore +- 代码目录：选择上述新建的OBS桶中的experiment_3目录 +- 启动文件：选择上述新建的OBS桶中的experiment_3目录下的`resnet50_train.py` +- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_3文件夹下的cifar10目录 +- 训练输出位置：选择上述新建的OBS桶中的experiment_3目录并在其中创建output目录 +- 作业日志路径：同训练输出位置 +- 规格：Ascend:1*Ascend 910 +- 其他均为默认 + +启动并查看训练过程： + +1. 点击提交以开始训练； +2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理； +3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看； +4. 在训练日志中可以看到`epoch 90 cost time = 27.963477849960327, train step num: 1562, one step time: 17.90235457743939 ms, train samples per second of cluster: 1787.5`等字段，即训练过程的性能数据； +5. 在训练日志中可以看到`epoch: 90 step: 1562, loss is 0.00250402`等字段，即训练过程的loss数据； +6. 在训练日志里可以看到`Evaluation result: {'acc': 0.9182692307692307}.`字段，即训练完成后的验证精度。 + +## 实验结论 + +本实验主要介绍使用MindSpore在CIFAR10数据集上训练ResNet50，了解了以下知识点： + +- 性能测试 +- 动态学习率 +- model_zoo：resnet50 +- cifar10数据集、数据增强 diff --git a/experiment_3/dataset.py b/experiment_3/dataset.py new file mode 100644 index 0000000..8896e06 --- /dev/null +++ b/experiment_3/dataset.py @@ -0,0 +1,86 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Create train or eval dataset.""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 + + +device_id = int(os.getenv('DEVICE_ID')) +device_num = int(os.getenv('RANK_SIZE')) + + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + Create a train or eval dataset. + + Args: + dataset_path (str): The path of dataset. + do_train (bool): Whether dataset is used for train or eval. + repeat_num (int): The repeat times of dataset. Default: 1. + batch_size (int): The batch size of dataset. Default: 32. + + Returns: + Dataset. + """ + if do_train: + dataset_path = os.path.join(dataset_path, 'train') + do_shuffle = True + else: + dataset_path = os.path.join(dataset_path, 'eval') + do_shuffle = False + + if device_num == 1 or not do_train: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle, + num_shards=device_num, shard_id=device_id) + + resize_height = 224 + resize_width = 224 + buffer_size = 100 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4)) + random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1)) + + resize_op = C.Resize((resize_height, resize_width)) + rescale_op = C.Rescale(rescale, shift) + normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) + + change_swap_op = C.HWC2CHW() + + trans = [] + if do_train: + trans += [random_crop_op, random_horizontal_flip_op] + + trans += [resize_op, rescale_op, normalize_op, change_swap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/experiment_3/resnet50_train.py b/experiment_3/resnet50_train.py new file mode 100644 index 0000000..0b422a2 --- /dev/null +++ b/experiment_3/resnet50_train.py @@ -0,0 +1,172 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet50 model train with MindSpore""" +import os +import argparse +import random +import time +import numpy as np +import moxing as mox + +from mindspore import context +from mindspore import Tensor +from mindspore.nn.optim.momentum import Momentum +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import Callback, LossMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +import mindspore.dataset.engine as de + +from dataset import create_dataset, device_id, device_num +from mindspore.model_zoo.resnet import resnet50 + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + + +class PerformanceCallback(Callback): + """ + Training performance callback. + + Args: + batch_size (int): Batch number for one step. + """ + def __init__(self, batch_size): + super(PerformanceCallback, self).__init__() + self.batch_size = batch_size + self.last_step = 0 + self.epoch_begin_time = 0 + + def step_begin(self, run_context): + self.epoch_begin_time = time.time() + + def step_end(self, run_context): + params = run_context.original_args() + cost_time = time.time() - self.epoch_begin_time + train_steps = params.cur_step_num -self.last_step + print(f'epoch {params.cur_epoch_num} cost time = {cost_time}, train step num: {train_steps}, ' + f'one step time: {1000*cost_time/train_steps} ms, ' + f'train samples per second of cluster: {device_num*train_steps*self.batch_size/cost_time:.1f}\n') + self.last_step = run_context.original_args().cur_step_num + + +def get_lr(global_step, + total_epochs, + steps_per_epoch, + lr_init=0.01, + lr_max=0.1, + warmup_epochs=5): + """ + Generate learning rate array. + + Args: + global_step (int): Initial step of training. + total_epochs (int): Total epoch of training. + steps_per_epoch (float): Steps of one epoch. + lr_init (float): Initial learning rate. Default: 0.01. + lr_max (float): Maximum learning rate. Default: 0.1. + warmup_epochs (int): The number of warming up epochs. Default: 5. + + Returns: + np.array, learning rate array. + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(int(total_steps)): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = ( 1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps)) ) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate + + +def resnet50_train(args_opt): + epoch_size = args_opt.epoch_size + batch_size = 32 + class_num = 10 + loss_scale_num = 1024 + local_data_path = '/cache/data' + + # set graph mode and parallel mode + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) + context.set_context(enable_task_sink=True, device_id=device_id) + context.set_context(enable_loop_sink=True) + context.set_context(enable_mem_reuse=True) + if device_num > 1: + context.set_auto_parallel_context(device_num=device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + local_data_path = os.path.join(local_data_path, str(device_id)) + + # data download + print('Download data.') + mox.file.copy_parallel(src_url=args_opt.data_url, dst_url=local_data_path) + + # create dataset + print('Create train and evaluate dataset.') + train_dataset = create_dataset(dataset_path=local_data_path, do_train=True, + repeat_num=epoch_size, batch_size=batch_size) + eval_dataset = create_dataset(dataset_path=local_data_path, do_train=False, + repeat_num=1, batch_size=batch_size) + train_step_size = train_dataset.get_dataset_size() + print('Create dataset success.') + + # create model + net = resnet50(class_num = class_num) + loss = SoftmaxCrossEntropyWithLogits(sparse=True) + lr = Tensor(get_lr(global_step=0, total_epochs=epoch_size, steps_per_epoch=train_step_size)) + opt = Momentum(net.trainable_params(), lr, momentum=0.9, weight_decay=1e-4, loss_scale=loss_scale_num) + loss_scale = FixedLossScaleManager(loss_scale_num, False) + + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) + + # define performance callback to show ips and loss callback to show loss for every epoch + performance_cb = PerformanceCallback(batch_size) + loss_cb = LossMonitor() + cb = [performance_cb, loss_cb] + + print(f'Start run training, total epoch: {epoch_size}.') + model.train(epoch_size, train_dataset, callbacks=cb) + if device_num == 1 or device_id == 0: + print(f'Start run evaluation.') + output = model.eval(eval_dataset) + print(f'Evaluation result: {output}.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='ResNet50 train.') + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--epoch_size', type=int, default=90, help='Train epoch size.') + + args_opt, unknown = parser.parse_known_args() + + resnet50_train(args_opt) + print('ResNet50 training success!') diff --git a/experiment_4/4-Natural_Language_Processing.md b/experiment_4/4-Natural_Language_Processing.md new file mode 100644 index 0000000..5880d42 --- /dev/null +++ b/experiment_4/4-Natural_Language_Processing.md @@ -0,0 +1,374 @@ +

自然语言处理应用

+ +[TOC] + +## 实验介绍 + +本实验主要介绍使用MindSpore开发和训练[BERT](https://arxiv.org/pdf/1810.04805.pdf)模型。建议先了解MindSpore官网上model_zoo上的BERT模型。 + +## 实验目的 + +- 了解如何使用MindSpore加载常用的NLP数据集。 +- 了解MindSpore的model_zoo模块，以及如何使用model_zoo中的模型。 +- 了解BERT模型的基本结构和编程方法。 + +## 预备知识 + +- 熟练使用Python，了解Shell及Linux操作系统基本知识。 +- 具备一定的深度学习理论知识，如Embedding、Encoder、Decoder、损失函数、优化器，训练策略、Checkpoint等。 +- 了解华为云的基本使用方法，包括[OBS（对象存储）](https://www.huaweicloud.com/product/obs.html)、[ModelArts（AI开发平台）](https://www.huaweicloud.com/product/modelarts.html)、[训练作业](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0046.html)等功能。华为云官网：https://www.huaweicloud.com +- 了解并熟悉MindSpore AI计算框架，MindSpore官网：https://www.mindspore.cn/ + +## 实验环境 + +- MindSpore 0.2.0（MindSpore版本会定期更新，本指导也会定期刷新，与版本配套）； +- 华为云ModelArts：ModelArts是华为云提供的面向开发者的一站式AI开发平台，集成了昇腾AI处理器资源池，用户可以在该平台下体验MindSpore。ModelArts官网：https://www.huaweicloud.com/product/modelarts.html + +## 实验准备 + +### 创建OBS桶 + +本实验需要使用华为云OBS存储脚本和数据集，可以参考[快速通过OBS控制台上传下载文件](https://support.huaweicloud.com/qs-obs/obs_qs_0001.html)了解使用OBS创建桶、上传文件、下载文件的使用方法。 + +> **提示：**华为云新用户使用OBS时通常需要创建和配置“访问密钥”，可以在使用OBS时根据提示完成创建和配置。也可以参考[获取访问密钥并完成ModelArts全局配置](https://support.huaweicloud.com/prepare-modelarts/modelarts_08_0002.html)获取并配置访问密钥。 + +创建OBS桶的参考配置如下： + +- 区域：华北-北京四 +- 数据冗余存储策略：单AZ存储 +- 桶名称：如ms-course +- 存储类别：标准存储 +- 桶策略：公共读 +- 归档数据直读：关闭 +- 企业项目、标签等配置：免 + +### 数据集准备 + +**预训练（pretrain）数据集**：下载[zhwiki数据集](https://dumps.wikimedia.org/zhwiki)，使用[WikiExtractor](https://github.com/attardi/wil kiextractor)进行预处理，然后使用[google-research/bert:create_pretraining_data.py](https://github.com/google-research/bert/blob/master/create_pretraining_data.py)将数据转为TFRecord格式； + +zhwiki为中文维基百科数据集，需要将其处理为具有上下文关系的句子对，然后基于词典vocab.txt对每个句子对进行token化，然后存储为特定数据格式（如Json、TFRecord、MindRecord）。 + +**微调（finetune）数据集**：使用[CLUEbenchmark/CLUEPretrainedModels中的脚本](https://github.com/CLUEbenchmark/CLUEPretrainedModels/blob/master/baselines/models/bert/run_classifier_tnews.sh)下载、处理TNEWS数据集，并将数据转为TFRecord格式。 + +TNEWS为今日头条中文新闻（短文本）分类（Short Text Classificaiton for News）数据集。该数据集来自今日头条的新闻版块，共提取了15个类别的新闻，包括旅游，教育，金融，军事等。数据量：训练集(53,360)，验证集(10,000)，测试集(10,000)。例子： + +{"label": "102", "label_des": "news_entertainment", "sentence": "江疏影甜甜圈自拍，迷之角度竟这么好看，美吸引一切事物"} + +每一条数据有三个属性，从前往后分别是 分类ID，分类名称，新闻字符串（仅含标题）。 + +本实验不进行数据预处理，请从网盘下载zhwiki_part和tnews数据集： + +链接: https://pan.baidu.com/s/1F2S9Wr-ND0LMfATjv7WEug 提取码: gent + +### 脚本准备 + +从[课程gitee仓库](https://gitee.com/mindspore/course)上下载本实验相关脚本。其中`tokenization.py`来源于[google-research/bert](https://github.com/google-research/bert/blob/master/tokenization.py) + +### 上传文件 + +将脚本和数据集上传到OBS桶中，组织为如下形式： + +``` +experiment_4 +├── 脚本等文件 +├── tnews +│   ├── bert_base.ckpt +│   ├── dev.tf_record +│   ├── dev_schema.json +│   ├── label2id.json +│   ├── train.tf_record +│   ├── train_schema.json +│   └── vocab.txt +└── zhwiki_part + ├── schema.json + └── part.tfrecord +``` + +## 实验步骤 + +参考MindSpore开源仓库[BERT example](https://gitee.com/mindspore/mindspore/tree/r0.2/example/Bert_NEZHA_cnwiki)示例，并进行实验。 + +BERT（Bidirectional Encoder Representations from Transformers），即基于Transformer的双向编码表征。其中： + +- Transformer是一种注意力（Attention）机制，用来学习文本中单词上下文之间的关系； +- 双向是指通过Masked Language Model（MLM）方法，随机的掩盖掉句子中的某些单词，然后利用前后未掩盖的信息来预测掩盖的单词； + +更多BERT的介绍可以参考[Link](https://www.jianshu.com/p/d110d0c13063) + +### 预训练BERT模型 + +[BERT](https://github.com/google-research/bert)模型包含由不同隐含层数（number hidden layers）和隐含层单元数（hidden size）构成的不同版本。通常情况下使用Bert需要预训练（pretrain）和微调（fine-tune）两个阶段。预训练BERT模型通常需要在大数据集上多卡并行训练多天。本实验先以部分zhwiki数据集为例展示预训练的过程。 + +BERT预训练阶段包含两个任务（两个输出）： + +- Mask语言模型（Mask LM）：预测被掩盖掉（mask）的单词； +- NextSentence预测（NSP）：判断句子对是否具有上下文关系，即句子B是否时句子A的下一句。 + +### 代码梳理 + +model_zoo:Bert_NEZHA中包含两个模块： + +- `bert_for_pre_training.py`：包含`GetMaskedLMOutput`, `GetNextSentenceOutput`, `BertPreTraining`, `BertPretrainingLoss`, `BertNetworkWithLoss`, `BertTrainOneStepCell`, `BertTrainOneStepWithLossScaleCell`； +- `bert_model.py`：包含`BertModel`依赖的 + +`GetMaskedLMOutput`接在BERT基础模型的后面，用于获取Mask LM的输出， + +`GetNextSentenceOutput`在BERT基础模型的后面接了一个全连接层和Softmax层，用于获取NSP的输出。 + +```python +class GetNextSentenceOutput(nn.Cell): + def construct(self, input_tensor): + logits = self.dense(input_tensor) + logits = self.cast(logits, self.dtype) + log_prob = self.log_softmax(logits) + return log_prob +``` + +`BertPreTraining`将Mask LM模型和NSP模型封装成一个模型定义，`BertPretrainingLoss`将Mask LM Loss和NSP Loss加和封装为一个Loss定义。`BertNetworkWithLoss`根据模型输出计算Loss值。 + +```python +class BertNetworkWithLoss(nn.Cell): + """ + Provide bert pre-training loss through network. + + Args: + config (BertConfig): The config of BertModel. + is_training (bool): Specifies whether to use the training mode. + use_one_hot_embeddings (bool): Specifies whether to use one-hot for embeddings. Default: False. + + Returns: + Tensor, the loss of the network. + """ + def __init__(self, config, is_training, use_one_hot_embeddings=False): + super(BertNetworkWithLoss, self).__init__() + self.bert = BertPreTraining(config, is_training, use_one_hot_embeddings) + self.loss = BertPretrainingLoss(config) + self.cast = P.Cast() + + def construct(self, + input_ids, + input_mask, + token_type_id, + next_sentence_labels, + masked_lm_positions, + masked_lm_ids, + masked_lm_weights): + prediction_scores, seq_relationship_score = \ + self.bert(input_ids, input_mask, token_type_id, masked_lm_positions) + total_loss = self.loss(prediction_scores, seq_relationship_score, + masked_lm_ids, masked_lm_weights, next_sentence_labels) + return self.cast(total_loss, mstype.float32) +``` + +`BertTrainOneStepCell`在`BertNetworkWithLoss`上加上了反向传播和梯度更新（优化器），接收数据输入，更新模型权重。`BertTrainOneStepWithLossScaleCell`在此基础上引入了损失缩放（Loss Scaling）。损失缩放是为了应对反向传播过程中梯度数值较小，计算时（如采用FP16）会被当做0处理，所以先对Loss做一个放大，然后再对梯度进行缩小。 + +`bert_model.py`中`BertModel`接收数据输入，经过`EmbeddingLookup`, `EmbeddingPostprocessor`, `BertTransformer`和`Dense`计算后得到输出。 + +![BERT Model](https://www.lyrn.ai/wp-content/uploads/2018/11/transformer.png) + +[1] 图片来源于https://www.lyrn.ai + +```python +class BertModel(nn.Cell): + def construct(self, input_ids, token_type_ids, input_mask): + # embedding + if not self.token_type_ids_from_dataset: + token_type_ids = self.token_type_ids + word_embeddings, embedding_tables = self.bert_embedding_lookup(input_ids) + embedding_output = self.bert_embedding_postprocessor(token_type_ids, + word_embeddings) + + # attention mask [batch_size, seq_length, seq_length] + attention_mask = self._create_attention_mask_from_input_mask(input_mask) + + # bert encoder + encoder_output = self.bert_encoder(self.cast_compute_type(embedding_output), + attention_mask) + + sequence_output = self.cast(encoder_output[self.last_idx], self.dtype) + + # pooler + sequence_slice = self.slice(sequence_output, + (0, 0, 0), + (self.batch_size, 1, self.hidden_size), + (1, 1, 1)) + first_token = self.squeeze_1(sequence_slice) + pooled_output = self.dense(first_token) + pooled_output = self.cast(pooled_output, self.dtype) + + return sequence_output, pooled_output, embedding_tables +``` + +`EmbeddingLookup`和`EmbeddingPostprocessor`用于将输入转换成Embedding张量，Embedding如下图所示： + +![Embedding](https://www.lyrn.ai/wp-content/uploads/2018/11/NSP.png) + +[2] 图片来源于https://www.lyrn.ai 和https://arxiv.org/pdf/1810.04805.pdf + +`BertTransformer`采用了下图中[Transformer](https://arxiv.org/pdf/1706.03762.pdf)中的encoder部分（左侧半边），包含`BertAttention->BertSelfAttention->BertEncoderCell`。 + +![Transformer](https://pic2.zhimg.com/80/v2-0e85f4d440e621803d11408b39834dd1_720w.jpg) + +[3] 图片来源于https://zhuanlan.zhihu.com/p/34781297 和https://arxiv.org/pdf/1706.03762.pdf + +`BertAttention`为Multi-Head Attention： + +![Multi-Head Attention](https://pic3.zhimg.com/80/v2-58d60594bc3e9cbe47faec82ef29fd76_720w.jpg) +[4] 图片来源于https://zhuanlan.zhihu.com/p/34781297 和https://arxiv.org/pdf/1706.03762.pdf + +创建训练作业时，运行参数会通过脚本传参的方式输入给脚本代码，脚本必须解析传参才能在代码中使用相应参数。如data_url和train_url，分别对应数据存储路径(OBS路径)和训练输出路径(OBS路径)。脚本对传参进行解析后赋值到`args`变量里，在后续代码里可以使用。 + +```python +import argparse +parser = argparse.ArgumentParser() +parser.add_argument('--data_url', required=True, default=None, help='Location of data.') +parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') +parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') +args, unknown = parser.parse_known_args() +``` + +MindSpore暂时没有提供直接访问OBS数据的接口，需要通过MoXing提供的API与OBS交互。将OBS中存储的数据拷贝至执行容器： + +```python +import moxing as mox +mox.file.copy_parallel(src_url=args.data_url, dst_url='zhwiki_part/') +``` + +将训练模型Checkpoint从执行容器拷贝至OBS： + +```python +import moxing as mox +mox.file.copy_parallel(src_url='bert_classfication-3_3335.ckpt', + dst_url=os.path.join(args.data_url, 'bert_classfication-3_3335.ckpt')) +``` + +#### 创建训练作业 + +可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。 + +创建训练作业的参考配置： + +- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore +- 代码目录：选择上述新建的OBS桶中的experiment_4目录 +- 启动文件：选择上述新建的OBS桶中的experiment_4目录下的`pretrain.py` +- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_4文件夹下的zhiwiki_part目录 +- 训练输出位置：选择上述新建的OBS桶中的experiment_4目录并在其中创建pretrain_output目录 +- 作业日志路径：同训练输出位置 +- 规格：Ascend:1*Ascend 910 +- 其他均为默认 + +启动并查看训练过程： + +1. 点击提交以开始训练； +2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理； +3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看； +4. 在训练日志中可以看到`epoch: 10 step: 10, loss is 10.741777`等字段，即预训练过程的loss数据。 + +### 微调BERT + +通常情况下，需要基于与训练的BERT模型在各类细分任务上做微调（finetune），提高BERT在具体任务上的效果。本实验在CLUEbenchmark/CLUE提供的TNEWS数据集上对预训练的BERT做微调，即学习一个短文本分类任务。 + +预训练和微调两种情况下BERT基础模型是相同的，只是最后会在基础模型上加上不同的任务层，用于解决文本分类（新闻分类、情感分类）、序列标注（命名实体识别、问答）等任务。 + +微调BERT依赖如下几个模块： + +- `finetune.py`：包含Loss打印、数据处理、优化器、模型保存等； +- `fintune_config.py`：模型和训练配置； +- `utils.py`模块中定义了finetune需要的模型，包含`BertFinetuneCell`, `BertCLSModel`, `BertNERModel`, `BertCLS`和`BertNER`。 + +`BertFinetuneCell`等同于预训练时的`BertTrainOneStepCell`/`BertTrainOneStepWithLossScaleCell`，接收数据输入，更新模型权重。 + +`BertCLSModel`在BERT基础模型上接了分类任务头： + +```python +class BertCLSModel(nn.Cell): + """ + This class is responsible for classification task evaluation, i.e. XNLI(num_labels=3), + LCQMC(num_labels=2), Chnsenti(num_labels=2). The returned output represents the final + logits as the results of log_softmax is propotional to that of softmax. +``` + +`BertNERModel`在BERT基础模型上接了命名实体识别（NER）任务头： + +```python +class BertNERModel(nn.Cell): + """ + This class is responsible for sequence labeling task evaluation, i.e. NER(num_labels=11). + The returned output represents the final logits as the results of log_softmax is propotional to that of softmax. + """ +``` + +`BertCLS`和`BertNER`在任务模型上接了损失函数，作为`BertFinetuneCell`的输入。 + +#### 创建训练作业 + +可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。 + +### 代码梳理 + +创建训练作业的参考配置： + +- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore +- 代码目录：选择上述新建的OBS桶中的experiment_4目录 +- 启动文件：选择上述新建的OBS桶中的experiment_4目录下的`fintune.py` +- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_4文件夹下的tnews目录 +- 训练输出位置：选择上述新建的OBS桶中的experiment_4目录并在其中创建finetune_output目录 +- 作业日志路径：同训练输出位置 +- 规格：Ascend:1*Ascend 910 +- 其他均为默认 + +启动并查看训练过程： + +1. 点击提交以开始训练，预训练过程约18分钟； +2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理； +3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看； +4. 在训练日志中可以看到`epoch: 3, step: 10005, outputs are (1.4425085, False)`等字段，即微调过程的输出； + +## 验证BERT + +在TNEWS验证集上对微调后的BERT模型做验证（evaluation）。 + +### 代码梳理 + +验证BERT依赖如下几个模块： + +- `evaluation.py`：包含Accuracy（分类任务）、F1值（NER任务）的计算，数据处理等。 +- `evaluation_config.py`：模型和训练配置； +- `cluener_evaluation.py`：中文任务基准测评（Chinese Language Understanding Evaluation Benchmark）方法，未使用； +- `tokenization.py`：基于vocab.txt，将单词token化，未使用； +- `sample_process.py`：基于`tokenization.py`进行文本数据处理，未使用； +- `utils.py`：依赖微调时用的模型。 + +脚本传参、数据拷贝等代码参考预训练BERT中的解释。 + +#### 创建训练作业 + +可以参考[使用常用框架训练模型](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0238.html)来创建并启动训练作业。 + +创建训练作业的参考配置： + +- 算法来源：常用框架->Ascend-Powered-Engine->MindSpore +- 代码目录：选择上述新建的OBS桶中的experiment_4目录 +- 启动文件：选择上述新建的OBS桶中的experiment_4目录下的`fintune.py` +- 数据来源：数据存储位置->选择上述新建的OBS桶中的experiment_4文件夹下的tnews目录 +- 训练输出位置：选择上述新建的OBS桶中的experiment_4目录并在其中创建eval_output目录 +- 作业日志路径：同训练输出位置 +- 规格：Ascend:1*Ascend 910 +- 其他均为默认 + +启动并查看训练过程： + +1. 点击提交以开始训练； +2. 在训练作业列表里可以看到刚创建的训练作业，在训练作业页面可以看到版本管理； +3. 点击运行中的训练作业，在展开的窗口中可以查看作业配置信息，以及训练过程中的日志，日志会不断刷新，等训练作业完成后也可以下载日志到本地进行查看； +4. 在训练日志中可以看到`acc_num 5437 , total_num 10000, accuracy 0.543700`字段，即微调完成后的验证精度。 + +## 实验结论 + +本实验主要介绍使用MindSpore在zhiwiki数据集上预训练BERT，在TNEWS短文本分类数据集上进行微调，包括以下特性： + +- model_zoo：BERT +- BERT预训练 +- BERT微调 +- 不同的优化器 +- 文本数据集处理 diff --git a/experiment_4/CRF.py b/experiment_4/CRF.py new file mode 100644 index 0000000..02f117f --- /dev/null +++ b/experiment_4/CRF.py @@ -0,0 +1,177 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +''' +CRF script. +''' + +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +import mindspore.common.dtype as mstype + +class CRF(nn.Cell): + ''' + Conditional Random Field + Args: + tag_to_index: The dict for tag to index mapping with extra "" and ""sign. + batch_size: Batch size, i.e., the length of the first dimension. + seq_length: Sequence length, i.e., the length of the second dimention. + is_training: Specifies whether to use training mode. + Returns: + Training mode: Tensor, total loss. + Evaluation mode: Tuple, the index for each step with the highest score; Tuple, the index for the last + step with the highest score. + ''' + def __init__(self, tag_to_index, batch_size=1, seq_length=128, is_training=True): + + super(CRF, self).__init__() + self.target_size = len(tag_to_index) + self.is_training = is_training + self.tag_to_index = tag_to_index + self.batch_size = batch_size + self.seq_length = seq_length + self.START_TAG = "" + self.STOP_TAG = "" + self.START_VALUE = Tensor(self.target_size-2, dtype=mstype.int32) + self.STOP_VALUE = Tensor(self.target_size-1, dtype=mstype.int32) + transitions = np.random.normal(size=(self.target_size, self.target_size)).astype(np.float32) + transitions[tag_to_index[self.START_TAG], :] = -10000 + transitions[:, tag_to_index[self.STOP_TAG]] = -10000 + self.transitions = Parameter(Tensor(transitions), name="transition_matrix") + self.cat = P.Concat(axis=-1) + self.argmax = P.ArgMaxWithValue(axis=-1) + self.log = P.Log() + self.exp = P.Exp() + self.sum = P.ReduceSum() + self.tile = P.Tile() + self.reduce_sum = P.ReduceSum(keep_dims=True) + self.reshape = P.Reshape() + self.expand = P.ExpandDims() + self.mean = P.ReduceMean() + init_alphas = np.ones(shape=(self.batch_size, self.target_size)) * -10000.0 + init_alphas[:, self.tag_to_index[self.START_TAG]] = 0. + self.init_alphas = Tensor(init_alphas, dtype=mstype.float32) + self.cast = P.Cast() + self.reduce_max = P.ReduceMax(keep_dims=True) + self.on_value = Tensor(1.0, dtype=mstype.float32) + self.off_value = Tensor(0.0, dtype=mstype.float32) + self.onehot = P.OneHot() + + def log_sum_exp(self, logits): + ''' + Compute the log_sum_exp score for normalization factor. + ''' + max_score = self.reduce_max(logits, -1) #16 5 5 + score = self.log(self.reduce_sum(self.exp(logits - max_score), -1)) + score = max_score + score + return score + + def _realpath_score(self, features, label): + ''' + Compute the emission and transition score for the real path. + ''' + label = label * 1 + concat_A = self.tile(self.reshape(self.START_VALUE, (1,)), (self.batch_size,)) + concat_A = self.reshape(concat_A, (self.batch_size, 1)) + labels = self.cat((concat_A, label)) + onehot_label = self.onehot(label, self.target_size, self.on_value, self.off_value) + emits = features * onehot_label + labels = self.onehot(labels, self.target_size, self.on_value, self.off_value) + label1 = labels[:, 1:, :] + label2 = labels[:, :self.seq_length, :] + label1 = self.expand(label1, 3) + label2 = self.expand(label2, 2) + label_trans = label1 * label2 + transitions = self.expand(self.expand(self.transitions, 0), 0) + trans = transitions * label_trans + score = self.sum(emits, (1, 2)) + self.sum(trans, (1, 2, 3)) + stop_value_index = labels[:, (self.seq_length-1):self.seq_length, :] + stop_value = self.transitions[(self.target_size-1):self.target_size, :] + stop_score = stop_value * self.reshape(stop_value_index, (self.batch_size, self.target_size)) + score = score + self.sum(stop_score, 1) + score = self.reshape(score, (self.batch_size, -1)) + return score + + def _normalization_factor(self, features): + ''' + Compute the total score for all the paths. + ''' + forward_var = self.init_alphas + forward_var = self.expand(forward_var, 1) + for idx in range(self.seq_length): + feat = features[:, idx:(idx+1), :] + emit_score = self.reshape(feat, (self.batch_size, self.target_size, 1)) + next_tag_var = emit_score + self.transitions + forward_var + forward_var = self.log_sum_exp(next_tag_var) + forward_var = self.reshape(forward_var, (self.batch_size, 1, self.target_size)) + terminal_var = forward_var + self.reshape(self.transitions[(self.target_size-1):self.target_size, :], (1, -1)) + alpha = self.log_sum_exp(terminal_var) + alpha = self.reshape(alpha, (self.batch_size, -1)) + return alpha + + def _decoder(self, features): + ''' + Viterbi decode for evaluation. + ''' + backpointers = () + forward_var = self.init_alphas + for idx in range(self.seq_length): + feat = features[:, idx:(idx+1), :] + feat = self.reshape(feat, (self.batch_size, self.target_size)) + bptrs_t = () + + next_tag_var = self.expand(forward_var, 1) + self.transitions + best_tag_id, best_tag_value = self.argmax(next_tag_var) + bptrs_t += (best_tag_id,) + forward_var = best_tag_value + feat + + backpointers += (bptrs_t,) + terminal_var = forward_var + self.reshape(self.transitions[(self.target_size-1):self.target_size, :], (1, -1)) + best_tag_id, _ = self.argmax(terminal_var) + return backpointers, best_tag_id + + def construct(self, features, label): + if self.is_training: + forward_score = self._normalization_factor(features) + gold_score = self._realpath_score(features, label) + return_value = self.mean(forward_score - gold_score) + else: + path_list, tag = self._decoder(features) + return_value = path_list, tag + return return_value + +def postprocess(backpointers, best_tag_id): + ''' + Do postprocess + ''' + best_tag_id = best_tag_id.asnumpy() + batch_size = len(best_tag_id) + best_path = [] + for i in range(batch_size): + best_path.append([]) + best_local_id = best_tag_id[i] + best_path[-1].append(best_local_id) + for bptrs_t in reversed(backpointers): + bptrs_t = bptrs_t[0].asnumpy() + local_idx = bptrs_t[i] + best_local_id = local_idx[best_local_id] + best_path[-1].append(best_local_id) + # Pop off the start tag (we dont want to return that to the caller) + best_path[-1].pop() + best_path[-1].reverse() + return best_path diff --git a/experiment_4/cluener_evaluation.py b/experiment_4/cluener_evaluation.py new file mode 100644 index 0000000..67c2d28 --- /dev/null +++ b/experiment_4/cluener_evaluation.py @@ -0,0 +1,73 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +'''bert clue evaluation''' + +import json +import numpy as np +from evaluation_config import cfg +import mindspore.common.dtype as mstype +from mindspore.common.tensor import Tensor +from CRF import postprocess +import tokenization +from sample_process import label_generation, process_one_example_p + +vocab_file = "tnews/vocab.txt" + +def process(model, text, sequence_length): + """ + process text. + """ + data = [text] + features = [] + res = [] + ids = [] + tokenizer_ = tokenization.FullTokenizer(vocab_file=vocab_file) + for i in data: + feature = process_one_example_p(tokenizer_, i, max_seq_len=sequence_length) + features.append(feature) + input_ids, input_mask, token_type_id = feature + input_ids = Tensor(np.array(input_ids), mstype.int32) + input_mask = Tensor(np.array(input_mask), mstype.int32) + token_type_id = Tensor(np.array(token_type_id), mstype.int32) + if cfg.use_crf: + backpointers, best_tag_id = model.predict(input_ids, input_mask, token_type_id, Tensor(1)) + best_path = postprocess(backpointers, best_tag_id) + logits = [] + for ele in best_path: + logits.extend(ele) + ids = logits + else: + logits = model.predict(input_ids, input_mask, token_type_id, Tensor(1)) + ids = logits.asnumpy() + ids = np.argmax(ids, axis=-1) + ids = list(ids) + res = label_generation(text, ids) + return res + +def submit(model, path, sequence_length): + """ + submit task + """ + data = [] + for line in open(path): + if not line.strip(): + continue + _ = json.loads(line.strip()) + res = process(model, _["text"], sequence_length) + print("_text", _["text"]) + print("res:", res) + data.append(json.dumps({"label": res}, ensure_ascii=False)) + open("ner_predict.json", "w").write("\n".join(data)) diff --git a/experiment_4/evaluation.py b/experiment_4/evaluation.py new file mode 100644 index 0000000..ab3eb3e --- /dev/null +++ b/experiment_4/evaluation.py @@ -0,0 +1,161 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Bert evaluation script. +""" + +import os +os.environ['P_NUM'] = '16' +import numpy as np +from evaluation_config import cfg, bert_net_cfg +from utils import BertNER, BertCLS +import mindspore.common.dtype as mstype +from mindspore import context +from mindspore.common.tensor import Tensor +import mindspore.dataset as de +import mindspore.dataset.transforms.c_transforms as C +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from CRF import postprocess +from cluener_evaluation import submit +from finetune_config import tag_to_index + +class Accuracy(): + ''' + calculate accuracy + ''' + def __init__(self): + self.acc_num = 0 + self.total_num = 0 + def update(self, logits, labels): + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + logits = logits.asnumpy() + logit_id = np.argmax(logits, axis=-1) + self.acc_num += np.sum(labels == logit_id) + self.total_num += len(labels) + print("=========================accuracy is ", self.acc_num / self.total_num) + +class F1(): + ''' + calculate F1 score + ''' + def __init__(self): + self.TP = 0 + self.FP = 0 + self.FN = 0 + def update(self, logits, labels): + ''' + update F1 score + ''' + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + if cfg.use_crf: + backpointers, best_tag_id = logits + best_path = postprocess(backpointers, best_tag_id) + logit_id = [] + for ele in best_path: + logit_id.extend(ele) + else: + logits = logits.asnumpy() + logit_id = np.argmax(logits, axis=-1) + logit_id = np.reshape(logit_id, -1) + pos_eva = np.isin(logit_id, [i for i in range(1, cfg.num_labels)]) + pos_label = np.isin(labels, [i for i in range(1, cfg.num_labels)]) + self.TP += np.sum(pos_eva&pos_label) + self.FP += np.sum(pos_eva&(~pos_label)) + self.FN += np.sum((~pos_eva)&pos_label) + +def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): + ''' + get dataset + ''' + ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", + "segment_ids", "label_ids"]) + type_cast_op = C.TypeCast(mstype.int32) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + ds = ds.map(input_columns="label_ids", operations=type_cast_op) + ds = ds.repeat(repeat_count) + + # apply shuffle operation + buffer_size = 960 + ds = ds.shuffle(buffer_size=buffer_size) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + return ds + +def bert_predict(Evaluation): + ''' + prediction function + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) + dataset = get_dataset(bert_net_cfg.batch_size, 1) + if cfg.use_crf: + net_for_pretraining = Evaluation(bert_net_cfg, False, num_labels=len(tag_to_index), use_crf=True, + tag_to_index=tag_to_index, dropout_prob=0.0) + else: + net_for_pretraining = Evaluation(bert_net_cfg, False, num_labels) + net_for_pretraining.set_train(False) + param_dict = load_checkpoint(cfg.finetune_ckpt) + load_param_into_net(net_for_pretraining, param_dict) + model = Model(net_for_pretraining) + return model, dataset + +def test_eval(): + ''' + evaluation function + ''' + task_type = BertNER if cfg.task == "NER" else BertCLS + model, dataset = bert_predict(task_type) + if cfg.clue_benchmark: + submit(model, cfg.data_file, bert_net_cfg.seq_length) + else: + callback = F1() if cfg.task == "NER" else Accuracy() + columns_list = ["input_ids", "input_mask", "segment_ids", "label_ids"] + for data in dataset.create_dict_iterator(): + input_data = [] + for i in columns_list: + input_data.append(Tensor(data[i])) + input_ids, input_mask, token_type_id, label_ids = input_data + logits = model.predict(input_ids, input_mask, token_type_id, label_ids) + callback.update(logits, label_ids) + print("==============================================================") + if cfg.task == "NER": + print("Precision {:.6f} ".format(callback.TP / (callback.TP + callback.FP))) + print("Recall {:.6f} ".format(callback.TP / (callback.TP + callback.FN))) + print("F1 {:.6f} ".format(2*callback.TP / (2*callback.TP + callback.FP + callback.FP))) + else: + print("acc_num {} , total_num {}, accuracy {:.6f}".format(callback.acc_num, callback.total_num, + callback.acc_num / callback.total_num)) + print("==============================================================") + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') + args, unknown = parser.parse_known_args() + + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, dst_url='tnews/') + + num_labels = cfg.num_labels + test_eval() diff --git a/experiment_4/evaluation_config.py b/experiment_4/evaluation_config.py new file mode 100644 index 0000000..cc6b966 --- /dev/null +++ b/experiment_4/evaluation_config.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +config settings, will be used in finetune.py +""" + +from easydict import EasyDict as edict +import mindspore.common.dtype as mstype +from mindspore.model_zoo.Bert_NEZHA import BertConfig + +cfg = edict({ + 'task': 'classfication', + 'num_labels': 15, + 'data_file': 'tnews/dev.tf_record', + 'schema_file': 'tnews/dev_schema.json', + 'finetune_ckpt': 'tnews/bert_classfication-3_3335.ckpt', + 'use_crf': False, + 'clue_benchmark': False, +}) + +bert_net_cfg = BertConfig( + batch_size=16 if not cfg.clue_benchmark else 1, + seq_length=128, + vocab_size=21128, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=False, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16, +) diff --git a/experiment_4/finetune.py b/experiment_4/finetune.py new file mode 100644 index 0000000..6397aee --- /dev/null +++ b/experiment_4/finetune.py @@ -0,0 +1,152 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +''' +Bert finetune script. +''' + +import os +os.environ['P_NUM'] = '16' +from utils import BertFinetuneCell, BertCLS, BertNER +from finetune_config import cfg, bert_net_cfg, tag_to_index +import mindspore.common.dtype as mstype +import mindspore.communication.management as D +from mindspore import context +import mindspore.dataset as de +import mindspore.dataset.transforms.c_transforms as C +from mindspore.nn.wrap.loss_scale import DynamicLossScaleUpdateCell +from mindspore.nn.optim import AdamWeightDecay, AdamWeightDecayDynamicLR, Lamb, Momentum +from mindspore.train.model import Model +from mindspore.train.callback import Callback +from mindspore.train.callback import CheckpointConfig, ModelCheckpoint +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +class LossCallBack(Callback): + ''' + Monitor the loss in training. + If the loss is NAN or INF, terminate training. + Note: + If per_print_times is 0, do not print loss. + Args: + per_print_times (int): Print loss every times. Default: 1. + ''' + def __init__(self, per_print_times=1): + super(LossCallBack, self).__init__() + if not isinstance(per_print_times, int) or per_print_times < 0: + raise ValueError("print_step must be in and >= 0.") + self._per_print_times = per_print_times + + def step_end(self, run_context): + cb_params = run_context.original_args() + print("epoch: {}, step: {}, outputs are {}".format(cb_params.cur_epoch_num, cb_params.cur_step_num, + str(cb_params.net_outputs))) + + +def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): + ''' + get dataset + ''' + ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", + "segment_ids", "label_ids"]) + type_cast_op = C.TypeCast(mstype.int32) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + ds = ds.map(input_columns="label_ids", operations=type_cast_op) + ds = ds.repeat(repeat_count) + + # apply shuffle operation + buffer_size = 960 + ds = ds.shuffle(buffer_size=buffer_size) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + return ds + +def test_train(): + ''' + finetune function + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid, + enable_mem_reuse=True, enable_task_sink=True) + #BertCLSTrain for classification + #BertNERTrain for sequence labeling + if cfg.task == 'NER': + if cfg.use_crf: + netwithloss = BertNER(bert_net_cfg, True, num_labels=len(tag_to_index), use_crf=True, + tag_to_index=tag_to_index, dropout_prob=0.1) + else: + netwithloss = BertNER(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + else: + netwithloss = BertCLS(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + dataset = get_dataset(bert_net_cfg.batch_size, cfg.epoch_num) + # optimizer + steps_per_epoch = dataset.get_dataset_size() + if cfg.optimizer == 'AdamWeightDecayDynamicLR': + optimizer = AdamWeightDecayDynamicLR(netwithloss.trainable_params(), + decay_steps=steps_per_epoch * cfg.epoch_num, + learning_rate=cfg.AdamWeightDecayDynamicLR.learning_rate, + end_learning_rate=cfg.AdamWeightDecayDynamicLR.end_learning_rate, + power=cfg.AdamWeightDecayDynamicLR.power, + #warmup_steps=steps_per_epoch, + weight_decay=cfg.AdamWeightDecayDynamicLR.weight_decay, + eps=cfg.AdamWeightDecayDynamicLR.eps) + #decay_filter=lambda x: 'LayerNorm' not in x.name and + # 'bias' not in x.name and + # 'layernorm' not in x.name) + elif cfg.optimizer == 'AdamWeightDecay': + optimizer = AdamWeightDecay(netwithloss.trainable_params(), + learning_rate=cfg.AdamWeightDecay.learning_rate, + weight_decay=cfg.AdamWeightDecay.weight_decay, + eps=cfg.AdamWeightDecay.eps, + decay_filter=lambda x: 'LayerNorm' not in x.name and + 'bias' not in x.name and + 'layernorm' not in x.name) + elif cfg.optimizer == 'Lamb': + optimizer = Lamb(netwithloss.trainable_params(), decay_steps=steps_per_epoch * cfg.epoch_num, + start_learning_rate=cfg.Lamb.start_learning_rate, end_learning_rate=cfg.Lamb.end_learning_rate, + power=cfg.Lamb.power, warmup_steps=steps_per_epoch, decay_filter=cfg.Lamb.decay_filter) + elif cfg.optimizer == 'Momentum': + optimizer = Momentum(netwithloss.trainable_params(), learning_rate=cfg.Momentum.learning_rate, + momentum=cfg.Momentum.momentum) + else: + raise Exception("Optimizer not supported.") + print("check steps, steps_per_epoch: ", steps_per_epoch) + # load checkpoint into network + ckpt_config = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=1) + ckpoint_cb = ModelCheckpoint(prefix=cfg.ckpt_prefix, directory=cfg.ckpt_dir, config=ckpt_config) + param_dict = load_checkpoint(cfg.pre_training_ckpt) + load_param_into_net(netwithloss, param_dict) + + update_cell = DynamicLossScaleUpdateCell(loss_scale_value=2**32, scale_factor=2, scale_window=1000) + netwithgrads = BertFinetuneCell(netwithloss, optimizer=optimizer, scale_update_cell=update_cell) + model = Model(netwithgrads) + model.train(cfg.epoch_num, dataset, callbacks=[LossCallBack(), ckpoint_cb]) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--num_epochs', type=int, default=1, help='Number of training epochs.') + args, unknown = parser.parse_known_args() + + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, dst_url='tnews/') + + test_train() + mox.file.copy_parallel(src_url='bert_classfication-3_3335.ckpt', + dst_url=os.path.join(args.data_url, 'bert_classfication-3_3335.ckpt')) diff --git a/experiment_4/finetune_config.py b/experiment_4/finetune_config.py new file mode 100644 index 0000000..616b93a --- /dev/null +++ b/experiment_4/finetune_config.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +config settings, will be used in finetune.py +""" + +from easydict import EasyDict as edict +import mindspore.common.dtype as mstype +from mindspore.model_zoo.Bert_NEZHA import BertConfig + +cfg = edict({ + 'task': 'nothing', + 'num_labels':15, + 'data_file': 'tnews/train.tf_record', + 'schema_file': 'tnews/train_schema.json', + 'epoch_num': 3, + 'ckpt_prefix': 'bert_classfication', + 'ckpt_dir': None, + 'pre_training_ckpt': 'tnews/bert_base.ckpt', + 'use_crf': False, + 'optimizer': 'AdamWeightDecayDynamicLR', + 'AdamWeightDecay': edict({ + 'learning_rate': 2e-5, + 'weight_decay': 1e-5, + 'eps': 1e-6, + }), + 'AdamWeightDecayDynamicLR': edict({ + 'learning_rate': 2e-5, + 'end_learning_rate': 1e-7, + 'power': 1.0, + 'weight_decay': 1e-5, + 'eps': 1e-6, + }), + 'Lamb': edict({ + 'start_learning_rate': 2e-5, + 'end_learning_rate': 1e-7, + 'power': 1.0, + 'decay_filter': lambda x: False, + }), + 'Momentum': edict({ + 'learning_rate': 2e-5, + 'momentum': 0.9, + }), +}) + +bert_net_cfg = BertConfig( + batch_size=16, + seq_length=128, + vocab_size=21128, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=False, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16, +) + +tag_to_index = { + "O": 0, + "S_address": 1, + "B_address": 2, + "M_address": 3, + "E_address": 4, + "S_book": 5, + "B_book": 6, + "M_book": 7, + "E_book": 8, + "S_company": 9, + "B_company": 10, + "M_company": 11, + "E_company": 12, + "S_game": 13, + "B_game": 14, + "M_game": 15, + "E_game": 16, + "S_government": 17, + "B_government": 18, + "M_government": 19, + "E_government": 20, + "S_movie": 21, + "B_movie": 22, + "M_movie": 23, + "E_movie": 24, + "S_name": 25, + "B_name": 26, + "M_name": 27, + "E_name": 28, + "S_organization": 29, + "B_organization": 30, + "M_organization": 31, + "E_organization": 32, + "S_position": 33, + "B_position": 34, + "M_position": 35, + "E_position": 36, + "S_scene": 37, + "B_scene": 38, + "M_scene": 39, + "E_scene": 40, + "": 41, + "": 42 +} diff --git a/experiment_4/pretrain.py b/experiment_4/pretrain.py new file mode 100644 index 0000000..7e6796e --- /dev/null +++ b/experiment_4/pretrain.py @@ -0,0 +1,167 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""train bert network without lossscale""" + +import os +import numpy as np +from numpy import allclose +import mindspore.common.dtype as mstype +import mindspore.dataset.engine.datasets as de +import mindspore.dataset.transforms.c_transforms as C +from mindspore import context +from mindspore.common.tensor import Tensor +from mindspore.train.model import Model +from mindspore.train.callback import Callback, LossMonitor +from mindspore.train.loss_scale_manager import DynamicLossScaleManager +from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepWithLossScaleCell +from mindspore.nn.optim import Momentum +from mindspore import log as logger + + +DATA_DIR = ["zhwiki_part/part.tfrecord"] +SCHEMA_DIR = "zhwiki_part/schema.json" + + +def get_config(version='base', batch_size=1): + """get config""" + if version == 'base': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + elif version == 'large': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=1024, + num_hidden_layers=12, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16) + elif version == 'large_mixed': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + else: + bert_config = BertConfig(batch_size=batch_size) + return bert_config + +def create_dataset(): + """test me de train dataset""" + # apply repeat operations + repeat_count = args.num_epochs + ds = de.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids", + "next_sentence_labels", "masked_lm_positions", + "masked_lm_ids", "masked_lm_weights"], shuffle=False) + type_cast_op = C.TypeCast(mstype.int32) + ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op) + ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op) + ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + # apply batch operations + batch_size = int(os.getenv('BATCH_SIZE', '16')) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(repeat_count) + return ds + + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + + def step_end(self, run_context): + cb_params = run_context.original_args() + print("epoch: {}, outputs are: {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs))) + + +def test_bert_tdt(): + """test bert tdt""" + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", reserve_class_name_in_scope=False) + context.set_context(enable_task_sink=True) + # context.set_context(enable_loop_sink=True) + context.set_context(enable_mem_reuse=True) + ds = create_dataset() + version = os.getenv('VERSION', 'base') + batch_size = int(os.getenv('BATCH_SIZE', '16')) + config = get_config(version=version, batch_size=batch_size) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = Momentum(netwithloss.trainable_params(), learning_rate=2e-5, momentum=0.9) + scale_window = 3 + scale_manager = DynamicLossScaleManager(2**32, 2, scale_window) + netwithgrads = BertTrainOneStepWithLossScaleCell(netwithloss, optimizer=optimizer, scale_update_cell=scale_manager.get_update_cell()) + netwithgrads.set_train(True) + model = Model(netwithgrads) + callback = ModelCallback() + # loss_cb = LossMonitor(per_print_times=ds.get_dataset_size()) + model.train(ds.get_repeat_count(), ds, callbacks=callback) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--data_url', required=True, default=None, help='Location of data.') + parser.add_argument('--train_url', required=True, default=None, help='Location of training outputs.') + parser.add_argument('--num_epochs', type=int, default=50, help='Number of training epochs.') + args, unknown = parser.parse_known_args() + + import moxing as mox + mox.file.copy_parallel(src_url=args.data_url, dst_url='zhwiki_part/') + + test_bert_tdt() diff --git a/experiment_4/sample_process.py b/experiment_4/sample_process.py new file mode 100644 index 0000000..7a7752e --- /dev/null +++ b/experiment_4/sample_process.py @@ -0,0 +1,100 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""process txt""" + +import re +import json + +def process_one_example_p(tokenizer, text, max_seq_len=128): + """process one testline""" + textlist = list(text) + tokens = [] + for _, word in enumerate(textlist): + token = tokenizer.tokenize(word) + tokens.extend(token) + if len(tokens) >= max_seq_len - 1: + tokens = tokens[0:(max_seq_len - 2)] + ntokens = [] + segment_ids = [] + label_ids = [] + ntokens.append("[CLS]") + segment_ids.append(0) + for _, token in enumerate(tokens): + ntokens.append(token) + segment_ids.append(0) + ntokens.append("[SEP]") + segment_ids.append(0) + input_ids = tokenizer.convert_tokens_to_ids(ntokens) + input_mask = [1] * len(input_ids) + while len(input_ids) < max_seq_len: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + label_ids.append(0) + ntokens.append("**NULL**") + assert len(input_ids) == max_seq_len + assert len(input_mask) == max_seq_len + assert len(segment_ids) == max_seq_len + + feature = (input_ids, input_mask, segment_ids) + return feature + +def label_generation(text, probs): + """generate label""" + data = [text] + probs = [probs] + result = [] + label2id = json.loads(open("tnews/label2id.json").read()) + id2label = [k for k, v in label2id.items()] + + for index, prob in enumerate(probs): + for v in prob[1:len(data[index]) + 1]: + result.append(id2label[int(v)]) + + labels = {} + start = None + index = 0 + for _, t in zip("".join(data), result): + if re.search("^[BS]", t): + if start is not None: + label = result[index - 1][2:] + if labels.get(label): + te_ = text[start:index] + labels[label][te_] = [[start, index - 1]] + else: + te_ = text[start:index] + labels[label] = {te_: [[start, index - 1]]} + start = index + if re.search("^O", t): + if start is not None: + label = result[index - 1][2:] + if labels.get(label): + te_ = text[start:index] + labels[label][te_] = [[start, index - 1]] + else: + te_ = text[start:index] + labels[label] = {te_: [[start, index - 1]]} + start = None + index += 1 + if start is not None: + label = result[start][2:] + if labels.get(label): + te_ = text[start:index] + labels[label][te_] = [[start, index - 1]] + else: + te_ = text[start:index] + labels[label] = {te_: [[start, index - 1]]} + return labels diff --git a/experiment_4/tokenization.py b/experiment_4/tokenization.py new file mode 100644 index 0000000..edc9abf --- /dev/null +++ b/experiment_4/tokenization.py @@ -0,0 +1,388 @@ +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import re +import unicodedata +import six +#import tensorflow as tf + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % (actual_flag, init_checkpoint, + model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + if item in vocab: + output.append(vocab[item]) + else: + output.append(vocab['[UNK]']) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/experiment_4/utils.py b/experiment_4/utils.py new file mode 100644 index 0000000..ceae53f --- /dev/null +++ b/experiment_4/utils.py @@ -0,0 +1,263 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +''' +Functional Cells used in Bert finetune and evaluation. +''' + +import mindspore.nn as nn +from mindspore.common.initializer import TruncatedNormal +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.common import dtype as mstype +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +from mindspore.train.parallel_utils import ParallelMode +from mindspore.communication.management import get_group_size +from mindspore import context +from mindspore.model_zoo.Bert_NEZHA.bert_model import BertModel +from mindspore.model_zoo.Bert_NEZHA.bert_for_pre_training import ClipGradients +from CRF import CRF + +GRADIENT_CLIP_TYPE = 1 +GRADIENT_CLIP_VALUE = 1.0 +grad_scale = C.MultitypeFuncGraph("grad_scale") +reciprocal = P.Reciprocal() + +@grad_scale.register("Tensor", "Tensor") +def tensor_grad_scale(scale, grad): + return grad * reciprocal(scale) + +class BertFinetuneCell(nn.Cell): + """ + Especifically defined for finetuning where only four inputs tensor are needed. + """ + def __init__(self, network, optimizer, scale_update_cell=None): + + super(BertFinetuneCell, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', + get_by_list=True, + sens_param=True) + self.reducer_flag = False + self.allreduce = P.AllReduce() + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + self.grad_reducer = None + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + degree = get_group_size() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) + self.clip_gradients = ClipGradients() + self.cast = P.Cast() + self.alloc_status = P.NPUAllocFloatStatus() + self.get_status = P.NPUGetFloatStatus() + self.clear_before_grad = P.NPUClearFloatStatus() + self.reduce_sum = P.ReduceSum(keep_dims=False) + self.depend_parameter_use = P.ControlDepend(depend_mode=1) + self.base = Tensor(1, mstype.float32) + self.less_equal = P.LessEqual() + self.hyper_map = C.HyperMap() + self.loss_scale = None + self.loss_scaling_manager = scale_update_cell + if scale_update_cell: + self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), + name="loss_scale") + + def construct(self, + input_ids, + input_mask, + token_type_id, + label_ids, + sens=None): + + weights = self.weights + init = self.alloc_status() + loss = self.network(input_ids, + input_mask, + token_type_id, + label_ids) + if sens is None: + scaling_sens = self.loss_scale + else: + scaling_sens = sens + grads = self.grad(self.network, weights)(input_ids, + input_mask, + token_type_id, + label_ids, + self.cast(scaling_sens, + mstype.float32)) + clear_before_grad = self.clear_before_grad(init) + F.control_depend(loss, init) + self.depend_parameter_use(clear_before_grad, scaling_sens) + grads = self.hyper_map(F.partial(grad_scale, scaling_sens), grads) + grads = self.clip_gradients(grads, GRADIENT_CLIP_TYPE, GRADIENT_CLIP_VALUE) + if self.reducer_flag: + grads = self.grad_reducer(grads) + flag = self.get_status(init) + flag_sum = self.reduce_sum(init, (0,)) + if self.is_distributed: + flag_reduce = self.allreduce(flag_sum) + cond = self.less_equal(self.base, flag_reduce) + else: + cond = self.less_equal(self.base, flag_sum) + F.control_depend(grads, flag) + F.control_depend(flag, flag_sum) + overflow = cond + if sens is None: + overflow = self.loss_scaling_manager(self.loss_scale, cond) + if overflow: + succ = False + else: + succ = self.optimizer(grads) + ret = (loss, cond) + return F.depend(ret, succ) + +class BertCLSModel(nn.Cell): + """ + This class is responsible for classification task evaluation, i.e. XNLI(num_labels=3), + LCQMC(num_labels=2), Chnsenti(num_labels=2). The returned output represents the final + logits as the results of log_softmax is propotional to that of softmax. + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertCLSModel, self).__init__() + self.bert = BertModel(config, is_training, use_one_hot_embeddings) + self.cast = P.Cast() + self.weight_init = TruncatedNormal(config.initializer_range) + self.log_softmax = P.LogSoftmax(axis=-1) + self.dtype = config.dtype + self.num_labels = num_labels + self.dense_1 = nn.Dense(config.hidden_size, self.num_labels, weight_init=self.weight_init, + has_bias=True).to_float(config.compute_type) + self.dropout = nn.Dropout(1 - dropout_prob) + + def construct(self, input_ids, input_mask, token_type_id): + _, pooled_output, _ = \ + self.bert(input_ids, token_type_id, input_mask) + cls = self.cast(pooled_output, self.dtype) + cls = self.dropout(cls) + logits = self.dense_1(cls) + logits = self.cast(logits, self.dtype) + log_probs = self.log_softmax(logits) + return log_probs + + +class BertNERModel(nn.Cell): + """ + This class is responsible for sequence labeling task evaluation, i.e. NER(num_labels=11). + The returned output represents the final logits as the results of log_softmax is propotional to that of softmax. + """ + def __init__(self, config, is_training, num_labels=11, use_crf=False, dropout_prob=0.0, + use_one_hot_embeddings=False): + super(BertNERModel, self).__init__() + self.bert = BertModel(config, is_training, use_one_hot_embeddings) + self.cast = P.Cast() + self.weight_init = TruncatedNormal(config.initializer_range) + self.log_softmax = P.LogSoftmax(axis=-1) + self.dtype = config.dtype + self.num_labels = num_labels + self.dense_1 = nn.Dense(config.hidden_size, self.num_labels, weight_init=self.weight_init, + has_bias=True).to_float(config.compute_type) + self.dropout = nn.Dropout(1 - dropout_prob) + self.reshape = P.Reshape() + self.shape = (-1, config.hidden_size) + self.use_crf = use_crf + self.origin_shape = (config.batch_size, config.seq_length, self.num_labels) + + def construct(self, input_ids, input_mask, token_type_id): + sequence_output, _, _ = \ + self.bert(input_ids, token_type_id, input_mask) + seq = self.dropout(sequence_output) + seq = self.reshape(seq, self.shape) + logits = self.dense_1(seq) + logits = self.cast(logits, self.dtype) + if self.use_crf: + return_value = self.reshape(logits, self.origin_shape) + else: + return_value = self.log_softmax(logits) + return return_value + +class CrossEntropyCalculation(nn.Cell): + """ + Cross Entropy loss + """ + def __init__(self, is_training=True): + super(CrossEntropyCalculation, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0, mstype.float32) + self.off_value = Tensor(0.0, mstype.float32) + self.reduce_sum = P.ReduceSum() + self.reduce_mean = P.ReduceMean() + self.reshape = P.Reshape() + self.last_idx = (-1,) + self.neg = P.Neg() + self.cast = P.Cast() + self.is_training = is_training + + def construct(self, logits, label_ids, num_labels): + if self.is_training: + label_ids = self.reshape(label_ids, self.last_idx) + one_hot_labels = self.onehot(label_ids, num_labels, self.on_value, self.off_value) + per_example_loss = self.neg(self.reduce_sum(one_hot_labels * logits, self.last_idx)) + loss = self.reduce_mean(per_example_loss, self.last_idx) + return_value = self.cast(loss, mstype.float32) + else: + return_value = logits * 1.0 + return return_value + +class BertCLS(nn.Cell): + """ + Train interface for classification finetuning task. + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertCLS, self).__init__() + self.bert = BertCLSModel(config, is_training, num_labels, dropout_prob, use_one_hot_embeddings) + self.loss = CrossEntropyCalculation(is_training) + self.num_labels = num_labels + def construct(self, input_ids, input_mask, token_type_id, label_ids): + log_probs = self.bert(input_ids, input_mask, token_type_id) + loss = self.loss(log_probs, label_ids, self.num_labels) + return loss + + +class BertNER(nn.Cell): + """ + Train interface for sequence labeling finetuning task. + """ + def __init__(self, config, is_training, num_labels=11, use_crf=False, tag_to_index=None, dropout_prob=0.0, + use_one_hot_embeddings=False): + super(BertNER, self).__init__() + self.bert = BertNERModel(config, is_training, num_labels, use_crf, dropout_prob, use_one_hot_embeddings) + if use_crf: + if not tag_to_index: + raise Exception("The dict for tag-index mapping should be provided for CRF.") + self.loss = CRF(tag_to_index, config.batch_size, config.seq_length, is_training) + else: + self.loss = CrossEntropyCalculation(is_training) + self.num_labels = num_labels + self.use_crf = use_crf + def construct(self, input_ids, input_mask, token_type_id, label_ids): + logits = self.bert(input_ids, input_mask, token_type_id) + if self.use_crf: + loss = self.loss(logits, label_ids) + else: + loss = self.loss(logits, label_ids, self.num_labels) + return loss diff --git a/project_1/1-Model_Optimization.ipynb b/project_1/1-Model_Optimization.ipynb new file mode 100644 index 0000000..2fc0003 --- /dev/null +++ b/project_1/1-Model_Optimization.ipynb @@ -0,0 +1,565 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

调优模型和训练策略

\n", + "\n", + "[TOC]\n", + "\n", + "## 作业介绍\n", + "\n", + "模型调优和训练策略调优是当前深度学习领域最常见、最难和最耗费精力的工作，旨在降低训练难度，提高模型精度，减少模型大小，降低模型推理时延。本作业要求在给定LeNet5模型+CIFAR10数据集的基础上，对模型和训练策略进行调优，以验证精度、模型大小和推理时延为目标，优先级为精度>大小>时延。\n", + "\n", + "要求模型在CIFAR10验证集上的精度不低于60%，最终成绩可参考`0.50*精度(%) - 0.35*大小(MB) - 0.15*时延(ms)`的方式评定。\n", + "\n", + "## 作业目的\n", + "\n", + "- 了解当前深度学习研发人员最常见的工作；\n", + "- 了解并熟悉如何使用MindSpore进行模型开发和调试；\n", + "- 了解模型调优的基本方向和常用策略，了解模型深度（层数）、模型宽度（核大小）、特殊结构（Bypass）等概念，及其对模型精度、大小和时延的影响；\n", + "- 了解训练策略调优的常用方法，了解Epoch数、Batch Size、优化器、学习率、正则化项等对模型训练和精度的影响。\n", + "\n", + "## 预备知识\n", + "\n", + "- 熟练使用Python，了解Shell及Linux操作系统基本知识。\n", + "- 具备一定的深度学习理论知识，如卷积神经网络、损失函数、优化器，训练策略、Checkpoint等。\n", + "- 了解华为云的基本使用方法，包括[OBS（对象存储）](https://www.huaweicloud.com/product/obs.html)、[ModelArts（AI开发平台）](https://www.huaweicloud.com/product/modelarts.html)、[Notebook（开发工具）](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0033.html)等功能。华为云官网：https://www.huaweicloud.com\n", + "- 了解并熟悉MindSpore AI计算框架，MindSpore官网：https://www.mindspore.cn/\n", + "\n", + "## 开发环境\n", + "\n", + "- MindSpore 0.1.0（MindSpore版本会定期更新，本指导也会定期刷新，与版本配套）；\n", + "- 华为云ModelArts：ModelArts是华为云提供的面向开发者的一站式AI开发平台，集成了昇腾AI处理器资源池，用户可以在该平台下体验MindSpore。ModelArts官网：https://www.huaweicloud.com/product/modelarts.html\n", + "\n", + "## 开发准备\n", + "\n", + "### 创建OBS桶\n", + "\n", + "本实验需要使用华为云OBS存储脚本和数据集，可以参考[快速通过OBS控制台上传下载文件](https://support.huaweicloud.com/qs-obs/obs_qs_0001.html)了解使用OBS创建桶、上传文件、下载文件的使用方法。\n", + "\n", + "> **提示：**华为云新用户使用OBS时通常需要创建和配置“访问密钥”，可以在使用OBS时根据提示完成创建和配置。也可以参考[获取访问密钥并完成ModelArts全局配置](https://support.huaweicloud.com/prepare-modelarts/modelarts_08_0002.html)获取并配置访问密钥。\n", + "\n", + "创建OBS桶的参考配置如下：\n", + "\n", + "- 区域：华北-北京四\n", + "- 数据冗余存储策略：单AZ存储\n", + "- 桶名称：如ms-course\n", + "- 存储类别：标准存储\n", + "- 桶策略：公共读\n", + "- 归档数据直读：关闭\n", + "- 企业项目、标签等配置：免\n", + "\n", + "### 数据集准备\n", + "\n", + "CIFAR-10是一个图片分类数据集，包含60000张32x32的彩色物体图片，训练集50000张，测试集10000张，共10类，每类6000张。CIFAR-10数据集的官网：[THE MNIST DATABASE](http://www.cs.toronto.edu/~kriz/cifar.html)。\n", + "\n", + "从CIFAR-10官网下载“CIFAR-10 binary version (suitable for C programs)”到本地并解压。\n", + "\n", + "### 脚本准备\n", + "\n", + "从[课程gitee仓库](https://gitee.com/mindspore/course)上下载对应的Jupyter Notebook（内容同本指导）。\n", + "\n", + "### 上传文件\n", + "\n", + "将脚本和数据集上传到OBS桶中，组织为如下形式：\n", + "\n", + "```\n", + "project_1\n", + "├── xxx.ipynb\n", + "└── cifar10\n", + " ├── batches.meta.txt\n", + " ├── eval\n", + " │   └── test_batch.bin\n", + " └── train\n", + " ├── data_batch_1.bin\n", + " ├── data_batch_2.bin\n", + " ├── data_batch_3.bin\n", + " ├── data_batch_4.bin\n", + " └── data_batch_5.bin\n", + "```\n", + "\n", + "### 创建并打开Notebook\n", + "\n", + "可以参考[创建并打开Notebook](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0034.html)来创建并打开上传的Notebook脚本。\n", + "\n", + "创建Notebook的参考配置：\n", + "\n", + "- 计费模式：按需计费\n", + "- 名称：project_1\n", + "- 工作环境：Python3\n", + "- 资源池：公共资源\n", + "- 类型：Ascend\n", + "- 规格：单卡1*Ascend 910\n", + "- 存储位置：对象存储服务（OBS）->选择上述新建的OBS桶中的project_1文件夹\n", + "- 自动停止等配置：默认\n", + "\n", + "> **注意：**\n", + "> - 打开Notebook前，在Jupyter Notebook文件列表页面，勾选目录里的所有文件/文件夹（脚本和数据集），并点击列表上方的“Sync OBS”按钮，使OBS桶中的所有文件同时同步到Notebook工作环境中，这样Notebook中的代码才能访问数据集。参考[使用Sync OBS功能](https://support.huaweicloud.com/engineers-modelarts/modelarts_23_0038.html)。\n", + "> - 打开Notebook后，选择MindSpore环境作为Kernel。\n", + "\n", + "> **提示：**上述数据集和脚本的准备工作也可以在Notebook环境中完成，在Jupyter Notebook文件列表页面，点击右上角的\"New\"->\"Terminal\"，进入Notebook环境所在终端，进入`work`目录，可以使用常用的linux shell命令，如`wget, gzip, tar, mkdir, mv`等，完成数据集和脚本的下载和准备。\n", + "\n", + "## 作业内容\n", + "\n", + "作业基于上述打开的Notebook进行，进行作业前请确保完成了上述准备工作。\n", + "\n", + "> **提示：**请从上至下阅读提示并执行代码框进行体验。代码框执行过程中左侧呈现[\\*]，代码框执行完毕后左侧呈现如[1]，[2]等。请等上一个代码框执行完毕后再执行下一个代码框。\n", + "\n", + "导入MindSpore模块和辅助模块：" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "# os.environ['DEVICE_ID'] = '0'\n", + "import time\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "import mindspore as ms\n", + "import mindspore.context as context\n", + "import mindspore.dataset.transforms.c_transforms as C\n", + "import mindspore.dataset.transforms.vision.c_transforms as CV\n", + "\n", + "from mindspore.dataset.transforms.vision import Inter\n", + "from mindspore import nn, Tensor\n", + "from mindspore.train import Model\n", + "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor\n", + "from mindspore.train.serialization import load_checkpoint, load_param_into_net\n", + "\n", + "import logging; logging.getLogger('matplotlib.font_manager').disabled = True\n", + "\n", + "context.set_context(mode=context.GRAPH_MODE, device_target='Ascend')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 数据处理\n", + "\n", + "对其中几张图片进行可视化，可以看到图片中的物体/动物，图片的大小为32x32。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAD7CAYAAAAFI30bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9e6xtWXbW9xtzzrXW3vs87r1VdevZXZ2229jtgMFENiEiAmwRHgkEgZKYIOFEAZIoBvGXo0RCQQkEEilSEhEiHrJACeAEQzBOLCVSCBEh4LTlAJE7NjZt96uq63Uf57H3WmvOOUb+GHPtfW51V5Wru32r7skZ0rnnnLvP3nvtNecc8xvf+MaYYmbc2I3d2I3d2C/cwgd9ATd2Yzd2Y0+a3TjOG7uxG7ux92k3jvPGbuzGbux92o3jvLEbu7Ebe5924zhv7MZu7Mbep904zhu7sRu7sfdp18pxisifF5E/+kFfx4394tjN+F4PE5FvFpH/W0TOReQPftDX89VY+qAv4MZu7Mb+f2ffD/wtM/v2D/pCvlq7VojzF8NE5GZzubEb+/rax4Cf/EoPiEh8zNfyVdkT7ThF5NtF5Cca5P/vgNWVx/4FEfn7IvJARP5PEfm2K4+9KCJ/VUTeEJGfuxouiMgfEZEfEpH/VkTOgH/tsX6oG9vbe4zv7xORnxWReyLyN0TkxSuP/XMi8tMi8lBE/pSI/O8i8ns/kA9xY4+YiPxN4NcDf1JELkTkL4nIfy0iPyoil8CvF5FPisjfamv3J0Xkt115/tMi8iMiciYinxKRPyoi/8fj/hxPrOMUkR7468B/AzwF/BXgd7bHfiXwA8C/CTwN/Gngb4jIICIB+BHgHwAvAd8N/CER+Y1XXv5fBH4IuA38xcfygW7sEXuP8f0u4I8D/zLwAvBZ4AfbY8/gY/fv4WP/08A/85gv/8bewczsu4C/DXyfmR0DM/CvAn8MOAF+DF+f/wvwLPAHgL8oIt/cXuK/Ai6B54HvbV+P3Z5Yxwn800AH/Odmls3sh4BPtcd+H/CnzezHzKya2V8Apvac7wDumtl/aGazmX0G+LPA91x57b9rZn/dzNTMdo/vI93YFXu38f3dwA+Y2U+Y2YQ7yV8tIv8E8FuAnzSzv2ZmBfgvgS899qu/sfdjP2xmf8fMFPgVwDHwJ9r6/JvA/wj8rhbG/07gPzCzrZl9GvgLH8QFP8n83YvAF+3RLiWfbd8/BnyviPyBK4/17TkVeFFEHlx5LOK74GKf/0W43ht7f/Zu4/si8BPLf5rZhYi8hUcQL3Jl/MzMROQLj+F6b+yrt6vr7UXg882JLvZZfGzv4j7r8+/w3MdmTzLifBV4SUTkyv+93L5/HvhjZnb7ytfGzP5ye+zn3vbYiZn9liuvc9My6oO3dxvfV/DNEQAROcLD8i+2533kymNy9fcb+1Da1fX2CvDRRqkt9jI+tm8AhUfH86O/+Jf35fYkO86/i9/EPygiSUR+B/Cd7bE/C/xbIvKrxO1IRP55ETkB/i/gTET+XRFZi0gUkV8qIt/xAX2OG/vK9m7j+5eAf11EfoWIDMB/DPyYmf088D8Bv0xEfntTRPw7OB92Y0+G/RjOYX6/iHQi8uuA3wr8oJlV4K8Bf0RENiLyLcDv+SAu8ol1nGY2A78Dz3rfB/4V/KZiZj+O85x/sj32s+3vaDf/t+Jcys8BbwJ/Drj1OK//xt7d3mN8/1fgDwN/FUeY30jjqM3sTeBfAv5T4C3gW4EfxznuG/uQWxv33wb8Znxt/ing95jZT7U/+T58rX4JTxz+ZT6AsZWbRsY3dp2thXxfAH63mf1vH/T13NjX10TkPwGeN7PHml1/YhHnjd3YO5mI/EYRud3C+H8fEODvfcCXdWNfBxORbxGRb2sU3HcC/wbwPzzu63iSs+o3dmPvZL8a50F74NPAb7+RlV0bO8HD8xeB14H/DPjhx30RX1OoLiK/CfgvcDnPnzOzP/H1urAb++DsZlyvr92M7dfHvmrH2cSo/wj4DTiH9CngdzVR6o09oXYzrtfXbsb262dfC8f5ncDPmtlnWibsB/FSxRt7su1mXK+v3Yzt18m+Fo7zJR5V7X8B+FXv9oSjk8HuPL1BcE2zfxcWjfP+/9vv1nSx5g+yl0K3ooJH/n7/Gs1k/0xs/36GYaj683VB2/LoE4NEQIiLBrf9/YLO909bLmj5vn/cwPzq/b/8/z/382+9aWZ33+0efQjsfY/rMCTbHA3+i/ntUoVaFTPjENXIl30XgRCvNMQx46oe2uwwXu8UHIUQiDFycnxCiJEQBDOj1IxWZZpGalVq9deJMYBIGzYhxoCIEMJBa29m1FL9/c1nUAgHnOF/HxCBe/cePAnjCu9zbPvNqa1vPbP//dFahC83ww6L9W1/v1+6V/59+2OIP/TIfJHDXyz+gCvLli+7JnvbT/bo28kVv/H2ZxwWNgAPX/3MO47r1+I4v9Jd/LKpLSK/H/j9ALefWvOH/vB3EWOHSKSTRCCQQodIIJIQZD9BqxRMDPV5TkqCiKE6IWYEIhIifTcQQkRibA5WQQxSc5DVXy9IRbWyGy+pWhjr7M6td8erCCKRdTwhhZ6T/piAUMcRq5VSCqaGFQOEFDsQwULwD26KqVLmGVUl1+ILUCtm8G9/75//7Nvvz4fQ3ve4rjc93/0bvpUgCa1weVGZZ+XB/R3zXNntZtQMswAIMSRcJRSIKXJ0eorEgKo7KjSDGQFFa+XyckutlVIM2y8sd3wxRo6Ojrh16zb/7K/5dZyentKvOrRm3nzrVS4uz/nMP/4Ml5c7Hjw4x4Dj02NiDIQgxBQ4OT2i6yLD0CNBoEItlQcPHlByYZomRIT1eo2I7J3mZrMihMBf+e9/+EkYV/gFjO3VcV2dPsOv+b1/fO9QJDSQY+58lhdT3KmpaQMN0tZxPGww7a3MwLRygDPmfkp8YzKMPFcMEIkgAQm+rusyP6ohCLFtXCG2VxJr79HWvdqjG3dzmvvnSQAMVQc6tW3Qi//5kf/oe95xXL8Wx/kFHi13+gheLvWImdmfAf4MwMvf8JStjnuGbk0IkUE6ggSi9H6j28IK4siwhoYR944TwCg5A9ClnhgTm/URMUZSSj5QnSFiEPxGl+KIJgZQrVxcGrlmxhpQjJqECsy1IhI4XvUMaeCpo1tEhPkioaWSdxNWDSvq1xsjBsyqGEbAr7+mRFVlnBU1perb78qH2t73uD711LH1YWC1GjAFqSNzrNTcsxszD8/OyLlQK/iG07vjlEBMibheETRSzReUVXecEaPWwnYaKbmw206YtkUsgb5PpJTouo6cM1WVqorWQqmF3bhjmiZSSgxDz3q9cvQojkjnOROq0E8dpQjjtENEGNKAqqINMS8LqZTSnjcjIszz9AgKfQLsPcf26rjeevEbjSCg9khUJqH9KoIhmApqoFZ93HRGWO6bUInLa7dIbIkY3dxfCcEcHRZTMHB/acTgDjGEAgoq/rraHDSqLSJokQn+/D0YfdsQiSliIMHawy3KMXWH7V77XW/k1+I4PwV8k4h8HK8j/R68PdQ7m7RdKwaiBGKIBIlEie0GLAPSUHvw7XBxnCEItIkvQIiBGAMpRWKIdCkiQQhpcZwLOolg4o5ThBgjhpKIVBRrDlqsoYkohBRIKRAJaIqogYWImWLLBLIWPqjvudJCPQlCMEdEYr6j7sOMD7+973EVcfTdpYQZpBQxg2HoUfPQuBRpSMCRiU9c9ojAlh2/oXZpk1nbY9pCdq0KCCEYZm+fvm0BLDRNez8P5QPBd04fwzZ2ZtJeH6yUwwb+FYbrQNXYnkJ4wupH3v+aBUwOTm5xoFc/9oEKE3d0KEL1dSHSogtpkbjtX0wa1JTqDhhzRIrm/RsLQiT6c0WxACiYCUhkH4qLHZBxu6qr17t/08PbI28bPLlyXe/uNr8Gx2lmRUS+D/ifcWnDD5jZV+zqfHgO5CqkujhQD9GX8MtqwSe6L4CKh9xa2k3W4DxljT4gdAipfUUCCTEhtAVj2gaidOh+VwyIDgQTglXM9BFILyE4R1etcWIC5gMXNUIVNCumRi6TI8viIX/qu73zVINSHAGVthM/CfbVjGsMkZOTW3S9YGqN3zRWm8g4Fi53lXE3tZBbMfWFhHi4TABFmeYJVSWKNaQBZkpIkWRGShGVgKo7w77v6bqOrutJKTWHqM5hEkldoiuJvk+oVWIMqCm73Q4RWG/WpBQJ0UO3nCuqyrTzCr4UU+MxfUwXdLlENrHN3yfFvqo125ybAYRH3Yk2R1fVHU1CSGJswoToxLx7A6NALxAFSQOEgEoHEpHYYwRq7TEFzRXTSqg7MCXVTAzCJvXEEJHQo0S22lMtUknYAidtyVk4SEYa0LxyyfIOv8eGnBfO++qYv5N9TQJ4M/tR4Eff13Ou7FX7nI4t3EQL1cx8B5EG6feJH9v/7nxL8C+k7RKHD+9v5ii2VtAKOaujhLYvCo50RWt7fX+tJaujWg+7JA3Ct0szVaxWtFZqzr4rR0EsuPM1bakoD02eIMT5/se1DaQButwgOYzTgvb24/02xIL539daHW2GtkYbtA+NRw4hvg1FxvYV9oka5z2Dc+hdopRETIFYQnOc4RGeMsRIaCHOgjSKVp8Hkf3fPvpxl9cIT5TjhPc/ti1fs//5sAppqL2tPROiKMGMUEZEt8TyECS3NRXAepBIlb5xlz0QEe0xgULBpCJhRKgMQUlBuJXWhJBQjKKJ3JK3uqz1/QXt//G12tCo7YfvUSRpZg0jX00TGV8+Qb/cHm/lkBjEiiQlBCUmJWCoZUd+TPiioJHF0qB+RAgkiW2R+oTtu0SMcU9a7ye5LOSwUCtsL41prLz22kMM5e7djtT1pCQEKrlkQBn66gkDU6xmpt0FUQKhiiejpGKiYAXTQi47Si1MeYcJ1LhyGiJ0mBg1FjRUiuYnynG+X6uqnI9bYvHwdRxnSlGmHcxzYc4TpWTmeaJWPSA1EcA3IKtQW1ItpIiFJbkQCKmjSkR7J/JrrYQQGIaBrusYhhXDMJCSO8uTkyNAmfNtLrcd5xdnIHBclFIq3TAAQt93xBjoug4RI4i/di3VObbmIGOMmBm5ceshREIIpNg9aRzn+zKBlm84hOLY4jAbQyUQYyII9FaRuTC/8VlCfcim+yKxy6TjABEmxKMRfGcMlnCGMWECORYQJW0qXQg8tzrmKPV8bHiaoIk3HyYuteOzPMNOVmiXUKIDLjPC4vdqBfOxNgxT5wB9PA9AbEkiiVxxqSIo4RFk+pXssZdcLutl/33PS+kB0tE+XEMnC5r07JxAEKQhEFmcqQ/z8i7te6MBNFIrjFPLntVAjLRkDgQKRtjTIQvY1eoIGBNEG2dmnvBZ8KRiKIoaBM/LIy25ZcEXusqBEL+OZhhFC6bOF9b2lWsl14bm5cBTOzr0rxAOiC60CRsa9o8tGyvRI4OUEqq2R4sL4vTXlCvvEdqCDsTgaDQE/9mCEZv8KQS5sunSNumm6tADpwn+c22fZZmYqof5ei1t72TaZ7T2n+3nvTvdo3L/2zKPSN5S9Rwsk2rnwZ+1DLyFRpzGxik7gky9IsHoVtDHxOmqspHK2nagiTAHpPQI1XMY7XLcAVaMZb1q40sXOZz/rrLMHb/ORebmQ39QDPxCQM5jRpxC7BNdn0ghsefw/UFS6pCGOEXEs2cS6IJnYbuQEAJERyIxdM2ZdiCBah3BICzjrIapAANQUR3BKliHtKxtlEpn0TNr1oEJoTrkzdmTBWKAGjIrVm3vOHMKlBDJ6nk5WRIQjTMpAjVAXmQa19TMjMkqQ3AEoSmhpkx1JtdKShEMNkcbTx61rLqZICGy6jtHNZsj1JQkRhBh1fscKcG5x5T6hjgVEWGz2TTHGfbX4Zx1wVBKyVTNPqeEZYtrf+uc2JIgMDtQBcOwwlTJU0ZVqdXff7fzcvcu9YQQKdnekwt70i2E4EoGc8fnJgcHJE5NAVQSaoHzyy06nnGubxD7zMlwinSRsYBJIHY9IQhddB77eJjo+8CdZ1b0Q8f69hF96rgbT4mjMX36dXbnhS99CbZ2xHz3WY8+QqWKovUCrPjGZga10XZ+qa6OaESSKyrccR42Qg4bJu1zfphCdRFxHrChg0MWzHmFRe+10EbS5D0i0mB2I20leBJmT1zLfiGgsgeeqoJpQDU4nyHdAZ3Kgb85YFX/SazpzrShSmuzpjnAhb+swc/hsNR4n+QhiETXh7mqojGccp0RJ76ZLLzScouDKxRS8vsRU8RaYkdoulkeRXYeDh6cGe1xBzvikYoJYS9gb3yzHZgqz3a77rY0EbuZodV1oTmXNl+UGKOH6qHNtwDBpCEjRyW1uOM0XYoalv8vvJds5Um2JXewp/zagtkXruxjNDcTwUSoEqkEtCqxKKviKpdcDBMI0SUVXYykCOtB6HthPQjdIKyGQBeDJwlDJetILpmigUqHiaNLq9nfszrVJk3GtFxTCNHXX/D5hy2Rj/++RIay+IM9X/2hQ5wQOoPkkFpLC8uCf191PSI+8RWjtJsQlwXTuE9i2z+seqic3YmGqARhrwfT3FOyMF5Wpmx06yNCKMRuQqLzlWb+HJ8kLthN7caa+YIpmj1bvDjMFqiXrv288gnW9S6LSjE5mtkZVpWqeS+uvZ5maK1ozo62zUOuzUlCa6T0gTJX5jxTszXUbmipzikZmAnTXD0UxxNCJVdPGukioG/CaPNNNufoi6NGSp6JAaIY8zhSSubhg3N2uy3zlMlzYdzuGKeJN++dUaoiIdF1Pc+/8CzD0LHerAlAmTPVClqUmivby93eyUY5JBDnebrW3LWvqc5Rujm4uCr4WdD6knAlBjT1yK1nkB7G+68QGOlzTzBhLDMhKMdHwnqVeOmF26yHxK1TI0YDJpytydRauDfvsLEwdWfkY6WLR1QqMpxhVOYxUy0QaCF4i2RC7/rwvuuBg/5WTVvy0N1ebf/vyWjnsgXxufYet+Yxc5yGiaMvWyAf7umDLJzWAgVtj0SBg2Zyie/FtZeOAn0BmhgqEFEwoeZKno1phLk0BJQCEgwRbbxjQyqyoBj/TuNj/O20IUfnYQ3XcmrwXZbGqUmShrICmDqyNryiIVzjBWYHacqCP0QgJt8UpQYwI8aAVW1cKHtUV6tXVpWyVJ7Q5oOA2L5iRKnNqcbDQlBQxBMAbbqb6h4p1qJo+6qlolWd5omBlFZ0fedI0wJBgm+84mgYa5t4Q5sSD+qN/Qe/xo7Toz/XtIo1bGaPPn41I72/GylBTKgkIFFLq6yrrdoL6AKsu8BmCKwHR6G1+hpekKCimBQYPFrtQnRHKRU0YzpjGrEQD8Cn5T5CCF5JSHPwZq6tRgjNce5lkLZEvLHxnB8yx2lAoVIp7hSDT9ouJSJC1y7HFScupDUMsrYEC04qJx8yU236rwJA7Bp6lYoV2D6YmHbw6isFlcjJ86ekHiQWCJmCZ/GsyU5S8iRCcuU8FpwRKxX33F0LHy34ou882UCqLqGJXp2UolBVCKk5AH10wl03MzM0GyF5UqeKPbJZdDGRusBw1hODQk1oNRjdaU55olZjmrQxIo4orWXqap3bJled+xzWXtCinjXVql5N1vKLYgHRgNSIlcC0q4zjzLRzve3LL32U1WrNM3dfAAncf/AWqoXYOM8oEROl5ErN2gozhEDwFGIrwuj6+G635ck3ESQkAi70bzkVp66WsJdlszTUCpAhKEQhhB6qMp9DjNANkV7gSCvHmrklE+tQ2aRAiED0MdfoG1cZDEuR9UvHaBX6+ZhdGXh4v5DHCWrANCHxFiF2dOtjQojEK0oHD1pjU00s+tvF7cVGA7njjE3OE66Auneyx444tWWlbanbomVaCdAcjFjwipv2gcycG7FW+iXL7tZ2jKWkUUxa/w+lFuPyYnZJzGQQEjEKXYIQDAl+Hdgit7Crs2CfaaNxafvfaTwt3pzCs7QVCbbP7C5IZKG/HKVcX9kK+GYXpGW47ap21ZqKQui6iFjAJGHBqE3eI3Wpdra9NvBqc41F2OwRC3t9aNtJ93PBzbPdqkYp7vzyXMizo88QA0ebDcfHJ7z4/AuOPsWY54mczz2xVFtCqNZHEghwmHMIpLBIqq6xySFAfwRo7+0wNssghJiw6GgTi2iuUI0UfLTKWMhiTJcz0ZR+6IlJXOQuwlJwYlX2kVsg0PU9VXr6FJlTaHOt6WlDbF+HdbY4+P3cWDhxe/TK967flhTIe4/p40WcZsylMCFYUGoYPDxKiYCgE00GktyPqZc4lurNPmoLwWP1Uq79AqlNqF49gWSaGC9nPvOPPse0U7ruGdbHG24dRVZHgVVnIMaYSwsJAkJ0lLSUbgl7UXVniYqiMWIiaAzubLuEBqixR1CieilhrZWq1R0zRorpWnNh7teEoVsTUyRSqFbZTpf7kNkMjk+PMBXqLGhRUiderbMNhFKZy9ziv6VbEbQ4EVi6IAVSlwgSsOJ3NS5JQyKqwjQp45i5d++c8/Nz7t07Z55npqmyXnU899QzPP/c83zXr/21rFYrXnvtVe4/uMenfvzv8ODhQx5cXDCO0765yFIptMhXqkJMgc3J0KqUrqt5waPtk6I0xNnos2VOL3pIEyKJzeYOVYQ5rClamC52iFXqJUxBGV+f6JKw/dLMZt3zwot3Wa0GTk9OiDFhKbbo1N+3mPeQiOtjBul56uRp+j7x8CIylQhpjYTUKtDswF1eVSdhiNS2rlvtvB7E/E7RtRLR8N6RxGMP1RfOqKKo+Jc7LxeZoy1LjicM/KtpwBoxJqrQWoc1D7tgGzBPOpXZ2F7smEbl9NRrn7so9C2cNsxrpRcic8+r+FdYdh8W3VeA4A7Ta9ub5MGsFco0BGRXUPWeP7nOTFibdDG2lm6LKuHAT9MQYj907jgFNCq1usohzcV1sDG2jWtxnAFEiRYwvJTSv6I7TqxVrKSmzfT5UquSc2W3mxjHiXku7qBrC+vnQs0ZUa9MuX16AlroYkdEqLlQcmnZedt/xgVpYrank66zzAzYoy+7gu4PafYWBdBCd1zxEtMK0gqkBxJlrmAFKhSMbJUuwMMwMe2MYRhZDUaZemKsvs4EB0oYSYyYAiub0SikWOhDoG89JKrQIlIXuu8VEIvjbPIja45zyaQfSn9p1FxoH1TeE3U+dsSZ58ykoMGY04wFbw8XLZJKW3QWMRNy8V2hamhi8uZksyeDYvJ2U7FfFqlgFXaXcHmmPLh3ScnGnVvGkIRVrKyiEFPL2odIVaO1FKBabhKmxmdJRAT6rscwZlHUrPWZhDrNVFOqZUSM2MWWzCgUreQy+3eth96f19BijJyenJJS8olZD5lKCZFh45FFH1dggTobtSj90DHPBZXANBfUIrWas0wipM47OlT1iR+C18WvV2tv6hJ7d7AEjjfHBEmYCvOUubwYef21Nzm/uCDPrTKtKNsy8lM/+Wm+9PnPszLh7t1n+LZv+6XcWg0c9T0XMTGNE9M4Oh1zRSO6tD0rpVBV2O52j/TwvG4mIsTQUckEjGKPNjXR5pkUX49dGAgx0cXnCXGNDE9R58r5+ReoeXR1jLjyIQicnW+JYcfnXr0khkAMCZ9A0hxxcZ3nUWC1SnzkoyesjtYcvfgxNvGIp9Z3GbuB+2Ok1Mg8Rgc23vLCs7cIgdjcewYxDiWzXXP2y+awR0vvRXE+Zo7TaAJmI5pSg+57ZHpE1kobW4b20HzjagaMVs+8yIdoGXbf1EyFmo1anHOLwRiGjqHvSALt1u45S69gWt7HJ4Zo69MXHtULLlVO3rbKsFbapcUdZxFHscVKC9XrHmHr9fWbLGWJe1zdkNheH7dU78TWV6B1oYopkCx68w6DlJaKkNCcljSOzR1WCDT97vKerfrIwiMOLufiG/Q0k6cmBWucKdXYbbdE4I3XX0O0cv7wIy47y6X1BIV33OfavBSMksu+qcu1tX3EdOifuahRTGdaCAgmaG1pFs2UnPfrtbaihX0jjegraVaXB+UmS5PWlSriFNmqM7pO6MKKIUEfC33I9GxR4KQ/IUVjWxNSI7VGD7/rwQELNA31klxuvvHgIw/U7ZWBf6/l+thLLq1CLgWCkcX760VmgkW0miNOiZhCqbUtQI/VtfoCkFY54v04HdWYwm4SygzjpVIm4ZlnniGGxEc/8hJHxyuGEAg1o1JBKjE5NC/BK0aWygNVz55KVM+KL7pOncEUK15RQq5QlXK5QzGmdcIi1OROeC6+aHPRa404ASQquYwYRjVfNHv5TkuiVJudNes6JBipd8H0aj0QYmWa2gLTtuuHZZEutAyYGmXOWFBiSITGfy4VPjlnzs4e8vDhGeNuxzSN++qepeJrmjJWCj/9k/8Prxxt2J6/RYiBV994nctxJMRA6jp207RHmosZ3g9W8Z6u19m8l21u6gUlUp0WqyOmM3l8C6zQda7NvXw4UebCdLlF80i9vIfOWwIBC6k1H6clgXCdtClTLg2IBKIIx13PapX4hpdPuHNnxa/85R/h5DhxtJ4wq5zv3qRY5JmTwmgrht0JF3Pi8290lNpBfQpISGwCeMm+EeM9Eg4NlqVdh12RxRlWynvem8eOOE29KURQ8UoO9EC6W2h8oXOWtZXHhT2SYclENERi+zrlqjBPhXlnbC+UeSzewCElUoqkGNt5DgbBvK/flQsTDjWtTqWK6wel1anLQRCvVtrupKDVuyRhrUO5NOmUUXRBnPbOCOZamAHqkcNBE9TUD8uBJYdQyKgNuy/ay+YU2wTe14C35FpVR/a0CrI907ZUlLVwa5n8u92Ocdy1eWWPXOeSrS8Uxt2WgHL/3lvElKgtyRhjIkZ95JqWn/3lnEuvRd87pnuCzXMLrf/sQv2ZYjohOpK4IITKcZ+oVdnlh+g4c/ngPponZD7H8njoW2rmlT77JH3TTbNslJCCcLTuON50PH17zdO3Vzx9u+doExk656nzVChWSd2OJMoJnsUf+rpPJvtFd01nvYTfsdFAkSV57hHOlw/ie3HXj5njhDpDnjNVKjkUJELGxccLv0VLvtTgiZdeWimmXSGLBbpQvTt0CORSePXzb3HxYOaNz15g1YA4uqIAACAASURBVDg+7pGNsZ13EI1hG0hdJRKwGKgUFCVoa2fWQvWqBW1JBgFSC92rjqgpuYVzixBaq1JMuZgzGr23pKFoLr72S+RKb6traEatI6LVOaw+EoCc3fmV0lJpnfvUWjJajSln8qyMU2GaKpfbC0pWD/nMa7S89nx2rrnvSbFj3W/ouo71sGrVHoGh7yllZp4nvvDFz3FxcY5ZISZpiR3Q4gi2WEVMUAq5jLz+2it0q4GT555nJUdMIoTdjvsPHlAa+rjaozF1vfftnOdrHUmYGbkWJHiJc9ACNaO714l2ybPHr3G0hm/4yIuUqfLjr36BN88f8Pqnf5pxN9H1rbCl5V9ioxOtKiEIQ9eRYuL2sCIFY5DKekj8ko/f5c7pim/5xG2ONoHT7pKoRqiCamWFUkWhf4t16nj5duGydKisudwNvP76wFzWFI4RSYTOI9RkHfveE3gRDFxNAje65xdwbx5/qN6aBKt4OaKK7kWo1pAaop5Nl5ZgYPlwjkQCtAoPrzJQ887g43Zmdzkx7iYwOD7pQLzkKudMyQlBCYPss/KPaA0bOnLlXqtMgaYTbccymFLNry+0jH81cx7Hlq+lr2hr4W/xeqfVgaudrUIQr0murZyySSEPt6BVbLWF5J3dq2ezi1KL7h0n5iWrHlolgiz15T2r9ZoUnOKJMZLzTCmVcdx6cidAtEazSGuQbV6LHqOwGgZWXWLoe7qup+86asvmLx2WlizsVQSyV1mIXPNx9TnsOmRDrGIUrI7Aji5s6WPgaCgUlHWqDLEQdEbqjGjvssGF21zG3JpyovMmwuu+o4uwiYX1kNgMkXUfWaVEH4KX5lbvPaEq5Cyef/BzWlh1BYuB0yPPmZwNzrtqjZgkJGh7X+fXr/KYh1SQfYgdp4FlgwIEqKVVB1X/4K0tEjkUXHDlk9wPsNhHa/TRJ35qOr9pzIyXmYdvbbl4MFOLkmJgte7ohsjZ5TnTNDFwxDAIxyvnWFRmTGi18EKKsSUGyr7vo9LCFVPm2bPouUH/UKAWZZeNbMa09m5I7jgVrHrBkaU9Or2WJu3kyEadDIPLjCQ6TZEz+yyph8LBQ6h1JIbKQ5vQUpl3E3MulFwfCfJrzS4zKf6ck+NbnJ6c8vJLHyWEyNmDh9RSeeO115nGkYuHD8jzzKaPQNjr+rR13epCZLNa8c3f8HFOj4549u5dCIG3LndcjpNv6FWJXYciaHWOrxNaWaaXeOYSsHC9exAgc4v0KqaXWN1heh+TC9TewlvE3SKFwLPPbOiANz/yErvdDCGSa+HNN99inov3ycRIwbv5b1aJvku88PQp6z5xMngEWaeRC6u8/sothr5j6G45LdfPFMu8uV1hqXI0BBKR9S3oo/Hx52E7Oby9GCOv3d9QqvPngkLxCjTVJVHklJ83NAv0XYtOwtKq8p3tsSNOjFa4drgwoVXniSOVBXl6TmYJ0xunBY2poGk+IU/KPFXK7EfApuTnEIV2BlHJBTFhnr0B7lKZECS0juV+ESF4/84QvNB/ucSGh/f6TDVnXa9mzLXxogati462nVV+QTvYk2xCy3abJ4Ki172SzLt0e3s/w3SJHpZTBp3f0lJbXXlt9eSNCllOLdTGjSr7ozlUjVwqUTy7XXJhGkfmaUJLAVO62HnWVpeS3I4YhaP1iuP1mrtPP8XJ0RF3bt2iGjy4HL0jVqNuQgwEDU12s+RlrXWk981b9fpuiEGMozgRpSBWCTJiskNkJjCDFmoRtheXUCNWK4LRdxGtHaHrSTkTQ0BlaVwm7gz7xOnJhpP1wO3TIzZD5Hgw74hk2cuw08rLNqVvfSi8BNrS4Drg4Ed5p9AjMXEkiUBks4oUbb1e1Su0g4B0gMGM0xB1vzAPvVxF8E39w+Q4BRgkItE7HqWYSDGyFu+vOQfxDvudV/SECmKBoD0Rocc7TXeGH/A0JuaivPnKyPnDmTwZgcStZzd0XSL2AyaBy91IlExH4Kj03LE7jlZTRWVmVhfGpr7zBTu3PpytH4zhiFPDcmplW7xZqdkoZlSHUpgodfIDx1pTvGvvOMHrf0NM7cjdjhCEvvPD26bsCG7e+YSNTV5kLTzfnu/YXkxM22mPNoF9k+Mo3tNASGgR7r31kIuHW15/5Q3fgJfz26s7zGBGlxKn6xUixs4mYkg89+xTnJwc8a2/5Ju4dXLCR599iS4m8li4uNzy2c9+kXxxwbzbUvJE30dignms7lBrQYOwGlYtIRWbuPp62ibO/FPHX6C3CbSg80SOE2/1D5jryHhRuTyv/IPP/QxaYDqHaayUsiPGwNN3jqiqTBcPmKaAZqPvOz7+sRc4PVnziY/f5XjT8+KdgSFFVl0ghcDQr4gxMXQbr9iyjFKYU0eRSgi3PLJbBVIXWA9HpBg4LrCzxJvrDdhAiBNBC7dCoU9wessbKj/YVeYCZ1tpIDi4YzaXRbYA6V3tMTtOz4AHbZ2521kxS1ciGtoktOM+q2fUHXE6qRsAabt8nY08KecPRs7PJuapoNXFtSE0HSienFBgzpkuR6x1nPbzvbVl2vygNtT2p1QuyHNfGREE0aWqyXnNfQsqWf6RlliWK8k6zzpfZ/MD1NrnB2hjbW3MFDsgT3HaJc/Fq3rm4j0ydeESFxXFkm1tiZmmoRx3Iznm1n0FUkseLkmILrr+dmnZGoAuBm6dnnDn9inP3n2Gk82m1c63EtmmyfW0pB/pkpLPVVLc9/yMInStWXVny+y8nhasstYzYtlhWinzjJZMqhO1eKu+XDLbswnNoHMkz8o0zUBgznMDEK1OPQV3kH1i1Xf0KdDFJfZswsoQW2SQCF1qFE3284XS4Rx1C4KFDpFIsA1BXZGTLNBJTyeRRMYodDYxiHB77RKlrg+M2cglM2drPVuX9nhLcvfDhDgF+hi9g1CIDENHnyKpTcTl+IzY2oet6IgWSMVDJsnNaWUvvzyfJ87OR37q73+Ws7OR7UUmpsi6F7q+w7YdJoGiXTt83jOquSZ66+j7YwgTWjKGEoNX/nTRO8JApZr44V0iDF1PUWUq3p+zFnfUEpeGHn67Ewla6znBECnXeHn5hNttd/RD5wejtdZ6IQTUjDnvyLmw3e4wFfqwJufKl159wPZy5OzhxZ5GSUmo9YqOTpx7RoRcKrlUpnFyvW32g8DWw4o+JW6dHLe6ck/sTPOENIqlH3o++clPcvfu03zjx15Ga+Wn/uGn2W135Ll6v4MEw6Zns06EpKzaWUN1iPvKoyDCZrMCEXYl7tHxdTQrI9vX/l92FztqqYy7QlVjNphK4bUH95nmzPZ8QlVIcc08Vz73hdfIpfLFV18jhcAqQIqBo9URQxfIu0sudOLnPnNO1wW+uI50XeTkaEPf99y5c4th6Ll16xQRuNyeQTA2d3oIgcvRMAmsj27RaY+cn4BE0B6pgb5E1qrc4j7FJjbTm5zGjl/27Cc5PjlmdespLqbK3/6Hn+fe+cwX3izMhVZp1Gqy3+MQvsfOcfphaN4ZPEY/mpV9BrPpM3H5SlRHnbTmt2Vu+0DwhE2eKvOuMG1n5t1MzdX7BtaK1QDiO9aCZkv1hZezUaqwEhdiL3XOwcuQSCE2uYp5v0eVpj90gW4IdX+gnHOjvgPuT0sMrYKhZf2DlGsdsC/aSzNrHKTrOA/1+i4rsv3fClphHgvTWCh10W4uPFNDmvtIRPac8oL0MUNLcY40VWoIe82lZ8GhlFaJZM5fbzYbjo6OWW82lHluB+3NzHOmqmt7JXpzkZikNfBofZ7ME0EShKF3R15lUaReT6uqnJ9v2V6OlFwZd5mqUEMkV2WehXmOTLMXoNQozFmo0qNSKCQMoRMnvqr6OVS7cSKXTNVAisI0JlIKjJPS9x1ZoR96puqAZLs9R4JxIkdIDOwUJHZ0HUQT/ESIRJCBFCKbIWFSeeZ0S5krqzFwshKOOzhOxjpVRAtHaWaMM+uOJp8TloYm7xWsP/ajM2IMdOLJm2GVWoYVwEjtgjsLHqZvgWyMD0c0Q965QD224+ymWpl3M4NGqiREChGlmyu9JFZHa0gdc+pQg7zdsZsy9x+MVAvcuntKlyoSR8xKa/NphH6FqjFrbefQeG2ylw7CepOaA66IGp1Eb2o8tGSTBW+mGzJBlD769+tsntvz3WSeiyPO2IKwVorad703UigdVmHcKdNOvZ0gAWtJmJR8Woa2O2mjRJZO3ksr4S51xODtxroY90qImkFb0igIrAc/1G21XrvzPD6m5MzJ7VsQhPsPHmIlU6lorMTe6Cus151zbCW1Dd03x9QPXkI7TdRrrOPc7oyf+OmZuXoJ8W689ERdjEjsiKsX0F6Yux2lVKaxYAh3PvICEiPDZoVp5eLBG8zTjgdn96Du6O+dEUQYhkQMgXWfiEHoo6+v2EWXi628F4FqIabA7TundH1iOB5Yb9Z84hs7To6POHlqIA2Bo5MVoVvx1PFtLAS+vTxPzTO7N19DauXYAuF8y+7sLeY8czs/oEtw/NzTTCXyxv3CXJSLevV8pa9sjx9xxtZVqNUcEw7a8NCyrlENqWBTRSdl3noSZtr5J+raVVcMirJKCRsUsepJJzOiKqGdExSkVYA0Dek4VYapotWF6aEps0NrjxWDpy8Uz+bH1mZq6QspIRDiUlJo7cxw71juiNgaInHEk9L+GKTraQ1RLKW+jjgPDSEOp1IKakJRo1SlZKMW2z9/6UC0IM0Yvb2Y1Uc3HeGARh9Bpa3fgGobF9GW/Y7E5InIGFM7FFAYVgPT1KOm3oylZooVPycpBOdABSxIa3Dc2hlHP5MolqWbzvU0JbDjhBIKlcIs3g4SCYTQk9ZPEYnEssVyQWxCJDCcnBC7jn6zQmthmmdMEvnhA9dwF9/QFPHG1uYKi3lp0ds22i5NewosxkAuQuoSm3Fmsyncv/OQMleO4orVkFEGUp9ZxURIiT6CRQibNZYrOmVqKczjQ8aciWVkRSQNMHfCvBN2WSg5UO1DFKpLELpVD6WgUclSqASXCwDrJJ5J3xp1rDz80sy8q5zfL5RZ2Y1eG7wZuqbTHEgEXn72DlWNkpsGb5ywksnnD7GUKEdHTverULJw//5IqZEXLiKQGFZrb4JqO1wI67KoGCI1KKWd1liK16oTnYsNKw8DU5dAAhUXfHtDZUWlIKKsV06bXFcTEbpu2CdvzLwVnOAVJ31aYQGSRfKk3H/jnMvzke12Zp6q15ynQ7ieUmp8p2flx3nCajvsrSWbsCtf4CWQtbpkqfrCXPWB1Do33To9ZVitGVYDm6MjTJW7z94lxMDPfPYfc355wVneUU0ZjjpC6Fl1yekXXRKUPt5qUMzouuDnW11TC8MJm098NypeHt1fnqG1UDUTU+Lk2ecIMbK5HKklM15cYgKroyNCSgybAauZ9e1n2J29xe7sPtkUaY6zCy4Pk5bB0xbxzVNtXcjmti85snr9jZ071EHoh8Qrr7zFaj1w96mnGbqB441XlN26dcSwGnjmmbusN0c8/+LLBOk5O3vItL3g3is/T62ZbrPmdBh46jhDFJ7pOy5L5PPjhlk/RI7TRdF+KL0fxlaJpsuBBETziVinio6VPBby5AdmlaLkdkTGHMA00ncJCcaQknNg0Q8NG6fZEc00UkukRD+XpOSExcA0Z/o5M81KNwf6XpqoGUcQy9kkOE/poufDCZfeTbRisWXbW0VlEwQcTuA0cw417Mtlr615XbNDzqVTvi2biMG+r6oa85yZ57znLEMrs10qN5bKnRCW00kDEpSgrQentD6olStIc1HbNsQbDqh1c+Tc5mo10PcDMUZUnHoxwbOrJVO0YmL7wwGX1wstkgit6sTMue8QudZyJEJC1qcEovcutYRqJWgmpEhcnRJCoNOOULwtoAFpvSbESOoTWlv/1NZYYznTyxPE7d6GJYK4sh8alGpLawoAivnRKa6yqExzRUJlO2ZXzpRKioEyXdB1HfM4s94cI9GPkb64d588bnlwdg5auJ0CRKFTrzI7WXV01jGu1uT3cI2P/ZRLTZBbJUYphajCMX5eelcTTMrDVy+YtpWLh4FahdAPdNG7ImmtjHkmV4jJy+/6fkUMgWHo0BLQCz+o7d6bbzKboUdH3so/HpO6gdCvqVTeevOMeUp0UUnJUM0ISrTE/hjituMZkOvMbIVL3VGlUlIGCeTUuiplX0RD8u9TLV4tY9rE3NfTVI3dbiJGrwpar5tzUt8kp8nLKMfLyrTLPLj/kN128s7+5k4uAMMwAAtX2qxl1Rep06FMjn2RQckZC4EuxSYmMgJCPwwcnxzziU98gueee5aXXvoot2+fklLHOO54cPaQ+w8ecL69YDtvSUc9FiCXiZyVPPumuRp8frGnBgLRbN+i8NpaEHTdowWwSNc9zdWuQiqtQU/0CKEPTfUSg3PM85Y6XXLxxiuMZ/cIZaSjsukDKcBmaHlhz83SNCgYnfPajd9ehOoRH+Nnbp+y3qx57uUXWK1XnJwcEwHOz6jTxJdee5XdOPPKvQtUIreeukvf99w6XpEipJBZ94FvihmdB86YGTbHPPfyP0l3csovf+mbiP2K73+XW/PYOc6KUpeKkFa1Y8up5tWwopRdJe+UXJwT84a2tj+1TovLldS8d+ei2wxN9W94/fg8ZybV5jQrdI5Mcx6Zc2KcJ/pZ/ahYabXlmHdjMoF4NRK8Uj0kxevVYzuRLzjSMpYKJzhoN+s+23xtzZZepqFV/DS+sn1mr/jx8tSSa+sb0HSb2L5rknOXcLWK2BEsB17c2iNX/4R23HAroVucWQiBrus4OTnh9PSU9XrN0A8sCadcCrlmajtmelFKqGk7CFA90lhER3utbjPhvZXST7KJa2gJLQprh515ZKBoLU0t4dpcmgrBlQxKmXbk3SV5t6XMo3PNnffFTQGXrQWalOJwK/f63xamaSu1TsETSk8/dYvjkyOee/ZpuqE1Iy6FuRZKntlud1xuR+7df0hW4XJS+r5nvHVE30XWgzGvOnbbDVGVCxFyUda7S0K/YpMKqX/3ZO5jdZzVlMsyYeKHm8W+LQYqokoYQS+U3f3Kdqdsa8SCsNl4Y9JoArVSx7ZYgmBR6AbXac7Vb95lNS6LcTEVplIJTBAzdDOxdMRdROMlDy6OsDBwZxepvSIUbxwSDsfE0qQtBqhUNGSKzN6Eed1kKUur/qUtXjCwgtYdWCGbHlrjXUcT37QWCU9ognSbvb1Xp0JQwaZCnTLTdmIa874HrlKWl2mvtWi9Wu1VC+OCNcnRUimELvUGgJFzRkVIoak3uo7NZsNHXvoILzz/PLdv3WY1DMx5ZjvObOeZXcloElDx8RUla0arYrkQQ+SoyZSkcXBaW8OXcr07+7tELzj/BIcSWPOO+rVsW2LtitPD6GxG55H7X/gZposzHrzxClZmbt95mogS5nPEKl077dZpkcPxOEvviBoj2loUhgCnx4lnnjrlN/+m7+Dus0/xwkfvMs0zn/p7P8HDy4ecvfU62+3Im/fP2E6Zi7mSq/FwfEBAeP31t0hBOB4CJ+ueUzWOVh0xKLHvOX7zkpM7d/j2UFgfHb3rnXnsR2eU1gDDQXlYekl7drshlpqNko2sikXPwiK05IsnYHyH8ka41mrcvUsRZDOyKbl69jYWf8/YLR2pR4pGik6UKlTtvau3eGllaGcY0bSce2QsC46s7Yx15958ER/gh1mF9mVWsXq964aEhvhDO3mw1aB7h3y8o35VrChWqju+uhz2xOFM9MYT29JgtvXlXPpBHkL0w79+AQ2xmHp4Z857xhjpUsd6tWa1Wrk+V8J+IzQX+TpnFqUhq/1V7T/c4rjbTHVlp9m+mui6mtDAw9XzvaBFF+2z7+8lCBWjYnWL5i15PKfMlz4nup7N8QlRjHpZoc6EokBTvcBBaiY0VYSj3Yi6TCn6SanHJ2tOj1dshgjVKLst4+Ul2+2O7XZiN2XGXFxzqk4NioFWbwwUNBCA80s/ljpECDkwPdwx0fPGG/fYXO7e9d485sPajGxTa2UPuQpFA5UBs0DsNliqTPk+uzFzlic0ROboCYI8KaYV5kwKwmY1ECQytjWYFWY1LnLlMhd2uZBLIQYjWWB1OhB7qHZG0ULVc4pWigpBDbMZEfUjMNqRDAZkUUow6FtmfvZEgnc/SnSyunI0sJLrFtOM1dlrnOl5JMS7ZhaCsFl1hK5rXFcENebthBUDFWpW8vmWvJv96O3q59crRtH/j703j7Ety9L6fmsP55w7RMSbc6ysqq7qgS6GblrQQsgYNbLaMkJgZGxsbLoRdhuER9m4AYGFJQ8IWR5wI9kNtmXLYIwFCFvGQgy2PPZAA3JDQ1fXkNU5Z74pIu5whr338h9rnxuR1ZXvZXW9fvny8VYqMl7EuffEvWffs/YavvV9vTWJKozLiclwTKkyrFv34EDCUnLlO5ViTbx4AbAuqmhSQggsugXr5Yqj9RHr5crGYAsGpfEdzWJJs1gSu4bsEgQ7b56cZZ3OVQo7+/nAspQzOauJvj3FJRgRqxOnZKOTM462lGJIA+0wFIo5TPQcTTv2p19i6rcM/buoCs+9/EnadsGtW7eQkrn3xhdI+w359E3II76WQ4SZwtyCobkcl5LhOwMBirDf7Tm9L2zvvs35/XN+5m/9NHfvbbh9nuinzOk4MhVlrHVSxNi4pjo8IUnJQ+ILt8/pugWray/gpEPvrQgbx5ubL9g04wPsoY5TRD4B/LfA81jg9MOq+p+KyDXgfwA+BbwK/JOqeu/BZ5v3a3Ocpdg8cWbmsXRkLTbhk02ZMEvCDQ3inKnlFcXlbIBpERupxKAiBSFjJYFcDpWpQ+0sBMFHsVl4yRi/XaqMRxj2kJmRc1ZXhCzF+EMPNdS5pifW+KECGBWMPzKhmg5bsXVenyzH+SjX1cpbF2zejtp9Tiasl8bCNBXGYSKNCc1aQ/A5urCamRzOZjZ3y40la+6mzgGmHFL7GnAezHnTMWrbjrbtTAXgkuSrc66eq2oW1Q6+OKVoLQvU+p537lBTPczaz21fyyKfKHuk9+vcCGLuSFyKNO1vAeClgCZy3qN5R5o25GlvnA8+0K6PadsVcXkMJREWaws6Nh6tUmpQo9ZLqrYzY5+rE4WlKGnK3Lt/Rk4jftixOduwPe/Z70b6URmKMhUlHdaJ99WiVS0znbKyGTKTU4Jf4sOK4o9IEjjtHf4ht+uHiTgT8G+o6t8UkSPgJ0TkrwDfD/w1Vf0jIvL7gN8H/OBD1gHnChOZrJaOJ3XsSwB13C+ZaUzcH7ec7/cWSqtjNyXbNaZywHvSRHAR9YFB62UXk55NKZFzogmO4JV22RA7z9FJh4+OFD1NK3ifEZdIKVkqsDD4Uc9o+MzatEoISQqTgwyHpkZOuX6wsi1yNuG5ofSgmaCmwKjJw0MAtR+BPbp1RYh4yJamNzg0Cds9DH3i3fdOGcbEdp9ISUlTMW7VWr+MTXPYjADEuzpaOVnDpxTDdVb1S4nByj51a5y3JVcjxOPVipOjNS++9CLPP3eLrmsJ0VcJFcskGuetsViUgCdIwImnaKKvPIHRebxzaLISQB7MaRhjlxClYg+fLHtk6zpbKTY5lNJwkN4VgS4EPIVGzyh5x+b8K4zjlmFzB1U4uvkCoTlife2bCWHBECOSRxbXNuhiQdq/CyOIGh/RhDcUxmQRfSnJSLCLaXZtdxPDOPJX/uqP0QTHuphA3N3TwpAbhmhOs+RacqMybWnm4lMCUzLH+s55phPP6tpn6I6v05y8aEqbeXpoCeahjlNV3wLeqv8+F5G/B7wE/Ebg19aH/TfA//6whTjUTA7nti0xFTXyjFlKF2PsziWTSqGkZLWpVAx2EGKNNCxiKIcuKnWXc4TgaNtAUSMTCY0nxoCLAr5O+dRidylG5uGESzLEHCIgIzPWQ598bgHKXPTUYiXNuaY5s9qLq9Hmk8ei8yjXFeYI0Bkrfo00x36i308GdJ8y41jIFZunM/pgLpBSf8ccgdZO+6GrziHadPVz83OuaI0Umxhp25ajozXr9cq4Watm9izKl9NEnlJlRaq11ULNIuyFuXkiqgBoxfnWKae5Qv9kLesjX1fVqgxVr13lU6w9igEnmYYtWvb0aU9Og3F4SyB0R8TmGB+XSOgozhjOfNNB6iA2aJkgJ1SruqWrfKeHrGLuHsxCfoV7p1uiEyb1CMJUav+DysL1VT7vULOWiik2UU6KRIprkWaFtCukWYLz6OQPWfEH2ddV4xSRTwHfCfwo8FxdJFT1LRG59dDnI0QfGWuEqE5wBYbRGiin046SMn7VEEtBN0OVdLXUWVPCeU+7WLPoWkJscN6TGA24HBzBBU6urVkNHatlaxFCcLgAzdKhHgYyIVQCh5Lpp4EsgqwziJKqK3Y14pwlv3JtCgkejxruNAt5GqFAqU4ze0DFajMYAYE8wUOXj2Jdg3oa10CG/qyn3/W8/rPvstuP3N+mmjpVso5ca5m1pik1jfZV77qpNG45jWjhQD3onbVfkxoKQ9RQf67KDZsP9qyOlly9dsI3fdOneO7mDdbrJU3jSWlgmpSz03NOz864f+cO52enlKkYF8KYKGS8M8bbWQiupFqGya6WDDxOwR+oq59M+0bX1aSWjdijlJo654Ts38WVHq93iZK42mSkZPx0zj7Bub9FDisWV74FH1cUb5N7Nt7sabordl8sr1FcYNpMqCacWLkmWlnSyivqSAJopqSptp9stt2GU5Ts1AhXciEX2xRNwtnVzbmu0ux3FLxEjq5/guXVW3RHN2gWR5QyUgpM5Ieu64d2nCKyBv4c8K+p6pnIh9tqReQHgB8AuHJrgavCbAU9aFJbSV7JFBvNayNhKvjo7XF+Bti5Kt3q8THY812NCARcsA91t2jJwSNqAF0VkGBdOfVYiuac9RzmCRFRZja+wwTKIU6sN3zFClqUaekaCJqMfKTUR9RkDQAAIABJREFU1KAcaiq2XwZ3aSD/CbNHsa7rdUcQh8dX1qrJ5Ez6kX6YGJOhHeboTNEqUVTRCvX7AXYklWG9Ihlm1sF5+koOKfpFDdtOfCgqAxCCZRm+Rpyl8m7udlt2uy3jOJAmk2kRTCzwwL8qClKsXlvnqb86wLw81fKk2aNY18XJTUSzsb/bLKtFm3mPLz1LPxJlwk8j5EzIyTZQ35JDh3cR5zwWgmIZGAouQIhIs0DyVEew8qVJPTDRkwpIq2vvK4taCNaEDHW2XSVZj+JA6S7ze6k/Sc1G7Ocsgm9amsWa2K1w3ng9KXmu5PKwhf1QjlNEYl2EP6Wqf77++h0ReaHuXi8A736t56rqDwM/DPCJb7mqQRrTKRelBJCsZBlJAtIaW9LJ89fothObXhnHjEhjb77YSNX6ZGV1sSBolfp13tGsbIb96vGaPCbee+s9pnEi5QIeXDTHGVFC6+kWkWYRcJ2DoIwYPi1VJmhLJ61BoMAwjUx5NCegghsNt5mGiUKhNEqWwqiZovMIqTN+z4fw+30U9qjW9YXnrumqXaGjY5gm7t3fsdnsONsN9GNizBabUVVDD/CeYjdUyQkEYpU6CbVZVKSgTm2e2TnEh9rVTlCbdTA7VBtOSAlOz+7RRMf5+SnHR6YGEJvAfr9lv9vz2uuvcnZ+zvlmwzCOxLhAJXDej4xTYrffoWoSLE4sS3JOaLzNrs/QmZTTQdr6SbJHta7XX/om7fKGJo1oTuTpDNIeGd5l4TPf9sIJviTe/MIXGYcJ1Fiqbl1ZUWJH0nNIe4q0II6srcEGg6fIAq69APsluj+F0RHVegvOCVkhu4IvJn6oAkerJaGJXL12jRAcDRM5T9y+f59+TGxHyCp431rKrxkHpnHkHUfrBc4FkiyR9oj1Cy8T19dwPlrwNY2oFsY5OHqAfZiuugD/JfD3VPU/unTofwK+D/gj9ftffNi55rqgkwpsRVGxjvZBTKsSgZTiaJct4jNCtOcWh/cOHz0uOJvYqf5IKwepeKyWKeCDZfk670TOId7IO0L0+Ma+i7ct7n2YTeZrN/cTTeTJuvVS9d+lEj/U96JzbbZUfZ15NK3W854ge5Tral11R1KbFhrHxDBOTLmQysxneRENzPyotjtd+q7FPh9YXbONHlQtuxCH+lBn3UdjlT9EFGZ6uYZZ0mEtZ6hYzokpjez2O3b7LVOaSNk2uXLpde/3E0WtnGMytmrd9YaqXFAzJX3yaIwf5bp6J5wsI3lI5EnZ7nbktEWHDUSlk2Ojg3Mmty0+oqGhWUVKDPR+JJNtMqs4hjyiKEkTkic0TWiuJNKlHHogs4SN80pKM8ZXODpa0bYNN65fIQZPYGCaRsZpJISJ8zyhkzLmyc6XC06UNnhicKyaiAsNY7CaZrNYE7qlRZvAXHz/MFN+Hybi/NXAPwf8pIj87fq7P1AX4M+KyO8Efhb4LQ8/lUAxULKI0uuI5kLKimRlJBFFWJ6siQvlel9IQ6YkuyEFY2JvFpama6xBtSjiCokBweOaaA7ZjSQG9tMARVjKCh8966MlzaJhddIR24hvAZfJvsLyxR8gEaBkElPJTCmRciaX2kxQrXIe1gAqxY4NeQIVWlrTiz+E/0+UPbJ1FQSvnn6c6IfE3e2e833PVgtJCgnrahpdX4X+ILXTpjhs5FXyZBE6jiY4jtZHeO8JsUHFMdAwTslkgNFD6m5rVUjTSHEmriaOSu6xIASjh+uHnu1uw+1773G6OWc7bBmmxLYf2Q8j77x3j92+5/75KaVkg695x3rVEqPn5KiliYGVX+JEKK42jJ4se2TrerRs+LXf8Un6s3c4P7vHj/34a5yf3WXz3htMwTFcj6zahheev4mII65XuKYhXj+iINw7P2McM2dniX5IvPnehmHMnO4H28T255Sph/vvEsksjhcsupaXnrtF00RCbMg5896de/jg+abPfIbVesmt69eJwSFpz74f+KlX3+Lu+R59/ZTz/Qj3z4wpbXufIHDzeEEXI1eOj5DYMa1fRrsT3I0XoV3jQgTmUU9h5vp5kH2Yrvr/xQf3Dn/dw57//pNx6EzWc1u2VowVPLtimC4vuCA0bcSJJ42zF6spnrPut4rVLxGrjWZNODU1PKQg3mqYZe7BekworkaaLtjfUW+RyXwPzEzuzKWtUiiajXiggqzdoQVcU0+laiFZiPn+aFWrxOqTY490XeEQNBY18bpUjB09V6CjVCzeBQ7T/vr8Aiyy88TgWS0a2uBZL2Ll0jTH6bTB+UQMHkpBL7G+l2LyDN57uq5l0XUsFh1t2x5q6TMH7LwJ9sNIP01s9wP7YWC3H9j3I/2QzHFmsTHSIGT1LJIHgU5zdfcPuIIfkT3KdY3ec/PqkjMNaBJTuSx7xn6P88J+u8WXQhsbQnAsukDoAkcnHSoQ3cg4JjqUvleGjdA7xSeDJO5cNgHE4AgYGqJpIuu1OdCua8m5kNKAD55rV9asVkuuHC3wXmBUvBMWyxVd9sRFIhSP8xtcmqkhrcTSxEgMDS52yHKNdmu0XUC0MgK156LFzSCPB9pjnhwy0HuerBGU1KRg/WQf/LFkcBOpTYh3NOuGMBWmncm1VvAQk44UUUvVBFz0CMo+TaTiWJYAooSlJ7hAGSyl94tIXEaalSd2HtpCiRn1BZVMKdYlDVVKI1VVyzFPxuSTFM124wmwamtbIQdQxRWPJ9MQzWVqolCYUi1QP6Vmm4PR7Km35nNykJxBWZrgLQUrdSg5WUMm+gYRA1A3wfPSc1dYLTpeeO661TuzPU4lVFLdSD9O7HdbhnE4jDzaRE9hve7oupbPfvNnePH5F/jkJ1/h5o1r+OARUZbrJVOZwAlDmnjjnXfYbHfcObXI8/7ZlilnxioCl9QYuca8oYkOHwtdGwmtJ3iPqH/AVfn4W9cGPvvKdV4Nb5Gc4hYDye+5f7bFpcxP+69wsur4phefY7Vs6YpnHT3f+qlbtIuOMScrb/WFcUi8+/Y5+37kzr0Nu77njbffpu97Tu+vkVI4WTSsuoZXXnme5aLleN1RSubKid2P1652NE3As4ek7PdbdkNh8Gumdkk4ikTZIu++DXkkCjTOEeOCEDtoriCLI7prL6OLE6bVFdRFwwprIUiDoBQXediO+NjZkbQoJVkTJVcGmrmoWIpNgaeScWps8ahDXL4og1FV6ERrzdIiAhFBsnH74axG5qKz0b42gBNC62t9VCqHZqHInEjP8+ZzmG7OPZdSxzLzYXIklzpfKzOcRpAihy6gF8OBHRjJtTzVjhPqmjit7PfvBxE4Z4MAvuL4LsoWdl0a71i0gWsnR6xXC66erPFOGLa91RF9WyM8Y7dqu8b+RqmTLFX9MDaB5XLJ87ducvPWdZarBU3bHOrgIQZ8CIbhU6XvB3b7nt2+Z5xMjyoXY66f+VWlmAqioIxTQpwwTgbMj3PY/JSaouTSM+QtQ96RykQpxo0rk7LrR6JzDH1PdJDHAU0drQ90TUN0jd0zsZDajI7CMEw0MbLvB5REPwyslgu0FJbBZDRW6wVdE2iio2SljdacdVhXfxoTWkolalEGbZnEHJ46j0lEZ4IzRiVjePJo6CAscM0SjUZers4hNRCSitt1zqYSH2SPmeQD0ljY5ZFJM4P0OFUaaQlOyKM5sR07vAS6uMA5oRcjDs5YBJd8Rjy0Ry0heLpFZ2xGpcWL0LUtmgtxH6Fx3FgukOBY3FgaAD4m1BeG3DPr5IhAnCUzkpK1sElbkmZSmihJSUMygS8dEQ+TeHCeGBp8cTR4ghrgLWtin3eUArm4p9pxqhRyGCgN6JTAJURs80MNphScY+0DFKUfduSi5Dzhoufa8THXrxzx3d/xbRwdLYnBbrAvfPENpgxttwDn6ZziAly9emyk1moNhdZD1zZ88pMvceXKCZ/7pb+Mk5MrXL9xxXSOxLqy3bKlmyw1m1Lh/umW07Mtm91obEdV+Mt5DpAkC5AN73t62hObRMlC03hO1t2B+uxptN1+w9/6+z/CF9/+O9y/f4/7p6fsdgN5DNVx2nq++dYdVp2HaU/qR/bnGfGOsoqoB/Uj0gZuPrdE1PEJ58kK316STRAWR8mFaXtOmUbS+W3y1NOf3mEaB6Z+i6qy02KcFZMyJuWts8yuRN5pTtgWYZs2DElxacDniePWE0MwFYjYkVY3YXmFsLyONEuKmDgbpRyCpwuQ24Pt8UacquRUyCmRKYcpgeA8AWEqJvaUisFT1F1UCg84T4pFNd54On10lX7f6MhcpQBDBIlGDxcbj3iHbxwSsKK+QCrJdLOLq3VNj6rRxOVSTC5DU5VksHh3npN2Yh30Oeo8fM2Fu0sR8jxp8rSaAlmyMRNRKomLXhytsEhXr82sXY4YeUYTbbprvVpwtFpUPkeDF+VsqIuZaO4QGThB1KSm2+hZLFquXjni6tUTrl+7wvrouE4MzS/BmONd5QoFk4OwaRQ9YIEPGMBLb07VkFNTsk/hMBrZxdQmq7U9pZZL4u7Zbe6f3ePs/JQ0jmjKNDFg8C8hFdgPI04dUz8y9SOpHyjjSOkMgFuKWgbpDMPtvUfFKOuKCEqglMLoMml07AbPWIQ0JcZhZBontBTmnK4fYUhw1jv2CENwTGqoiZKnytGrNN6kU4xhO0BcoHGBugjOH/DAoKaOWz8n6M+dPvpqe6yOs2RldzbQlx51ymrd0LjASXMEufDe+dZSpSAEV4hltMkcTSQyA4Ph+jqHbzzdOuKdR6d5ttRGM4nBPvALA56HxiNB0FVCBcZUGRzHHjc5fGNA3egiAoxpYtLMtph4F2lCihKCx+M49o2VA5zVXJMYMjo7g14MeTT9dhVEHUtp8U/erPojs6KFfR4ZMvRlsM3mQJ6i5JwQ9RQfkCqg5wXaRaRrG1brFcvVitDYNFgThJwz/W7Hvp8Qv8AHi16mKbHf9EwpEWOE4ImLyKLpuHZywvWr9rVYrWx9NEOpMsEiiGaCCNE5vIlCGG5USpVFuQRHq92fogaL2g8FNyml7Gmis7rYU+w49+Oen3n9p3j7za/Qn++Z7p0TB+VTL55Age1+YsiZO+db+uh4/mhB6nYM771OnM7w+SqEQNFAUaEvgzmokgzgJ6n+JbuGLidCznQhgxS251u2mx1375yTc4HQk3GcTy0jLXfcdSbXMhTPmEam7V3K7j4dFe/dNvimZVqukMURHN9AuxNGaZBi8hu+7sYKTHneJdNDo87HHHGa8yzZOt4eRxCPl8pCpHVkSgtOC1kzUqqmzAzpEbVuuZ8DvEoOoRbpaMWGKljHThUJAsFIj1WUXC9QLphWdrHic6kMPWWOduZJocoZ6DwgdRLIVabrOu2AFKvbamXrrDriTsRuUHm6GwlZC0nt+0XiU2NuQ4zbGNzMKuRguWjp2vYwS27pMjRNQ5cyi65FVYjeRN9K1X6ao8BclQCMEMIwlsFbZz54f6izypwBlAxa6NqGZdexWi5JqaBiGlV9HslaGLPlNrlU9v56F5ViSJCpbrzDkMhPsQqfFmU/TIw9TKMFAU6URWc66rthrLhZo2IsUyaPE8PmlOAzPoKEgLoOcDZ2iTnO2iWsP9fJu9rzGIeJcZrox8J+LOxGIWVB1ZHFsy0NIy2DX5BcYxDBNKFpD6knCLYxhoALEYmtdc9ji4ZolfWaBcH7q9QfllPisTeHJAs6FsQLC21oCGhWo5IrlsIHCpTMMA5IdgYnmhsPHiQatGhIvaVe2Yr3zhvHfqpQkZ49qeZgDiE6Y5IfsK4pqgfqOKcOVydbrCaWQRNKAldQERvdE3ssUgmPtZCYKAKDy2QyYwX6ehcI2AhaeIodpwJjKQw5M+SJqWYIM1zMB5vZH6eE1DLIomn49Cdepusak9IohTun5yCOz3zykwjK7nxku+3Z9plUYCjCfpi4023Y7vecnp0DiksZMuTJGKM9QhSxRkGdUiol0W+35HHk05/4BCfrEzZnA6dnG26f2gTR/c05wzTy3r17DNPE+bS/4PikllxEYLS6p6bhqRbhU3VM0wl5FDQNdIuIbyfWS0VTZrubSMXA5kyOtBnoVXnr8z9N23m09UgItKuruNjg1kvEe2JsjX4wATkz7M9JKbHZ78kqTG7JkJSfPYVd33J7dKQiaFhRXMO0ukHxLftwQimFtHmdtLuPnL+F77csWiG6FtcdI+0Sd3QLFieU7ggJS6MOVAh1iGXeWQVD47hZQ+UB9vgdp4H9DEZUU9kDTlK0Rmo1wixGKTa/B+cFgn2f02Srf1VHWCeJbM5HKZLJUpDKEh8qoak6rI6BYT6LkcWRdKqMOFbLkgrMtogSg9uIWLo5R1OiBmUCEvb35tdvTOg2afIkjlw+MqvDADo7mMoBMH8mpZJ3aLbHOCfEEDg+WtF1LZvtjhg9KWVSykYP5xzL5RJwJO3xWaFYDbrrOnJWTvXctIPGzDQlUsrklCk5UXLGXd6r1ETdSs6sFgtyUm5ev0YTW4oE+nHEBUc/DgzJur0pF6aUGKbBsp0DVtX+l9LT3FO30YRclha5uwEXdzgdwPXgTb1Vfa03g/UFUmboB1Q9og4J3hiIKjUkPuKjsRq5pOY4dxMpTZzvJ5IKKRTG7NiWjj3FIkvvrD7pG1JcU1wkuUjRyUZw84gnIa4QxNmMfGghdEhcWn3ThQMT10Ud0/r1Fx2Ji3f/IHvsqbpmRSaD75jOr7KXgaQT2Rdr3GC7fCrZKL6CLU67bJEAYeHAGRuRoOCyyY12Du8hu5GkSokWEZZg6Xp23li9F6E2GqrDxEiHcx4RhVjLxr42OaYARZRepqoHnRAKTo0OK7lMUmVfxzKRYnri4gwOEeSpvsNU62hcZWmPTaSZCt6PIDaD7nCUyl+6Xi65enLEZ155meWy487pqUHRUma/79ls9jQhEGOk7ZRFhlyUhTRMCyWp52yzY7Pb0w89/Tix3fWcn52x7BrO799Dc2Z1JdgNVBJlnNid2d+5de0qV4+PIQmbbc/r79ymH0d2w54xJe6cnrLd97z62utstlveePtthmlizMkoyay1xZieNLLAR2tFW/r0aegSzo8Ud0JJO876t5Cyp+0GWm8MZkFhWwokZZMWLNqGoysLCMLpCOOonO8aJo1sU7AyGIPByaZIKYFhasl4UlyRJbL3K/IykNaGqvCtBT5JHSkXdvsenXpi2uOZuHrUIZOgE4iLaHcV7Y7wJ8+j7ZoSIjjBFbtvp6R1FHqGIdZW0YcgHn/s0hlaKjNOsQikOIsMk+Zan3xfP9ZMAFcjTQ94OXReVRXq5I9UShU7h1b2B0FdFfWS2hWf61IzqP5Ac2N6myatNl/Ui70oifGBBpm7cKVyihpnfKqRrj9Mxtg7yJd4lp5aU6vnOueMcd2nCy2ZuSiJRZ+xCXRdy9F6wXK5oE+DYSWNtIBxMMmRXKnoZsYki/qFrm2YUqbtWkvVRqtFjsPA0O/p9z0hRsJ+j/cBL8WoxqocsdTmVNc25FxYtI1hTX2VrhZqJLylbSKnZ2e4vifvjYD7Utn2Q0FXPr7mUFmhfrLOd1qBE9J0hKgnNFtjOkqg2cgxQin0U8YntfQ8ehqvSPHsx0DJnlwcY1b2yf6dJyMwngqoeLIuUYlMYY1KRMMScY7izUfknE2joep6+crB60MANZxpEY+EFgkdhBYJDWD3ugHerWsCHFiUrBwzO84H2+N1nEXJY6piScK0n8ijMIY9SRITxRybc0ah7wFRiivgFQ2ChNqkcbVRNM9FOtDoUCcUZ87PLSJOC8kVcJiWkAPf2kVMg91wcxdBKi6wzKm/JRSkYmUE8RnnlDCDoypD+SQwCuyLDXcuxID7U7aabc4bDijsp9S8CsFHJDpOViu8Os78udWuh5GEw2kgBM+VKyfcuHGVl5+/yXrV0TbKME5sz3u8c9y9ex/vPJv9QC6FMSdQRUqyVL0JQMfzz91gv9tz7+5tgoO7t+9AHnn37Ztszlf4d+8QY+TG1ROTwigZzZnT2++SUmHcjKRhQtKEy4nGCU2IXDm+hQIv3brFZrtj2TTcPb3Pz3z5VfphrLPMD49KPvYmAZprpHKfnITJnUBeEeI1XJ5g+R6adqTuTWTccv/+O+ynPe2773GlX/HKt77M0dU1n77VIsGxTYX9JHz5TuR07/j7bx2xGTx3h5asDt9Go43sLFNwvkPE46RBS2E/3ifniXE4Q0vCjT2ezGq9xiXP1J+QdMf98Rxxjqur67jlCdKu0NAarJBciZPVmoVcOEw3E2p/iHV97DXOUkrlYrSuaFZIkskuG26z4iFFxGqadRJInUVuNfg0LKBkG9kD4+Ws0WYRq0OZRlD1i2Kz05UUDLkEdH1fV01hvpwzo3TROm9eHzkPf4rW6SKsE29deChqUJVc6eUy6anGcQpSUQOCFyH6YKm29+DBQFyOIJE2NiwWC7q2s+53cJW4ODNW3tVpSkYOMsvw1khUSq4RqOF1u7a1Uk/b4sTwweMw0O92ABR6QogsGk/0zkYzc2YYJlJSpjHVzrxlCTjje+3axiQ9QiSGwLUrx5SSaUMgTRPTgQ3r6V3T2VScEU17j/gWJBqHg0sUEoQO0g71kbTbIHlkl5Q4Znb7idhNHNOZLO/CERvh6uRw0XNl4/HBs0uRsTiKj0bS4gLqPOIDIh6pGeAcLc7d98PW5SK4TPEdxdukEr6FZgGxQ8WjswbxpRRBL9WtD7envP8xH2SPHQBfkunyaFajgfIKa6X4whQKXmwsMrhYWZSEySUymV3ZUHLBTyZkHxrFiyMsWqt9VJdnabMyaOVsD0ZnlMoIZHSw4rTPCafQEJAKpUF1BkmQpomsRkigXiqPKAylGGmHmq726EzrZEimkRKnQC7CmM1VF81PdaruxLGMC8aU0QLL2CINnCxXpEZZdscEH2mbFV2M3Lp2xLWTFSUlph6kJDyFNnrD+/UjijCo2IZUDGUh02g15bqNXTlZM60XNpKXR9At4zBw5/YdvL/P7fvneO/Znb3Moms5WjSghbPTc6ap0A9CViG4SAgNoW3w3tG0EXEO5wMn6yXDZz/De3fu8vrrryOqnO33tUb3dFvRQhp7ogSid5TW5rqTsxLV1FwFMn79PJJ2lMUtxn7Dm7ff4M5mYvwbX+TkqOVz5y9xfGXBi5+6xkkX+MUvR1IRPnszc94rP/XOyP2948t3E/vk6EsLBLwLlflswmVDuVAyqgHF4bxHKeyLQzQwNS+jIbM69jgf4cotSmwNBpXBu2BpuTNuiXRJW0hECNXfmD1BNc6D1fpkylazcCqHzeB9dU0LYmwSCCipHLCWRv9mnH1zjTNXYtxc++IzonBmTyrkyqQzAa6m5vMF0kOnPVNJcUs2UPShnlpFxGqNBC5kNQpUBcRZPlUMJ6pSififXscJUBLksVCyEl2kNMLx+hgtsFqeHBxnGwNH6yWLRUcupntvkhqull6ErDMD/6z3RAUw1OmO+rhQiY8Xiw7NHp8mfPCHaaNUHXnfjwjQBmtGTsmi2YLBlUIIiPc0TVNrtN4ULoORPayWS7b7nhiNrcmKrXqhmfS0mlqG6CqIWcSkbrKz66o+gBbUrYzdvT0BPDTnlDxwv5/IkrlzrycV4fjawCIr62gZwLW10Lbw3FBoWzidPNtRuD84kmKNXbQGHlZuE2dSNDZua/ewakadQLNCVE1SJ0Sk6cCZk7VpP3eQWNF6D8slx+l9OEyVPcweq+MUEWIMBpgtmXEcUFUiLeYhbUQrl0wSIVRpjNwaqDxLhflEhwTBLwMuCCWauxv73m4w5ylYilxEccG0hIytBcZpxIkn+MZ0hRRMhMsA83sdKxjfXp8RkpsI2UxTZ9h3o67KtTygpTqQqZAL9KUYk/VT3hxKU+HuOzv2+wEnnitXb3J91fDZl76VGCJHR2ujhxMjmG6dEJ2wS54+K2NujDxC8sUmpHad51KJqoBYbVqcx2EEuoJn3V1FgChX8Q58NCq56zdWOBGyBvoRdJuMwT2s8FFYuRYRR4gd4h2xaWxgwVtmYWlqZrU+YdUnmnZFiANFdxS1UU95ioGcM/OUb6KNSoZobEdpsIGHSrM4FA+lg/Z5XEiEF2+R08C729vcLQNnnz9lGc958+1Trhy3/KLPvcjyuOPo+SNW0bO+numT8Kl7S053kZ98teVsr7x5vmcsmbGZLI5qzCESusodbs0hHUwmp+lObONrFyCuYmZs1FfEEYIpMYg3shjKxWiwlWaMMPvDsI4/3ohTjMw2V03VUuq893y4/lfKXDus3XAsyvivf8+P8z2/+zN8+ruuMQuRiMw653pItZ1KZZWfL4zdjkWTObIsFZwdDNdZLs2qqilu5pqG2/nsFAdEgMx4VPvbWar4X515zlU3fKrNosQlwZ2n0EpRht7GKkNoOF4d03UdJycnRvy7WuK8q7hdcFoqy7t1NsUVRDxe52bdxbSOO4ztKH/qf/3LXDk64tf/w/9QXX936YaAxhnZSwyupmT2PThvUZMYS5aXGlH6xjbQaJFm8NFUUitvZxGHKzMO9aCAA/VGbGLEP8WOE7jA56q+f/MXvShXVXyuEFDnDDYUWqY0klPP+TgyTYk755C0cOfuwFgcf/CP/RlefPEaP/h7fwNNExnLkrZpuHm1JcTMnd2AajahP6wkZJmAq47TgTpyahAtOF/VTENr618F/YCDXIsFlBZ1mu+4+DcysyI9YY7TO8+iWzGcT+SSTQMZxevcA3JIFsZhIrtMcgkJgmsCxc3QHvtAiziUQFZhrBMjZcrm6GqKXCZzvJKtQZSS6Q9NY8bhSdEjlevRapEmVdwPYx37NGII8Q6KkEZqOuksRUlQVJgoJm+8U5PpGJ2Nq2UDxue5QfWU2jQl7t3Z8s2f/SxXr1zhc9/+7axWS5bLzj7IYhtanpKRpZQCxaZPVEutNVl8eRi9Y75psTlloF0sWK5WPPfCCzW1MocluQzJAAAgAElEQVTog2kBtZXp3aEzHoz5/5dTMal3kNW8HN4HLm6WCugWQb2NCY5Toh8mE56bMk48sYm8cPMmTQx86bU3H/9Ff0wmIkaeUYz8pgLzEM343CM5MQ69jV5inWlt1qgIZXmCambIPSlPfHl3l+b+wLs/fsrRcsvp3cy6LeThRdrlildeep5Ey9XnF9w9G0g/+nnunu55977psIVFRLwjxAAiNbApFO8rX29ls6pZpGCfh5pAAJX+sM6il3xRGhSZYYlPIjsS4Gr39XBjHEBxFx9cVYtiRApSyW/nqRS49PC6O8wwTK3gujlynJ8g87nV2JlULXU+/P0DIKHGpnXM7uDr1G60OfjR+j8tUjvudY65fpX5NZR6zF2c/2k0VZNmjSHSNC3LxbJ+2c5f1PgTE3ZNqIxElhVURiWDyFvkUgmK53NLnbpyYg2BGGMt5vv63RznrDX1QYCSixqWOQRXmwWujhjN6zs71zltmz9PZcYNV5alGIxZ/Km1GnwdJmwON1pFtVgYUxs21ekoBqh2DnXByl/OIzIy6IaSM+e9BTopKTkBJeKkpYkdwTUcaceYMK0gZ2qyNmxY18W5Q49DS+XbBLu/S64TlHJpK5yz7/k16gFfM9vBZ/ChmurIhYjWL7yJyHvAV76BU/wSTJ3vOtAAZ8CXsfd6A3ge2ww29e9M9XnfhemsPIddx58EPgFcqz+PwJeAvv78EnAVC4TvAa/x4a7nw+yTqnrzEZznibJHsK4fZAvgU0AHnNbf9cCbPHi9j4FX6rG79Tx3gNu/AK8Rnq3rh7UHrecJdt819XdfAfb1MRFbzzVWuXuHC5XOF+v5FLiC3auPap0/eF3fV8N4wr+AV4EfqxfrGvD3gN8FfE+9WL8caIH/DPg/Lj1Pgb9Sn7MAvhf4iXqhBfhFwAv1sf8Jpgh4DTgC/mfgP/io3/s/aF/YDfQV4F/Hbpx/AnOM/+6D1htzqGfAb8Yc579an/fPf9Tv6R/kr4es5y/HHOF3Y1ia76v3eosFLz8B/Nv1HN+EBTnfW8/7h+t5flN97OJxvJ/HGnF+oyYirwJ/UFX/u/rzH8WiiwjcUdV/q/5+jUWK36yqr4rxv/06Vf3r9fj3AP858NuBH1OtzIuWx22AX6qqX6y/+1XAn1bVTz++d/rMROTXAH8GeEnrh1RE/h/grwMv8AHrDfwa4Her6q+qxwTLNv4dVf2Tj/2NPDPgoet5Hbitqn/o0uN/GvgBLPr8H1X1lUvHfj/wLar6O0TkDwPfo6q/5rG9GT4qHOc3Zm9f+vcOiz6vA39z/qWqbkTkDhb6v1p//dql439dRH4I+OPAKyLyF4B/Ewv5l8BPXABhD4jSZ/Z47UXgDX3/zv6VS8c+aL1f5P1rrSLy+mN4vc/swfag9fwk8H0i8i9fOtbU52TgRRG5f+mYB/7PSz+/xmO2p2WA+k3s4gMgIivMmb5x6THvC61V9Y+p6ncBnwO+Bfi9WPq3Bz6nqlfq14mqrn+h38Az+zn2FvCSXNrBsDoXPHi93wJevnRMLv/8zD4ye9B6vgb8e5fuuSuqulTV/74e+/JXHTtS1X/s0nkee9r8tDjOPw38DhH5DhFpgX8f+FFVffVrPVhEfoWIfLeIRGCLpQO5pux/AviPReRWfexLIvK9j+VdPLPL9v9iFOH/iogEEfnNwK+sxx603v8L8EtE5DeJSAB+D9ZEemYfrT1oPf8E8LvqPSkishKRXy8iR1hP40xEflBEFiLiReQXi8iv+IjeB/CUOE5V/WvAHwL+HLazfQb4rQ94yjG2WPewdOEO8B/WYz8IfAH4ERE5A/4q8K2/MK/8mX2QqeqINXi+H1unfwr48/XYB663qt4GfgvwR7F1/XbgbwDDY30Dz+x99pD1/BvAvwD8UD32hfo4VDUDvwH4DgxBcxv4k1gX/iOzj1Vz6Jk9s6/XxECgrwO/TVX/t4/69Tyzp8OeiojzmT2zyyYi3ysiV2oa/wewBt+PfMQv65k9RfbMcT6zp9F+FfBFLK37DcBvUtX9g5/yzJ7Zh7dvyHGKyD8qIj8tIl8Qkd/3qF7UM/to7eO+rqr6h1X1eu2+freq/uhH/ZqeFPu4r+2TYj/vGqeIeODzwD+C1ZB+HPinVfWnHt3Le2aP256t69Nrz9b20dk3EnH+SuALqvql2jH7M8BvfDQv65l9hPZsXZ9ee7a2j8i+kcmhl3g/Yv91bNb0A225XOrx0RHb7c7IjFP6KoaSmakImJlQZBZRkgs1OqWyKpm4m/cOL0IXIqJKGUe0KEmNCXxQUBGi9zjvWHQNzgdiu8CJIyJQMnlzhuaEU0WBrMZJFVZHiPfkECv9XME5x2q5rK/tMm/TBcuOdw4thd1+h5bCl1/90m198skgvu517bpGj46XxGhaPaVkqOvmnKOJrbEMVYb8nBLKrCKq5DxRtJBzAux5ttbzvj5/Fnxli7vAUNv6e1SVaTIpBO9i1SUy2rgYA0phmvaUkplyDyjO2ce/FKO8FZmZj2Yy2/fr1MgsFj+/rspL9sYbtz8O6wpf59pevXZDX3rZ5gw+kNtL+IDjX/2br53Z6oMPf81f66UDeum3evidfu3H1fu6lELRwjQNlFyYUqU1rIoU84nuv/fuB67rN+I4v9a1/DnvU0R+AJs55eT4hO//bb+dH/3xn+D09Ix379wl5Uys1Fw5pfk5OCfE2OC9p207U7/zEURISpWF3eCdcu2o5aht+bbnXiCmwtnnv8iw33NnHNhm5QsjFOe5ebLmaLXgF33rpzi6coXnPvPLWHUrXu7WuN2G+z/y18jnZ/i+pyhsJOLWxzz3K381HJ9wd3WFEeF8c85iseC7f8UvZ7VcIlULXtXU83IxWv9l29Lv9vx/f+sn2O+2/M5/8Z/9hWAQetT2da/rat3xj/+WX82tW5/AeU8/nuO8cHRyTNctePG5T+FdYL/pSdPE+dk9iiacT5QycXr+HuPUs92dAspytcL5gAtt5VRs8S6yWl7Fu4BzxuA9pQHvHdeuXiHnzOuvv0ZOhdXqOjEsOFo9R9ssuHHjOkVHXnv777Dr73H77EsoiePjawBszu+RNeHCgHNC2ywq/WGDqlAyII4mrnEEnK5Mvni/hVL4/b/3v/g4rCt8iLW9vK4vvPgJ/uxf/L+pyszMGuSzvMT8s6uncJQD1aOd+AOcp1wERxdB0FeHH/X5OhOSX1IGODjA+v2raP9S5ebMxc6dkon+DUNPyonz3ZZ+2PP2O19hv99y5967TONAvz2nlEQuI6rKX/ihH/rAdf1GHOfrGDXbbC9jo3DvM1X9YeCHAT71yit669oJy9bTR+iCQgzcuHULH4Jpa4uwWq3wPtC2jbFvO9PrbrvWuPfEozmzO72DaOa4geNFxy/5zGdw48Tfee0NdEr4Ak4Un82xDdstsYwM9+/SUhh253Rty5VXXqIthTZtmTYbdve2mBjYCd3xMZ/4zu9iipHz199mt9lw+5136NqWs7v3ISV8tKgzpdHes7eP0ub8lO1mw5tvv81+u/0GLvVjta97XW/cPNZC4Wx7hogwpi2xidx47gZt17Drz9Gs3HvvDnmaKGVCS2FKPSlNnJ7dp2gmNi0+eGI8No3saDKxbbOia1e8/Py3E/yCNATGceTNt34WQbh+/ClA2Z06pmmk7VqauODW9edp2yXr1TWmccClt/C54doqEqJw8+bzaCm8kV5lmnZkzlCdGCrzeNGMiuBCrNIfGcQRBKQ4fLOogkgfG3vo2l5e12/73Hfq/f2An/WGnPHSXuQB9n/n5n9ZeD4zXR5opA8Zgn6V6555ct8ftpZD36XU18Thu3LhOC2C1AsSY73g1azCqJSijGNP3+/5yutfYrffcPvebcZxz/nmDjlP6GjSHMtoLP/D+PBl/UYc548D3ywin8ZmhH8r8M886AmqSsnlQGkfvEO8Z7ns8CEyDKNpfzQR77wJY8lMfWcExEZQaldlTpoWbcuibYnBV1bx+eLKQRrYMvtMyUKeBqZxYLfdENsFWQRtGvzRMcUFyuRQFWS9RpZLcnAkbAH2+x3nZ6cMMXL/3h1yGmm7iAjmOAWksQh6Gga2mw2bzTn7Kln7MbCve13nKKNkS9FLyZQi5nxKYhh2lFQYhx05TaCZUgr9sCOlREqmK+qlMcVJF3DijT8axbRlCk4CwTf4dokw4l2Lc4pIQFBCMBayGBqaaITKbbOiaxYIDi8tXgZUFgTn6eIJirJor+KlNUUCnRANFEw3Sg96C87SOcyhOrypsP4CL8Yjtq9rbRVlTMWuMRgzPrOcyYVncX52rM6OH/ygAHogGpY5rPxagahcnFIvnft9aTeXKyd2ZI5YZ6eql56hKpRS2Pd79vst290pu/2Gfn/GOA2kqT+U5pxAqK8zuwvS8g+yn7fjVNUkIv8S8JcxtpL/SlX/7oOeM44Tb7zxJufn5wxjz2LZ0rQtN25cw/nAnbv3SFNiszlH1WqgOWf6fo8qhKa50H4pShm2rLqGV259G8vlgnfee49pu+fudse2H9gq9GoPR01zKGVlt9sw5cTrP/m3Ob52kxde+TRX1keE42Ny23I+FlLKnOmE25/xzuf/LkPO/MzPvsH90zM+/1N/HwFSf8p6veLGjWu1zpYQ71idHIMTxmFgt93xs1/5Evv9xwNG+PNZV0GIRHQyUTxhgpzZbe4y7D1jP1FysU0tm0hfmhJn52eUUmiahhgjy0VHExsWTQsCY9pRSmKzuUvvlyz9LZaLazx36xptbDk6XpDzxGZzH1CaEGh8ZL0+YrFY8/ytl+jaFW2zZN/vWC/XqGbOtgOqkWV8kRAi7tYVpjSw3b1L0QkXTFVxSj0pj5zt3yOViWmcgMSohehbrlx/keg/PgzwX+/a5qKc9QNpGq2em01e25cR1UzOGRAkdHgfOTq+QgiRJpro3ezinF5O9Tn0Kubj80M/UCNNLvlVvXg8oiBKrtpfpVxK5dVUdPf9nldf+xLb7Rl3771GmgaijgRX8NGjwdFIrrXzKtHSPtwtfkO0cqr6l4C/9GEfX0pht+9JOZOzUqSQS2acRqTeUNM0MQyjaRKNU3WcVsz3k6kSCt4izjSYMJczqv7dMDIOPUPOjEVJUjWKtNTaYyYXmFJG3cSubHHNgvPNuZUCxoE0DWzSQErJdJCy0G8yY8oM+3PGfsPQbxFguzlFyKwWkRBM49l5R+hMNCpNiZJGvDMBsY+Lfb3r6sTRxq7WHjNj8TggTyOahP2uR4viq2xBzlZzSilfIoctpDQhAmEMiJsj1kIaJ9SPjMNAE0ZETAsqRA+SGKYdlELOk6kxOkcMga5t6bq2KqsGgve1YSdocQTf0sSWRVeIaaCUnqITyETB6tWK6XFr3XhLbW45sdfowseLcfDrWduiyna/Yxp2aMlIHk1rSEdgbqYIEhI+NIR2QSxqcstycV20hpQfHJ1/rTDU7FIgevlNXESXOmekFyn8nM6nNDFNA/vdhv1+Q0n/P3lvziRZlt35/e72FnePJZeqyuqubgC9oGFYBhzakKPRKFKjRpnSyDSjwFGpUaUKjTSjQIH8AjSaUaDZcISB0WYwGLDR6G70UlmVWyy+vPfudiic+zyyQaBryKnOKiRvmWdERYR7eLzl3HPOfzkRWzOdkTbnTVtsndVsuLRK1lr7hX5L79SPM6XMi1d3LFmIRbjd3yIivLq5AwwxJWop+rHWcwlnsBjj6LqNjob1AWsNXYABS3SOCcPr/Z5lf+I2FxYRFmuIVFKZyCWTUiQVzyHt8AamdKLub/nRT37EZhyR6Z6SFm5vbxEpjKPebJtx1J1ununyzMZnjLFsOth0QhcqPuh4YeugcwnrPNs+UEfH8IPvUHJ5l4f6na4Qej756Dt4ayg18ubul5QSSffajL+53VMFhs0Waz3ejRhbcb42lDsTy8z98TXGwHbcEkLH9dVjwDIdK85lpumI8x1ZDjjvGTaOeRFev/o5KS3kJdGFgatHl4Qerq5HxnHUWVBi6HqHnywpauDbbkfGzZZhEyglMcyGlGfu9i/JMXNajuSy4ILD+A6yoeTCIS5IicSSH+bdvIdrWWZ+/JP/i2n/EqmZ4KxOE/UOawzeKfugisf6wCEmxnHLZvwm3rqHHmjLGG1LKdeS/qG4pv3k+lX5lVD6K40BA6WuiVBFBMrfBIuAXAq39685HO64vXlOmk9cOCF4T+8cUFnauHHvOq1wWHul9QuPzTsNnCsCZqwHa0m5aEArGlRqqdTa6AG1tql66NhWYyilYsQgVvuXtR20mAveZWIVEkKxlmotOB1H65zRGdl4XBcIw4DvenwJWGuZphMiBRtP1JIQMljBOsF6wVp9H10wlM4yjtoy6HtP6Bzeg7e0iZgKSFlTcaZiLYx9QMLfR8/of7tlMHjX6TFAsGJb2VSRUqlZqBikOozrGDdXGAzW9pSSyfVEqQv7wx6kkmvRyaeuxxpP13n93HrAUqXqOFgfsDlQ20jmlCvW1NbbMqScMfPC8XBkv99zc3PL/f2BZc56TsXgjKXrekp1pNqDkYbcrxQ4h/cDgp7TYgspKFhk2nTU93XVWjmeDsT5qAPZfBuzXD3WGmp1LXBWbBXmuOB80NESKwLfXuuM/5h2XdAC49s1/FsgEIhWijwANdLyz4dA+rflo6LDAEuipgnyRE/BucrWO7yB3guIxYkGSmOdvqbRkeBVzNcr47TOMV5cs52OVOdYXrwgxoy3bXR8aeNHa8WA8i6NJXQDxjjEaAlgvQbSJIVTLnz65g3bcWDX9Qoe7TaY4PBOb+RHO49Y0wCDkd/53e8TQsfN/ULFMh/3lOXEo60j9IGPPnxGCI6LyxHnLFZAijDtI/MSCZ1DMHzw4TVdHxhHvdGcozEAMkYydZmhCgFBzBecib/Hq1ZhOiU6Z6g5E4+JXDMu6E3W2QsET+eesttc8od/8O9rJmgKpSRu959yON3wb/7yn5Ny5OLyKUO/48kH3yf4DdZvMMbj7IDzHdNicAVCeASyoe8WpMyc4h2UwDxt2BvHX/3oOfM083/8s3/G69dv+MlPf4xU4dk3PuHpU8t0LAy94fLqmkol5QURS99fgAkMQ6ZSGYYN1no6NwKQnsY2YvaA8P5WEiVH7m8+Z7AVZ0G3CoOUTCmGVEAJuw5TK/VwQkRbISvXdc0AG7xGrUKKiwZXr3Psu6Agm46Hbs+plXmaqFKpoji9NVani3adZq2uFfm15aimUktiPr2BODHsP6eLJ/7wSQ/icKYBylXfWaobsgjHnMkihJrOidgX5ZzvNHAaDC54QtcROuVoWudwdh0Kr29XeWMG57R/GUKPsY6CB2OxXQe0eduN+By9g67DBs+w3RA6T7FQjGBNQawh9CNjP+C6HuM8xiao2oszYpHaYyo4A66NgDWYtkMKxhl88IybAdAem3OWFXg9E/ZXSK4RpH+FI/+eLu39VXLKTNNMqYmhna+uG3Gu5/LyCRcX1zx59CHDOFIkE/PMYb7FuhnrNljx+LDDhwus22H9Bu83gKNWi2TLaUpYWxqToZKTp2RPyR2IYzoValmo+RXHw4Gf/vXPuHnzhs+ef4axlnFzSRd6bm5uMcbS9R3WGUXtbYd3PdULXTciInRhi7OBzm8VCHOJUiKn6dTI8+/n0nI4g1v7ldJKYw2FVWS9WTFVmGPEuoXjPEETJhhjsE57hjllai0sDew1Iekc9loxRvvfazpZa2FZZkoppKq/1xoVVAxSMcYirdftnNfcsyQFsNKETSdCjQiFvnMt5W1jp4sgVUhGz521mn3aFqCNNQ/38N+x3m3gtNCNjsurC3ywPHn8iBgjvXdKBzAGZy1d3yvxvesx1lBb5pyt1cBpAjlnXr94oSBEjFRncZcbhqHj2R99V4GIBg4drKEaA66jlMqbuyNpOXB3cwARrncb+uCIDIhz5MMB4yw3QwfW6I5pDV0/Yoxhc7HVoGqEWksDNQwiCj4ENJiuvR0T7Htc0OmG4azltD9wOu756U9+TqmJj7/5IZvdBd/85re5vHrC7//hf8Dl1SM+fPYJguWzFy+5ubvlF7+cuT9ESv0G1kPoPsaHkTleE5OnFtv4eJFahVL21FqIadH57BmkCiUPIMIP//I5JSfubl+y39/x53/+pxyP9+z39zjr2O+PXF5cscyRJ0+f8Mf/3h9zdX3JRx8/pRtGRCCXSNeN2hawXjNetlqel8IST7y6+wUxzV/14f+NLdNm17vQYQ3ksmgrpVTAYr1vF7mjCNzub/B+j7GW7Tjy4eNHypO+VIHCz37+S5Z5YZlP2ke0Bu89Tx49wjuHtNYcom27ZT6ScubueCCXihiHtY6Ly622UozQhcAnz75BsBb2ryBNhONnmBIZfUacIdUNpVZiiprJlkQslZenE4KwGTqw0nqkFWfrF+Y57zZwGkPfBTabAWOEq8tLUop0Tnsnrt2A/dDjnKKiGENBU+eypnbiSEti7y0G0QzRCs6BD5bdZqOcLGMobQdLIixFVQTH00ScIzllnDXt+QYjSqYtqUI25FIQC9UYrHOEfsQ515rjuvvZRgy27b3blqlaA85ouWJbIH2fl4iQcyHnor1qUR6YNY5h2LDd7ri6vubi4orQ9eRSiakwz5n9fma/j6SivcNaOnIOxNjQzlw0cM5KT1uWWdkYcUZE8LZDKWqatcw5ktLCm5sbDod7jqcj0zwR04zFcTzcI1X47PlzlmXh2bOPiHHh8mpH14fGFzUMvdKSBIvohQBVsyGphVLyWSb6Pi5jwBvBIr9iaqFJSavIzvckOCqmFuJ8wptKXEYckGNUatA0Mc8zOc4gVZOi7Dgdu0bnW0t6kFqVtpYzcZkptVKxWOuI01qNCq4G8nLCWoMsJ0yesSVhpGg/VqC0d++sPTMlVFnUJJZoa9C03rk0Fs6vW+80cHrvePbhE0Qek3Pmw6ePKEUvToBqNGXvgoI2odPeR8q5pfG6K8QpMZ8m0v0rRAqPLgLjGNiNlk3vePxoQ3AeZzypiqK708zPf/ZzDocjL56/wGD4xgcfs9uMfPTBY/rg8STVume9Ie72B4pUxDuGzci3vv0J4zgQMGdU0bx18awXkjeKIPqmudYm+PsbOWutzPNMjCpVe/r0AzCVy4tHbHcXXF0/4eLqEcPFFjsEbg97jseZH/7lT3n58g1/+RfPOU0zw2YkBI/NBe8jXXerx7ZapArLPJGaZLOURKla6o3byzNwJFJJMZJy5DgdmNOE6xy+Blx0SKnc3b3isL/lsL9js9lyc/uaJ0+fcJr+MY8eP+KbnzyjH0YuLy/AQqmJnDP7+3tSXrg/vCTGE6XeAvGrPvy/seWoXJoFV0q7nh3GBOwQAEtumIN3CqQ9DgPGQF9nQiyYowa1u7iQSmE53lJKZrAGZ8F7BfpevXpJqWCcx1nL1lusBW8qjsq2UyWQYh+VId7jrWHTe1wyLC+ORCOUNGGNsO0CzniQClRsTlix9M6TizInqhQ2VulwffMqKCmSSqHEubUj/u71zjPOLmjfowTHVbmglkJtNIAiqjwJXsGWEILSD5IGTiN6ky44nFSGzlOr4C2t8VsxRgjB0XuPNQFyVeFQLkzHE6fjkTjPOGMxtWARgrN4Z6ipQK2KINZCyalRHiqSA8FZgnfoZdOy1LfIvLprNZWSacIz88UqhL//SxFQHwIGYbvdgakEH85yPTCUqhn/aYocjxN3d3vu7u45HWfmJeL9gDVCihWpBZGIwWDEnsGClCKH+ztyjggZ6xzWeZwPWKtk9FyzcgzNWm4GvNespgja4xLNWI2BN69fIVJ58eIFJWeurnbUOhL6C6XQCCCVlI/M8cS0vCHGEzHtKSV9lQf+N7oMMBgwpilrXEsYvANjqc30xFvTevxKQfLO4K3Qk3Fi8GWBUtl6oRjoncFZg/eWUoWJgjXaw3RGA7YTMKZgROiccn1X9ebglMUyOMFZwdSC9i+Ve21aWFM5ppBLbiwdrVhiSaSctHKQSimakdZSFPjKmfp1CpwWobdVuW8OhsuNMv5bapyKutsYqw41Xded020jYBtNoGwrp9ORu893LHHCSkJKIS2W4oTN2DH0AxQHSyZPM/P+wP7Va46nE74mgnOw7BFfIV9TxHD75iVSCtu+VyJsji2AgvGGXgobqmqVEWpOze2nNP5XI+K2Y24a4TuvGfN7ukQqYgvPPv4G1lqW6TE5Rw7HWxCYTxPOHXj5+R3Oz9zvZ+7uDvzVj37Kmzc33N/dUktlux2gQs4ztSpHExGsGGrJHO7vmKcjP/vrHxGXGZGEDx3Pvvlb9OOWy6sn2FbyYS3juMO7juvLhbk7kedEsYkuaBslBJVovnjxKTc3r5imA1dXV7z4/I949PgR3//B7zKMPa7PxHzi58//jGm+5Xb/c1I8sb9/rTLT93QFa/iwDyo/tgbfqaxyCA5nHX2n7BaVwxqGrsdbR2eUrmScYGwieDC94Xe3OmW7ltpKZiGLcLczVDH0jco0pROpJO6PRyrCxRB+hQN6uVPJbN/ohuSsCUvRUjxGdduqNZNy5sXda2JOzDGRa+EUF3ItpBwBoZ8a57QqHTLm5esVOI0xqvRp2l/TVBerDj3nBw2BsWoHtiLbAFYUnauuQskMfY+h4Kyc+yK18T9LqZSkgMJ0PDEfJ0pUKWDvHcG51pPJpDiRDZyOB5BK3zIVKZnGnkZyhpyRnFV1sAZMUST5HDiB0thVsgbO9J4HzvavCwFvHVJHpYuc9Lypdr1or1Iyx+PE4XBgv99zOBzU+KNtRMUZUoxY52hXAgClFHJaSM1nIMUFKI3fm5R/W9WUQ59lsNbhnCeEnlKKumuJsiyM1T4ZGJao5+fm5oaUEp999hnLsnD16BGb7cB4AbmeOJ5umJZbpuUNKU0saf9eo+pAU2m1zN0anDN0LWPsGt0vl6JcXhvwVuhc04tkek4AACAASURBVKubql83RWmFXquPbFqCATiBsenNB984nCI4YGl0o8Hbs5GIs5ahVYgPeWVtbbyimWVOTSWYSTmRciRl/Ty3CtKIYiOggX+90szKqPkCOPfd9jid5fHltu0e5mwa4KS2jLKVu283neH8saBlb8qV3js+fPKUaT5SZEKMUKsnRvj81R3e7jkdZo7HEz/8ix9zOJ2QFBmd5cn1js47Ns5iZeGzn/+EVDKvb96oF+OHHxGc1xRfBDGQneN4d4dElVCKCClG9ZFsdnjW2HPgXDNP1dDW95uOZIRqs9JGMFQ8hUytAqZgKFgqwVlKrXz26S959fINn/785+wPB1KaMAaOhzfMs+NwOuJ94OLqCc55VAGnmUBMC84Z+j4wDlu6LrDxlmAEUxb9XcYpXqHODYzjiDUwjCPLYpine6yBftzgrGP0gVKFV69e8vr1a+7u7tmMIz/+yY/YXY781nevcX3mVH5CqgeOy2eUEsmt4nhfVxHhtijrxVtHF3qsc3SdwxlVieVaeXM6IsBFXuh84Hq7xbWyXX0MnCZCdNCSCTCIM4pk5wmDEIJiHNZbxAauvAolXGPdnFOoKkgplIaSq6dA5jAdyaUypdzEFxmRAjURjBB6pS152wPSvq9J1rr5q69rt7Lw/871jjNOmmyrGdPahopXdTxyK6pmrG4E7Xma1K/ZC5oBimaXtQp5ZTGI7pDzoqDBtCzMS2SJCzklOu/wzrEbB4J3dFUPXIx6E6QYsdY2LbI6q6yvW2ulpEwOuakgRIEtqcgqbXDt5LZ4b61K/ax7f2V5wHm3L6VgUBXG+fwaJcjnUjieTqQk3Ly54eb2hnk+keIMrbddclRKF/7stuOcw1pBqjv3jH3owDs22w1dCAx9h/MOIwUp2iuv6AZbi+rdRcBaf85kz8oTY7DOI03FJhX2+73q4ofAaRrYXCb6Uaj9RDWRUjJVMs0h8qs66r/xJQhlZRWY1aHMgpRWTZVW0raws96XzWDcrNVulmZOrdXmesSKKEfaNNTeNsWRNwZphHuMsl4wpuEHmohIrdSSNQYUxSSkVkQK5hwx9A0Et7JbTAucGneq0Wo3oiBRpWqBqTfurz0275gAD4M3Z4fvsyFqKdqrqHoQjX3QXklzBq9oxply5c2bE/f7Az//xWecpkkDlrV0/YauE6xfFOmrlaUKvvNs7Mh2t2PoAh8/3uGNEPe35BTPbirW6clNiO5qopIuYwxSVadcYkYa6d01Y2Xb26ZwCs2ERP+zTfmkpgfvL6peSuVwf2IXjnQh03X6tw/jFoBpEU7xwE8++xccjjP/8s/+nNPxyHxq5hFoxVFTgurxLuBNz/XFBV03QANyblwghJ4nTz/Ce8uzD5/Sh8AY1NLuzc0dS0rcHGZSUT8EVaqo94ENPZ215JxQQMth0UzKuYr3J1LOHKcjp/nEaTrSdZ67u0s2O8/HvxPoRoMfO6W2uIX3OXBqogPBKmBT0gTZMJeg5azXJOjpdsQYS3DqIeGlQCna268wV8Usxn5pnp7qvHScM8YYrrYjwTpcbW080yqMZiic10wwK/1svV9Xt3ZjBONgtxnekm2KKsHWlEtAcivpaQ7wWSugKplSC0t8KxB/wXl95wLqWrKqdIwGOyMgda1tm9KmCVvr6ppTspKSMeRcSSmRUobGMFvmpXE2PVUMpSry5ztLFQhBm8ubYWDoA9txxFNxMRCN4JPD14rzKqXMuYBon2Mld6sxgRLwTQv86wagAVIzF2NM421qj01RSH9ubr+XqxGWa+tVGxXsY6yyHuYlkUvm5V3kcDhxd3fHMs8tYIpmnJhmAOIY+oGhH+i7ntAFpBpqVeTcF2VCeK+KMh+c6tNNbW0RdV9SAMKuQpRzhmmwWBcwVKx1KuF1HqyyAgTOROllidSaub935Op5NCk9yQ0Wi12F11/hgX8HS2p7GGpT6pTS2mjWYaw7q+xcK6cfiOwa5FJWhZG1oj/bWjbTsmCN5dFmgzVvucGvHTrWyQrNeKOByGt7ROQt02RjcNa1563npcWV1jctVpAKRVq4kWZ4zNsuS8rc/VrxOHPO3L55iW0pu3cOUFmXalFlVXABUPKDyF+AVLV/OC0FMY5vfvu77A8H/tW//j+ZloXQz2x3F/zW97/PxeUFHzy+JM0Tt69uSfPMk6tLNkPPR4+v8bZyDLDEhRQcIUYO1RJj5vXtPRa42m6UsL/baFk4bujGUUnS9qEkX7PJKg8ngCqQVRZYmsHv+7sMTrxSSix4b7SkM56UE59/+pLjFPnZ89dM88Ld/S1IZRxVOjvHSHCe3dNrtpsLPvn2dxmGLZvdtVYAccZby9XlFfM8cDrtAThOkXk2mBqVv1kt1Xb4XsUUu2GDCEynEylF7m7eUIoQulFHZAwD3jn6sfW8gJQSh/2enDMpLiw58+LlDcPBs7m27C4d3U4NuNfr8r1dVZBlIdlMsYZgHWItixectXhrsBVsTVjjsMEqnawFWGMdlcqb6UipQmc83lp2fU+uhV/e3uKd49n1NdZY5pIQBN8APt/K+tQCcQh6vSiwtLbKpPE1FdSBRgk0OlNKqpBqoVRhzpUshZgzuRamZaY2XwypSl2qor/va5ZxNgf4dmBry6G1lyjqeiTK4Ncspu0m1mtAKpWShZQztRp2F5cY5xmGgVwLMSd8WshFe1zDOBKc42K3I3lP7z2dUxqFt5D7HjGiYxGcI3QdVQyTTICh6zxDHxiGnr7vwDQNRVMlrbNT1mC5ujlZ/bOQ2soEk3mfI6e16jDU7GQx1rRWB+SiHqzH48J0mjSLW81cZGVR6E3W9xuGccs4bOiGAffWkD5teXSEULVFIs3FBqDZxon12iPr1LWn6wf9dtXhej6oUz+yKr40Y2pNWbzX26EfOly2WFseenZO2l+3/lubccv7HDpVSVeNXYcosOpsTFMPSVPnSes9YpS8blg14Pq5iLDkQrZC71VZpmq7xoJoWaFyUZrnQwuctvUcndXWWanK6V7BIlm5tmf+jTQKozlnrLVlkvXtUn+9/tZT2Pqo0n7u1613y+M0bW6QcD4wxkBQMhGmZqgVSRFEGFzAOk83XlIE3tzv1ZH7/g7re37vD/5AA5Y58vrNK/70z/4196fIpy8/I0nh+9/9DuPFFf/gj/6I5Xjkzae/xBvDdtw0WlTBThPl9S1VLLuLa/qhkJdK5yzf++1P2I49/bjB+o5iYC4JIphmoXamGzUqFMYQgj9zOAFMfb+16l3X881vfMLxeGo0oRGMYV4yx+PC61e3nKaFmrSX7dBKI2edUDpurxjHLU8+/Dbb7QWh3+GsIzcXndpaOZtxQ/ABUODONIpK9R0IeGPxxtC7dbqlEvIvdjuV9nqdSTSdjo3poODi8aTu/F3vGcbA5dUOYwvYe4RE4YQLcP2BIQyF4vdgC2519XpPlwikIvhg1egjBMRaSgiIs/iup2I4JVXopJI1owwe5xxj8FQnPB1HppT5xd2JCvjg6L3j+88+onOO0EyrQ6MeOtMkyisFScMj0oQNc5o18ILiey1+1EYdLFUz0dI2vSVn/bxqL91bq88NHWfrQ6lYIJuCJPl68TgxD7uHZmItRymKkklaMFJxObXeoDbhO2spgkoZqeS0YEUBGRscH3zwAcYZhqFjSZnj6cR4PKoRche4urwihsD05rX6OFrtU1rrWu9SQSrvPIihH3oG79huR7ajendig84mEnWHWa2pqFoy1HPgpJXxD6HSrL5a7+my1tL3I8dTc70xBuRhvHOtmVryudKw1rXjpDfYdnvBZrNjs9kxDDqyWaqQUmwZn/YunbVU7/AhqDNWSzSs6I0ja9+xqZUe+l+q8nJeHZb0GqxamTTzh9VR3lrwncd7ix9Ck8BbrBPCUPFdBVMQSkOJv9JD/xtdgmZzpQE6q0fSw+XcwF2Na2cu5NrztOi10DlLLra13XQKZRDL4H3zfRCMCKZVnpiKNDaGBsT19de5Y/VMW1wTVf0o5+vlnGW+1bNc70kjtJ6soWJRFfyazLWPX3Bs3rmtnLWu0UNUoV5y4ebVDWlZmG5f44zw0dWOse95tLkg+IA3lSLCzgvVFvJ0T2bis88/4+Lqmn/4j/5DjtOBz9685uWr1zz/9FPub+/4we98h2dPn/D7v/UtrFQ6U0nLDKaQ2ixvQQjW0DXnpa5z7D7+mLHvePr0A4Y+UEXlZYvxCA1dV0LGGbx6cKzXnqYx5gxKWfdA4n8fl7WOYXOF2+8bvURdrXabHVKhHyqpFs0ciqVzagS92QxsNiM/+MH32G53XF99hDGW6XgkxYWbm88pNRNCr2T2MGBdoNZRj3ejgTmrG3DJio4eT4eWpeqNWEuT2s0n5dyWSM2J4+GeXDLzoo7+MWW6IbC9tITB8PQTT+gF21nEZGLeAxXj9Oe9ucTigJ99lYf/N7esIQ2e7NW0ZigZXy1bo+fRSVRTnpaAdG00ycY02LYqqDsGA1j6YIlF/S5dY9MoOg8gSJr1fhKVQ86nAzlnTsuCwXC51YpjGLdgPcl6isCSYlMKrcoTrVBWsrsC0aq1FxEkqcjbtPeJ0daLoWBMZfQO+YLQ+c4d4NdeSG09wRQT+/2ROE/E44nOGmS3UXClFDCZXDU19xS80T+QapjnmX6TGDc7rHdcXV0zzQvPn79inmdOpxPzvMMHT7CGcTPinUHSRC1vKSKco3MCdvWP7Bj6oNxAazFiz4FPRBpquCJ6a7/kAYlbe2rr5mDec+TVnBkFjS9n/Xnj6DpP1zv66ijGk6shzjpwT12lOANtOekNklPUR47UqoR2ceDDis5qT1MzHTk/SsnNTWdSQE5ayVZ0dtUy69dLTkpxW7l/VZqAQgGCRibGeYMPBtdrPz63jEeBA6s+nebvz7C2/y9rzfAQi2ssXY8CN1YEJ4I36mEb7Ophq9f8etmfM7n2enNUifRp9tTgUIZlJU5KT8ui52c67smlMKeMNZbBW6iF0HWtYnWsdcUZkZeH3rm89R7WO/AM+siasZpW+TZ0Hb0ev8jZ/50GTh3WNqn5bCnc3h84HY/81Q9/RI4L153nYuxxHz7FCxzfvEZqZZomjLNcfXjNaAsXA5xyZX9/C85TqqPrd3zvO99nt9nx0x//ghwjL1+8wBtY0nfwQ8/l42tKXCiTJ8eFshyozvDk8lJ5f6YD6whDj3OGJVVy0flBYixFkoLljQe4jkF9O2iuS2/Y0kwi3u/AiTFY34FVZcaw2bTRzgkbhGfProhxgwk7cja8enFkWTL7/Z7peOKvf/wXeN8c3p3n8uJaL96qGuRakvJ5cyOvF+2FU3QTnuJMTpn9/o6YIvv9fQucrce1TJq5nI6ICH2nrAjnLN47ejTod50OdsMoGBmTQazBdfo3imuIbjb40PPso99lCFvgf/9qj/9vaEmp1OPMpuvonePJMNA7z67rzlJoYyCIBsbOqj2kSKHI2nukBbNKKTpB4cXdPdTC3RvLxhs+GMBRiMc7ctGpDrlUDsuCsYari2t1iZdC5z2lRHzo8JtHYD0OFb6oectKi1uhrIfpl7mmlrhpJoxTimFFJ2XGlLR36sMXtmDeOY9zHRafS+U4TRyOE/vDiZISO7vRvkbL6lKM1FJYTkdccHhzRXCmuSFVUlqIi46adWvPs3G5pPWu9GBouu6cU328c4jTaYjO6UREUKNUaV+zVmccieh+KFRyrS0RTuhxf8hC3/5Y33K0Nq3f876v2jyuDLTxIWtg0vaHMRXfB3KC4A05qUNRzZXTad84lfN5vKz3Dqmr16XmFVqKGdSeuj2/VnJK5BSbjl1VYGdVSS3M04mUE/OsIJBr14Jv2ukQ1B5N5Z1+vc+VPypGe6imOZ4LWNMT/Iar3QeM/cVXc8DfwTJAQOiBHuiMIRjQQlbvCVaRCA+cynOWVx804CsftNasNKCSuEmFyYKL4KmU+UAphVOuZBHmlHHOkWvBFstpWRSA8pZQK32XmpGI1/djmrOmNBK9vP3OZH0n5wF7K8quHWuoVml01Zzf9d+53r3JR+hINRNL4tPPVRf88vaAFeHpxQXVeGJK2FoohztqSaRlZtgMXO16sjGMHSwpc7e/pYrh9cuXGGt4/otPefnqJTlGzRiWSIzxPAWPJu43DUEb+q6RqU9nvz9EyClq6XhO6xeqCDG3xrPm+bhGgF8DpmsjBlY3pFWyuX58X1eRwqnsWcqEwRJLBKND7LoAXdBLt+uU4yd1oZSFnCM5F07LAdDhbN5r/7PzQctBa3BtOJo1Dxc9UklJM815OpFTpqQFaiZ4NZ9elkRJC/v7G3JO5FKxxpKcBRPwfkMIgb6/UlqSUTAoxYixQqlW5Z/WI1TmU8EazwdX3+LJ1cf84ff+Iy63j4D/+qs7+L/BFazhm2PH2Kk94NDQ6JRmGrqHaeOYjaBTP+uDNHKlFgXnMDVjaoSycJhumeeJl4c7nBS23tI5w3WvrkoFwVrDJnR4rxvZnDOf7e8xxvB0vqDveq6LpesGLi7VQd4TqCLMJlNEyCvf04A61utG2XWOUguv93eknDllTYRCNwLCqakJf9165zzOUpSAGlPicJo4TjNLKjggpswSE6dpojoDy9QcilR/GrzOsg5eB6PltBCXiTjPGGvJWb30pDaaUJN9qVtSIadEiQtpnqkpUlJqQA5vcQ/VQKA1ZM59ExHOc8AbpPeA7P2Ng7wi7CsI9j4bQegSKkkzELQUqvWhf6mPVk6tc19EQSXXJLgA2IBzeknWRhWSukKcOjfdrNlfm6NeGmJfm+8Aa9aLpo3SspxS1HpLzqRBpUI555S2ZD2CR0w9T81cFUg0nqc1gWB7Lrcfcr17xtXuAy62j971wX5nywLBNGNuHuh3tcia1Gm/uVbqShEzb8/rWQFUe25d5aL3Y66FJUX10qyWZA29UwcmsaoerGG9orT/PCUttb0LDLnSjZFqHRdWBQnWWHU9KoBUBX5kFV025N60nqZByfCSiZLbzyjyv9Ty9aIjlVy4ubnh1d2Ju8ORXzz/nOPxpM1fhJ+/eMWbYJHDSzbB8ChAHxxPHl0zjB3jZqQ6y/XVjihHTr+8YZkTh5sbQtcxeM/QddRSSLlwPB45Ho+cpgkrlTefPieeDhxffU7NEVB7MTOM6kZ9iuRSVY8uQszaC1kdrtsdhHG24Qd6EtbguTqgr0svJLRv9h6j6kKhyAEbMqZalmWCktkM2kPsOg+0UdBZKGLABS6vP8CFwNOPnmK9I+ai4ExU+755f4e0Ms06i4hSgErrYcUU1T6wRP0euslSm8VcTSAZQ8UqBwXnLF0X6Lqu9TR7un7QEp1AlcIUJ4TMcX8k50LYJEIwPN5+xOXmKf/4D/4Tnl5/k289+QFDt/2qD/9vdEmtLLnoaOSi13OlSRx9xVZDFGlgaGmadc86+aARwshVeH08cTtNFBPADYgNGlqdlsmLqGGPaf6e+7jQ1UpftlSB/bywpMwvbw90oed7buT6wvH0kxHfD/jGi/J5oUqha2260krylNRu7hhPLDlxV04sJXGoSWeVJnVrmqblC6vEdwsOibAskdM0czpNzPPCEtN5bvKSEqbC7SESvSGMjlqDklehBSm9Gb1zippWzSKdheAdfQhnDl9dd7iWeU6nE9PhwO3NDTVHus7igmczqkmBtaoz16pdoPH0TON6KundYNr0PveWecc5s5TG7uchcL7vJh9KWlbiMsYgueoYlOZHcwbRVq6rtA3IOp1h7r2WZI2/Ka3EWgUhpWaqGEzUjapUnX+tyLi0jFLpYKVkYlp0OmKOpJwaEl7VI2El/bX3fR5tQhOD6TsGLCUZSgBTPQ7P5fiY691Tnlw94/Hlh/R+Q7D9uz/g72g9INPrZJ7Sus3tiNWKGEs12nOutWCtVgumnf2Vqw1vpw76Gn8zNK38UL/yfxulKJVCESGVQiyFU8wULBnlmFarIJ6YhqGYSpHaZpW1z6WSRI2M5xJZSiJJJlMppvU6hRZo1RPq1613jqrf3+95+fI194cjh8ORJWbGYcQaw5JnUs789Hiit8LxInAx9vS7LQwDU0zYAMF3DF1i0xnEFtJ8hzMbnlxf4ExlOwROkilpJi0TaVmYS+Hzz19y+/olf/Xn/5JaEh9/4wO2F1u+9egaHwKbi10b2+QaqppYpV7GGMI6Y8i6FhybCUgLpGeC7UqUb4O8lGT//i5vPE/8E6IkJaYvFikWeo8prumAFQiSrNZfUoQsajAbn/8S51RV5r3noh+VLbHtSDGyP5zIuVD2pbnaZBVRqFsu1jhKKdzf3xOjouq5ZPXLrFWnYQoY43ECMWbAUwrYAjFGrCtYnddA328wtqdENYYIueNyu+OPv/ePeProGT/45B+wHa7obfdgnfY+LlEOrGsS4+rWTXIdx6vy6VU6rUKQJok0yptUIxwtuB9vBgzCi/s9tWTmnDG1cOHPYRZnDZvmLJ9qIdfKi/t7sginJRJrZa6CMZbt9SXbqwuyTUxSofF4D9M9qSTmuJClsBRFy2MpFKnMbfhboiJW6KwG7TkVjBG6zv26owJ8Bag6ojZfy6IOR+qijvaancNUYSmamRyjxdjMYYn088I0L7hayUknKVrUIiqnhRIcrg86y8TpQ913miejaTNvinCcJmpOai6RekXerSOLQ5omWhFFgDVworNVMKq3pM0SMuYcNNde3aqxFXn4//c94/SmI9igUsX6kF1qyXNWFbefb9h4KRQpCipZyHkgeI8XpR+nnEhF++GllHOptYJvZ3tCNHDO80xMUW+MUlrfU94qu5qRQ5XWI624svZBDdiWJRtVIlECpkJnLxnDFY8uP+LR5UeM/ZY+DJiytnDezyUo19pLy/6bN8BqEfAw5VIz9EaMbg/bKjOLdQ4nsB16Yq2N06k9bFULtUC8Kn2qgFGifMWAzaqHRx+18aPrev/XpNlv63kvJSoAXRNFCrFqyZ7adIhU0xn80eGKhirKtjAr+P51AoectWzGLTEmptPM6TiRciFYzzB0PP7gMUYqr19mphT5/JS4jQt8+orXx4WnP/0lXd/z+cs7DtOiY0Zt5rh/Tcknhs2GPB8ZA1AtQzAEK8R5xnY926tHxFxYjCPVRKxwionnn7/EeE9ErcuefPARIXT0Q99G/TYHleYIvyzxfAPCuTI/bwDeeUCa9V2jQb2/9xeIwcrAdnNFSom4ZFIunKZIlah2cb4nJXWC974nJuVVzjHy5u4FpeQzTehyd4F37kERtEojY3wYvvXWhZ1SIufCfr8/k+DP35eHcO2aP23JlWQK07RQiuB8h/XN8tAY9Raw4NwWx8DTy+/z7MlTvvetf8T15SOC3SD1PTenRh3g7+LCtQ1457SFYQxF1KDD9z3OOmzwOmitDVn0oWsziYbmomQZEX5ns+HqeOCHL37JtGRq1oCH7RBrSCWTszAtM0WEw6IKvIsLnaNug8dXocaZlCNvbl9R6sx4YeiCJ5dIqZU5LVpquwcSkqkG10Qv3rWWTTPh8a0954yQTWVKX7OZQ2t2Vso6k3pFntXAeOh7jBF81wFCiYVUhMMU8WHh9v5IPxTmORGjoqkiQpxP2jGRSpxmrKg1lW9jRud50kFMANbifKf9GKf8vZgyFB1Ub31mM83UIvQNxV+zpdW4o67vu8oKuGqCsmY2oW1aLbBK096+r0toIi+jXFptUVRSEkQSZtWcNHaCMcrrLEU5mMs8q6hAipaFpTa0252Bt7eH3q12Yusjxtgy0nh2e1/7l2JQFZqhacsfTLSlZTdVRE2023hqleIZdZ0vDqTHyIh3W7wbHyqOZvTyPi/jnFaCzmFdOIu5jbVKFHcWF5qBd6P7GO8Ubnfae6ym3T3B4jpLN3j62WPW2V3KXcc1FkvMWW3gkg523KLYhroDKQ9ce56ZVBKpJHAqVqlV+5lCM0SXtU/bVF8rT1y0a7vyUFsq2yoltaH7devdBk5R+du8TMzLfKavWKPzaB5fXxG8Q0okzjOH2zfUlHi9X5ii8Bc/+UxNIHwg5cx0nKgCr58/1+mG1RBzhhjpneNiM9IFx1//7KeaBVZDjInHTz+Cmrm8usB7R6qVFDOvbw8UgdvbE5vNlt/73e8x9H3z7GtWeMVQUnmYI9Q4GVIr02kCBIYR5xxd3///gvxeSuVwUARaQbg7asksp2Z6woiVnhhPLKlgnI6wSOkNMSbSspBybPxb4e7mrh1WnUsVVsDPPPSUNfNfzh4BZycqY+j7vvELmwJoSU1K5/Des9ls8T7oiJaGttpSVTNtWr+tAYKLFd68jPQ2cdgLfYCxB5xg3PsdNJ33PHr0hF03qjig6/W4OKsiB6eGKUOv3gQiOriuGkMBZhbtk1YdwXGoM7OLfPTNRwwXgRevX7FMC8dYWIxhExSUfX1cSEWo1tEFz9OwwXlHTCdyFVI1UAzZVBKZKU8k40hJQapVIG1bm0YltoU5Kj9TZ2M9eEt48UiFKUZSUyydk6C/Y31h4DTGfAv474FnaA/1T0TkvzXGPAb+R+C3gZ8C/5mI3Py615KGdOscmPLWxa43SRe8zgLqOs0mG/ImVYgFYhZchcF5HJa+77VPYpTUXnJFcqELvvG6QKQyzRPWWGoWckyYJg0bho1mpaVgbKbrIrkIOWaSi8216a0TYRRZdw3sOb9/DLURgaH5EDqr5c3XNHB+uee1EmUBW6i2UE2imkQWwYgB46nVEJOQklCy9ppzKeQ1e2xou7Qeto6JbptV0/67t47nmoFqr/ItEK6px5x1OBdaaa6ZrvMB7z0hdE0lZM+vD2Bqy5TM2odtjNBqqEVL/JJ1PrexVvm+X7OM88s8r2AQ76jOYpxuYliDeSuTVHBI+ZpF1pET2mLJbzm2F4RUM5XKduwpOXOxG3Xo21TIIlScBtxcyaVqhipCqWBbdUfDEqy1nCfa1oKpaiOn5YZtZ27tXz8g62t70wo0lwAAIABJREFUTSv1xuCoCmotuZxR/C+DAJ+B/1JE/tQYcwH8C2PM/wL858D/KiL/jTHmnwL/FPivft0L1Vo5zSdiXFQW18wyOm8ZgmXs9cJ2PoALTEnISehsoO8G+ovH7C52PHnyGGsNTz94qmVbUZ++OOvOsrscyQixLMQpYanEWPjpTz6lpMJuGLjcbfjgo2+w3QxKehbhk1SZ58hPf/ZLjDHkmMhtuJt32qsRhM4Fai0sLUNaGUgXWx0B4FqJvzrdO/e1RNW/tPNaTOK+e0Efgnom1n1rxSi6nk4dy1x59ebEPGf2x8I8Rw77I0ucGnPB4nxQ043SQJw2RCuldM40z2R5OPc+lfhuzkGzC4rOD706wBtZAMNmq0qh3cVOh/K1PvX6+r1TZ9iVGN8PHePYsd1uGIaBZcmcpol+ihhXOMWv5XjgL++8ItyysCBYsbgSMdUQxJ0pZsaATxYRZS8AZykrq1DENBaKETrg46tLnm42+N8Xbu+P/Ks/+zExZiQM1Fo4LplcChsfKLXy6m5P5x27ncaGq8sB6xynJWEsxDQgxrGU0s63nMn7VYSl6FjgKCoK9s0RtlDJUrmbZ1KpTDE3mlv5wu3wCwOniDwHnrfP98aYfwN8E/hPgf+4/dh/B/xvX3QiDOZv6LubgsNavLM669xZzTQbhytXIXin5ht9RzcMDMOgF7cFqYWSFmqpdE5RV1s6klQOOSHG4K0lm8w8z+RU2PQ9GEvXKwCE0d3M9eB9oO+7BqppP8Q2vfl6LThnWilozhxUnRmvN7ZdaUvnESH2a5d5fpnnVajkMGF9Qmqm9Au1FEieaixztSylsqRMjIUUy4MKi5WupQYeFd1w6tnE9KGX+f/gzLZ1Hvpn3n6s6HiTwjYTDx+8ghjGgqkYo5mvPv98r+tfJZlSM7kkclmY44nTAu5UwCbuTzcU+XoFzi/zvFaEuWRwqtf3KDG9Ngd907i2uahuPeYIgG9TStfr3lrDOrNovZe8c1zuNkgVxrHXnnNzFBsGbY9d7kaVUrbWQG1MinHoMNZSciElxUxc+VUu9TrcsVahiJyzy9UBCWgc0xWdX5F61FH+C9b/qx6nMea3gX8I/HPgo3aSEJHnxpgPv/CXBc/jp4/xQQ9sFdoQtY7tZmC3HcAYUilMS+YwZ2qpDMOIH0auH19xfXXJ9nKDc5ZN7c6EWQvYZlc310Kqlbtp0iavGPaHEz90Vg/y0OHHjjB4Qu9xzd+/N56+93z87CklF7wDIxlrld8lbRaJSAYE52obadpcYoIoZ60Z9jqnJ9r7r2XGeV7/rudVfGa5es6xRt3IwoIUg8w78mx5+fnEfKzsTwtpqcSo0tgueKzpcfZClT5Fy+6UKph8lry+baKy9jT/tgxUy2+lwIgo2g4WH5Rgv9vtcD6cBQxdcHobNZln8K5xQ5XKdL+/Z148n750zPWaj1+NbKZA/vwVqUy8vH1OLunLPRlf4vp3Pa+5FF4c7hhCwNqW2BjorYJnrnnFrd4NpvE9JSWsseyGAWcdoW2KUbRCm5cFawxPrkYuR8/ye9/m7jDz4+evqMbyx3/wPcYh8PTxBQK8vtkzz5GXr25wInz325dYY/jFz16Tpsh03CE1YLo2LNHUs/KviLQH6z7MIppRViNUZ/BdwFSh2tqoauXLoyMZY3bA/wT8FyJy/2+bQRlj/gnwTwCePLpkHHVAln1rnkwInq5TykNFTTJy+6P1/etJ6YeOYejoeq/jWUUR7847LODbjuKrOhmJd01vrFMYu+AptRK6QOg7QucJncNZzW6qsYhYus6Tmz5aKLjGC5XyQG3R3a0RsO0KWrSUhTWTWgfQ6ePruL6M83r1dKS6iWIWqk7vguIgC9UKsWTieTqpEtjXvuQ6T2idNyWi2YVZaQhvHba/6UK1Pn9tlaxEv3NfreiG51z41bne69MNSqWyq4N8bY5IOmsopYlSHPv9HT7Aze0rlhQo9hWpztzt35BL5uu4vozzurvakUrGWIOrOgPdGdOua6jNN0ADJzij0uJa9WeiVLyYNgLaqB5HVAyxzkq3DoaxYylFkXqB3XZku+m5uBgAYYkL1kpD1htH+y1udEoV5ytd1/rSLevM9WF6wFkxtv4//Mqm3P72RkP9kjJOY0xoJ+F/EJH/uX35c2PMx233+hh48bc9V0T+BPgTgB9859vy7KMP2V2M3B+PLSA5njx5xJPrS7ousMTE/v7A3f2BInqGighYw6NHG54+3rHZbLQHWZQ+8n+z92axmmVZXt9v7b3P8A13iCkzKysrq7KG7qYRk4C2MWqpZWMZ46Gx7LbcthseaPGAkIUlGySEBBKW3H4AiRfLshrk4YE2D6BGFpKxZCELP5hW2+BueqDpqq7KzMoxIu7wDeecPSw/rH2+eyMrM6OyKzoy4xJLcXVvfPcbzj37nLXX8F//f1f5AX3FZRn8AW5rJeCIhYdnl/zC7a+zG0buvXiHW6dH3L53ymrRUtJoNHbFtNX7zjGJkKY93jUcH/W0bUOajKE6F9PMmQvIM05zJvVIyZoGIVDTmc9mVPKk1vVzXz3W7M/RJuIQGums6zl5kghpTIy7yfgy95H93s6jOGGmGrNmUAXTNx6cksZHZ//r5x4izjnanCPMXKFI0zTWNN0UB9arUNmOIpoKuRikxpcaoVR851AZ4ne7reFBpxHvhHi55d2jBU53rI5bTl8UJBSGvHss3u/TsCe1rvdevqsxJxLGK9snhxdBZj5OvWrsHuB3AsUbBG/Ke8iCYM3ZZWNA96QRUSXudtaADZnQG8ZbEI7WS5aLQOgy4uDeCz37vef8bFXpKAecE45uLRCEzTYxJbi36PFeiCmTS2YfrY8Sk61RqI5TU23+1PQ/RoM/5WLItVIeN3D53XXVBfjrwC+r6l+99qu/C/xx4Kfq95997Hs5oe87+q6lawPBG9NR37d0XSXniIkpWnHYh2DF+mDciYtarF8sGhBn8AMR2rqQQSsPipsZne2G0ikzjpFF11JU6Wvk2nWBtg1k6rx8tomXEBw5wZgTORsetGlmLKIQ9Po8O4fmz6x14n2trbnPXm1ztie5rqCoZBBTORSRunsJFChzNzqnOgppNH1zvbtk68aKYJAWuRoq+NBP+47I8wrsPk8riVhGcOiaZkdKEyKeXPWQSq3ZOV81iKaJlBJjnTzSGFEnTNNIGITLywuK6+hvtSY3J/p4cZqnbE92XStOtdb753uq1BBf6/3W1AWbs2Hqd2MSs3KLsSRVlEKN9KaKrNGqK+2dMbpT8bcZIwtvGk9OSte1uGgkxw5H05koYtFZEbe+v85Bp17RQH7gsZlbtVz7v9qw+6EW+nH23UScfxD4CeAXROQf18f+fF2AvyUifwITXfmxx72R946TkyX37p4wxYFb71oX/e7dU9bLJe+fP+Rys2O7M5b4O7dOaULgqBdu3z7hhRdPuXf3lNVyBQjjMKFA49tDDWZOHRTDF1JAc0Gccvt0TT803Ds54vRkzfFRx6JrGPc2y+x8S5waLi86NmTee3dDnBpC6+iXDdSb2zkTdYspH7CcIkJbZ2znC81kcGfC48+cA31i64oIHBhxHKINWhrSAHGvTLtM3M8QtMg47Yygo9LAzVIkM93bHLl/mF13mlfPseaOqyn+NM3plsFWtjurhw7jUJt3gUMnCEM9qCrTOJFzYrfbAUrfGt1caB2ugcvpkjwMrDmiD4Fu2eD8zV1XJ7BwldxGhMbNG04FoJdi4naNu0YdaE5FmLHyV6gSp8Zw69WaMNEHqzMGBe8IBIoqFykxTMowWmB13HQ0bcu9e8axen+7QYpy78XKcXGZDhu27dlCRuyxiptXVcZkROTR+UPzOSuMOgdaBtQv+VCT+0j7brrq/5CPvuv/tce9/js+MDiWi47VsqPvAsF749d0VsyfplmWwlLwrm1YrwLr1eKgb240bZiUgireyyFymKdFbDcU1JnUp/fWDEjJE7wVtn1dbO+N+dl7hwYlBGfv6a7ql+LkkJKbrndllj50hk2X2znrOBp+MBp+8DMIR3rS62rwk7k2KTX6M+5GqynZpjM3Ew4z5DrDieaq03d2za8d8yOf5z4ioj9oz2DHYNC3fMCDztIIc4ySq4hXjOmALxYql6gXfOvxrUMagQY0FDQUfAfu8XwQT9We9LraWllgXbgKsBWLuK8pb9mjil0HmNqliNgM+NyZUa69pk521bq48zVDUSMRL3iMkcI2uRB8Lc+ZQzS5FWFyuabdBVHTeb9SBTNHXxQjNla1bjtUNqT575FDxnjVW/loe+pExt4VXn7xNkEKb73+OlqgwZQHx2FPjCPLrqVr4dbRktVywWuv3uOFe6ec3r7Far20HaRUYc+ilDJVIlXbeQx+MjcOClMamfJI13qrT6ZIiVPdWYzI1s6UkB00jZUP7t65RQg2d15Kpl82FdjeUwrEtEOLHqQ2xDnEGYAfYNjZQnzUDX5zTPDqKK7WuGpjb6pcjQRFgsnzeh+qDHNNm6A6q3JwWh/8+qDNYPj1el0767bO+/2+UglarXNOu4YhH6IR5iYectBlr5yFOGcM5Iu+Nb33ZUfTB1a3l/RHHaevntCvHe52hH6iuc2hLHMTLatyMUVyRYmsuh7vld5XwmKxxDtpNpIUjE4uEFDnDBfrpCpJGp6zUEmiAdSba/MR3yj9urWJnQA403UK3jFVIHvWjDhlvWxwwbNqF4gIsbVSz3YawcE+ppqKO4oKQ7QRylgjzoqNORAMWZZktHW5FHbRrt+Ps6fvOD2sFj3rZc+qa8m5VJU8pQmOtgksFz1FYdl3LPq21kSrfILz6AGgKjOudg5y5gpKrZVwiHLEiald5lxTDw6gvQOkRa3DG7yNei1XC4IP+GDYTO9M/tQ5W3CLQM1xyuENZ4d9VSW56Y5zVnVx9cxb16CAy4gH3zpCW8cYD91tm2sHu91m3nYAmyuu7y3fybA/Q5G6rquTQJ5SssljiJVorr/iygHXqLd+jqvsVTM6w64Db41G52ttPeDbQOg87SLQLByhy/hOcY3i/GevOfTETLGJHrFIc1Z8miN1V6/5Us9n0is1S6j70VwirfXKguE5qed8nrpzzvoIrlxlg17mmb3r14fdx76Oe0K9RrDpJLR+Rq1356LEbN+THoAXlAIpz/jgqkhbI9A8H/vH2FPXHFosWl556S7L4Hn9zi3iFDnqGpq2pX3xLlMsdP2KXJTVckkIHk+m5Ghz4sVd7RQ+WKo9OybnDycGqLgyR9svWGTl9M4p3X6gaTvarsMH+zJtbGrc7jk6PmKxLJye3qp12WNCE+g6X3fagDpYr92h4HxIPbGSQ1El1u7vsm1x/jOW0z1BEwoNe/K8eQTbxbu1ReG3XuzZdZnzd3fIlHG+wVcYkFZuRa1yCgYDGusF/ejlO4Pg27al73vu3btL3/ccrZfklHj9jTfY7wYeTudQchXus0jIbozqkK+9rYgYmbJzhApbcmIbdNut6PqGftmwWAbWR47FceD4tCW0ShOG7zjGm2QKJl7Y2H1mabtNduEc3ncgQqznVbzhOzup4wwpWRe9UhQ3wSIZY5CHYG+KEPCN49aRZZI0igtCi+BxBOcprrCThPOO1aJDvM2q281uDjaLAwe+adCcubzYmKbQNFmZoQmH0nZRZb+vpDDzmK0zWknx/rE9v6frODEnt+h7VssFx+sV0zTRdy2haXEEQqMs9xFV4eTk2NKqPIAqU8xMMc+84hjLjlbQs00eyFyrmD9TBO9tVKttG3IuuODtywUjJT40JwRxStu2Bm6XWZOmtZl2uYosD7hNHm1m6NwsUnPcVnv1B/XNm2perONqzbl6fmxyltB5QgfiDV42j0bO3UvvvE0MYVKuMJPfGr7yEMFXxznPKofKHO+dQ52z6+IQ2D+mISdXE0a+8kZ6723NamTsnDEDhcYRWqFpoWmhbQK+MUGxz2DT74laUVPiURUKFrWlRzrRs+bozKp5Laqr94Q1vNWGReBASDRnhDN2MniHipGTH9j5ZZ4Cqz5SBPEO568xvnPVMadCEK2mWeoMfD0e6lVxPbJUmEd2D87S4B0fe16eOpFxEwJ3bp/Sty0/8P1fYRonlqsV4owPc4iZfSyEpuF3/o7fjojya7/6S4Dy7nsP2Y+Rl198kSY4UhwR1cNol4rBhOrZOEhXtF3DosB6vcJ5GwPs+gVdv6ZpAtvNGVoybWMO7vj46Kr54Bxd2xpuNF2N113vos8A21lzyHtj5QmNjZKFpq032c00h6N3KyYXUISkltt1vUmRdIuGNBifom8amq4jpww5mYSsmGSBJkFyxvmEaq2NQdUbusJvzmWPkjNxGnk47Igxst/tmMYJkpGLuBpJqNT2RZk3R4ss27Y9zLbbBtlVVIRUKrWAazyLU1jdKqxOJxZrZblY44KQidjtdzNNMe7TMhrsb6iTVTsRKDY2ixjTe3COU2fX+eygsspcFLGUvT5fTa4UlUrkU2vbrvFoMTyuILhaIkMMAmVkhda0w4M6I+cYauNPJvuorEbAnATUO/rQo0CspDCxUgnmNtQGZuXNdebRS/7w2vp1+xR01aVGfy3Hx8dMU6Sts+OSHUXSISJZrpZ4B32/IOfEZrsDcaS7pUrxyqG2CTCHG/PudKiDuspCHQLOG+mtDfIbZMUinLoFOjl0A11t+hjOT2oHWK8FGTPzuxyaQGD0ZbY7zrO6n12WpCdhgsPL4tB5ZT6zvoKjr9WSjZnKG9ohfyBimzF31x69DnSfL+b5XNpkkFDyRIrpET2jAxLvkeu/Vszm9a3YwRDsmGb2JZ0dZ6W18w2EFkKjhKZ222Wu7d3cdYVahqqe0PhnTXgNqSm4VK2pa45GawMm1wizXP/FHEFSDl1wLXMzT2qpxmSGu1KgahilcsVuNHf6dWY9quxGXu04itp7W/pNJeeBUq4m0rIaoXFxSpqVGpwdg/8u7tWnLNYGU1Scb2iXjpe//Bo5mzxoLsrFdoTNnjG/SU6CbwN93/H5L77KdrPl619/nbZtOT0+ZX20YtEaS0uKqUIJ7UzZ/VPJCBAQjzhP0zW4aeD+uw/ouo7dbsJJQ7dYG0BBI1C11wWcq9AH7AKKyaIW52zqJWdXYUw2Qtr3fQVcVzBwxb+5my7WJh2d/wJTfoCR80xWlvFC8Y5cQek+tDSNEamgkIaxsiiZVswUxwobqryKMnMANIfa5vVu+2a7rQzjNlJrDZ46zluu6drPssFyncijOs7gjDfV+Vr+EUJoccHRdI62F5YrYbFy9EtH2wMu1vZjQfn4yOTZtzrSCrgsVWPJHmhd1T8PJj/hNKNFmOfkpnmDs1eQqZN+JRDEIakYYXRWNFtFLsbIO2fvIx5keQcfPOM0kaKyG6w95XOFR8VIjoVpMFamxXJRGz3m0JuKnuiDlfCS5+DgVWFI1m0fm1nEr0oIu/DY7fApExkbFEGCObOm6/BFKRkkF8IEoTFJ3lwKU0qE0tD1C1LKjJOJge32A6EJ9M3CaiK11iI1xJhT51xZ312dRJqJV8dppCjsh5GmaVguzLFpjrVWUp1nAefqjfpoaHvAKULhqhYjhw6frc/VFMNNNuuQ93WzqjPfgFNfNxKrXVptcibiyMys3Lk6yxlrqTrHrcqsyW2fI49sQKWU6lzdAW87H4/McidXR4nIla729Vrp7FBnxn57Ngdd+KYNpoga5vHMShJBghvtOOXwzcqUFa8yO1LAc6W5Xoo5rHmFZu0tQ02Y40QET0ZFmcTo30zV2SgkY4xMtak0pYSnsB8nUixMydYvl4CKULKB2KcYoUbC4qiM8orWdXW1RhrU1VTU/IIXKykF5+xerkxX342S1FNWuVSGsdAKqAi5acwpJcBDu2ppChA8sWS+/fY7HK3XvHDvjnF04hnHxJvffpvN5oj1175IF7wx8VAXtij73d70tivLyUx42vcmzLbdbim65fXX3+D05ITXvvR5msbAtSUnUhyAQghWX6O3cUvvTb5NnEUwKcVHmhZd1wHClGY4zUwqodzoiBNPcEumODCVPVkmUCHogpSUcVRiFILvaZvMGAYbw8xG/DGMe0pJpGKw57mpYCXJKyLjprJqzYMGIQTapjF6slwYdiM5FULw5OIM74ttgHMR4BBpVjiTr8xIOZeqfmmbe2gDS9fRtp7bd445uu3p+wkJyjhtrCbr9jweuPJsmzh/EMtVMaRDK4adDV5qepspBYZam+zmjSxYRujEWdshWaq90xEExuAt8xgLJRXiZZ0tDwUXHJfjCCPcv78lR3tOCA5ZQ2g8oo44Fd57eEEBxpXHt8Gww2I1yyACvjE2p4oKiHE8EGAX1ao5ZFhtLUqZniA70hOxiqHTuuu7KoGgCBQIxdKy0ARKVLa7/SENbpqWvuuYRBjHkWE0ktNSIxrFWKJnItJSHadeP0He0zbNgTleuYKoqF7Nrc7pV1E9jGvN9RJq53Z+zRULz1VkdNXSn/9wvdmBycEyaMJo92wX14KxUxVqHdHE7Iq/Kl/M8+VatWKYI84PCDXpoX551W11ztGEYDruj0T+VAUAQd0MW/kwzs6rjr3RiUF2GVdcnRZzNKEhNA6RCaRCcjSjpc7n32DTa19zF/zATSvWibbyx0x6Y5He9db5obxyXe9HZ1KWqmaZCrGSvZhqnxDr73MxjKbzIJ7q5K7qp1pF/a5wo7UlVdeYMk/ymbz0mCyzSbW2Ojchr/sBysffsE8XjuQc3bKveCqh8z1gO7yq0mbrlt29c8pms+W9d99lu9nwpVe/SN8veO21V9jv92y2l2y2ym4YUKzGiSpNZSXSbHraOdbvxRzbcrmkDYHf9dt/EAVu37pTo8RMTnPhWfBNi4jWVM3BQebULoJcpR2sJGDRioj93zlH21g319Xmlb/hPQSlkMse7ye8TFWN0kFekaNQokOj4F2LNMrR0TFT0xiagYIMtQlRGd/npo6ly+6KDauiGur9SfCBJjS0XWf63/MEl1QFDGeXt2Qrvxgspk4whWAjuk5Ik+H57Doyv20jfc0B89sEzFHmjLflfWSTvYlWVBliqjPoji7YAEgXbFDEhkE4zLDLrPrqXEW42Ez4vpK6ULvsXUU7FLUO9hRNuDE3tvk1SxOF206Wsi/WPU6k9jRsdl6ptWsvnLx0igJN21mvwdlUUycgRdCN/R33Lx8wlcTg1VAfbYMXocUc/0wAMn4X6/rUcZxU7NW8c4FFBEVNkj7UyaGcjLE9JyO9Ld6xXC3wXpjiUOfV5SoKkatv8w1kkYHDl3kyocH7wOnJCQoslwuaMGvP1G64Su2w60FjhVqjqeHnI+k5cKixXXlHrf/mCSa9yX7zUO91eLwEgi8UHJLkwI5Usjtg8wxXW79qJ1sOBcmr9ZTKYhTq85zLhyhfuYoQDuH8tdolM8qhoh4M7mTvLWJ4zVCbdonJ6qqlIi2qdo242mmlIjXm5oG4ivkLc5xzY83ur0rSUbGO9jVPgsv8xPrTVSR/eI/6dUBX1A6dqvUQbPKurrfjUJJJ2da1dcEcXOMrD6jOpUqrQXfmC2ZYoq+RsC+K5sI4ZKYpsd+b3vrYAt6mw9QJbs4UK3+nfhcJ4lN1nKrGKl2qxsscSZRc5TzF0baez7/8EtvN1jqlIjx8+JBl33N6eoLICae3jnHOxNp8CIRK2tB6Ki3/7MTszw8hoGBdcVVOTm4BcnCYpmioKCYbvB82qJYKXL8i89Bsut65sh61TVu7vRWvGYx0ZJpqDSUlTBSuvxrrvIFWijLuhUV4iYWPrPs9uRQuH9i423CxZ7x0uLTAq8c1DVDoFguUQtgGlMxMpm4brK1PaBpWqzVd19F3C3LOvPPOe+Rk+lBaYD9M9S4SXBNo1Wpn8+DBYtFTtHB2cW4YwaajaRqOj08Q4DxOpJKJJSE14/AenM/gTTojJislGVA7AB7kCG70lmgoheA8XoReBVdAU6LUTaqoEK3ng68BRHtQeLAMsO+a2hyyEctKHoevSfVq0WNkHPZ4W4mlcy3ZhDrZpx8gA5+B+KH2OFpXHXyy6aY0TExj5NtvPWSYErs4ki3kxQVPEyGJsK0X3txIaoPJHX+cfQo1Tg7R2lwTLDrPFttO37UNue9YLRcHvFZRK/obqcJVDTLnfNAx0XlQ9noNC8Nx2ksSIIcd7YMNm0NtxLm6G7pHnnd92ebo6fq8bA34DWibc8WnSU1Tbm5KB9YgauQEXCJ7ECKuysNa1uCs0KnX6pO+6jM5d0W4wXVXdP0n24BKKbRtZ3VlZnynPcP5gFepDR9rAAUfODpaUbQQc7GNWzyhaQzwTsGLI4scoqdagjc4mZuZ/OdRTEOEWD+54TNHyPlEzeqSKnrFYoRUuRhrxj6yQhV+N5/J+XdeHCoczq+bgfH1sSaYK831GcayeYWv9FiNMn1A32nu7M/de2q/IqdMSSamOI4TwzQyxkTMyebuS0ASZJ8RsWBuzlBUlCL+cKwfZU99Vt3GHm0HyZXANqZ0rdFS8E5YLnq+8uUvXatnuYPkRgiBUgpnZ2egSts0eOdow9wsMBKJprFIcxqthhWzdWfX6yNCCHRdV52kHj7b5umXB1wXcJgzVyk2S5vNAYsYMH6WKLZOe+bs7IJSMoveIuImpcfuYM+yOedY9ycctV8AiZzzjxn1AmGDlolp2rEfBJk6HIFF11rt1/sDccsMU9Iic/8bVdMXujg/J68yr7zyqjUPQ2eTQvu9bVh1ZrpdQu88t2+9QBNa2q6nbRru3buDqvLmW28yDAOb7Q5RPcjUXrpzskAIVd52hqm00HaO5WrJah1YLDziCnHGCZfATS7ClFLYb3dkHwhOaHpP6z1HvsNhygyKOVU3l1Scw9VpsBkGNJub/zPfV9iQQR96RCDHGRdrDaY53inZmrhDNA7X2SkfApps/YnLlEipsNvuSTGx3+yIMbGrxHDmAAAgAElEQVRNA0kLQ0Xyd8VkP8ahjoCWSv7ibXBlN0yPzdWf+uTQrDV0vQM9y7xaJ7wcJFfbpgKlkwHcc7auWEqJnDPjOPL6m+/wV/67/4W33rnPT/74H+HH/u0fIVW1yVzF1Yb9cGBK8d6zWCwfcZizSd0GRao0Rs718criMnfVnUVOTmbE19XkgzU15oLrYYlBbu4NBnbhNW4JYiqjQgPFOtCm4SOUHGttciaFlqvZ8LkDKrU+PF8eNauYNdRVoW1tZDJnY4t3zoiJ225BCC2377xA1/X0/YK2abl79zb/1V/+z/nRP/oTnJzerscyN4Lma+4KCjFHmSGIzak3zqaLfEBE8VhNvqi70ka6iaY2WKBSrNmj5sA4oBuuztn8kx4aK1d1zKvo05713tsP+Nt/4+/x4L0z/tCP/jA/8of/FUM/zKiWmdCauQk0q1B+cBrsGoqlBj45Z2KycdAp5ZphmCDBnNdKzVW0DkjM15u6iuYoj88Qn3JzyNKoprHoy26IyH67YZomhsEcnAH/HU3b266FUDI8PHtIKYVxHMk5MwwDf/Pv/O989Yuf4y/+mf+Erm14+933rDkkjqZOmux226uOd9vS9QvmEcnrqXZofPVvlU2nFqed1AYGFsqrt78mhLY+Px2aVN43nJzeso1As+HdmvZG1zhVEzldENoGJx1tuU1KkKa3SVM2CI8o47THkVh05jh9zSIsg2i4vBwJoT0A4GeLKTGMI++//97BIYYQuH3nDiJCru2K5brn6OiEP/Av/zDHxyesVpZZ9AvjR33ty1+jaTreeP0bbDcb3nnzTXa7Ledn56QUEW+sPK4JNAs4uh04vh1YrgJ9H3DOpKQbPxNclMehVp5pMwhRwYdK8u2ELLAtEYfDuwYLgswzabKyWQmuOrLZUV13nvB//v2f4/NffYWf+HP/KU6E871tqFFTbaiWw+tQDhwUhQpkrz2JktOh0YyApoKmgqtBjfcBxeHVoblAHBGgd9ZNt0zXPLuKWP1bhMY9nnj8U6lxzk7ERuKkasJkUowHB2RyvIq62r1WIx7NJRNjPEQh7z845/f/ru8jpmS1TieUUiNOtbG9WOeYvTc4hR5wg1cRp8hct7QD/SDb+OF7bQ/ONc4PPofaNXYuE1Opz7vS+L6ppnPpXx2isz4T185jxT/OccNVsHKoSc///U5fZBjLmBLeW1ln7rYjtqmKOEJoaNuO1fqI9dExR+tjQvAHxYCu62iajiYYt2vO2eqvlcDW+7q8la/Ve3lECWC+gG0TVZDy2Bvsxti1P3QmAnZzSaU+PmOyZ5KPcu2loldO9PzBJT/we1+291HqRJJpDOWSqWglrnpBjzIY2PnnOy8UvXpsbjDOk2JXGZ8Rz3zo67l68HEDK/I4FpAnaSLyHvDNJ/iW3wcccXXKzrDR2LY+/s+BCHwRWNSf3wDO6+s98Fp97lAfPwJ+9Qke43X7oqre+y1670/NntC6vgbc5motvw28Ut/3c8BUH3sN+P+uve53AL8BXNb/vwTcxTo3A1fXwO8FfhEYgTXwZeAb1173vdjzdf3u7Vm6Zz96XT9MquBZ+gL+AfCT9ef/oZ7IP4g13OaF+PPYwvyr2I3y/fX5P1O/lsAPAq8D//DT/pv+Rf3CHOAfqj9/Cbux/idghd1EPwK88TGv+S+BXwC+Hws6fhdwp/5Oga8C/0Zd5x/6tP/ef1G/bsI9exPzx59V1f9LrVD2u7Ho4qdUdVLV/wP4X4EfF+sA/fvAX1TVnar+EvA/fnqH/dw+wv6Sqm5Vdf9dPPcngb+gqr+qZv9EVe9f+/2PYZrhf0RV/9FvydE+t9+MPXP37E10nK9f+/ll4HV9tNvwTeDzwD2sxvv6R7z2uX027JOsyReAX/+Y3/8Z4G+p6i98b4f03J6wPXP37E10nNeLtt8GviCPdmZeBd4E3sMQ8a9c+90XfusP77l9jH1Ywf36Y1ssRQOgRiDXa1CvA1/5mPf/MeCPisif+V4O8rk9cXvm7tmb6Div2/+N3Wx/VkQaEfkR4N8BfkZVM/C3gb8kIksR+QHgj316h/rcgHewps1H2T8DehH5t0SkAf4C0F37/U8Df1lEviZmv1NE7lz7/bcxbfH/TET+1JM++Of2ROyZuGdvtONU1Qn4d4F/E3gf+G+BP6aqv1Kf8qeBE+Bt4H8G/ibWdX1un47918BfEJEz4D/44C9V9Rz4U5iDfBO7wd649pS/Cvwt4O8DF8Bfx5pK19/jW5jz/HMi8pO/BX/Dc/se7Fm5Z58qHOmzbiLy3wAvqeof/7SP5bk9t+f2ePu07tkbHXE+zkTkB2o6JyLyQ8CfAP7Op31cz+25PbcPt8/KPfvUZ9U/Y3aEhfovA+8CfwX42U/1iJ7bc3tuH2efiXv2e0rVReQPA38NQ/P/tKr+1JM6sOf26dnzdb259nxtn4z9ph1nhYL8M+Bfxwr0Pwf8eAWlPrdn1J6v682152v75Ox7SdV/CPjnqvp1ABH5GeBHgY9cBO+chjAzj8iBTmzm6TPhNK0sJUYr571jsegrGbGNO81ibEPl2ZzlgOdN4IqU44NHUMmtKuvVFdmIiXQ5N2uoG/bWSJMhVcZ3oz2rxBV6JRIn9b0WixZBGMeJUkxkyugd7bjGmN/Xz/5M8yde1zt37+qrX/rSgYrr+nmff7zan+XDuRUO9uhvr95LPvT39hsjX9EPvrZSm11/3XdDyjEfazlo6FzR2n4Y+cPP//zPPwvrCp9wbdeLXm8fHVUdJ/cdBOAzUU7OphWV6vecq5L6zJdZicpnDk3vK3Fx5cEM3vSfFn1nbGju2vvP30uleRRTdADIaaYbOXAQomoSwzqztVBXXq9Uaam8nN57IwJx3u7rUrk+633+5rv3P3JdvxfH+XkeRe2/AfxLH3ySiPxJ4E8CeOf43N1jmqr3sVqsCN7TNUYzthtGUs7spj1t6/nyF1/m+HjFl7/yRdqmYYyRnBLb80t22z2//o3X2e0GzjcjKRfSZCcsBDv5wcvhhFC5pRU5UMwtVz1Q2G4vAWXZ90b5P404Jxwfr1GF++c7SlGarso6Vd7Q/Wh8nX3vWS46vv/7voAIfOObb7PbT1ycjZRcaIJtBL/25sMnTZjwW2GfeF1fefVV/sE/+jkoGcH0qo0h/FG1yqxXOtswuzJllic08imt1GKmJ+OE72Cz+aCDdJUBdZYVVp1fP2/MWtl2HqWrOzjBg1evN14qpJQ5v7xEFRaLJSF4ln1XWZLgugsOITwL6wrfxdpeX9db6xV/9j/89+i6lrZreemlF+jalq7tQE2FNk6RhxdnjNPE2fk5U4xcbi7IpeAak90d4whaaJ3QNg0v3r2HiONsM4AIt06OWS0XfN9XvsSy71h0LSqwz5lUA6QYIxdn5ziEO7duI86x3Y+kkpniiJZMGkdinHjn/fukrEhYgji0GKfv+dm7xDSRRiPAXq6O8E1DWNTnTamSrbc4Ef6Lv/bTH7mu34vj/LDN+zvCAVX977H5YBrvdL+f8OsO78GHjBOTVtACpVLD3Tk9Yn205Ad/21dYLnpSjmw2A2+9c5+cMp13pJhN/qAtFB2MZk6rDGyxmy20LSgM0XTWpzEiCF1vmjIxTogUTM5WmeJUuT+VRjxNaxDAwo6YC+NmsEjYmdreqvE0jeeFe8d0fUMjpgvddZ6cPSXHKvXQ4p8dPs5PvK6/5/f9PtvgK2+izhFJ3fGzKqqmuDpT4ih145/ZbgHRCCjiSg1WTJVGMab9q8+eD8kcplYqwCtqv3oZVDYxL1WL5juo/YywNpdcia6NAPdys2Wz2fJPf+mXiSnxwosvcnS05ge+9lX6rnuWOakfu7bX1/WLL9zTRgqaR+O5rDycbVge1iAX48Udp8k2LhHENzhXyAJKAWdb36oRjlcdv+drX8B7z9e/+RbjOKHTFskD599ShrZhvVrim4BbLQnes2gCQTwbsaBnNyXatuNzr3weVeXdd99iGPbstpfEnDg6Xttm7ToEj7iWosr6eEmME9vNeSVPt+NNlQpydXoXL55wLcP4KPteHOcbPDru9Ao2mfGRplC10AuKhf7BOcqUyGLM7+KEEHq6tuX46IjFouNys2GKid12T4wJ+u6KT5MrXe4yMzpX/zmLqpVspKo524lqiuKccTyKKL5yLc6qm0Zw7E1/XWe1RLu57IQ6U+PzQuuFzgutA7KlAjMvqKXwJl41p/3PgH3idYU5HaqR5MxpikWHRU0OI+vVc1WrA6z/EYrJ71KQkq9KOOIo15wkzI7z6tIu9dMe8QAyazNaJDrLv2r12Pb5ds3EKq4XUySlxOXllsvNhvfvv0+MCR8CKSVLRecj+XDi0M+6feK1NcLgTE5CjCPeCaXPzLvT4Z4uJiujlQwYXBW2F5NFoeDIeGDZBpoQOFo0NFKY9glHQqc9uUSyUyQ1+DYgGigOJGW0OrspRiSYbpQING1DzgnfNCBCaGaCZSMyxplMcdLGshnvTS9Js4k21BRfnEdcOJTsPs6+F8f5c8DXROQ1bIrjPwL+4497gWLCSLthRJzwwgt3aIPn4sEDhv3Iw/MtOOHYr+kXHZ97+fOsVkvuP3jAgwdnvP/+L7Lb7bh9tEZEGIeRaYwM+z1TTAfJDbDoJ8VoRLdFURXUmcBpQUmlMI2JEISj1cokSl2wi6EoXdfx4ov3KKq8++CMUpSSI1Js5w04jtuONigh7hF1jGrEgl48bXAsugZxwr17t2ibAL/yWP/zWbBPvq4KKYPTR0OaMv+uWLSZc7nmLUGKIApOC1oSabhES0I14rzDr4/BN+QDEe0s9HZdQE9NBI5aktGa6ovQNi0K7OKElkycBkrOxHGi5Myw2xNj5PLigpgiu/3OJFmmxGa75Zd/5VcYp8jr336LF+7d4wd/8LfRLRaH7OGDJYNnwD7R2ooV/Bn2WwYBNNL1C14Wjw9NFSG0NHiaJoZxIBeTvHTO4drWItDSITmSdw/Ibo8ftiwWPV++uybGjvMH2bLNuEUSSNpBCDSaURcYixJLIW52RFV2eaLPSz4vL9G0DcdHaxaLjm7RP+LAx/1ILoUxKVNKnO1GxjSyGXbknGhd3Zij3fNTmghBCaF/LPH4b9pxqmoSkT8N/G8YtOFvqOo/fewLrfqKOIs4m1CF0KqqSJXDs1qkCkUhZRNBm6bINEbGLhpFfy4H3RBj7HbzwT3yXWpaMUcKOoc8mBxGExp8MBVDVcjRSgbeOUS1CsU5kq/U4AetEgub0pQo2VLTIgJqSpzOUcXjwoGF/LNuv5l1VcwxMjvOWrMsYo+XUqUmysz+ft1xKlJM07zEiOZISiPOO3RxZI01p4cC/0Et8dp3V/Xuc9UlGieTSIhdhyCkNBqD/GglnTiOlJwZdwMpmsNMKZmoX20cOHE47xFJ7Pd7tvudadkUhXoZWO322XGev+m1rYs7xYhzvjZfhJzVsrwPOQUiDu8tcBC1JkzJSo6ZPEU0BNrg8aKMXUN2UMaIoIgWpBQ0RtSZFlnJBUkm46txRFOAkhD1tF2LDx4VR86FkiIlJ3PGWUiqeFXTjHIZ8cE2+VnbKCWTuB73lJzpQ1PlcT7avqe7WVX/HvD3PsErcF44OjlmvVqYjKsqw7RjiIMtpXNMWdkOE996813atuWdt97m4vzCBOVjZre14m6x08xq2aOqdG1AgBRzlQ62jntKhaxKvt6DE0/fdXRtqE0qayjlnLkc9qSpsNtcAMKiCciyJ7Q9ORfG3Z6sih8zbkqcXUacE1anSyR4koeUCqhJZwRvaf2zYp90XVUhJfC1gyO1i1pqul7KVS2RUqpQWkUcKGjJaI7E7YY0jZyfPyQ0DUdHt/HeohbFooZSMlM0FcJZVXG16Chq67Xb7fiN3/gGMUWWsyifl6uuaimUaJ8v5YCzwDnHrTu3aZqWk9NbTDHiu56z83N+/evfIObCZox0Y6SXqhtexeOeJfska3tYQxfq/zxFhd1uwPtITErMGe8dTdvQpoZUqu6Pc6wWa0IIhODJ48D7b3+bfRx5+N4Z5Xji5IXbNKHn6PSUHCPD5RYthcbZ+Z3GgVyUKUVyUbpcCEAskU4KujlHWHP71m3UefZjJMXI5dkDUhQ01rTft7RFiW7BGCM+LMhxIm0viXFke/6AmCJnvEfTdTRf+Apt033suXn6KpdiEZ4JdCmKaV3nXFXtq4jSME7cf3BGCJ77D87YbbdWP3SOpuqrO+9R1YNTMhEwiFMkl0KMVnuZYlW4TOWQXlSxzUPToijIQcDPfrHfD4eU0InDBwdSUGfiT1ELriiaCuIEPxVcEUrAoEjMiIxnx2n+Zq0odRuzpo/WiBNghonoIY3KBvmqgfsccSomcJcqnGQcJrIKyY1kNZG+lDPjMACwWq5omkDfekvNhj3DsGMYdkzTxDRaSSi0DeIdwVVZ6lIQVTxX6b9zjqZpaNuWfrEgNC0nJ8fWHXaWiUwpM6ZMUDV53MIBAncTTTAJ5642xLq+t8xJrG4cUyLlVAXrBPHeEA5iEadzJproMF10yXajmRqlUiTgg6dZOFyITPvJoksx6OGUkilcqiEu+sZTAK9Ko4ky7ikhIMWyu+Csnmmfa1mDOsVVXEbbOHCBvp9ILuBiRBQa762UkwZKFnKaSI/pAD51lcsmBBb9gi50XO4iJUd2QyLGUusm8N7DC94/2/D+g3NLuYYdDmW16DlaLnntlXss+5bjRUfwjsbbiWp9QFXZjyM5F/ZTJmdlmEwq9GxzwTBF3ntwwZQy+3HPNLmDRC1YN7fvelDlW6+/ZfCmfoG4QNN2uKIwDGSUXbKIybsegOEygUtIsL81OCF4Ty4Q88eemmfbFHJWVKwJI7WUUWa1LTWHmUaDjZQ0oVpIxUK+xlvl2TeCU0cWYZwmvv6t1ymqnF0+ZJpGLi4uiTGy2W5om5avfuUrHB2t0VdeBi18+81vMo4jjbfP/o1v/BqlKLdefIH10ZqvfvX7aJqGYbNFc7EbueL+xEFoG3wTQBwuCHfu3sOFlrb7dXCOh5sttD3SLvDekeKzF3F+EvPOcetkxWL5IqEJLNYrBGE/jEzTxPb8nBhj1TsH3y6QoiSdEBG0iuDpZqAMe9pxogsO/ILcrNk2pzSLjlvHPZoi+6TEYWQzZGLOnA0DgnL7uKdvAq+crHBaOH94RtHM9O7b5O4crw7fdkjX44oSCige7y2QogjOeY6O1uQCjetJKTJ1C3IcOVp1THHg/sP3LNDaXhDl43P1p+s4RfDioOofT5M5TquVKPia9sWEImzyBifQSME1nuP1gtWi42S9YNE1HPeB4B2LtsE7R+MsAt23QirKMiqpKPsxMcUMEtmPgWGKjFOqdVRrWEkpqGa8E6RrLLWLkaLgmg5HoaRMLlp1lzGHK1IhKqYJXrQwxoGiajsgkFJ+FBV+E00/2CzRq9rX4bs5Kr0G/1FVcrKIU/Zb4jhyud0SUybvJnIpnF08YIoTm82GlBK73Y6u69jtt4Tg2G42lJI5O3tIjBNt05JzYtjvyaWwHEe6vsc3gaZtmfyeguLmcusHIuOihiQNTUvTtHhvsKhxnBjG0YDbzlACNzriFBtCWSx6QtOw6HsUbG1ytmDD1TxDADG0ifPmVrRkqxlPAxpHOif0weODR7wnFcUVrZmjabdnFTZTZkyJizHhHKyKEnQGpilBC6koeRjQDMPFBb7rCWWOVi1wqerptnFDhaRBCB6HIl1HCYJjSYjCMCxIVXIY/fhI56k6TueErnFcnj20HXu9RFBiVEoRcsqW8mWTohfN9F3D1770eU6Ol3ztSy+z6Bo6V3CaYdzjJLNuHd4rPthFvFo4wxL6llyEi+3ElDJHR54pFe68cIcpFc63E7v9yLdef5NxnJCSrNbZGXC6b701NtLENE1s9ufmONU0wY9Pj1gsF7z66ufpFz3r9Yr9fuDnf/6fsN/vQCCVwtn5xiafbrRZd7uW26/2idowEiyVstqzUlJmGPZMceLBw/cZhz3333mTYb/nwfv3mabIbozkksl5ghlGUtPq1WLJsN8iKL+2uWTY7/mlX/klima+9KUvoii7/ZZclOZyi4TWuvwo+/2GXBscWpQco0VIooRccH02aIrv8Y054pQzD96/T87K6e0X6PDErDc64nTesV73rI8WhNDgfECB5XJB0wamkg2/eXZJzIWkdo13UlPf/QWkiWZ3RpMzL510rBYLTo+X+L7h/OIhcQiULoEWhmngbL/nF9+4z3ZKDFpovJA0c9Q63PaSVkDGqaI1EoUND955gGsa1p97kdAtaE9u48UbqJ1MStaTyCmi4vHqaYKwPj1GNbMfIKcVq2XPNE08uH9pwc7H2NN1nAJtcObVizOvXrGOqKPoXHuy6HTZNqwWHbeOlpwerThd9/RtwOUJCuRovHiNA+/A1UkhmYWxgydlYJtqlJNrtGi1OO+sW+4dBA9t8DTBs+g8XoRGOnJWdpM1ekqKaFG883gEL4J3YsXxxrNcLXBe6NqGFAMew7RZ/fQG32FwCDDngcrrkZgWw8xO00SOke1mYzCg7YZxGnlw/wH7Ycf77z9gHPZcXF6SUmKMyWqhWsBBcDbYgBqovqhhDIdpYrfdcn5xbpHn+RkC7Pd7ikI/RaYpofNo7uFaEIswS6kojUxxVm8XPOAR8XgfrKY3jkzDYHW4ojWAvtnrqlBHKAUpV4865+j7Dpzg/Na00XPNxMQ61loSlGjdb4qhS1pvevUOGi0EMr5ESsXUppw52+zYDJHooAuOYeUIWdhMhQalyYaQUbVptLFkaBr8ZkWTC359gnh3wBHnnEilEIui4gi+sy5/dazzGoYQ0AJNCDwOAv9UHaf3jtOjjmEwkPjxwgq5rbO50+1uQKXQO+jbhu/74sscrxe89spdln3Dic94LYgUECWFpr5viw8O33vEC21jMU5SR9yPvPXOW1xuBt57ODHGzPkuktVSChG4tQ40vuGlWyv6JnCy7PHeE0LDFDO//q13uNwODJeZLFZrdd6jcWTYJN5841ssV0uWqxYR5e6tNas+MI2T3agSuMkNohndZUgHUPLcdoFSiOPEOA58+43X2W4ueeP1b7Lb7Xhw/4HVyvZbYoxstxuUQtcFQggcnxxVfK01cHzboqoM+4HQtjUCEi4vN1xcXPD+/fsMw453338XUOI40bQdNGuC78hjQR24Aq5CpFBL/0SVPI1W004FpwriEd+wXKwYhj2biwtEHCVGaLpDyeamWi7Kg82WBxdbRIQgnhACRycnhLbh1m1DHzy8uCSXTByGGliIgcuZECISLFApvaf0DnymbQqvHXU0Qehlz5gSThPTOPIb33qDs82OthFWXeAl7lGCsN9c4ouydB3BB5b9CvGO5AoaHDvNhOWKF9cnhL6niFDIDLsN4zhxvhtRHMul4cDzNFI0UzCYU1sd5tHxyWOX9Sk3h8A7oW083jmWvUFNpimjTuuBK4vGs+xb7pyuOVotWPUNfetpvUV4joBqQXI5TIOUonVSpHbxRGqdgwOoveRCTpk4RQrQOUcIjuPVikUbeOH2mr4JHPUt3jvatmecEmfnWxrv2e0mclYWix5EGHJGRYnTxOgdu+22fqYSHGRnWNRUnkmw9CeyR6qbdVLIoagWxmlgt9/x8OwBlxcX3H/wgP1+x8OzM2sujCM5Z4ZxskShDXgAcfZFBcCL1axKqTXx+rgTh3f25cRZSqY2iWaRpZFEkBVyMRxu0dr+V6QkAAo2PaKloF5tTl6cwZ58IKdImqYKQ7L3uMnrqmo9Ak2WCWYsGl+khHhnafNhOAE751oQdZah1cwu1Wm60jSUpiFjG2zwxifhnPmF4D1NcLRB6ILQidKhBM1IFlKyyaHUYD2IimopFEpS0jCQxbHb7fClkEXIKdlXTsRpQhFS0yBi+F4jILHMMEupaAD/oWQu1+2pOk5LrQq3To5Y9C0vv3SXlAuX59+CnC1SW3R8/5c+z3rZ87k7RghCGQgOjlcdoXbltSibyy3TFDk7v0CB3gVcAJfUitTOEWi5d+suy25EZMNujCDnFIXjkxVHR0t+92//CkerBafL1sYoXb1hXCCmwudevMtuP/L2/TOmlIglMU6JN956yDAmNntl3Az88j/+VbvxZbJ72jWownY3HuBJN9VU5tJ9odTRy4Kl52+89SYPHz7g//l/f57Lyws2lxc2K5yKTRRVGJhre5wTQrtAvGM7mENL0bq07cLQDrvNlrRQtAhNaLl37x7r9YqLy/vs9ztynsg5sdnscC6walt6cchUJ8l2IyWNCBnRTI6TpXSugRxxZY9XcM4TGmGxWlFQdrst3jtyHCm5pdzwGqcNwtqUEFoY9wMxF8JuT4iZqELMmRzzAaurKFNlLIsloFnJo9IFz+1b92iXSy6kI2ZPMxb64rl9siQE5dZpxnnPD//OrxpWe7vFU7i1aHAogywRERZHJzYWOdUNcbJS3LBNlGHk4a/9Kuo9k5oPOF2vAUhpohQYvA23tI2v/AoOVYip9lYcn61UXdUAs6rlEAFqKQTvcK7hzq1jjtZLXrh7ymrRcXy0wDslTYoXxYdQd3+POkWcpXGqOg+rWO0z2ayzBBvpW/YdTjzDVGi7SCoJVTg+WXN0tOT2yZr1asG6CxbRlowDgvN4XzheL2nbQK4g7CFHhjFyuR1p9pFcEjFldrsB1YJrDNepWM3EcKo3+A5jjjivkXnUzmRMke1uy3a3ZZhGYjIiD+fAtbaxSLJNdUoGbFdxIFYqsRq4Neu6bmER7H7Ee2/lFB/wjWUgi0WPUChZSDmQUrYapQhCYdrvkRwZ9ztyGvHOmhJpGuwakoBPiWnYELTQiEdLwnuLZlOKlZosW6dWbzYcyeqDxaZy6qiyTWZFcgHXVMD7jOMUm02vKC8yDlUhqiEpJxcYxTNmQRLsR3O4y2mmi/Ms+o6X7t5iWC6IXspUA3gAACAASURBVJCcaEnWe2wacI7QNdbHyBGwe02KUpLVMnebDUkck5rPWLT2/FJso44pUZzDO6vVeh+smZytOa0pf7ZS9ZwLDx5umYaRpvFcnF/SNIGTkzWnJ0f88B/4vZwerzjtHdbnsVoZcqvCVqxmOBQlp8yQIjEnXDC2osYb9urystLELRYEH3jl3i0Q4Qufu0UqhX3MqAhdv6QJgdOjpdXSfEPWwuV+h5ZMWwnQFp1BjtZHSzIwqjDFxMnxLfb7iffeu7CI9O33mKaJ/f7SUgcPsVAvtBt8h4GlS6WmUilTSmLYX7DbXvLtt99ku73k5HTN+miBq6mRDx0lK2cXe3a7Pd/61uvGJrVY0nQdn/vCq/Rdz6Lv8MHTL3tSnHj9G79B6z0nJ6esFgucZhovnKyW9EHREsg50/hAUaH1mTJt+ObXfwkvcPHwbSgTR4uC1BpYzoUhelzTcWt3Qbc85tYLr1LwNB6aIOy252iJaJ6gRDTnG+04cypcXOw4vXVskzxYo2j34BznAifRyiciLSEIziWL4EsmK6TsKBrQZkkKnvfGzCWRIpHOJS4uRloHZxcbuiZwfHLMyemS3//iF8hT5P/n7k16LNvSNK1ndbs551jj5u1tI+JGRGZGFgmZlJBATED8AEYwQzVAqgkTJAZV4hcwQmKaEgOQGIAEEkxRSQyQUFGZWaXKSjL6uK235tacZjerZfCtffwGqrg3IB1PD1+ShYVfNzc3P/vstb/1fe/7vC9+8nPG3Y4Xjx+TUuJks0E5i+lFNx1SpiQwyqFiIu8nfEpc7wcCillptLFEH2VzdNIanIcDUBi0yK3u33+I1pZYAilEbm6viektmqqXIlBgmVrCPM8oCl3XsF6vuHv3Duena1YmoopANUoV2uWSiSnKRQlRjnp1hGeseTXJK0VcKCVTUqAsAmutsc6RKTQJUArXdPLEUTKdz0U4jPthJMVIUxJGKdq2EdipNahKf1Ta0DQNOSu61lFyZr1qcKZQsiOmTFj0qcC7PRySqXmMkZJzdYZE5nlimkbmeSIEL7a8YkUBoTWu6cVSl1TVTsrAxlpH07ScnJzR933dODVd3+K9p+97rFY457DGEEMQmG6KlJRQKsvAp8oLc0nE6Dnstygyh/0tikijM1olchrkSBk0OQemwzUpB2y7omCZhgPzNBL8jLUGP43MjWOY/JEB+q6umDIxZbSWuUFR4ENEq8I8C4xlgegsa3GJpSj9Za2NWKlzRsXEOHuyVmidyAYOo7xnVpuMtopV25KVpnENwTppSWcwxqG1HP9zLkJKy8L51EYmHKpI76cAGEfOwvPUJosiQylSDmJ8IBGjI3h/lDIWaksx53/Zy3Fcb3zjzFnT2JbGahqV6Yzmwb1zHjy8x3sfPuL0ZI3NM+RMrNAGP4mOcrq5YZ4nDtstlMy673DW0bUGcqk9jIgzsnHGMFCyIXkN1tK0DirguChARVRR5BBJKLwXOOpf/fhXjOOIjhFnDY8e3cM1DtP0ZBSHWYg/UxCwRFIF12m+8+E5OSf2+xMOk+cnX16TQsI0Fl3e4Y0zZ+bpwDAI8Fkhx9oXz55yOGwZhz0pBU5PNlijaYzGWMvm5FwePP2Om5stv/zlL9E6ce/+PS4u7vKDH/4efd+TcxSHkTOi9zw9wQB932K15sWzl+x2Nzx78hUhjKw7gUrMU0XZKTEq+PEGcmI6XGF0wironOKkF/OlNRDzzHD9c+JLw2e//CkhanaDIiaYUyT6Ez7/7Jc0bcezF4Kde1dXKa/ULtpotJMj7xQGSvYUNEYbsJpUYTcpJUqUym3a7aBkTs56eeCEWe7R3YHWWN6/d05BczN6nE9ks6dpAnN2lJjYzYnRF/ZzgaI5UStUVtw8vWKePTe7W4zRvP/BA4wzdPXn26hEUJrcrYjAzTBK+8DOIq+KEzlH4rjDGYOfPF2/5vTOfZTWYi19mzZOUL/2ZFqcJCmJEyFWp4EypvYtFkzY0jOTjwUSUVKiGI1RSphmtfekZajH4hrIJaGLQlVZkFaLhOYVobyAyIa+9t8mH4mpcBhnXMrYBLlo9lMmJBi9xHxkX3VqTpxR1llsLChlQBcMmnf5pF5KxvuReaqOKeOIweP9TPDyMCs518mpxlqDtQZnLbmIbs4eTw2SDmCtxTmHc445VB1tzkff8sI6hcI0jYzDwDyNpDhB26NQldEKRUkPfE4yRBimCWcyOVvQmqa1aKWqqBpKjJQc8cPMHGDca1JWRGvws+Pm+gpjHS+eP3+nN07qCcB7qSydFvbtcniKMZBUQiGOvQo+QxWp/FII9bq15KykElRyX2WTmOMrPkXJisPk8TGj9A5SZpw9c4ikRSFjBfvofWKeAvMU0VbjQ5LUh8rBNVUlVoyuvowiMO0adXMkdRUZZPoQ0Ebo8UqbI+v3m9abZ50VRfABlRRtU5jHzKe//Jyb6y0fffwB9+7d4XsfP8RqzWEYSCEwDwdyjHROYYshOKlohv1WSOyrrsIl5EI5a4//cK2VEHFq41e+LtfGt6dQoxe0pu0sdtXxo9/7AcMw89lnj8VJsJtQasLYHTFnrrYz45z46nIkF8XZuqexmouVSFhiUEyhYNyKRmUo6Z2WrYTgef70K6ZxBjTr9Qk5RcI8EcMMKaKyVCJFgbYaXcSRlQvk5Ck5YI0M81Lw+Gli2O2JPrCb9kJpT4HgZ4ZxT+csSglu7Pmzp9xev2S3vcGozObBGUobbg/Sa85V3TDFRAye26s9XQOffOceq3XDvXs9zmlUlgfcOGfGMRLHG3YhcrMP+KQIrsPPE//0z/8JqcDzy5fE+C5vnJBi4DAeADCNALlXnURn7He7upEK66FbrXFYgbOWwnQYSDmhGoVrHetGHo77w0GO7420YR7dPcOQuXp2SQkeHWYBd1ROZmzlIarP1uSYuZkTwxAZJo3W8Oz5Dc5p+q4RMWIQ7KQlyUZqFLlodNuKTRqDIdPdWQk+0loymcvL5xwTB75lvfGNU0KdMhoBX2QKcTdgrGO/H1ivepm2K0VKkRQDKQVKlr6KMZIpRNH48Iq6swg29UIjUpWVqNTRf5xrCNxCfJaQtkJWCoWRaboS7ztZeq+oQkiBUqXdFMGipRgZx4lUqJlJhlhEfzjHhI+FWjC/w91NWTlnxv0e7yNaG6JrhARexA++sD60qu7hnEmk2ooppCwOIa0k7iKEgJ8n9vsdzk9sxz0pR2IM5BgIIeCqKL4o8N4LOSlGlAFjLNqIuUJiOPTRCrr8vLnGdiglWt7WyeRX2kmZFApWZQwZXTI6Q8mRFGC73RJTZr//dmve7/6SrSSXJezMoDpx3mgtVfoxJ+pIApP7pOQkvW7vyarQVWbD5CWi5nZ/wMfIybrDKDgMEznMlHFA5YwpZrHAk60mak3W4AuEoyS3ME2BlMS9V+RHlhMJ8sEilSpidDBGdOTOSMqAMtJHnb2vxLa3bOMsgC+Z7VwwGoYgYvHkB/ZD4vryllXTUkK1KvqZFDw5TiILUQFtMpt1R0qOUQvMOKVUe1RmAYXXv3CJ1khVxkKVMGlepeEt2HyFywm0RhVLowsP766JscMHTyaDSoQok9TGzrQ646OkWUYsenNBKYWrl08ZR892JzxB16h3mvER5pmnX3xK03RY49BB+KStUeBsRS0o+qZHG8142JNLYRhmMjDOAe9HnNOUkri5uWQcd4QwooxmP43SzokBqxV31iuaszMa15BVZJ5nIfZEUNpguxOsc6zWGRszIRsR5DeeGArz1uJMJkyJ0EYaEq0utFrkN4MfwUdMTjSqsG4lo2oKmRg82+1EzJnJj8d2z7u5CtYo2m4tsJXdLSkqUi/OndOzMwqFqYYkFq3JKePDhA8TRQVy8Vxdjxjn0E6cfi9eXBNj5KvHz2gax831IwGaL6T+aRKyUhRGgbWK1aqnfe89NJqD00xW4XOGlIjXE9Zq5pTQVlMai7IGtMw+yjSTk8gatTO0pyuBDSlQxtKvL4gx8fTF58yzkJ++7bq+4YpT+JuLzsugUUW4hjHVCi3zyiVSd5tlM5Tjd0Gp2ufSClWUOEKqf3aZpJb6oRDcGYiUQuUsPtbyKt60lCw/RwqorGVwlMX9o4y4kUqtYo0WD30MmcYIBTumhImRVIQjKOkqi9Mh19TNN/tKv8lVcsaPg/SRXSbFBoyR04FW2Aqsdq5BKfA+CJHKCI5sDuLq0FqqmBg9hcx2ewtaMfpZcqpiFHZi31UXmkFbamhfQ9v1tI2l7TdYZ3HtSNERskFRaFQhhszQNxiSpJXGhCoZg6KxhaTAmIzW9UPJ9bMFdABKJoVIzOkV/ecdXUop6UW37qiEKdRCpf5eqVV7oVS9ciImeV20UagEYQ6kUphnD4D3UU4bRXziPsTa27bC02y1FDtzVUTkACGyHScMmikG/PLalyx7R8kYH9FFH/GEupLMVM6QcrVWLv1NQ8GAsrimQ5lM03Qy76h7wjetN75x5pLIytTKYC2DncbRnWxo+g2u26CbFbox6HYlUzsyOXn8MJFjYJ6nKp5PEplhteDn6me5wjK4KUVcQLlITGkur9we+mt5yoVCKAv+TRia2Qtiqql+aWsNpRiae5aTbuLx5prb/cSL7Q2DbWhPpNJp16cUPRGfXBNjoDU1uOodXTlHpv0VxJnsHL3NaNfQrDeY1nFx5w5KKe7cuSD4wM+uf85hGOoNVgh18OOsQavCOI3Mc2EYtiKMdk4A2EqjaWi1pbMNq36F0ZqPPvoO5+fn7B/cp20dH33/90EphmyYp5mUwJrC/VPRYF60swS0zQfmfURFS4MTLiSKORmUSdh2xqRMtwa8wgaFqWSnnCK5LC2cd3MZY7hzdkrTd4QY2R+2+BAYx0HgzrWY8HWz3B4GkQzOMrVebTqM11weDkQfyJdXMpWfRW7Y9Y6+77lzcZd+taLrNxjn6E82UDKH20vG4cAXn/6SQwj8+Ge/gpQZLm9RMbEpIiUsxUAqDLsBLDgarLOstJUCLSVcyswxkaJmHi05yf1q3IrN6X2Msaw256QkBoiSM/yv//g3vjZvnMdp7UJwN9UxpFBYAf6mdJQglSJSh2IMWUvjnlqRqsr9M1b6Z3rpZxxjPaWx+P/cqhaBdqpJfLpWrIvfVin5cwVqFrd0O3RtmS4hXV2jSZ3jtG/JKfNy78kpMk0zNmUBqZZXejbKt8eN/k6vUkhxJkVDVBK8RjEopM9ZsiRK+dkz+5lpnJnGiZCE8J0WB07JRwSdQkwOKHESFV0k66dkrNY4Y7FG0hLvnJ/jnMjNrLW4VqJUrG1ITo56zhRWvaYk6HuHJjKMmRAkaiVGLdlFGlyjaFtFvzLEAm1QZKUwup52lvNMWc417+ZapuSLomVhP4QQRStr5rpxZkJO+CiVZAxBKP91mp1qVThMM6BIoVasWWRM2lqsa1htNrimYXV+BhSMSnI9m1Z6mbOXxqZCiP7KoDOkJDrvIA1XjCo1GlreSUYrkgadynForKqOu2k7XNNhrKQE5BSw+i3TcVqreXT3RCZkckcIlsNZWpu5fPY5puzZ395Fbdb0jSabhv2oyVmTswxh1qsVWhW0qpKCUHsSKb6yctbmtNyPlUAePDElQojiiTYbsWR14pEuejniS8WJEsdPqnxJa0UP2DvDyhr+6PvvcbMbGOYv2I6By2ePKWjWrifHhIpRjnio36Ld/Lu7SsnEuGc4TDRtC2xAaWIWnNuLFy+Y5pmvvvwK7z3Pnz/HBy+av9pjRi09as3KSt7LApXNtZeVSqIow7pbsVmvWXU96/WaP/mTPxE83fUV3s+M44F5no4Rz6tuTWMLm9WBHBRtp4kRDqMnzJkXLwXgcXEBrjVsTjSua/lBueAwZr54EtjuMi92k0RnGJFFhZTf6aN6zpnDfiDtDgKdToWS4LAfiDExv3gpj42mAaXJxhBzZLvdi1TJT8QUOfiMj5HdzWXFtjm01rSz4TQVkjK4rufhe4/oVh3dyVoGhXfWDPsdu5cv2W13bK+uUapw//0TGqXociL5xPMnEqQXTGULnPQ0XcPJ6qR60D06BKJPoA1909GtNjx6/yO6fkN3co5SmuQdKQVSDKId/ob1hnmcmlXfSG9KKRrj5LNVdK3Fqghpwo97gsloa8g5kbwnhyiT75qOKSffACWRspanWwIo1UNcjk/KV09MvpZ5rir9ZumhigCtbuUYDcZaIcPnX/+zTik0htO1VDan65ZU4HY3k4rCZ0VJr/qwkn/yLtecS0Y9qOKq/tKSUyaEwDAMTNMk1X49USiEarScDLRWlbauaRqZ2sYo0/ZU7W8GzarvOdls2KzW2Fp1rtcbnHPs93tySsQoR0drLaXo6jAqMhDQup4qhPyjSmGaM9NciKlgc8EoRWM1m1WDUpmuld8Xb3OR04haui/v7nVdBPCxpBq7CxSxTocYmcaJjEioMQZqn3OYJD00BS+4uVwIWYZIORdiluudlMGMM/vDQNt2Qu9PDn2MZK/cXCOMirbrMBTWJ2ucAusnvFJkBUkhmUfOsDpZ03UtfbciR/Ha65zQWrCC8o+DnNUycAe1hAsuG8L/T/HA/19W4yzvPbhgHkYa5/jhJ99hs+p5eHeD0ZkwXGJ1ZPvVT5mdobPSoxiHgFKWzdl9rHM0qwZUZp62lFzQVpGjCHIlSrRGMtTPCyJqQf+zCK2tyB38PEuT24ie0zgJfRLXCjBPQME5Edu3SoGF9uKE03XHPiQutwO7v/6C/TCz309QZUraaEnffIdLzlIKJWRs29C5jvcffYC1Db/64gk317c8e/yEcZrpuk7I+euNWGCNQRvDqpUQsM3mBOcsfd8L5qu2VmRT1qxXa1Z9zyff/S4nm5PKIjAo1RJj5PrqmsNwYBwHCpnzszsVPFFQyPAuJvAp43NmLoaYFVc30p+ZDhaLwTowSnN33bOymcs24JuMc2Ct7BE2Q2vtO3xQR7oRCUDLdFokyceo7u3hQC5FXDbGkK1h9p6vnjzDV8kRSpGdJeTCYfbElKH4+rRUtNsD6v/8c+6enRL9wMXdc37wB9/DaM3l46eMh4EpzJjG8Z3vf4e2sZytG1SO3D59QtyNDCQCiZPNirM7a/7OH/1IZI3BMQ4jV7trPBEl6kOmyRPSgc8//5J2tebRR2LHTkH4AznLnOOb1hvXcarKT3TOcnHnjPPTNQ/vbdBE9tcHVI7oHCghEsJMyYUwBJR25NUpWStQbrGOwKuOU02rFBGX+MmlopBfS29UcDuFJYIBIMUoOrSvVw/lVfdKHT+r2vOUrzROk4Hz0xWxwKpz+BAZkpcjSduIF95aodO/s0uuaeta+rZjsz7BWEfbtLRNy3q1xlrLal0/9z1aG9zXN07rODnZYJ2j6ztexe9WQrvWbNYbuq7j/PwOq77H1CgNpdWRlKWU+JZB03WdTHxzhhxIM/iQmebI5JPoiItiDoU51F5ZjebUCOMgVmakrbwDXXWh2oDT7lsdJr/LS0wiwoRIuWZupco3rdrolDMqimc8pUQIgvQrOWNdI/eo1sKeWAa2SxS7Bkrh5nZHyZlnL16QcuTB+/donGWa5wrIEUL/et3SNg5jBYw0pcQYI2OSltqps5i2ZX1yynq9Ik6yJ2hrUEZkiGRpAaWiKPs9PmZOhgOuaSiVb6hrZf1N641unDFmrq4Heqdp244f/eH3uXdxytlKoXNkumnJwcMk+drbl8/x88x+uwM0yQ+4fsWp+xDbNKimoWSNHwZSSIQMKEPft2KhqwL3pRZPRfSbsXqfm8bWkl3iX41dBkUigxgnOVqUKOJsSR/VGCttB6xh5SwffXif0zsTX13veP5yx8uXAz5k2q6grGZzcoJ1BvjiTb7cb2xppTnpT/nw0fuc37nLRx9+D9u0KLNmHEc++uhjCoWTkxOcdazWa3n4LO4uZEjYNi3aaKyxx+uAWjYsTdu2xxhfOepXuVoG5xoePHjENE0cDjsKApzWWkDEfhr41Y+/5OZ64NMvrzgMA/tDxmnF1ZjRTWZKijYLQFfu90y0ha41dF2h6TtcTLjWoxJ0XVNtn+/myikJO7WI3GeYRpEc1cGJtRZSYh7Hr+HmEmfOYfqO+w/vg9Y83+3ZjxOXV3XeUOoIMEPImadXt7y83bM7HLhzfsqUMmenJ5yuT8jF4VOksYqHZ5Jq+/mXT9jtB37++XN2u5Evr19ijOOi/wBzeofVvQ/ZbHr8fqBYS9f1hBCZA5SQ2O5HQhqYL68xTUMonqbrafpTjLF0Ff7zTesNC+ALIeaj59XWrJ7GarGaNy1ZQfSWomLtDSKSAzLJjygNYRopFJquQ6HJRYCr2krP1DV141w87XVqW7KqVGqRHWlroSiME12n0rlumlXUnsTZoqpXs1SNp1IWpV9pTF2toLu2oW0acbQAy/ujKH7do/+OLTl1CbPS1k3OGlsrTRkEoGBdK86+79FK6NwUOQIqkA1xOQko6lGPWlnqY8slBIk6SDHVyt8Biq5bobUh50gu+fj9jJHgrnnOjFNimBLDnPBJ+lpjyIyhMEZoI7QJdFVE+FiIuT500aIJPqY56ne6d11Kwc8zCan6Qwi1fSLDMa1EiaCWWUIl61sjbRhrDGXxj9fWTLKZtNQzVZkQosjRbrYHSoHnz6/wPuFML+qKlCla4VTBlESYZ8Zx4nY/sD1MjDHilGIKkdEnDlNAWyun1viqP7skB6TaB5/ngMmJw2FPzBnVrCmqpm++TTrOUmD2hXk4oJXixfOX6Jzo7kqzN/lCDpVPWiybs3vkHNmcnBBC4PbmhvkwUb7S2G7FxXsfoowlmx5lChd3OqwxNI1BFYhhlmFB8II6m2ZUyXXj0zSrE7SxtAh+aph24oEfD8SUGL2XHmpKNbwtgbOYdYsxilDkiDB5RfCRzq1YtZIPr2OhKIkp3u6Gd7oykV6G57C/xujMy8unrDdnnJxuWJc1beuOx24FFfyRORwOpBgJPhylW0djAqKOgIUDoV4lharlpvY0Tcv3v/8JXddz7949cs7cbjuJ5BgPxBTZHXbs93ueXY5cX3uud4bBW3yO2Jx5ekjMKvHpS8XZrLk7Z7Qq+BiYfeHpreb2ALsJBg8JS6YwhyQStnd0xRh5eXVJVuLASxUE7HOp3Ad5KLVtJ+0UY0g5E70Q9W+ub8VCWyvUO+dn+JBkUh8lArggCVU5w+04M4TE//Fn/5zzs1P+7h8r+rYhT4EmKcrtllwK0/Ut25sdP//iKfvRAxarEj//8ikvtnvavuN03XPRW7L33F7vGMaR7X7Ex8w0izwqxEBMkZfPn9OvN5xcSH68/y34A38LWLlCCpF5jozjzDzNlLSqZaV8lCI9KmMbTDHi4DBGPOoxE6cJiiLNAd0otJGqpulkWGA0QJFgN139qzphkuR3myJVkXES+IVqSCVjokwJizaC5NcyKVzQRvK0kiS+TMYnT8iF2Wtmn/BzlPwkkL6qFveT5La/wxsnErKQ4oz3I7vdLTFlQYUVOOz2RwKWeJp13Tj3pBjxPlSOZzomV5ZSZWVwdIOhK3OxWmYXNud77z3CGMN6vcIY4aQqpYjBv+p/50JMECKkrKU9ozIJ8EkxBrjaJakuY+VN+sQc4Gpb2I2ZcYr4kI9RHynnd3imLg+nGONyW8prubyg9XlxVEfoamculeqQhdtZtEI7i9OG1UpjQyT5SNCRWLL0PJOc6FIuEBPb/QHQ3G73pFVHmxNFa4oP8v6RYQazD4w+YK30sQ/DCMCzpy8Y1j2c9pAi8+yJQVIfpAdu0GR0qhP0kqveWv6Bbx0dqVTb1TQErJ25fHFLqw2fPLqHNZakqqJdOVAG6zqULrSmI6eAKgk/zlw+2zLvJ3btGc16w8mHH9B0LeteUib9NJJzQncWXQq2k6OA6SQioShpFNu2l+GBsmLp0wYbIz47dExomykpU6aZkiNzmPExE65uKCozxImQC7uxY7sP/ORnj7nejiIQNhrXtBitCTHBO+ww0QqciczTLSEM/It//hckNM8uD8xzYLcbXpkbqrxIVA+itw3VG3yEftSdadFIKqWqtEseeH0dHsUYOT8/5/z8nEcPH3KyEatl4xrZQJ0jpEBSBR8SyjRgWnTV7xYzo1RhzIYwKP7ixwesKph8oGQ5ysVUOHiZxu/nQiqKjAwnreWdrjhBBqGpDk6NNYDCFCX65gpyNs5AUYR5IsTIYZYTRdYW1zQ8eO8hxjlSFj7EZXfJNM08v7nGhyDfJxewMmzdDiM+Jn78i19wul7xw/tnbEpD3E5C+u86hlWsD73EXIIAkieP05r91TWrxvGdB3dorcYS0caw3pyCNrRJSXbSYaAA682Ktu0ocSYigORvm/m98al6zvKUCTEzjDPj5I9+V1VTDZWWC6S0rr70jNIZax3JJhHHFskYKSlJXrNrMK4RJZYJMlBYHo05i+Ok1FRCJbxPbRwoLRtp0Vgr+DGjHVkrQolCKPeJkqRc0SqLUkFlfJYKZpgjhzEwTJ7ZB0nYXKCRavG+vulX+s0tqQ5j3fwy89VLfCw8fnzDOAeGYTr2lUoRovjCFihFwtgkd31Jp/z6iyWDvUUTa7TBr9corfBBvM+77ZaT9ZoQQm3qSxWknRL6VhTTg3ipM4s+QisDqrIFimI3CAkpe5GleB+EmpMUGSWbZtHVECGGiXe65KS6hqpaYXHi6Np7Fq6tWr7waDTJRU4Kxhisc6zXa1zTEFPGe8/WOWKUGcaxRbx8G6j7Q2R3OKBLxp/1gpIMEVOkj95YQ982DCExRTlRhJTIwCFncmPZrxqis6xaI3ZdbVDWiTU7LflnrxyBYZ5JMRF/i9jnN7txlkyKEyFnxhB58uIaYyWNrjcWjLDyur6HnCuwOBG8iFUTDcoqVudn5LLAGCK9MzStxfUrUIrWOkrJGCV2v+AlBrRtV/XnEI1WrsJ2Y5WAPqLGZENn/ohZGwAAIABJREFUHSUWttc3DMPE06+ek2Kgc9B1lg8/vo9tHFhNIPL46iW325kpFaLS2KYBvchpjlSrd3bllNjebHFWUkG/fP6Yw+D54vENPiQWTNdSc5caBXt2dobWinn2pBg5HA7kXLDWCt+xEweRDIMEemuNbHwlw2E3QFZ8+qtf4aeJRw8fsN5sjtIk1zQQE0+ePOPyxUteXF2y3+4RhJ2S94GqRocCo4+kWBi2kBPVNgrKyCbftaID3h8G+beYKrd5R9dCEHNdW9sjgJKrqRS0zpBKxqcAMZL9SE4JUxLGWi7uXbA5PeEHn3wX17Tsh5H9fs/zJ0/F1bdEnSD7pjwwax88JV68eMHQtdxvNWpecXPq2HSOxmrO+5Y/+MH3uNyP/OzTp0zTTJiDtGWsJeXC4TCRGkfr1tgitCRtLOt+DUrRn2yIMXHY3jKPI8NXXwrYuutf9dN/w3rjkA9V85ZjzuyGif0gkaOxZHJ9BRcpgMpCjBcIigJtUQaarpO2o1PVZVAzZrRUrLrGmSoKSmd09c0ef4pcGzZJmtM5irc2+ZkUhPmYYyBFYUCGGEkx0VgjlQcGsmbwhcOYud3P7IYZtMY4I7lDVE1peRWd+q6unCv5pih8SOx3B/aDZxhGQswobb8mhlVC+FdLFaOrflaRUq6aTVMBtPL2jFEqP0qWXld9LVNKxBAYh4FhGAghEGOsxgYtA6Jp4ubmluubG8ZxZK4PUTH96K/pe2XgkbNweHNSx6mxbBSKxlkKRXpqpRz1vO/yEk3s0tYsxz6g/PvlBKeRFFqrEZ6EFcp/31r61tE1DussB175xNWirV5+XUSRvVSwqshJIYaAT0lAIiWTSsEocMZwvlkTlWGzqsT/OdThr+i4l9tOa4Oup1LbtBIzrTQ2Z3n/7HakmktVUFUL/jfcOJVSHwH/LfBIXhb+tJTyXymlLoD/Hvgu8CnwH5ZSrr/5ewmlJqRIGgu/+vIpPkUud3uUU3QqoU3G5GpXTEqOUUHS8kxzju0K3XmikIkxYKzDpAkVNUrdkVLc2DqlrUT4CrIttakf51ip8J4UPcP2BcFPHLYvCTEwTImQpNemdWJzJk6Ws5M11hnm7Jj2mZ98dsnN9sCPf/ElKWfu3b/LymgmHwkxcnuzr9iyt+/2ep3XNabE5dWOzQn4kLi83h9bMFpLdb/ME5SCpsq21htxEBmt8fPM7e3tseJs24azM4E9XF9XeETtgYoVs+D9jFZwe3vLerViHAeck75aypmrmxsur17yZ3/+51y9vOLm6pqcMtbao+xJ1XhYSiGpgFJFcGha0xh5MBudcM5wcd6htGbdSxjf7N++lMvXeV2ttZydnolDKAYwgo9bOJlaK4yCVSPDNl06KRKUUNVP1w1dazDZE2fPi2dPOAwTzllWXUdrjDAINCQgihdWYCoIzFFrTTQab424CZ1hYzUbq/nuw/vc9Rljem63e37+018QfKBxDmclXto1DeuTM9brFQ8ffUDb93RnJ6AU4zQzTRPTbs+sFArPHAJPHr8gfMtk/bc5Z0TgPyul/Aj4N4H/RCn1h8A/BP5RKeWHwD+qv/7WtUhLCjDOnsMwsTuMDONErqgjYenkI61IyEVGpue2wTQttmmxzmKswG+LnK2q71lL6l3tmaIWh5E66uEX5l6poXAxBnwIhBBFdpGlghVyi0VbI83oWNjuZ263I9vtyH4/174ZtG1D33es1z3rVY+rpOlfG0O+Peu1XddSBImaZVYpkpT0qq9b6v9If1l6X6YSskydlP/LvmeumsEjd+D4e6VS3OVDJvYCeFkcLTEltttbbm9u2G637Pc7pmli9rMAKBY9Ys61sonH9NTFlGZdzUZqLI0ztK2VKqpzdK0TzsHbtnO+zvtVbiYB49QedtUooChHR5XV4rJqraVvLJu+ZdN39K2jdQZyIseIn2f8PB0dOkYJuchojVmYDsdfy4de+AJGScqs1UcnoFUKpzWbTv6+1lkaa2TTNKYyKZYsIkPbtbRdi3PifZf3YP27jBgljKl8jL9pPHAp5QnwpP7/nVLqr4EPgH8f+Hfql/03wP8G/INvuxLGNqw6Q8pCAL+63vGTn33BfneHf/1H72OdIs5bVElYIyFapmkAhe3bKnuIcvHqkS7mDDHggpcSvOkpquYuZyo/MRF8pSfVqIaiAkVHilFkbfFqTdIZ3Tc4NJui8TExhBvGaeaXX10zTp7nL7Z4nwhB+m4f3L9H0zbce3ifpnHY1hF8wCnNOExst7vj5vq2rNd5XRdNbLs5Ax9RtgWdSVmmpVrrrz06NE3jaNoGW4+B3kuKKciDNQRPrNxVKKQkT39XCeKv0IM1UngRZBstYndgmid+8rOf8vz5c26ur9jvdux3W0rOhKYVSUrbUlCEIN8vzQMKWPUdzlpOTzsxadhC4wx376wxWjNOiWny7G73RP92ZQ69zuuaS+HgZ6738rpdnJ1ISmkjG8561aMU+Gkm50znOrQxnN65g3UOu+pR2pKDJ84Bv9sxDwPjdocPkdZorGowWgwnU4gUoLFW6qWcaRuLbQy2dbR3NjStI+8OeB853AxMCdbWQd9wsWnxTrFpRc3SVmni5D0uBFarnq6XRIeYE3EaSH5GW4UrDe1qRRcjh2k+mix+0/p/1eNUSn0X+BPgHwMP60WilPJEKfXgN/yZvw/8fQBXyeumisGXELXtbs/JqpE+ltKVVlQqFk6htZXepTEoTSW6KwyOhVycM0zThI4FVyQeNkYJ8/JzJOckb/JSK1kqki5nvE9Mc2R/8IRYwCiK0iQMISZGn5h8ZpjFmeCjPHP7vsc5x/ndM9qu4+LeBa5xdKueaZp59uxKyNlK8RZWnMf1N72uJ5sVbdfRdj1KR1arlUyj5/iKVMXSHpPTRMmZWJ1dIYYaeiZftVR9smG+6osta6kUS+UvLkF4kgCgxDPtA7vtjt1uR/C+xqssCZmiO10E9zHI+yPHdEzidE5cbdZqGguNMzKN1RqtZKCRUxL301u6/qbXdVOzhVzTyECtEvqNWSpFVfUJ0rPMGqng6sfyvk8xSuJlSqiUJYitFGEVKFX1sBlTT6K6usZUpYqZIkKW5QQnCo3ANE7MGXKjUDnhjEY5Q2MNtrrYjH6VOZZyIqaIr6eOcTjgvReSU84oLChF49y3OsJ+641TKbUB/kfgPy2lbNW3fONllVL+FPhTgK4xZQ4JZw0NCqMMjYEvPn9MnEbGf/X79F1Lt+opceZwfU3J0Pd3pJpo64aYar+s4se0c4SY+fznn5GKoj+7i7YNyjaVKbij5ERbsujATnu0gjAXpjHw+ItLbm73/PgXTximiC+WBS9VqgyloEhaYWzPRz98xHrV88lH77Ne9dy9e07bNaxPN9imYX16zvXNjsvLW0qBp08vmedvfoL9ba3XcV0//OBRee+D9zm9c0FKBXTD7e2O/+uvfsLhMDCMop9VRpNzYRgO9c3rKaWw34kQPqZQRdcScZCr1dUae/ScA/gQ6hE91yQA2Tqtc2hjuL6+5urqiq8eP+by8hI/jagcuThdYbTCObm+gy/EWKdBRW481xjOz1e0rWPVNVgjiDlbP0ouTNPAOMwM+wPzW1ZxLut1XNeH56el26z44aMHUDLT9aXYmHOUB1v9t0sPv3IgUmZ7OKCMRU8BlLB0Q4jYXOi04bTriTkxWUfICTVOYirJiVTykYW5yITcHOnmyHlWrFLh8XbP7X7k6dMbxgRmsyGXQuc0GMdp21SLp4T2GWfIJF68eI61mnEa8H7m8vIFPgjwBaVp+41QuLoWaL/xdfqtNk6llKsX4b8rpfxP9T8/U0q9V59e7wHPf6sLgzxRjFasWkfrrHiCQ2R3GGmdolsZjJVJ7FHHqaSvUn8g+aRt5Staconsdgd8zAxRo4yjWCFH73ZbVMnc6SxdYzk76cRrXntlfg7Mo2c4TBymwCHI0VIbU6fAjfQ720ZCqs7P2GzWnN45Z913nJxuaBrHerPGOKFZS69EjiBUnerbtl7XdVVaCTAFQEHft6QYOdnI0Xax4pX6tUv/aeldanlDVH7mK+cQC/eyCt+Xm//Xep5VGG/qxqqNIYTAPEvj38+zEK2sYb1q69HbkAukHETYrRUqa7TKdWOtfTKrJNqlnlByZbOK51n64OVbSOF/G+t1Xte+7zk52UDOlMOWFCDHctwkKQW1DNuRAWyMCTLoIvOFknUdkkrqqVFyorPWQAJbh06mxpwuOtDlnGJKwRZo0DRFU5K4ymYfmFPBeU+pe4pSkiyx9EgXpkTOmXEcMUYxzyMh+PrwTghqXN53WhfMApn5hvXbTNUV8F8Df11K+S+/9lv/C/D3gP+ifv6fv+17oRTKSqRC21p+8MkH9F2LUfJG/Yt/9lPunq/5d/+t32fT96w2Y5UFjaTsKVokSQnhZeqmQykD2uLHA19+/oTb3YGbqeAT7AKEmNjudnSN5d/4w+9w/+KEh/dPsMYSVEGVJPne0SM3R2S3H0kZupVY+EQ8a3GNYb1Z8cnv/ZDVeo0phRl4uT3gnPAIc4HHl19w+eKKTz/7gpubW5FBWAd84xDzja7XeV1FnB55+uwLci50ruf8rOOP//jvEGNhmETStd3t5KgUQ92A5MY7Xa+PzqFUYRJLFQkc78qvj9gUHKuKvutYrdacnp7StC2Hw57b2xt2ux3TNHJ2sqZvLB+/d0HjJLgtxMSzlweGKUBCJGdZcqsap2gaaBs5ao6HA0tshEIkcihzRKa9Tet1Xte2afnuxx/Tdg0pRsw84qeJ/W4nrZD9nlIKXdtKi6QetXMKoCLaREBRlJXTQciUmEnVOtk2ViJ9VcZHqTZDSuQg4W6hRByK3hg2xnGhGjqt6ZTDYphDZAyJaCbh3roGoyQbTCtVZ8KFED0xR168eFYfwvLwbVxL0yi6rgcUwyQnHqMy3yY0+20qzn8b+I+Av1RK/bP63/7zegH+B6XUfwx8DvwH3/6t1HEnV0rRdS1d15Ci9CCub3dQErvBo1UjE3SlKSVA1VeVnAhFnvxxDiiVyCUyjBOzD0yT5/Z2YgqZ6zHho/RQV53jdntB1xhCjDirgIzW0LWO1arhZNOBVmxHT8yFttHVI68wVm4qa1QFuXpGH1EUWlurlEYCyJ49e8HLl9eM00QMkjX+Fir+XuN1LcTomf1EydAah9aWzWZNKYp+XX3LCkJVLxzzmJDJdCmSMZRzPv7+q29ft6zFtF44QkJWfc9mvaHvOqTXLe6U2XtyEqVF2zjaxgrVXy89dk3jDCllnLXyTeNS2cqpSML5ln4qzLNUqKlCLo6swbdrvcbrisi0YiRXkpU4hCplKFZNthVeatbqa31sqnZa1JlpEckWyWySeCe5xkarmoYqCpSQtGh2i/Q4xScvm6pV5aj/FeKZPICVEsrZMiFXyJ+r/4j6YE41vfTVvaiVnIDEbSb7jLOvoeIspfzv/GZj2b/3bX/+62t5YtczuMh8jGUcRoKPPNnteNY5zk433L97wh//wcf0a4MNIzlFDpMn+JnrnQirhzkRU2EcPd5HQpDeyPZ2z27wPL45MIfINHu61vLzTxv2w55PvveAs9MVnSm4zvKD3/sI7wP33n/I7jDy0199yeyDCOmR6TxKY7uGTOSf/pM/k0jjKOToRw9kmt42T5imiX/x459yOAxM+xFdoO2+vdn8ptfrvK4xBF6+fM4c5EY5lELjOu4/uEfT9HTrU0JMuC+dPLQq4Hlpwts6CV9QY5IZJVNdEZ43OOe4uHMhGkxkALS93aK04sG9C9brNbv9jhgjl5eX3NzciGbUOU43a6zOXD5/jFaF05MNShs6Y9CNZdM1+KiZQpEo7jp4Wq96KIXdjaQ7HvY3Mqw0LTEWtLZY9+5e1+A9X3726ath3jiSY2IaZmIU4EmmkGZfoTmuAllkg41xqkf66p6LoDM0SpFVkThuCjZLptdJK7ZMo2qRpBWNNQzF89If+OnLJ2waR1ICANJoKIoYpRfeNS3OGlorT9e5Upq0fpVSCxxZCTkljFnkVYIINNZy7969+jD9zeuN05EW+nOudBMKjJMn+MA4CRzg2eUtucD2EChoOtuglEEZ4f3JwKYwzXN1qgyEkKpzRDzGPojzJ1f4X86Z/WFiuxu4vt0DBbVusAbarsU6x0VStG3D9e2WyYejzczHSEaBdfhYmK/2zCETssXahm3f46xFq8Q4jVxf3zKNEzoXjNY4o78VjPq7vHIpeB9ISYHSRCJKyfFcmYhNSXST5VW42eI9P9Lbq16XAtkUFrwcSuHahqZpODndCJNASyLqQknSxpBz5vrmhhAC+2FgmiepGo1UMloVphjRZEL0GOPEUWJkUqyPXmypqESyKD9PyZBTYZoDpSisM6S0uFzero3zda6cM9M4AuK8Ew9/PkaaFOpJoQhujqXyU5JuqqqiQhVVK0xk2Kde2W8VFf2AomiNBqKRWtQ0DdZWpUTJ7MeRkgImGWLOx9d/6Tkv5KblZLL0N0WNI9WsvK/KkR+xfCgNrmmwxtG2La4WTb9pvXE6UpiC8P2y51efPcFozTxMlFwwpmGcEn/2l59ysl7ho+HB3TP+7r/2Cau+oW9nmhiJqmWcZm53IzF4trfXzFNgv5sZJs9+v8OHxKZvyEodX9DnL/bsdzN981Muztf84e99wOmm49G9U7rG8OHqlBgjp6tOoBBlaYKLRXQ7iLXy6voKcgRW5OT54qsvpfk87AkhcHNzQJXC3XVP4wyn6/bd3jhzYX8I5CJv8qBBT4nd+Llg/bQl5cx2uxWAtXPiJbdVp1stmIL4qzdDkQerUorVqmPddzgHfddLCFeB0Uvl8/L6kpQS41/9JTFGDocdYZ5xTtFYJxSjwqupfvDYnOnaTm5aFSlEUha5RvCSoDoc5Hg6z5lpTNxcDaScMU6OdD7kt8459DpXzonxsBcrZSniBa+hiLlI1lcBYs3tTkGSElotx/DG1Ejhxsr1mir96qjrNeSSGXOVCylNxtBU48nFw/sYrcnjHl0ST589o9WavlkxhYLPhViUmBdSZjwMlMbSOVFPtJ3IFZte3i/73SBYyCgNc2M0pWhCAmcsD957QNO0bFbrt4sAL5Wf+IKiykxzOIqgdaUIyfTLA4rnlzeUAje7SWRGvQGjME2HTdA0DTFGnJPm86If01qha6VRlOTI5Fzwc2CaI9fXO3LOvLzaEUNksxKXw7qTqeyq70lNJtUnaog11S8Iwr9rNCkbVJYXfZw9PgT2h4NsuFHAx1CqgkB+rnd1lSIsxaUvWbLkoesSQGeUFeKNDxKml0pGK01O6WuedTkmKaUoytQTyVwLugRZnEDBizi5FMVuPxBCPD6wdvtdzcSJ5BTFcIKuFSesVr3oNYvog5fJbcriPEpJfu4QE1prpsnLezYtFSiSIlBhFMcq511dtYoUrfRSpS09Rok8Aak45UuX04SuJ62a46U1OcNyC6hSh0hFft9pLdAeLf5yrMI0jtP1Bms0SWVMCthyQNWEzZCQMlEvAOzaf31VdrLArxdf/NKvzqnmRmnJvDLVmtnU7Ctp+b0mHefrWAWYUxK/cNaEkHDW8fHH76FK4cmT58SYsK4hpcRf/tXPWK9F6X/37jl/9K/8Pv2qxXYNqybynrb4eWJzsmKePC9f7tkfJvahcJhm9j4SkoA4chYdYcqFL756zrNnhqtnV2w2PT/6g+9werrm/Yd36FpH37XYxrDpW9EZHnaUeSbcRIoqPHp4hxALMVumOfKLWTKSdocDIWSslg0ghpmoElqvMe9wWFsBUpJ+UykwR4/ShtXJiqZpubh3X2Jj9wM+eckYUgqrDYXCPM8iRzG23gySATXNswwnxoFBw3B9KcTxbkVGcbMfBRk3yyBo9jNaK042Pc5oNo0QfE43LV3bcOfOh6QU+eyLL5h9IOTMHCLb/Y5pDkwx1uN6wmrNsD/Uyb1GG0nhzLkQi+g/94fdt1rzfpeXUgpnDdoUSlEYZ+QBghyFjRISvqk5RH6aJFRt1dFYw7oVO+3sJa45BSUQ6DrYmeeAKrBpGkCDltRQXwpN3/PR+x+y6hrO/IgNM/blY+m7+kgoGbPqcCGR9gc0kqLrKogll3wc3EVfnWEx1oozoY2lW/V0fc+9R/dp/u/2zqw3jjM7w8/5llq62SQtSxqPx0mQucgEyP//E8lN7oIgGWScWLZ2iUt3bd+Si3OqSQ8mNoQojIfTR6BEQM3uZlXXqbO8S9PSb3bM88K//9vvmabpJ4/Ng+txrguAWlHwundsNhudidgCxdtgdpomOIy8fP2eacl89etrzqYNu22rqjU+EptC1/U4cfS9QkrOzjZI8KTDgCyZ/aB3RKyqnedMXgofUADzy7dX7KcF7x1913B5fqZVrN1R9+PCNC3MS1Zl8xCUy5yFWoQgRec6Yte9u2NNVFO5ftQK8PVuHLLOkNZN+VqFrqiCddkTgje7Zj3vK21Sk7B9bwrrTQxqEFD0At2X/ZGyq4lzsW38rPjgLhKciTxET9sEujayO9uQciKGoD+XytGDPZd8rFZK1m5jXnRGLdIA2tqJs7ZfTBf2EZ9W0M9zRdl6sv7+OLuGyxGXXQXzCLsz13POI1IJXj8Dwau+bra55LovFafaqCE2uoR1jqbv6NqOrolsKAQKOE8RZ/hovc681+cNTjsWbxtyPZXmZ2UKZev5FSe44Gj7jrZrtdNxcsT/Hg57xnH8yePy8PbA3uFRqEcMka7tefLsVzrj+PYFKVcuznZ472jmhWVZ+Md//hdiCHz38h1fXJ7zD3//W862HZe7RqEMsUckcLbJNCHyd79tGOaZH96942Y/sL99w5gLxVqCXISaK7cfDriPB15e35r6zZa+bfnVsy/p2obzix3OCfOiUmSlZpwT+i4SnKkvsdDJxOIXvrzomRMsSYfh+EwW1W/8OX2/P/uo3hTdK2p9AuMwkZYMvNVZ8TwTveevfvM12+0Zz549RZxwdX3FOI68fP2aYRz5eH2rfPRc6NuG3/3tb+nbSMkL0zzz7YsXTMvCunHIpiqfTMEnRkfXRi4vtmzaqOe1a7g837Isib7vWVLl9sMVh0F5y6tcnM5czTDOqRDyMI0IQnCB6D2b7ZaUC+M0sqRfJiPsc8QKyxpLoiK0nZogOhw1F273eyUrpIQDtlFdP3MuLMAcVLCj61pqUQhSSoX9OCttOlaoQjEpyItnvyK2Le1ui4+RdtPjqOQ0UpaReR4Z5pmbKTMUaHwgOM/2fEvwjvPdFh+c3QQLaVTuu6BWN6VoR9Gf97T9hq+++ZoQI8Ow53DY8+HdFdM4cvXhvVJEfyIeNnGa+sk6F1IfmMxhmlV8IRdS0a2dLVgptXIYZ5zMvH7znnlaePblJcPQQz0jBkeUTEmqPr5yw73ztE1kXrLNuAo2E2bFlqVsZdKUWHKGj9DEGfC0TcNh0kSZ1D2OGK1tQ+c1khMlKQfai5jnsyCzXtMelSUrFfVJecRxf1u5bi9ryeQM8zTesWy8J/pAE4MCp52wdJ2q3fiAd96UxLV8dXZc26al5FWLU1lf919bGxjtYJoYaJqgn43jlzJKSnHHLWsxtfLj/NkqpmDuleKdzTWzCQVpUg1BHQTaJvwCYZyfP5T6Wg1FaPNCWeGF2Dwfq87F2HJ6PR9BiKJOqM4pQ6giRAlWyABOcNETuoazy3OcVYElJYZpoo6j0iOTYrNTFZwS+1Qlbd2cyzq7vvuIiFN6d9u24B3tdkvb9bigQtbTNKlyk83PBX5ZQsbOCZu+YRwmKoXrm1v2h4H3V/9ErZVxGBDB8FXOQK6Vpm2opfLi+1e8evWG12/esOlbvvn6OdtNx6+fX+Kkcv3+Pcu8cDvo3aI96+k7r1vYMjHsJ3I2PXIRXFDZsMsvL0Dg6sM16WbPy9c3+l63G02cKensbNPSNJ6nlxti9PSNSlClIvgY+XJ7RsEzzsppljQpVc1amscatWL2FHqDWaW8StYlzTSOto1NBIFlHplHx/7mGh88qyhw8CrOgCVZ7zwOxzIvStOr2TRSNfnVlKkFhQZV2PQ9fd/w7NkTNl2g8zNNKMQIIaxLAwg+EkPEB3VEfXJ5dlwugAlpi5p66ZZdlZuaEPHOE3zFe8ezp5d/ZPPxyKJWypKU6SNC8QnE62KtVprYUIPZ0dQ7W5z9YdQlW2mJQdX2DUGJl8q2DbpA8pFUKm+uD5SaWVym7R2/+d1f44Lnh//4lv3+hpffvWA5HCh73Yrvc6WIo9loe77dbnDitONIUCyROxdwTmeuMUY2X+yIbcPmi6dUhPdXNwzDwKvvX5BzZtfv6PsNwYUfEzD+RDxo4tS7tjtWndmGypMB1wW7G+VF2z3zj/HOUVCsYMmF65s907zQdx3jONM0AS+V6yvdau8PMzhh573OsbIyF44Vkd0tndcKUpW9bclRKvOStKhxoy55Uj6yDdpF1VdidMyNXvTTos5+LlSq1LvqKt/b9j3kgX7w0HnXis4TqYaNNKyjraSdaQ+kNDPPnnE44IymOi8LyRTcc1YGkRdHqYXDOJm6kW3my1rRHg1IESNUxBh1SRA83l5vJWuuuEO71I/vKUZNkLJ6aauEEqsyfA36/97oeuvzBUv6jzlk/aPtH1UKVe66RlUwcj+aZ68YzyVXEO0CjfOiMDTjgnvR6zYGT3GKfNAOI5iebqGWxM0wMO4H6rhoZ3Pvsauep16ndl7NjLGJgeA9m01PbCJnuzPVkQiRJWeGYeRwGBinWQucXnU7Y9v+qKP5U/GwidO2dLWLZCPpV6NiOYRdqxiuJ43gHeSqCtuzaygIMTaUqj4oy7Dw7fdvCN7x3au3AOwPIyllhsNEFeg3ERD2+2SQEh3ohwjBOTablhgCvYdcq/qxewhdPFZRtRYyBcz32U2O60lnJSqeDC5qdSxBT+w46oVPVkOqvu8e9YxTHSt5XfnwAAAKz0lEQVQT4ozp0+hyYKWulaKJZ4Uh3Vy943AbuLn6qF2FUxrs6zeqInV7OGjC7XpKKfzr7/9gsC5Ld6LoiHnSGy8o3Otsu2PTN0qNdRBcwTsoZSaZs+GSii765hmpCS+FvlVuc+PMqiMrpXL1A28MDF2z4XrTqO9P/M9S8/6cQ6FCDZtoAuBJXdCPnPSy+kDdHYdaTdS6Qp0LKcNBZpognLU6Iz7bbpEqTPuZnCvPdhdU52l8YFOgux1wXujywm2aefHuA9c3e8osRO/55uk5XRM5M/ig1EypKg6COIUWNS1fPX9G27Xq3NAE2rMNc8r84bsfuL498If//I55mhEy3nuVlGwcZ7vdLwvHud7lQww4V5jmpFguG5g4dC7Y2F2kCpQqykjBbBhKBadsoJQLuVTSXqldh2HWhDwuIFDM4GuZjR1Qi75IVaaKKuC4Y4txrEWOHjRrhQIKylZu7JLvtq8IBES5ZFl/ZrIt78qkUMHlx5s45V41cfxeQCtQuaObGv6vpEQ19omIwlCWlJkmFTRWbU45KiqlrHOn6Di6kxabqZbjdvVOVV5xfdb6G4Y05cQ0z4qMMEaZwJFdpMK35nUlttVPmiBWBkquepPQKtfmfo89asWhHd/dtXD396qA/yMEheiaPRfVPS3mHeaDKq/HpkEQlkW33i0BvKdpO5oYSNOECORpJs8LS1Y0S8Urp91m2E1UucEs+j7qOotFl3ux7Wjalth2+FURK2e1yxgH1WHNxZaCKq4uWWez/mdO7cMmTnOuPP/i0pY+CgPxwRMATyXg6HxD20TdXpbKf7270QMXKnghepWFa6LqbV5dXbMsSdVNSiWItWk2IZaiCW/OSvBvGgjO02w2iAjvr2+Yk3Lhc6k6z6mQ0yo0sMIrtIVoWhtcFwPtJ/VfX5b5CItRyIS26jeH4SEP84OHiBCjw3mFYIkUalUxYQBvlZkPVpnUTEmq/F4RcK1ax44j87ywjCOgds3OmTuhyNFmI81ZPbonvTFtd1uioTQEOBxukRL44vklwaPWGcPIh9vMPBdub26Z5pnGCxLUPjZ4x7ZtEWCcE6lURrux15UK6mwRZa3qbB3JY41SK8M0K5RMOCLYV9rsekNcVtEWS5xNaEilMkwLxUFuHYhnuzunbRuazYYqDndWQDxn3VZnlf0WqZVXr14zTyOv377i+maPF4WuNU3Ppo08f3rBpo1srTP8uFcf9pwSVQRfbBHsI8U3TNWTp8S7H35gGAfevXnLsiTO+x56EB8ooO93mHl/dfOzx+bB4UigtCzdtBnmS4IWgjVZyX1PuktW9eZCratO5o99nlNWr+5VHAJfj8wQxdzVIxdaWyxlFGWbP05z0k1dXudg61Zu3Rs6apXjvNJ7rSCrF6tAJ0qtpt24CpDdqZ7/0mwz/i/C2dwJA5DXtSrj7l87+keesIrlCFS981PrUSCiUkxnwJ5CVOWfCnnRxFly0RFIXdWVEglYlkqK2JJKrWaXpbDkgWVR+bpaTNFc7quW67nz3oEUgtfP4pFXbYmiWCvqRFlwjzWqMf2U+i1HYeljD7Z2Zvc0SXXLHZFcmJeM2PVbynpNilbuAsVwmVWH31SnbLL94cA0DOxvDwzDSHCOGiNnm5a+Ub+n1b6jlMLxiqv2GnUdGShV+nYYSTlxc7tnmkaSMdi8KOazgH6ussLaRit8fioefjlUKstwoFah9YEogvcbKJXD4Zp5qXRtpkmO0SVyrdxMC6lktQF2griA2r8qgDmnxfyc63HIL1W9sUWwnxGCD0rBii0JeGXUy/1hUm+cVbjYEnIIjopesKUW0jgp3vPJjiZGgg+UnLia9xQp9GcdpQrv90HhTbbIcFLswzY/5OF+sBAR84m5W7voHUu/D7alttXrnV9Q0cSUk7ZZmxhIIqR5IpdCnvZk0WpFEFYjjbxkuxgVNrIn470w3GRiEPzTDa527OdE8I7rfVK5wY8fFK5mCVxHRgphEndvGRm9ivA65czrNSSIBGM0zUrddD+7Q/izjgokKq5ospxTNsFgXfA2ptqx4ndDCITgOdvuqFVo9gdKSizLnkMtfLi5JU6RaMrwi0Qywm2+Qbzn4nxHLYWrly8Z9wdeff+KnDK7tqNpIl89f0LXBM470UXhMJCypmMn7lgYiXHhrw8T5TDz8u0b7QaXA1IzEc0POSdKgUGnbKSsi+B3Hz/+783aPmfIOk8ypoFfC7r1DlEAVymid6JUIZVKrvrlVrgIANV+cYP71IpbWwobqdnTHRkK60XSWEs2jXrgVemGY2INtvUT8frhScXGDIUQPG3T0jaR4D0le6YuUkuh7zRxdklwKZMLZMlW2T7kkX7Y0Fmhmmrr6dRKRL19OBqorTMyOcIw19JeP6TBqxJ3GzypCBlYVeDvHqsdhEdH0foaWZXDy0KRNZvpeYVCzpCzLvtyLvo6bq2S74aypa5bePu9nMPVejfKtMcoU0aOeMTHGqsE5JHXr7QhZF0G2bHWyr4eGUMreyfGQJbKtOi1nauuAkgZRAU6coU5Z3COqW2hqNbtYkUHQBujOsi2kSZ6vCtGVXJaJDlBqlJ2nYmOizgd+xSFPS7LjKuzmmVGj2AVZqlMc9YqGK8ssvwZXC4/Z3jveHLRsz1TPcPhdlAJ/CVTMuS00DaB/mJH00RSrcypkMjkmunEEUTogrbN0zCwpKzUvGpWFwjVV6qoasu6ABBQRknX8s1vvkZEePnyFeMwMgwzIo7d7oKma7m4uMSbdl8plXEYKSVDnYkx8OvnT4kxqiBuzTx5orqN3nXkDM2HiWGaefvuHfM8sV9ufpEWC58vDJICIOvIQkdiTtT8Cu5M1uZ5pohZQFur5xDaNgKBTRdU7CXp7Gz1Zi95XUToq1ZrHcU47zlVYoTdpmPTtqSlkJMAHicGkkYtbZ3pMzoTsChFOdX6/LbUcivF0FSAslbGWgA42kaVwx9rKMSrIWfrKkxHtWsboJKnAUpls20NaogB5BWy1PQtKXsO06TteDyD4NXKOWemcSSVSvEO8Z45HHShmgqN8zx7+kSZPptOfcqCx4vY+/H0uwtyhWU/Qa6ctzsQTzQ1pPdv3jJME69fv1aqrVd9z80X51QRbueFeUm8v7ol10IbjVorqsv6U/Hgrbo37CTVPITq6jZ4t7UT5xHvqTnZrPHHz+FEq0mtNMvxAUdIhOhmqN6bNa6FhS6HIqs6ilh7tlZGwQe1rjW+vIoCJGrR52qCYgVjjFoxs3KZK46GlCFGxY6us9C/iJA1hYhh9sQE0rUNXmdQx9n0fRiPnV9nSUqcpxRhqdkSpy0l7pXttUJ19fjatVqHIaqy49zKMrIHrF9yf/Mvd+8Z7rbl9+azwt17/WPM5n0fpEcZwr3fT47XpRYoVbUcpNyp5vPj06qVoO4Tql58VqXqHLLkbDc/oUo1v/WykpRUtNrM9fzK5sK2+WihVKt2BlIrXgKIxzv1McspmUZvIqWEYK6X2MzWZqBLVrm74Eyti/qz51UeEsArIm+Abx/sBX958Te11mf/32/ic8fpvJ7O6yON//G8PmjiPMUpTnGKxxB/IX3kKU5xilN8vjglzlOc4hSn+MQ4Jc5TnOIUp/jEOCXOU5ziFKf4xDglzlOc4hSn+MQ4Jc5TnOIUp/jEOCXOU5ziFKf4xDglzlOc4hSn+MQ4Jc5TnOIUp/jE+G8E7ro9hFDVsgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "DATA_DIR_TRAIN = \"/home/share/dataset/cifar-10-batches-bin/train\" # 训练集信息\n", + "DATA_DIR_TEST = \"/home/share/dataset/cifar-10-batches-bin/eval\" # 测试集信息\n", + "LABELS = \"/home/share/dataset/cifar-10-batches-bin/batches.meta.txt\" # 标签信息\n", + "\n", + "ds = ms.dataset.Cifar10Dataset(DATA_DIR_TRAIN)\n", + "ds = ds.create_dict_iterator()\n", + "with open(LABELS, \"r\") as f:\n", + " labels = [x.strip() for x in f.readlines()]\n", + "\n", + "for i in range(1, 10):\n", + " data = ds.get_next() \n", + " plt.subplot(3, 3, i)\n", + " plt.imshow(data['image'])\n", + " plt.title('%s' % labels[data['label']])\n", + " plt.xticks([])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "在使用数据集训练网络前，首先需要对数据进行预处理，如下：" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def create_dataset(training=True, num_epoch=1, batch_size=32, resize=(32, 32), rescale=1/255, shift=0, buffer_size=64):\n", + " ds = ms.dataset.Cifar10Dataset(DATA_DIR_TRAIN if training else DATA_DIR_TEST)\n", + " \n", + " # define map operations\n", + " if training:\n", + " random_crop_op = CV.RandomCrop((32,32), (4,4,4,4))\n", + " random_flip_op = CV.RandomHorizontalFlip()\n", + " ds = ds.map(input_columns=\"image\", operations=[random_crop_op, random_flip_op])\n", + " \n", + " resize_op = CV.Resize(resize) # Bilinear as default\n", + " rescale_op = CV.Rescale(rescale, shift)\n", + " normalize_op = CV.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023))\n", + " changeswap_op = CV.HWC2CHW()\n", + " \n", + " # apply map operations on images\n", + " ds = ds.map(input_columns=\"image\", operations=[resize_op, rescale_op, normalize_op, changeswap_op])\n", + " ds = ds.map(input_columns=\"label\", operations=C.TypeCast(ms.int32))\n", + " \n", + " ds = ds.shuffle(buffer_size=buffer_size)\n", + " ds = ds.batch(batch_size, drop_remainder=True)\n", + " ds = ds.repeat(num_epoch)\n", + " \n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 定义模型\n", + "\n", + "预置模型为LeNet5：" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class MyNet(nn.Cell):\n", + " def __init__(self):\n", + " super(MyNet, self).__init__()\n", + " self.relu = nn.ReLU()\n", + " self.conv1 = nn.Conv2d(3, 6, 5, stride=1, pad_mode='valid')\n", + " self.conv2 = nn.Conv2d(6, 16, 5, stride=1, pad_mode='valid')\n", + " self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n", + " self.flatten = nn.Flatten()\n", + " self.fc1 = nn.Dense(400, 120)\n", + " self.fc2 = nn.Dense(120, 84)\n", + " self.fc3 = nn.Dense(84, 10)\n", + " \n", + " def construct(self, input_x):\n", + " output = self.conv1(input_x)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.conv2(output)\n", + " output = self.relu(output)\n", + " output = self.pool(output)\n", + " output = self.flatten(output)\n", + " # print(output.shape()) # 仅Pynative模式时可用，Graph模式时请注释掉\n", + " output = self.fc1(output)\n", + " output = self.fc2(output)\n", + " output = self.fc3(output)\n", + " \n", + " return output" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "使用Pynative模式对网络进行调试，比如打印网络中某一层的输出shape：`print(output.shape())`。" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "context.set_context(mode=context.PYNATIVE_MODE)\n", + "x = Tensor(np.ones([1, 3, 32, 32]), ms.float32)\n", + "y = MyNet()(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **注意：**调试完毕后，需注释掉网络定义中`construct`里的打印语句：`print(output.shape())`，并将切换为Graph模式进行模型训练。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "context.set_context(mode=context.GRAPH_MODE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 模型训练\n", + "\n", + "一般情况下，模型训练时采用静态学习率，如0.01。随着训练步数的增加，模型逐渐趋于收敛，对权重参数的更新幅度应该逐渐降低，以减小模型训练后期的抖动。所以，模型训练时可以采用动态下降的学习率，常见的学习率下降策略有：\n", + "\n", + "- polynomial decay/square decay;\n", + "- cosine decay;\n", + "- exponential decay;\n", + "- stage decay." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def build_lr(total_steps, decay_type='cosine', lr_base=0.1, lr_init=0.0, warmup_steps=0):\n", + " \"\"\"\n", + " Generate learning rate array.\n", + "\n", + " Args:\n", + " total_steps (int): Total steps to decay over.\n", + " decay_type (str): cosine, square\n", + " lr_base (float): Base learning rate. Default: 0.1.\n", + " lr_init (float): Initial learning rate for warmup. Default: 0.0.\n", + " warmup_steps (int): The number of warming up steps. Default: 5.\n", + "\n", + " Returns:\n", + " np.array, learning rate array.\n", + " \"\"\"\n", + " lr_base, lr_init = float(lr_base), float(lr_init)\n", + " lr_per_step = []\n", + " if warmup_steps != 0:\n", + " inc_per_step = (lr_base - lr_init) / warmup_steps\n", + " else:\n", + " inc_per_step = 0.0\n", + " for i in range(int(total_steps)):\n", + " if i < warmup_steps:\n", + " lr = lr_init + inc_per_step * i\n", + " else:\n", + " if decay_type == 'square':\n", + " frac = 1.0 - float(i - warmup_steps) / (total_steps - warmup_steps)\n", + " lr = lr_base * (frac * frac)\n", + " elif decay_type == 'exponential':\n", + " pass # 尝试实现\n", + " elif decay_type == 'cosine':\n", + " lr = 0.5 * lr_base * (1 + np.cos(np.pi * i / total_steps))\n", + " else:\n", + " raise\n", + " lr_per_step.append(lr)\n", + " \n", + " lr_per_step = np.array(lr_per_step).astype(np.float32)\n", + " return lr_per_step" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "观察不同学习率下降策略的曲线：" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3gU1f7H8fd3s+mBACEQSehNiiAQUJogSJWmomBBmoAU60Wv/SqWn1jABkivKiCigAIRaSqIElSQYqSFjmKA0Ek7vz9muTfGICFkM1u+r+fZJ7szs7vfmQc+mZw5c44YY1BKKeW7HHYXoJRSyr006JVSysdp0CullI/ToFdKKR+nQa+UUj7OaXcBOZUsWdJUqFDB7jKUUsqrbNiw4U9jTHRu6zwu6CtUqEBiYqLdZSillFcRkT0XW6dNN0op5eM06JVSysdp0CullI/ToFdKKR+nQa+UUj4uT0EvIu1FJElEdojIE7msv0FEfhSRDBHpnmNdbxHZ7nr0LqjClVJK5c0lg15EAoAxQAegJnCniNTMsdleoA/wYY73lgD+A1wHNAL+IyLFr7xspZRSeZWXfvSNgB3GmF0AIjIb6ApsvbCBMSbZtS4rx3vbAcuMMUdd65cB7YGPrrjyHLbs3cpHyx7lYNHrMAFhOMSJAydOCSbYUYRgRwTBjiKEiPVTRFutlFKepVpMETrVKVPgn5uXoI8F9mV7vR/rDD0vcntvbM6NRGQgMBCgXLlyefzov9qz5wcWOA/AmfmX3NZkBWAyipGVXgyTXoys9BJkpZUi63xpTFoUEJCvGpRS6kp0qlPGtqB3O2PMBGACQHx8fL5mQunYvA8dAs+RufTfpLd8krTGg0nPSudsxllOnD/B8fPH//v4/czvHDp1iIOnD3Lo1F6OnN3w388JdARSMbIi1YtXp050HepE16Fq8aoEOgILZmeVUqqQ5SXoDwBls72Ocy3LiwNAyxzvXZXH9142uW4QzgOJOFe9SmhcQ6jS2lpR5J/fdzbjLLtSd7Hz+E52HNvBjuM7WHtwLYt2LQIgJCCEmlE1aVC6AY3LNKZudF2CAoLctRtKKVWg5FJTCYqIE/gNaI0V3OuBu4wxW3LZdhrwuTFmnut1CWADUN+1yY9Agwtt9rmJj483VzTWTdppmHQTnDwMg1ZDsfw1BRljOHT6EJuObGLjkY1sOrKJLSlbyDSZhDpDqV+6Po2vakzLsi0pX7R8/utVSqkCICIbjDHxua7Ly5yxItIReAur8XqKMeZlERkBJBpjFopIQ+BToDhwDjhsjKnlem8/4CnXR71sjJn6T991xUEPkLITJrSEqMrQdykEhlzZ57mcTDtJ4uFE1h1ax7pD69iVuguAypGVaVWuFa3LtaZmVE1EpEC+Tyml8uqKg74wFUjQA/z6Bcy+C+rfC13evfLPy8XBUwdZuW8lK/auYMPvG8g0mcSEx9CxYkc6V+pMleJV3PK9SimVk38GPcDyEfDNm9D5HWjg3nu1jp87zur9q0lITmDtwbVkmkxqlKhBp0qd6FipIyVDS7r1+5VS/s1/gz4rE2bdBnvWQr+lEFv/0u8pAH+e/ZOlu5fy+a7P2ZKyBac4aVWuFT2q96BhTENt2lFKFTj/DXqA0ykwoYX1fOBqCI8quM/Og13HdzF/+3w+2/kZqedTqVC0AndUv4MulbsQGRxZqLUopXyXfwc9wIEfYUo7qNAM7p4HjsK/Iepcxjm+3PMlc5LmsOnIJkKdodxa9VZ61exFbMTf7iFTSqnLokEPsGE6LHoQmg+H1s8W/Odfhm0p25i1bRaLdy3GYGhboS19a/WlRlQNW+tSSnkvDfoLFgyDn2ZCzw/h6pvd8x2X4fDpw8zaOot52+dxOv001191PUOuHUK9UvXsLk0p5WU06C9IP2c14aTshAHLIbq6e77nMp1IO8HHSR8zY+sMjp47SpMyTRhy7RDqRte1uzSllJfQoM8udb91M1VwURiwAkKLue+7LtOZ9DPMSZrD1M1TOXb+GM1imzH02qHULlnb7tKUUh7un4Le/8bqjYyDO2bA8T0wf4DVBdNDhAWG0bd2X5betpSH6z/M5j83c+cXd/KvVf9i34l9l/4ApZTKhf8FPUD5JtBhJGz/Ela+bHc1fxMWGEb/a/qz9LalDK47mG8OfEOXBV0Y+cNIjp87bnd5Sikv459BDxDfH+r3tu6c3fKp3dXkKjwwnCHXDuGLW76ga+WufPjrh3T8tCPTNk8jLTPN7vKUUl7Cf4NeBDq+DnGN4LMhcHiz3RVdVHRYNM83eZ55nedRN7oub254k1sW3MK3B761uzSllBfw36AHcAZDj5kQEmkNgHbmoqMne4Sqxasy7qZxvH/T+zjEweCvBvPQioc4cCqv0wMopfyRfwc9QJEY6DELTh6Cj/tAZobdFV1S09imfNLlEx6q/xDfHfqObp91Y/zG8ZzPPG93aUopD6RBDxAXD51Gw+7VsOw5u6vJk6CAIO675j4WdltI87jmvPfze9y28DbWH15vd2lKKQ+jQX9BvXug0SBYNwY2zra7mjyLCY9hVMtRjG8znsysTPol9GPEdyM4mXbS7tKUUh5Cgz67di9Dheaw8EFrIDQv0qRME+Z3nU+fWn34ZPsndPusGyv3rrS7LKWUB9Cgzy4gEG6fBhGlYfbd1ryzXiTUGcq/4v/FBx0/IDIkkgdXPsjw1cNJOZtid2lKKRtp0OcUXhLu/BDOpcJHd0L6Wbsrumy1S9Zmzs1zGHbtMFbsXcGtC29lxd4VdpellLKJBn1uYq6BWyfAwZ9gwVDwsPGA8iIwIJBBdQcxt9NcSoWV4qGVD/Hsmmc5lXbK7tKUUoVMg/5ianSC1s/B5k/g69ftribfqhSvwocdP2TANQNYuHMh3Rd1J/GwGweNU0p5HA36f9LsEajT0xoPZ+sCu6vJt8CAQB6s/yDT20/HIQ76JfTjzcQ3dRgFpfyEBv0/EYHOb0NcQ5g/CA7+bHdFV+TaUtcyr/M8ulfrzrQt07hn8T3sObHH7rKUUm6mQX8pgSHWjFRhUdbFWS/riZNTWGAYzzV+jndbvcvB0we5Y9EdLNq5yO6ylFJupEGfFxGl4K7ZVk+c2Xd5ZU+cnFqWbcm8zvO4usTVPPXtUzz97dOcST9jd1lKKTfQoM+rCz1xDmzw2p44OcWExzC53WTur3s/i3YuosfnPUg6mmR3WUqpAqZBfzn+0hPnDburKRBOh5Oh1w5lUttJnE4/zV1f3MXcpLl42hSTSqn806C/XM0ehTo9YOVLXt0TJ6dGVzViXpd5NIxpyIvrXuTZNc9yLuOc3WUppQqABv3lEoHO71gTlswfBPs32F1RgSkRUoIxrccwuO5gFuxcQK8lvdh3UueqVcrbadDnx4WeOBGl4KMecMx3uigGOAIYcu0QxrQew4FTB+j5eU++3v+13WUppa6ABn1+RUTD3R9DZhp8eAec9a1Ju2+Iu4E5neZQJqIMw5YPY+zPY8kyWXaXpZTKhzwFvYi0F5EkEdkhIk/ksj5YROa41n8vIhVcywNFZLqI/CIi20TkyYIt32bR1a3ZqVJ2wMe9ITPd7ooKVNkiZZnZYSadK3dm3MZxDF0+VMe5V8oLXTLoRSQAGAN0AGoCd4pIzRyb9QeOGWOqAKOBka7ltwPBxphrgAbAoAu/BHxGxRusNvtdq+DzR3yi22V2Ic4QXmr6Es9e/yzrDq7jri/uIjk12e6ylFKXIS9n9I2AHcaYXcaYNGA20DXHNl2B6a7n84DWIiKAAcJFxAmEAmnAiQKp3JPUuxtueAx+mgnfjra7mgInItxR/Q4mtp1I6vlU7lp8F2sPrLW7LKVUHuUl6GOB7F0v9ruW5bqNMSYDSAWisEL/NHAI2Au8YYw5mvMLRGSgiCSKSOKRI0cueyc8wo1PQ+3usPwF2Dzf7mrcIj4mno86fcRV4VcxePlgZm2dpf3tlfIC7r4Y2wjIBMoAFYF/iUilnBsZYyYYY+KNMfHR0dFuLslNRKDrGCjXGD69H/b9YHdFbhEbEcvMDjO5seyNjFw/kue/e15HwVTKw+Ul6A8AZbO9jnMty3UbVzNNJJAC3AUsNcakG2P+ANYA8VdatMcKDIEeH0BkLHzUE47utrsitwgLDGNUy1EMqjOI+dvnc9+X9+l0hUp5sLwE/XqgqohUFJEgoCewMMc2C4HerufdgRXG+pt+L9AKQETCgeuBXwuicI8VHgV3zwOTBR/cDmf+1lLlExziYFi9Ybze4nW2pWzj7sV3s+v4LrvLUkrl4pJB72pzHwYkANuAucaYLSIyQkS6uDabDESJyA7gUeBCF8wxQISIbMH6hTHVGLOpoHfC40RVtm6oOr4H5twD6b47lED7Cu2Z1n4a5zLOcc+Se/jhkG82WSnlzcTTLqbFx8ebxEQfmerul3nwSX+o2Q26TwWH796fdvDUQYYuH0pyajLPN3merlVydsxSSrmTiGwwxuTaNO67yeMJrukObV+CrZ/Bl0/bXY1blYkow4wOM4iPieeZNc/w3k/vaY8cpTyEBr27NR4G1w2GdWNh7Xt2V+NWRYKKMPamsdxS5RbGbxrPk98+qT1ylPIATrsL8Hki0O4VOHnIOqsvEmOd6fuoQEcgLzR5gbJFyvLOT+9w+PRh3r7xbSKDI+0uTSm/pWf0hcHhgFvGQ/mm8Nlg2O3bo0GKCAPqDGBk85FsOrKJexbfw8FTB+0uSym/pUFfWAJDoOcHUKIyzL4bft9id0Vu17FSRya2nUjKuRR6Le7Fb8d+s7skpfySBn1hCi0O98yDoAiY1R1S99tdkds1KN2A6e2tYZD6LOlD4mEf6VGllBfRoC9skXFW2KedssL+7DG7K3K7qsWrMrPjTEqGlWTQskEs37Pc7pKU8isa9HYoXctqxjm602rG8eEbqi4oE1GGGe1ncHXU1Ty6+lHmJs21uySl/IYGvV0q3gDdxsGeNTB/AGRl2l2R2xULKcaktpNoFtuMF9e9yNifx2pfe6UKgQa9na7pDu1fhW0LfXLSktyEOkN568a36FalG+M2jmPEuhFk+sEvOaXspP3o7Xb9YDj9J3zzBoSXhNbP2V2R2wU6AhnRZAQlQ0sy6ZdJpJ5P5dXmrxIUEGR3aUr5JA16T9DqGTiTAt+8CaEloMkwuytyOxHhofoPUSKkBK+tf43T6acZ3XI0YYFhdpemlM/RphtPIAI3vwk1u1p3z/78kd0VFZpeNXsxoskI1h1ax/1f3c+JNN+baVIpu2nQewpHANw6ESq1hAVDIWmJ3RUVmluq3sLrN7zOL3/+Qv+E/jqJiVIFTIPekziDrRmqrqoLH/eB5DV2V1Ro2lZoy7ut3iU5NZk+S/tw+PRhu0tSymdo0Hua4Ahrhqpi5azpCA/5/jwtFzSLbcb4NuP58+yf3LvkXpJTk+0uSSmfoEHvicKjoNenEFwUZt0GKTvtrqjQ1C9dn8ntJnMu4xy9l/Ym6WiS3SUp5fU06D1VZJwV9lkZMLMbnDhkd0WFpmZUTaZ1mIbT4aRvQl9+/uNnu0tSyqtp0Huy6GrWuDhnjlphf/pPuysqNJUiKzGjwwyKBxdn4LKBrD+83u6SlPJaGvSeLrYB3DkbjiXDzFvg7HG7Kyo0sRGxTGs/jTLhZRjy1RDWHlxrd0lKeSUNem9QsTn0mAV/bIMPbofzp+yuqNBEh0Uzud1kyhUtxwPLH+Dr/b49aYtS7qBB7y2qtoHuU+DABqs3TvpZuysqNFGhUUxuO5nKxSrz0MqHWL5XhzlW6nJo0HuTml2sES+Tv4W5vSHDfybeLhZSjEntJlGzRE2GrxrO0uSldpeklNfQoPc2dXtAp1GwPcEa3jgzw+6KCk3RoKJMaDuBOtF1+PfX/2bRzkV2l6SUV9Cg90bx/aDty7D1M1j4AGRl2V1RoQkPDGfcTeNoWLohT3/7NJ9u/9TukpTyeBr03qrJMGj5FGz8EJY85hdj2V8QFhjGe63fo0mZJjy39jmdrUqpS9Cg92YtHocmD8L6SbDsOb8K+xBnCG+3epuWcS15cd2LzNo6y+6SlPJYGvTeTATajID4/rD2HVj5it0VFarggGBGtRxFm/JtGLl+JNM2T7O7JKU8kga9txOBjm9AvV7w9Wuw6lW7KypUgQGBvHbDa3So0IE3N7zJ1M1T7S5JKY+jM0z5AocDOr8DJgtW/R9IALR4zO6qCo3T4eSV5tZfM6M2jAKgb+2+dpaklEfJU9CLSHvgbSAAmGSMeTXH+mBgBtAASAF6GGOSXevqAOOBokAW0NAYc66gdkC5OBzQ5V3IyoSVL1mvm//L7qoKjYa9Uhd3yaAXkQBgDNAG2A+sF5GFxpit2TbrDxwzxlQRkZ7ASKCHiDiBWUAvY8xGEYkC0gt8L5TFEQDdxlpn9stHWGf2zR62u6pCo2GvVO7yckbfCNhhjNkFICKzga5A9qDvCjzvej4PeE9EBGgLbDLGbAQwxugcce7mCLDunjWZ8NV/rNdNHrC7qkKjYa/U3+Ul6GOBfdle7weuu9g2xpgMEUkFooBqgBGRBCAamG2MeS3nF4jIQGAgQLly5S53H1ROAU64ZYJ1Zv/lMyAOaDzU7qoKjYa9Un/l7ouxTqAZ0BA4AywXkQ3GmL+MSmWMmQBMAIiPj/efzuDuFOC0JhvPyoSEp6xmnOvvt7uqQqNhr9T/5CXoDwBls72Ocy3LbZv9rnb5SKyLsvuBr40xfwKIyGKgPqDDDxaGgEBrxMuP+8DSf1vNOI0G2F1VodGwV8qSl37064GqIlJRRIKAnsDCHNssBHq7nncHVhhjDJAAXCMiYa5fAC34a9u+creAQOg+FarfDIuHw7pxdldUqC6EfYcKHRi1YZT2s1d+6ZJn9K4292FYoR0ATDHGbBGREUCiMWYhMBmYKSI7gKNYvwwwxhwTkVFYvywMsNgY84Wb9kVdjDMIbp8Gn/SDpU9AxnntjaNn9sqPiPGw8VHi4+NNYmKi3WX4psx0+HQQbP4EbnzGr26qAsjIyuCpb55iSfISHm3wqIa98imu65/xua3TO2P9SUCg1RvHEWjdVJWZBjc+ZQ2j4Adyntk7xEHvWr0v8S6lvJ8Gvb8JcFo3VQU4rbFxMs/DTS/4XdgbDG8kvkGABHBPzXvsLkspt9Kg90eOAOj8LgQEw5q3rSaddq/4Vdj/X/P/I9NkMnL9SAIcAdx59Z12l6WU22jQ+yuHA25+EwKCYN1Yqxmnw+vWcj/gdDgZecNIMldl8sr3rxAgAdxR/Q67y1LKLfzjf7XKnQi0/7//TV7y+UN+NS1hoCOQN1q8QYu4Fry47kXmb59vd0lKuYUGvb+7MHlJ8+Hw4wyrV06m/4w7FxgQyKiWo2gW24zn1z7PZzs+s7skpQqcBr2ywr71s9DqWfhlLsy9F9L9ZyTpoIAg3rrxLa6/6nqeW/Mci3YusrskpQqUBr36nxuGW7NVJS2GD2+H8yftrqjQBAcE806rd2gU04hn1jzDkt1L7C5JqQKjQa/+qtEAuGU8JK+BGd3gzFG7Kyo0Ic4Q3mn1DvVL1efJb54kITnB7pKUKhAa9Orv6vaEO2bA4U0wrROc/N3uigpNWGAYY1qPoW50XZ74+gmW79Hx95T306BXuavRCe7+GI4lw5R2cGyP3RUVmrDAMMbeNJZaJWsxfPVwVu5daXdJSl0RDXp1cZVawr0L4OxRmNIejvxmd0WFJjwwnHE3jaNGVA0eXf0oX+//2u6SlMo3DXr1z8o2hD6LISsDpraHgz/bXVGhKRJUhPfbvE+14tV4ZOUjrDmwxu6SlMoXDXp1aTG1od9SCAy32ux3+8/ZbdGgokxoM4FKxSrx0MqH+O7gd3aXpNRl06BXeRNV2Qr7yDiYdRts+dTuigpNZHAkE9tMpFzRcjy44kHWH15vd0lKXRYNepV3kbHQbwnENoCP+8L3E+yuqNAUCynGpLaTiCsSx9DlQ9nw+wa7S1IqzzTo1eUJLQ69PoXqHWHJY7D8RfCwyWvcpURICSa2nUhMeAyDvxrMT3/8ZHdJSuWJBr26fIGhVj/7+r3hmzdg4TDIzLC7qkJRMrQkk9tOpnRYaQZ/NZiNRzbaXZJSl6RBr/InwAmd34YW/4afZsGcuyHtjN1VFYrosGgmtZ1EiZAS3L/sfjb/udnukpT6Rxr0Kv9ErKkIb34TfkuAGV39ZsiE0uGlmdJuCpHBkQxcNpCtKVvtLkmpi9KgV1eu4X1WU86hjdaNVcf32l1RoYgJj2FKuykUCSzCwGUDSTqaZHdJSuVKg14VjJpdoNd8OHkYJt0EB/3jQmWZiDJMajeJkIAQ7vvyPn475j93DyvvoUGvCk6FZtA/wZqecGpHSPKPoX7LFinLlHZTCHIEMeDLAew8vtPukpT6Cw16VbBK1YD7lkN0dZh9F/ww0e6KCkW5ouWY3G4yDnHQP6E/u1J32V2SUv+lQa8KXpHS0OcLqNYeFg+HhKf9Yi7aCpEVmNx2MoCGvfIoGvTKPYLCoccsaDQIvnsPPr7XL7pfVipWicntJpNlsuif0J/dqbvtLkkpDXrlRo4A6PgatH8Vtn0O0zvDqSN2V+V2lYtVZkq7KRr2ymNo0Cv3u36wdXb/+xaY1NovxrWvXKwyk9tOJtNk0j+hP8mpyXaXpPyYBr0qHDU6We326Wes7pc7fH+KvirFq/wl7Pec8J9ZupRn0aBXhSeuAQxYAcXKwge3w/fjfX5AtCrFqzCp7SQyTAb9lvbTsFe20KBXhatYOeiXANXawZLH4fNHIDPd7qrcqmrxqkxqO4n0rHT6JWjYq8KXp6AXkfYikiQiO0TkiVzWB4vIHNf670WkQo715UTklIgML5iylVcLjoAeH0CzR2DDVJh5i8+PkVO1eFUmt5tMeqYV9ntP+McwEcozXDLoRSQAGAN0AGoCd4pIzRyb9QeOGWOqAKOBkTnWjwL84zZJlTcOB9z0PNwyHvZ9DxNbwRHfHiumavGqTGo3ifTMdPom9NWwV4UmL2f0jYAdxphdxpg0YDbQNcc2XYHprufzgNYiIgAi0g3YDWwpmJKVT6nb07pIm3bKuki7/Su7K3KrasWrMbHtRNIy0+iX0I99J/bZXZLyA3kJ+lgg+7/G/a5luW5jjMkAUoEoEYkA/g288E9fICIDRSRRRBKPHPH9ftYqh7KNYMBKKFYePrwdvhvr0xdpq5eozqS2kzifeZ6+CX017JXbufti7PPAaGPMqX/ayBgzwRgTb4yJj46OdnNJyiMVK2tNPl69IyQ8CZ/eD+ln7a7KbbKHfb8v+7HvpIa9cp+8BP0BoGy213GuZbluIyJOIBJIAa4DXhORZOBh4CkRGXaFNStfFRwBd8yEG5+GTXNgclufHtv+QtifzThrNeNo2Cs3yUvQrweqikhFEQkCegILc2yzEOjtet4dWGEszY0xFYwxFYC3gFeMMe8VUO3KFzkc0OJxuGsOHNsD41vArtV2V+U22cO+f0J/DXvlFpcMeleb+zAgAdgGzDXGbBGRESLSxbXZZKw2+R3Ao8DfumAqdVmqtYOBKyGiFMzsBmvf9dl2+6tLXM3ENhM5k3GGvkv7aj97VeDEeNh/nvj4eJOYmGh3GcpTnD8Jnw2BbQuh9m3Q5V1rZEwflHQ0iQFfDsDpcDKp7SQqFatkd0nKi4jIBmNMfG7r9M5Y5dmCi1jz0bb+D2yeb7XbH/XN0SCrl6j+31Ev+yb0Zfux7XaXpHyEBr3yfCLQ/FG4ex6k7ocJLSFpqd1VuUWV4lWY2n4qTnHSL6Ef21K22V2S8gEa9Mp7VL0JBq6yxsv5qAd89TxkZthcVMGrGFmRae2nEeoMpf+X/dn852a7S1JeToNeeZcSFaH/MmjQB74dDTO6wsnDdldV4MoWLcvU9lMpGlSUAV8O4Oc/fra7JOXFNOiV9wkMgc5vwy0T4OCP8H5z2P213VUVuNiIWKa1n0ZUaBQDlw0k8bB2UlD5o0GvvFfdHtb49qHFrDP7r1/3uUnIY8JjmNpuKjHhMQz+ajDrDq2zuyTlhTTolXcrVcMaJ6f2bbDiJfjwDp8b8jg6LJop7aYQVySOYcuH8e2Bb+0uSXkZDXrl/YIj4NaJcPMo2L3aasrZ61tnviVDSzKl3RQqRlbkwRUPsmrfKrtLUl5Eg175BhFo2B/6fwkBTpjaEVa/DlmZdldWYIqHFGdS20lUL16dR1Y+wuJdi+0uSXkJDXrlW8rUg0HfQO1bYeVLML0LpOYcg897RQZHMrHtRK4tdS1PfPMEc5Pm2l2S8gIa9Mr3hBS1mnK6jYODP8H7TeHXL+yuqsBEBEUw7qZxNI9rzovrXmTK5il2l6Q8nAa98k0icO1dMOhriCwLs++CL4b7zBj3Ic4Q3mr5Fu0rtGf0htG88+M7eNq4VcpzOO0uQCm3KlkF7vsKlo+A796DPWuh+xQodbXdlV2xwIBAXm3+KuGB4Uz8ZSIn007y5HVP4hA9f1N/pf8ilO9zBkO7l+Guj+HU79ZYOT9M9IlhjwMcAfyn8X/oU6sPs5Nm88y3z5CR5XvDQqgro0Gv/Ee1tjB4DZRvAouHw6zb4MQhu6u6YiLCow0e5YF6D7Bo1yL+tepfnM88b3dZyoNo0Cv/UiQG7vkEOr5hNeOMvd4a/tjLiQgD6wzkiUZPsGLfCoYuH8qZ9DN2l6U8hAa98j8i0GgA3P8NRFWGeX3hkwFw9rjdlV2xu2vczUtNX2L94fXc9+V9HDt3zO6SlAfQoFf+q2RV6PcltHwKNn8C45rArlV2V3XFulbpyuiWo/nt2G/cu+ReDpzynfsIVP5o0Cv/FuCElv+G+5ZBYJg1ONqSJyDNu5s9WpVrxYQ2E0g5l0Kvxb1IOppkd0nKRhr0SgHENrD63DcaCN+Pg/ebWW34Xqx+6frMaD8Dhzjos7QP6w+vt7skZRMNeqUuCAqDjq/DvQshK8MaL2fx45B22u7K8q1K8SrM6jiLUmGlGLRsEF8mf2l3ScoGGvRK5VSpBQxea53d/zAexjaGXavtrirfYsJjmNFhBrWiajF89XBm/zrb7pJUIdOgVyo3wRHQ8TXouwQcTpjRBRY9DOdO2F1ZvkQGRzKh7fBtppkAABK/SURBVARaxLXg5e9f1iET/IwGvVL/pHwTuP9baPIA/DjdOrvf/pXdVeVLqDOU0TeO5taqtzLxl4k8u+ZZ0jPT7S5LFQINeqUuJSgM2r5kTUoeFA4f3Gb1uz91xO7KLpvT4eT5xs8zpO4QFuxcwOCvBnMizTv/SlF5p0GvVF7FxVs9c254HLZ8Cu/Fw4bpXjdPrYgw+NrBvNLsFTb8sYFei3ux/+R+u8tSbqRBr9TlCAyBVk9bY+aUrgWLHoRpN8Mfv9pd2WXrXLkzE9pM4MjZI9y9+G42Hdlkd0nKTTTolcqP6OrQ5wvoOgaObLP63a94yevGu28Y05BZHWcR5gyjX0I/vtrjndcf1D/ToFcqv0Sg3j0wLBFq3wZfv24No7Bzpd2VXZZKkZX44OYPqF6iOo+uepTpW6Zrjxwfo0Gv1JUKLwm3jod7F1ivZ3aDub3h+D5767oMJUJKMLntZNqUb8MbiW/w4roXSc/SHjm+Ik9BLyLtRSRJRHaIyBO5rA8WkTmu9d+LSAXX8jYiskFEfnH9bFWw5SvlQSq1hMHfwY1Pw29LYUwj6yw//ZzdleVJiDOE11u8Tv/a/fn4t48ZtGyQjn7pIy4Z9CISAIwBOgA1gTtFpGaOzfoDx4wxVYDRwEjX8j+BzsaYa4DewMyCKlwpjxQYAi0eh2HrocpNVrv92OshaandleWJQxw83OBhXmn2Chv/2MidX9zJb8d+s7ssdYXyckbfCNhhjNlljEkDZgNdc2zTFZjuej4PaC0iYoz5yRhz0LV8CxAqIsEFUbhSHq1YOegxE3p9CgGB8FEP+OAOSNlpd2V50rlyZ6a1n0Z6Zjr3LL6H5XuX212SugJ5CfpYIHtj437Xsly3McZkAKlAVI5tbgN+NMb8bY4zERkoIokiknjkiPfdhKLURVVuBfevsW64ujCj1VfPe8VQCtdEX8NHnT6iSrEqPLzyYd7f+L5epPVShXIxVkRqYTXnDMptvTFmgjEm3hgTHx0dXRglKVV4nEHWEAoPJEKtW+Hb0fBufUicApmePZF3qbBSTG0/lc6VOjPm5zEMXz1cpyj0QnkJ+gNA2Wyv41zLct1GRJxAJJDieh0HfArca4zxjr9blXKHIjFW75wBKyCqKnz+CLzfFLYvAw8+Uw4OCOblZi8zPH44X+39il5LerH3xF67y1KXIS9Bvx6oKiIVRSQI6AkszLHNQqyLrQDdgRXGGCMixYAvgCeMMWsKqmilvFpsA+i7GO6YCZlp8EF3mHkLHN5sd2UXJSL0rtWbsa3H8vuZ3+n5eU9W7vWu+wX82SWD3tXmPgxIALYBc40xW0RkhIh0cW02GYgSkR3Ao8CFLpjDgCrAcyLys+tRqsD3QilvIwI1u8CQ76H9q3DwJxjfHBYMg5OH7a7uoprGNmVup7mUK1qOB1c+yNs/vk1Glmc3PykQT7u4Eh8fbxITE+0uQ6nCdfYYfP0GfD/eGv/+ukHQ9CEIK2F3Zbk6n3meV394lXm/zeO6mOsYecNIokJz9r9QhUlENhhj4nNdp0GvlAc5uhtW/R9smgvBRaHpA3DdYGsiFA/02Y7PeGndS0QGRzKq5SjqRte1uyS/9U9Br0MgKOVJSlSEWydYUxlWcA2U9s61sO59yPhbz2TbdavSjVkdZxHkCKLP0j5M3zKdLONdwzb7Aw16pTxR6Zpw54dw33IoVQOW/hvebQA/zfK4LplXl7iaOZ3n0CKuBW8kvsGQ5UNIOZtid1kqGw16pTxZXDz0XgS9PoPwaFgw1Jrw5KdZ4EHTABYNKsrolqN59vpnWX9oPd0Xdee7g9/ZXZZy0aBXyhtUvtHqf9/zIwgpagX+u/VhwzTISLO7OsDqgnlH9Tv4qNNHFA0qyqBlg3hrw1s6CqYH0KBXyluIwNUdYeBquGuudYa/6CEr8NdP9pg2/GrFqzG702xurXorkzdPps+SPuw76T1DNvsi7XWjlLcyBnYuh1UjYf8PUDQWmj5sTYYSFGZ3dQAkJCfwwtoXyDAZPNbwMbpX7Y6I2F2WT9LulUr5MmNg1ypYPRL2fgehJax++A0HQLj9fdsPnz7Ms2ueZd2hdTSLbcYLTV6gVJjeN1nQNOiV8hd718GatyFpMThDrbP7xkOtbps2yjJZzEmaw6jEUQQFBPHM9c/QoWIHW2vyNRr0SvmbI0mw9h3YOAdMJtTsBk0fhDL1bC0rOTWZp9c8zaYjm2hXoR1PXfcUJUI88+5fb6NBr5S/OnEIvh8HiVPh/Ako38xq1qneEQKctpSUkZXB1M1TGbtxLBGBETze8HE6VeqkbfdXSINeKX93LhU2TIcfJkLqXogsCw37Q/3eto2ns+PYDv7z3X/YdGQTTco04dnrnyWuSJwttfgCDXqllCUrE5KWwPfvQ/I34AyBOndAo0EQU7vQy8nMymRO0hze/vFtDIah1w7l7hp343TY89eGN9OgV0r93e9b4IcJVjt+xlko3xQa9IUana1JzgvR4dOHeWndS6zev5qaUTV5+rqnqRNdp1Br8HYa9Eqpiztz1BpSYf0kOL4HQotD3TutZp1SVxdaGcYYEvYk8NoPr3Hk7BG6VenGQ/UfomRoyUKrwZtp0CulLi0rC3avhh+nw7bPISsdyl4PDfpAza6FdhPW6fTTjN80nplbZxISEMLQa4fS4+oeBDoCC+X7vZUGvVLq8pw6Ahs/ssbSOboTgiPhmtugTk8o28gajsHNdqfuZuQPI1lzcA1VilXhsYaP0aRME7d/r7fSoFdK5Y8xsGeN1WNn2yKrLb94Rajb07qIW6KSm7/esHLfSl5b/xoHTh2g8VWNeaTBI9SIquHW7/VGGvRKqSt3/iRsXQibZsPubwADZa+DOj2g1i1u7aaZlpnG7F9nM+GXCaSeT6VTpU4MqzeM2IhYt32nt9GgV0oVrNT91nSHm+bAkV/BEWgNpVzrFutmrNBibvnaE2knmPLLFGZtm0WWyaLn1T3pV7ufXrBFg14p5S7GwKGf4Zd5sHUBpO77X+jX7GYNqxxavMC/9vDpw4z5eQwLdy4k0BHI7dVup1/tfkSHRRf4d3kLDXqllPsZAwd+hC3zrSae1L1W6FdqCTU6QbX2UCSmQL9yz4k9TNg0gS92fYHT4eT2arfTt3ZfvxwdU4NeKVW4LoT+1k+tM/3je63lZepBtQ5QvT3E1Cmw3jv7Tuxjwi8TWLRzEQESQKfKnehVoxdVilcpkM/3Bhr0Sin7GAN/bLWGXvhtKexPBIw1UUq1dtaZfvmmEBxxxV+17+Q+pm+ZzoIdCziXeY6msU25t+a9NL6qsc8PmqZBr5TyHKf+gO1fWsG/cyWkn7aaeMo2gko3Wu37ZeqBIyDfX3Hs3DE+/u1jPtz2ISnnUqhavCo9qvWgY6WOFAkqUoA74zk06JVSnin9HOxbZwX+rpVwaKO1PCQSKt5gBX/5phBdPV/NPGmZaSzevZhZW2eRdCyJUGcoHSp2oHvV7tQuWdunzvI16JVS3uF0Cuxe5Qr+VVYvHrCmRyzXGMo3hnJN4Ko6EJD3IRGMMWz+czPzts9jye4lnM04S/Xi1elcuTPtKrQjJrxgLxLbQYNeKeV9jIGju2DPWmsu3D1r4dhua11gGMQ1tB6x9SG2QZ579JxKO8Xi3YuZv30+W1K2IAjxMfF0qNiBtuXbEhkc6cadch8NeqWUbzh52BX638HetfD7VmuqRIAiZazQL1PP+nnVtZe8Wzc5NZklyUtYvGsxySeScYqTBqUb0LJsS1qUbUHZImULYacKhga9Uso3pZ2Bw5usrpwHf7R+Ht35v/VFroJSNaFUDShdy3oeffXfxts3xvDr0V9ZmryU1ftWszPV+ozKkZW5oewNXBdzHfVK1SMssHBG8MwPDXqllP84exwO/gSHf7G6df6+xZosPfO8tV4c1mBsUVWgRGWIcj1KVLa6fDoc7Duxj1X7V7F632o2/L6BDJOBU5zULlmbhjENaVC6AbWialEsxD1DPeTHFQe9iLQH3gYCgEnGmFdzrA8GZgANgBSghzEm2bXuSaA/kAk8aIxJ+Kfv0qBXShW4zAzrTP/3LVb4/7HNav8/ugsyzv1vO2eINTpnsbIQGQdFYzkTUYqfzVl+OHeI9ceS2HJ0G5mu5qLYiFhqRdWiVslaVC1WlYqRFbkq/CoCrqBraH5dUdCLSADwG9AG2A+sB+40xmzNts0QoI4x5n4R6QncYozpISI1gY+ARkAZ4CugmjEXGtX+ToNeKVVosrLg5EFI2Wn9IkjZCUd3W719ThyAMyl/e8vpsOJsiSjBltAwNjthC+c5kPW/XxbB4qR8WGnKh5chJvwqSkeUoXTRspQuEkeJ0CgiAiMoGlSUwMvoNZQX/xT0eZmBtxGwwxizy/Vhs4GuwNZs23QFnnc9nwe8J1YH1a7AbGPMeWC3iOxwfd53+dkRpZQqUA6HdeYeGQeVWvx9fdoZOHEQTuyH1ANw4gDhp36n0ek/aXT6TzjxJ5w+yvHzx9ntdLA7MJBdQYHsPn2S7ceS+dYZwFmHI9evDjaGCAOBCE7AaYQbilTksdsXFPhu5iXoY4F92V7vB6672DbGmAwRSQWiXMvX5Xjv3waQFpGBwECAcuXK5bV2pZRyr6AwKFnFevyDYlmZ1Dt7jHpnj1nj9qedgvMnMedOcuLsEf44c4TD51I4nnGGk5nnOJV5npNZ5zmVlUZ6ViYZJpMMsijtpv78eQl6tzPGTAAmgNV0Y3M5Sil1eRwBEF7SemQjQKTrUdWOulxy/5virw4A2TuTxrmW5bqNiDix9islj+9VSinlRnkJ+vVAVRGpKCJBQE9gYY5tFgK9Xc+7AyuMdZV3IdBTRIJFpCLWL7UfCqZ0pZRSeXHJphtXm/swIAGre+UUY8wWERkBJBpjFgKTgZmui61HsX4Z4NpuLtaF2wxg6D/1uFFKKVXw9IYppZTyAf/UvTIvTTdKKaW8mAa9Ukr5OA16pZTycRr0Sinl4zzuYqyIHAH2XMFHlAT+LKByvJW/HwN/33/QYwD+dwzKG2Oic1vhcUF/pUQk8WJXnv2Fvx8Df99/0GMAegyy06YbpZTycRr0Sinl43wx6CfYXYAH8Pdj4O/7D3oMQI/Bf/lcG71SSqm/8sUzeqWUUtlo0CullI/zmaAXkfYikiQiO0TkCbvrKUgiMkVE/hCRzdmWlRCRZSKy3fWzuGu5iMg7ruOwSUTqZ3tPb9f220Wkd27f5alEpKyIrBSRrSKyRUQeci33m+MgIiEi8oOIbHQdgxdcyyuKyPeufZ3jGk4c1/Dgc1zLvxeRCtk+60nX8iQRaWfPHuWPiASIyE8i8rnrtV/tf74YY7z+gTV88k6gEhAEbARq2l1XAe7fDUB9YHO2Za8BT7iePwGMdD3vCCzBmtzmeuB71/ISwC7Xz+Ku58Xt3rfLOAZXAfVdz4tgTVhf05+Og2tfIlzPA4HvXfs2F+jpWv4+MNj1fAjwvut5T2CO63lN1/+RYKCi6/9OgN37dxnH4VHgQ+Bz12u/2v/8PHzljP6/E5gbY9KACxOY+wRjzNdY4/xn1xWY7no+HeiWbfkMY1kHFBORq4B2wDJjzFFjzDFgGdDe/dUXDGPMIWPMj67nJ4FtWPMP+81xcO3LKdfLQNfDAK2Aea7lOY/BhWMzD2gtIuJaPtsYc94YsxvYgfV/yOOJSBxwMzDJ9Vrwo/3PL18J+twmMP/bJOQ+prQx5pDr+WGgtOv5xY6Fzxwj15/g9bDOaP3qOLiaLX4G/sD6JbUTOG6MyXBtkn1//ruvrvWpQBTefQzeAh4Hslyvo/Cv/c8XXwl6v2asv0f9op+siEQAnwAPG2NOZF/nD8fBGJNpjLkWa/7lRsDVNpdUaESkE/CHMWaD3bV4G18Jen+chPx3V1MErp9/uJZf7Fh4/TESkUCskP/AGDPftdjvjgOAMeY4sBJojNUsdWFa0Oz78999da2PBFLw3mPQFOgiIslYzbOtgLfxn/3PN18J+rxMYO5rsk/I3htYkG35va5eJ9cDqa6mjQSgrYgUd/VMaeta5hVcbauTgW3GmFHZVvnNcRCRaBEp5noeCrTBulaxEuju2iznMbhwbLoDK1x/9SwEerp6pVQEqgI/FM5e5J8x5kljTJwxpgLW//EVxpi78ZP9vyJ2Xw0uqAdWL4vfsNosn7a7ngLet4+AQ0A6Vntif6y2xuXAduAroIRrWwHGuI7DL0B8ts/ph3XhaQfQ1+79usxj0AyrWWYT8LPr0dGfjgNQB/jJdQw2A8+5llfCCqodwMdAsGt5iOv1Dtf6Stk+62nXsUkCOti9b/k4Fi35X68bv9v/y33oEAhKKeXjfKXpRiml1EVo0CullI/ToFdKKR+nQa+UUj5Og14ppXycBr1SSvk4DXqllPJx/w95HRZLg0n4rgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "steps = 3*1562\n", + "plt.plot(range(steps), [0.1]*steps)\n", + "plt.plot(range(steps), build_lr(steps, decay_type='square', lr_base=0.1))\n", + "plt.plot(range(steps), build_lr(steps, decay_type='cosine', lr_base=0.1))\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "采用一定的训练策略对模型进行训练，并观察最终的验证精度。这里采用Momentum优化器 + cosine decay学习率下降策略。\n", + "\n", + "- cosine0.1_epoch100:{'acc': 0.5056089743589743, 'loss': 1.3536554261659965}\n", + "- square0.1_epoch100:{'acc': 0.5385616987179487, 'loss': 1.2873077663855674}\n", + "- const0.01_epoch100:{'acc': 0.5464743589743589, 'loss': 1.3035117800419147}\n", + "\n", + "- cosine0.1_epoch50:{'acc': 0.43900240384615385, 'loss': 1.5275637297294078}\n", + "- square0.1_epoch50:{'acc': 0.5453725961538461, 'loss': 1.2635320337154927}\n", + "- const0.01_epoch50:{'acc': 0.546073717948718, 'loss': 1.2912015158396501}\n", + "\n", + "- cosine0.01_epoch50:{'acc': 0.6571514423076923, 'loss': 0.9970117075703083}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.system('rm -f *.ckpt *.ir *.meta') # 清理旧的运行文件\n", + "LOOP_SINK = context.get_context('enable_loop_sink')\n", + "\n", + "def test_train(num_epoch=2, momentum=0.9, lr=0.01, decay_type='square', check_point_name=\"mynet\"):\n", + " ds_train = create_dataset(num_epoch=num_epoch)\n", + " ds_eval = create_dataset(training=False)\n", + " steps_per_epoch = ds_train.get_dataset_size()\n", + " \n", + " net = MyNet()\n", + " loss = nn.loss.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean')\n", + " if decay_type:\n", + " lr = build_lr(num_epoch*steps_per_epoch, decay_type=decay_type, lr_base=lr)\n", + " opt = nn.Momentum(net.trainable_params(), lr, momentum, weight_decay=0.0)\n", + " \n", + " ckpt_cfg = CheckpointConfig(save_checkpoint_steps=steps_per_epoch, keep_checkpoint_max=5)\n", + " ckpt_cb = ModelCheckpoint(prefix=check_point_name, config=ckpt_cfg)\n", + " loss_cb = LossMonitor(per_print_times=1 if LOOP_SINK else steps_per_epoch)\n", + " \n", + " model = Model(net, loss, opt, metrics={'acc', 'loss'})\n", + " model.train(num_epoch, ds_train, callbacks=[ckpt_cb, loss_cb], dataset_sink_mode=True)\n", + " metrics = model.eval(ds_eval)\n", + " print('Metrics:', metrics)\n", + "\n", + "test_train(num_epoch=50, lr=0.01, decay_type='cosine')\n", + "print('\\n'.join(sorted([x for x in os.listdir('.') if x.startswith('mynet')])))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 模型大小\n", + "\n", + "统计模型参数量，包括所有可训练的权重、偏置的参数。" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('conv1.weight', 450), ('conv2.weight', 2400), ('fc1.weight', 48000), ('fc1.bias', 120), ('fc2.weight', 10080), ('fc2.bias', 84), ('fc3.weight', 840), ('fc3.bias', 10)]\n", + "Num params(M): 0.061984\n" + ] + } + ], + "source": [ + "params = MyNet().trainable_params()\n", + "print([(p.name, np.prod(p.data.shape())) for p in params])\n", + "num_params = sum([np.prod(p.data.shape()) for p in params])\n", + "print('Num params(M):', num_params/1e6)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 推理时延\n", + "\n", + "模型第一次执行推理时需要编译计算图和算子，通常时延较长，通常需要先进行预热（warmup），然后再循环推理多次，取多次推理时延的平均值作为模型的推理时延。" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Latency(ms): 1.290428638458252\n" + ] + } + ], + "source": [ + "x = Tensor(np.ones([1, 3, 32, 32]), ms.float32)\n", + "net = MyNet()\n", + "# 预热\n", + "for i in range(5):\n", + " y = net(x)\n", + "# 多次推理取平均值\n", + "start = time.time()\n", + "for i in range(100):\n", + " y = net(x)\n", + "end = time.time()\n", + "print('Latency(ms):', (end-start)/100 * 1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 作业结论\n", + "\n", + "预置训练策略，以及预置模型的精度、大小和时延如下：\n", + "\n", + "| batch size | number of epochs | learning rate | decay type | optimizer | number of parameters(M) | latency(ms) | acc(%) |\n", + "| -- | -- | -- | -- | -- | -- | -- | -- |\n", + "| 32 | 50 | 0.01 | cosine | Momentum 0.9 | 0.061984 | 1.290 | 65.7 |\n", + "\n", + "在预置模型和训练策略的基础上，请：\n", + "\n", + "- 尝试调整模型深度（层数）、模型宽度（核大小）、模型结构（Conv, MaxPool, AvgPool, FC, Bypass）等，并评估其对模型精度、大小和时延的影响；\n", + "- 尝试调整Epoch数、Batch Size、优化器、学习率、正则化项等，并评估其对模型训练和精度的影响。\n", + "\n", + "调优模型和训练策略的结果（请填写）：\n", + "\n", + "| batch size | number of epochs | learning rate | decay type | optimizer | number of parameters(M) | latency(ms) | acc(%) |\n", + "| -- | -- | -- | -- | -- | -- | -- | -- |\n", + "| x | x | x | x | x | x | x | x |\n", + "\n", + "### 模型调优总结\n", + "\n", + "请填写\n", + "\n", + "### 训练策略调优总结\n", + "\n", + "请填写\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} -- GitLab