From f25b7ce40778d03281f4da797d5c73fc228d12bc Mon Sep 17 00:00:00 2001 From: u010280923 Date: Wed, 8 Mar 2023 18:00:42 +0800 Subject: [PATCH] debug --- forward_demo.py | 331 +++++------------------ src/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 152 bytes src/__pycache__/model.cpython-38.pyc | Bin 0 -> 17322 bytes src/__pycache__/model_run.cpython-38.pyc | Bin 0 -> 5827 bytes src/__pycache__/utils.cpython-38.pyc | Bin 0 -> 3889 bytes 5 files changed, 68 insertions(+), 263 deletions(-) create mode 100644 src/__pycache__/__init__.cpython-38.pyc create mode 100644 src/__pycache__/model.cpython-38.pyc create mode 100644 src/__pycache__/model_run.cpython-38.pyc create mode 100644 src/__pycache__/utils.cpython-38.pyc diff --git a/forward_demo.py b/forward_demo.py index 28571d8..da7df70 100644 --- a/forward_demo.py +++ b/forward_demo.py @@ -1,273 +1,78 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2023/3/1 11:54 -# @Author : clong -# @File : train_sft.py +import os, sys, torch +import numpy as np +np.set_printoptions(precision=4, suppress=True, linewidth=200) +# current_path = os.path.dirname(os.path.abspath(__file__)) +# sys.path.append(f'{current_path}/rwkv_pip_package/src') +# Tune these below (test True/False for all of them) to find the fastest setting: +# torch._C._jit_set_profiling_executor(True) +# torch._C._jit_set_profiling_mode(True) +# torch._C._jit_override_can_fuse_on_cpu(True) +# torch._C._jit_override_can_fuse_on_gpu(True) +# torch._C._jit_set_texpr_fuser_enabled(False) +# torch._C._jit_set_nvfuser_enabled(False) ######################################################################################################## -# The RWKV Language Model - https://github.com/BlinkDL/RWKV-LM -######################################################################################################## - -if __name__ == "__main__": - from argparse import ArgumentParser - from pytorch_lightning import Trainer - from pytorch_lightning.utilities import rank_zero_info, rank_zero_only - - rank_zero_info("########## work in progress ##########") - - ######################################################################################################## - # - # example: train a simple L12-D768 RWKV on dummy data - # - # python train.py --load_model "" --wandb "" --proj_dir "out" \ - # --data_file "" --data_type "dummy" --vocab_size 0 \ - # --ctx_len 128 --epoch_steps 1000 --epoch_count 20 --epoch_begin 0 --epoch_save 10 \ - # --micro_bsz 16 --n_layer 12 --n_embd 768 --pre_ffn 0 --head_qk 0 \ - # --lr_init 6e-4 --lr_final 1e-5 --warmup_steps 0 --beta1 0.9 --beta2 0.99 --adam_eps 1e-8 \ - # --accelerator gpu --devices 1 --precision bf16 --strategy ddp_find_unused_parameters_false --grad_cp 0 - - # example: train a simple L6-D512 RWKV from scratch on enwik8 - # - # python train.py --load_model "" --wandb "" --proj_dir "out" \ - # --data_file "../data/enwik8" --data_type "utf-8" --vocab_size 0 \ - # --ctx_len 512 --epoch_steps 5000 --epoch_count 500 --epoch_begin 0 --epoch_save 5 \ - # --micro_bsz 12 --n_layer 6 --n_embd 512 --pre_ffn 0 --head_qk 0 \ - # --lr_init 8e-4 --lr_final 1e-5 --warmup_steps 0 --beta1 0.9 --beta2 0.99 --adam_eps 1e-8 \ - # --accelerator gpu --devices 1 --precision bf16 --strategy ddp_find_unused_parameters_false --grad_cp 0 - - # example: fine-tune RWKV 1.5B using 8xA100 40G = 1.76it/s = 115k token/s, VRAM 37477M - # - # python train.py --load_model "/fsx/BlinkDL/CODE/FP16/out_1b2/all-8040.pth" --wandb "" --proj_dir "out" \ - # --data_file "../data/train.npy" --data_type "numpy" --vocab_size 50277 \ - # --ctx_len 1024 --epoch_steps 1000 --epoch_count 1000 --epoch_begin 0 --epoch_save 5 \ - # --micro_bsz 8 --n_layer 24 --n_embd 2048 --pre_ffn 0 --head_qk 0 \ - # --lr_init 1e-5 --lr_final 1e-5 --warmup_steps 0 --beta1 0.9 --beta2 0.999 --adam_eps 1e-8 \ - # --accelerator gpu --devices 8 --precision bf16 --strategy deepspeed_stage_2 --grad_cp 0 - - # example: fine-tune RWKV 1.5B using 1 GPU fp16 (VRAM 16G) NOTE: fp16 might overflow - # - # python train.py --load_model "/fsx/BlinkDL/CODE/FP16/out_1b2/all-8040.pth" --wandb "" --proj_dir "out" \ - # --data_file "../data/train.npy" --data_type "numpy" --vocab_size 50277 \ - # --ctx_len 1024 --epoch_steps 200 --epoch_count 1000 --epoch_begin 0 --epoch_save 1 \ - # --micro_bsz 11 --n_layer 24 --n_embd 2048 --pre_ffn 0 --head_qk 0 \ - # --lr_init 1e-5 --lr_final 1e-5 --warmup_steps 0 --beta1 0.9 --beta2 0.999 --adam_eps 1e-8 \ - # --accelerator gpu --devices 1 --precision fp16 --strategy deepspeed_stage_2_offload --grad_cp 1 - - parser = ArgumentParser() - - parser.add_argument("--load_model", default="", type=str) # full path, with .pth - parser.add_argument("--wandb", default="", type=str) # wandb project name. if "" then don't use wandb - parser.add_argument("--proj_dir", default="out", type=str) - parser.add_argument("--random_seed", default="-1", type=int) - - parser.add_argument("--data_file", default="", type=str) - parser.add_argument("--data_type", default="utf-8", type=str) - parser.add_argument("--vocab_size", default=0, type=int) # vocab_size = 0 means auto (for char-level LM and .txt data) - - parser.add_argument("--ctx_len", default=1024, type=int) - parser.add_argument("--epoch_steps", default=1000, type=int) # a mini "epoch" has [epoch_steps] steps - parser.add_argument("--epoch_count", default=500, type=int) # train for this many "epochs". will continue afterwards with lr = lr_final - parser.add_argument("--epoch_begin", default=0, type=int) # if you load a model trained for x "epochs", set epoch_begin = x - parser.add_argument("--epoch_save", default=5, type=int) # save the model every [epoch_save] "epochs" - - parser.add_argument("--micro_bsz", default=12, type=int) # micro batch size (batch size per GPU) - parser.add_argument("--n_layer", default=6, type=int) - parser.add_argument("--n_embd", default=512, type=int) - parser.add_argument("--dim_att", default=0, type=int) - parser.add_argument("--dim_ffn", default=0, type=int) - parser.add_argument("--pre_ffn", default=0, type=int) # replace first att layer by ffn (sometimes better) - parser.add_argument("--head_qk", default=0, type=int) # my headQK trick - parser.add_argument("--tiny_att_dim", default=0, type=int) # tiny attention dim - parser.add_argument("--tiny_att_layer", default=-999, type=int) # tiny attention @ which layer - - parser.add_argument("--lr_init", default=6e-4, type=float) # 6e-4 for L12-D768, 4e-4 for L24-D1024, 3e-4 for L24-D2048 - parser.add_argument("--lr_final", default=1e-5, type=float) - parser.add_argument("--warmup_steps", default=0, type=int) # try 50 if you load a model - parser.add_argument("--beta1", default=0.9, type=float) - parser.add_argument("--beta2", default=0.99, type=float) # use 0.999 when your model is close to convergence - parser.add_argument("--adam_eps", default=1e-8, type=float) - - parser.add_argument("--grad_cp", default=0, type=int) # gradient checkpt: saves VRAM, but slower - parser.add_argument("--my_pile_stage", default=0, type=int) # my special pile mode - parser.add_argument("--my_pile_shift", default=-1, type=int) # my special pile mode - text shift - parser.add_argument("--my_pile_edecay", default=0, type=int) - parser.add_argument("--layerwise_lr", default=1, type=int) # layerwise lr for faster convergence (but slower it/s) - parser.add_argument("--ds_bucket_mb", default=200, type=int) # deepspeed bucket size in MB. 200 seems enough - # parser.add_argument("--cuda_cleanup", default=0, type=int) # extra cuda cleanup (sometimes helpful) - - parser.add_argument("--my_img_version", default=0, type=str) - parser.add_argument("--my_img_size", default=0, type=int) - parser.add_argument("--my_img_bit", default=0, type=int) - parser.add_argument("--my_img_clip", default='x', type=str) - parser.add_argument("--my_img_clip_scale", default=1, type=float) - parser.add_argument("--my_img_l1_scale", default=0, type=float) - parser.add_argument("--my_img_encoder", default='x', type=str) - # parser.add_argument("--my_img_noise_scale", default=0, type=float) - parser.add_argument("--my_sample_len", default=0, type=int) - parser.add_argument("--my_ffn_shift", default=1, type=int) - parser.add_argument("--my_att_shift", default=1, type=int) - parser.add_argument("--my_pos_emb", default=0, type=int) - parser.add_argument("--load_partial", default=0, type=int) - parser.add_argument("--magic_prime", default=0, type=int) - parser.add_argument("--my_qa_mask", default=0, type=int) - parser.add_argument("--my_testing", default='', type=str) - - parser = Trainer.add_argparse_args(parser) - args = parser.parse_args() - - ######################################################################################################## - - import os, warnings, math, datetime, sys, time - import numpy as np - import torch - from torch.utils.data import DataLoader - import deepspeed - import pytorch_lightning as pl - from pytorch_lightning import seed_everything - - if args.random_seed >= 0: - print(f"########## WARNING: GLOBAL SEED {args.random_seed} THIS WILL AFFECT MULTIGPU SAMPLING ##########\n" * 3) - seed_everything(args.random_seed) - - np.set_printoptions(precision=4, suppress=True, linewidth=200) - warnings.filterwarnings("ignore", ".*Consider increasing the value of the `num_workers` argument*") - warnings.filterwarnings("ignore", ".*The progress bar already tracks a metric with the*") - # os.environ["WDS_SHOW_SEED"] = "1" - os.environ["TOKENIZERS_PARALLELISM"] = "false" - - args.my_timestamp = datetime.datetime.today().strftime("%Y-%m-%d-%H-%M-%S") - args.enable_checkpointing = False - args.replace_sampler_ddp = False - args.logger = False - args.gradient_clip_val = 1.0 - args.num_sanity_val_steps = 0 - args.check_val_every_n_epoch = int(1e20) - args.log_every_n_steps = int(1e20) - args.max_epochs = -1 # continue forever - args.betas = (args.beta1, args.beta2) - args.real_bsz = int(args.num_nodes) * int(args.devices) * args.micro_bsz - os.environ["RWKV_T_MAX"] = str(args.ctx_len) - os.environ["RWKV_MY_TESTING"] = args.my_testing - if args.dim_att <= 0: - args.dim_att = args.n_embd - if args.dim_ffn <= 0: - args.dim_ffn = args.n_embd * 4 - - args.run_name = f"{args.vocab_size} ctx{args.ctx_len} L{args.n_layer} D{args.n_embd}" - if not os.path.exists(args.proj_dir): - os.makedirs(args.proj_dir) - - samples_per_epoch = args.epoch_steps * args.real_bsz - tokens_per_epoch = samples_per_epoch * args.ctx_len - rank_zero_info( - f""" -############################################################################ -# -# RWKV-4 {args.precision.upper()} on {args.num_nodes}x{args.devices} {args.accelerator.upper()}, bsz {args.num_nodes}x{args.devices}x{args.micro_bsz}={args.real_bsz}, {args.strategy} {'with grad_cp' if args.grad_cp > 0 else ''} # -# Data = {args.data_file} ({args.data_type}), ProjDir = {args.proj_dir} +# Use '/' in model path, instead of '\'. Use ctx4096 models if you need long ctx. # -# Epoch = {args.epoch_begin} to {args.epoch_begin + args.epoch_count - 1} (will continue afterwards), save every {args.epoch_save} epoch +# fp16 = good for GPU (!!! DOES NOT support CPU !!!) +# fp32 = good for CPU +# bf16 = worse accuracy, supports CPU +# xxxi8 (example: fp16i8) = xxx with int8 quantization to save 50% VRAM/RAM, slower, slightly less accuracy # -# Each "epoch" = {args.epoch_steps} steps, {samples_per_epoch} samples, {tokens_per_epoch} tokens +# Read https://pypi.org/project/rwkv/ for Strategy Guide # -# Model = {args.n_layer} n_layer, {args.n_embd} n_embd, {args.ctx_len} ctx_len -# -# Adam = lr {args.lr_init} to {args.lr_final}, warmup {args.warmup_steps} steps, beta {args.betas}, eps {args.adam_eps} -# -# Found torch {torch.__version__}, recommend 1.12.1+cu116 or newer -# Found deepspeed {deepspeed.__version__}, recommend 0.7.0 (faster than newer versions) -# Found pytorch_lightning {pl.__version__}, recommend 1.7.4 or newer -# -############################################################################ -""" - ) - rank_zero_info(str(vars(args)) + "\n") - - assert args.data_type in ["utf-8", "utf-16le", "numpy", "binidx", "dummy", "wds_img", "uint16"] - - if args.lr_final == 0 or args.lr_init == 0: - rank_zero_info("\n\nNote: lr_final = 0 or lr_init = 0. Using linear LR schedule instead.\n\n") - - assert args.precision in ["fp32", "tf32", "fp16", "bf16"] - os.environ["RWKV_FLOAT_MODE"] = args.precision - if args.precision == "fp32": - rank_zero_info("\n\nNote: you are using fp32 (very slow). Try bf16 / tf32 for faster training.\n\n") - if args.precision == "fp16": - rank_zero_info("\n\nNote: you are using fp16 (might overflow). Try bf16 / tf32 for stable training.\n\n") - - os.environ["RWKV_JIT_ON"] = "1" - if "deepspeed_stage_3" in args.strategy: - os.environ["RWKV_JIT_ON"] = "0" - - torch.backends.cudnn.benchmark = True - torch.backends.cudnn.enabled = True - if args.precision == "fp32": - torch.backends.cudnn.allow_tf32 = False - torch.backends.cuda.matmul.allow_tf32 = False - else: - torch.backends.cudnn.allow_tf32 = True - torch.backends.cuda.matmul.allow_tf32 = True - - if "32" in args.precision: - args.precision = 32 - elif args.precision == "fp16": - args.precision = 16 - else: - args.precision = "bf16" - - ######################################################################################################## - - from src.trainer import train_callback, generate_init_weight - - args.vocab_size = 50277 - - from src.model import RWKV - model = RWKV(args) - - if len(args.load_model) == 0: - rank_zero_info(f"SFT must load model, please input ") - exit(1) - - rank_zero_info(f"########## Loading {args.load_model}... ##########") - try: - load_dict = torch.load(args.load_model, map_location="cpu") - except: - rank_zero_info(f"Bad checkpoint {args.load_model}") - exit(1) - - if args.load_partial == 1: - load_keys = load_dict.keys() - for k in model.state_dict(): - if k not in load_keys: - load_dict[k] = model.state_dict()[k] - model.load_state_dict(load_dict) - - trainer = Trainer.from_argparse_args( - args, - callbacks=[train_callback(args)], - ) - - if trainer.global_rank == 0: - for n in model.state_dict(): - shape = model.state_dict()[n].shape - shape = [i for i in shape if i != 1] - if len(shape) > 1: - print(f"{str(shape[0]).ljust(5)} {str(shape[1]).ljust(5)} {n}") - else: - print(f"{str(shape[0]).ljust(5)} {n}") - - if "deepspeed" in args.strategy: - trainer.strategy.config["zero_optimization"]["allgather_bucket_size"] = args.ds_bucket_mb * 1000 * 1000 - trainer.strategy.config["zero_optimization"]["reduce_bucket_size"] = args.ds_bucket_mb * 1000 * 1000 - - model.deepspeed_offload() - - seq = torch.randint(0, 50277, (1, 100)) - model(seq) +######################################################################################################## +# set these before import RWKV +os.environ['RWKV_JIT_ON'] = '1' +os.environ["RWKV_CUDA_ON"] = '0' # if '1' then compile CUDA kernel for seq mode (much faster) +os.environ["RWKV_T_MAX"] = '1024' + +from src.model import RWKV # pip install rwkv +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-169m/RWKV-4-Pile-169M-20220807-8023', strategy='cuda fp16') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-169m/RWKV-4-Pile-169M-20220807-8023', strategy='cuda fp16i8') +model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-169m/RWKV-4-Pile-169M-20220807-8023', strategy='cpu fp32') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-169m/RWKV-4-Pile-169M-20220807-8023', strategy='cpu fp32 *3 -> cuda fp16 *6+') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-1b5/RWKV-4-Pile-1B5-20220903-8040', strategy='cpu fp32') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-1b5/RWKV-4-Pile-1B5-20220903-8040', strategy='cuda fp16') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-1b5/RWKV-4-Pile-1B5-20220903-8040', strategy='cuda fp16 *8 -> cpu fp32') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-1b5/RWKV-4-Pile-1B5-20220903-8040', strategy='cuda:0 fp16 -> cuda:1 fp16 -> cpu fp32 *1') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-1b5/RWKV-4-Pile-1B5-20220903-8040', strategy='cuda fp16 *6+') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-14b/RWKV-4-Pile-14B-20230213-8019', strategy='cuda fp16 *0+ -> cpu fp32 *1') +# model = RWKV(model='/fsx/BlinkDL/HF-MODEL/rwkv-4-pile-3b/RWKV-4-Pile-3B-20221110-ctx4096', strategy='cuda:0 fp16 *25 -> cuda:1 fp16') + +out, state = model.forward([187, 510, 1563, 310, 247], None) +print(out.detach().cpu().numpy()) # get logits +out, state = model.forward([187, 510], None) +out, state = model.forward([1563], state) # RNN has state (use deepcopy to clone states) +out, state = model.forward([310, 247], state) +print(out.detach().cpu().numpy()) # same result as above + +print('\n') + +from src.utils import PIPELINE, PIPELINE_ARGS +pipeline = PIPELINE(model, "20B_tokenizer.json") + +ctx = "\nIn a shocking finding, scientist discovered a herd of dragons living in a remote, previously unexplored valley, in Tibet. Even more surprising to the researchers was the fact that the dragons spoke perfect Chinese." +print(ctx, end='') + +def my_print(s): + print(s, end='', flush=True) + +# For alpha_frequency and alpha_presence, see "Frequency and presence penalties": +# https://platform.openai.com/docs/api-reference/parameter-details + +args = PIPELINE_ARGS(temperature = 1.0, top_p = 0.7, + alpha_frequency = 0.25, + alpha_presence = 0.25, + token_ban = [0], # ban the generation of some tokens + token_stop = []) # stop generation whenever you see any token here - import ipdb - ipdb.set_trace() +######################################################################################################## +# 1. set os.environ["RWKV_CUDA_ON"] = '1' if possible, for faster preprocess of a long ctx. +# 2. Reuse the state (use deepcopy to clone it) when you are running the same ctx multiple times. +pipeline.generate(ctx, token_count=200, args=args, callback=my_print) +print('\n') \ No newline at end of file diff --git a/src/__pycache__/__init__.cpython-38.pyc b/src/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4539aeb7f04ac5f61f8ab34db9f0cc4a587076b9 GIT binary patch literal 152 zcmWIL<>g`kg4O8^DIoeWh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vOKeRZts8~O_ zI3-U%peR2pHMvATEZ8|eB{e=gzbL!7ATc>rKRF|@1gxUCC|N%~J~J<~BtBlRpz;=n SO>TZlX-=vg$k@+7%m4r^KqIUG literal 0 HcmV?d00001 diff --git a/src/__pycache__/model.cpython-38.pyc b/src/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e0528786137ec673289dabd5b468d5151d43db3 GIT binary patch literal 17322 zcmch8dvILWdEb5Qd$CvmO8^8Pf=Ei{DkMlr3G29w^&myr2?@y}Dajkj>&4!?i(Tvs z=U#x|cF|@`Nmg33mJ`=aY{dh#8OL5SV({r$ds_XUB8o*{{B&X@9`X=r4uLY8S2w%*VR!Cf!tk~a#5+|7b1cdHpGM0B<>8dft}h)G&+#0v?OnT=#KRY*0{g|w7g zjfrNakO|vN7ADaq(#STa3RBJL!nCe+bIyz%b+mfUjvmx*8M^j@)|%Bd{_R*$eqQ^i z)-qny?6{qHMJpU|BK6tAoO7TsFZIcTnw?rR)NeRu{a}3oHM}1}?O~};qhGyn1S3y0 zjyCTr+^1{KeRjq<@&)au413bfRt81j@yUq!eh?)O4tmd{?0xo0l$~@E^@q^vdu;uxme1eims`&3ezLOBDtondtJKIF zenypAYsIdk+QnL{(jLvXTaBA&KXuV@I#)Z6V_)@D$#beV@mO|Hc;VUWPuQiVpSZN) zI<_PypL^klo-bbcXz|)pSFe3|`DtG(13?Bqrg>e! z^MNzhT}QcR%C6ly^Q>yuow9f4`KuS&wo`net=8O5sqCC7ua-P%;i~ePCh8hXotu87 zSZuXBvmC?#&~%jn^iwjqAN=sO;xo&>{%|2F`D?{1PrQiXtwy_K7pxm=o5k*AdBZN9 zVY1y>Dt9{F?4W>&rSitLZt}Ep+|wIwsp@p6PM>a;wv=+J<@QF)TReBRJ9!#)PO(yQ zy<)TEt#-}R&pg^qoqn<7ZI#?bW@4u=Uc+NQkhgrRRcbnZ%x!O|vg7)(O-HS?U8gYR zY+>b#9JE-jG)h&s%oiH{naDhj%RK?$Xe7Ua;h45=8?O+7Eu>A`l61t503o8jiLX?S zqkaFEK!Ox6ALcARQBUt_MXhH5>OB+C=oy~5rn^Ty>k~R^&79^%dKl)GF$Iam5(pqUBm(^qPK56`iLKp!lld4 zJOK=Q=HgQzov7_>*2<1Q(R7+^b+cG$E1-sNRXPu!^R1Q2!{_ojUw_iqulf1~KO&gv zXP$6D2&CGllxi#AY}Q(!6(EVz+N`N|%a3?%RbKT?XRG5U%k7p|t8TP6Tt89)N_+m4 zTiOI7wpDSZR9?GLQnnv)njP<^I>@uiqR}vzlO&(*R4jTmsFA>innm1wEr@^*H`XBZoZy zS(MJB+>)BC-c12xm%_=Eg{avFe(aglEtvaNVy;%#wt=^GeH(aNH-L5K2<93+OEA~k zi@9hUeaYI6Y)7|az?N7&4lJ@NMo+INZ1c7un3}jpc`_^q&H?X6pGe)aCT&|@s-CtZ zLMy-LP3&Zdw?pjg=^V3{=p}opUV0ajv2}9aR+9nMjJCw6*3Zg=+5Tf{<#C1@i!Rbl1raIN8y&AkNh=h5cFxZH;T#tC2^=)NGN@St%Lh)~J zL0}Y%ezI7E*xP6@oh}yFH%g7LChdA9uU2k4-fG(x+EW~VWKJS%g)$GIg4F3!Cd73^ zPc9g`Vre-B(El+_Ng=ex@8)SckQ1X^br)Y&QIg~hdESqdIvpqzu|L7`jZBTR^5f54 zUoKv_e({Mia(zku6w=I({md}uHjwDcz)xxwU8G_{)px?nUrYR4$#=d>A(uovsa(Zf%8>75CPTFyU%-X;U(df3-3eOI@W zISot)Er;V#{Mr~d2SzgobpX##hb?9xzO7wD%5vjnb}E>I)EyYs*-5hT@R`{cwC~eC zqCKa*sG-H2otD<~Ub=peYas75VP|erjak^Y^w7Sghr`kod%>3vdN^WFdPnQ`*;%sf zHw^V9s342>6yEgy@NM}e<$ZQxmdLZy3)*&aFb1(G=_ zpZ-B8J;6$kuH%FIl)Z1fY;0{$4%bi^^UU&HYBGO_3%_qaCLH?i|9boFx9^fiR4elP z=3~`={&oKT`(tBzJ_R(@`>VKIa_%a`^2@nx;HnK=vURA8z~Rpr>lSFpP*0=DU6P5g zfIEh7ES>fB+Fpi`Jm09fUb)@ueDD@~<`7ZM(7O3i8eA&Eo~e1o;?ns>yIg9xA4H=6 zT=+uYc&z%U{C@MXJC?j#0AwO~$+s^-_*g5olI!aw#W(ijC6AiCyLLB=Y(DQt+>MT- z$cuyFe5<6Yt{-odZaS)1v;7z}q+-Kq`O#Lb~>zSV9)=a0Y~ ztHK6=#4BPb-)Ob`#Iq#@KJQ_eqz4XDw4HM4rXLLioNv*v@XbcM8dg_o%JqD+)4mZF zG;3SMwUPAZNLu-c0z~7pCHoOOEOu9G70-`euC<&JG|e>!w$x^+vEleh<&>R{S873h zjkY(u&W1M>r2Zr$BD_o-KOq;=}%Qkb9W9Qlr^+y|YjhY+Eg}ZA#Xt=+bQU z_0>YOTdQ_U)k4HBuCHz4vnGnga--zB#bSV8A^|^&N1M3Z4*_U;Qgl=3q((Xv#uLv8J0u5>_8hvk5Qz^bRhZ*1`bB5VN>U?&eWCSO;+;i|oa98$u7I&h}}G$RQOiT`By-?O;=C&reP z(s!V+@XJD_)^@d*%xz=agkeDS1q+P4sQtZ&M+r-_!K6&4Jg!rc#%{$R(c)+iDYy%# zN}%Gzfv7d8m$%WgqGA%D>aQ}HB3FglUa2)2>UlnXlEwNZIz%cSTqOBuh)U}t3uS^! z1`9gy8i}3hn;-e1=YI}8xTGk}H~|XHz`E>a$Ce9?#&CsPoy#Rl^=%~ZwZGfsN3w+8 z5|X7Ak|i`XF88235|+oLJUS?khUEbsl8h6WV_e8MiBv+!IOQdGQax&TX_TdVQKTl2 z60L1JgH%R{e6p7ogK?sl=}lImz-jWF{N%oL{ufZCdIZ}9`;fhQl@&J#wg_$l5CZhigAt+ zaE|W$7#fUpFoq6k^fO=`6=d$gI--4#!FFOh`NuGj6c#_mPlG!@Vgd>`tovPj$~kQt zAFa<=l0q2jCo$~1^A*))lb-;{r$^TMT{+C2kEA}$F2+en_|31OJRl*E&#b-(0f#{z zXX_^jo+O~$98m<4S8iSkearz@Jk5H}PhQ5K2xV{CX*BxsSBCxsn@kW~0-%6#X<)|n z^w|vU(NsJt_ZZSs(x!~+v7Z^{Tyl^7j6la4Izq*O1JdUsp`-4?YN^$78hdRxas;9Q zJ9obcu)huGSvwIM3=|edlwuTmc?Yh=z+Ov;y=J03DYh8xFGwFRP0J*-*T6`bgS}?K z5`#li?6qs?cLL94$ZkVxXL6si>`;66tUBDBb=oY`v~tDjnHecf4N7xi=``C9N)J%~ zzY|+lAHmxO-27>#XbxOb)adiG>P42ww;w2JiXTL^2t-qOfT@oW6bU{~@G?P(z$T~@ ztPzMZ(LpNEe=C(%z+*(KpnefbfT2#P{t}=25dx}xN)Sj;NwA1>`)yoeef0IYG?WNB zR*23y{is3REo2A7&p$99#@Im&eMN9xV4s{2$oJ1G9kd-&l&}b~2A8N`;L=pjV#33t zC=D0{R5V}}WI&%kCr9|RteWs*F18yMZz-p-5ol$vqFhunIADef$dDD23k%yV z82c+7e~1Qd2B6PWlUTo`-klRBI#1p@g_d+7d6RQwh$mm~+e zpN0pDWkjp7A4!qFKP{cduGHMsr!QY0^U?g->HFvZf715|{q9fI!_s?D^^il5zSU1* z0pCkHR$s&bzELXq#>$G|;yoz%HMF}Y1^+jGX8I4tC>XXDC>VPMInXaCI2(T0ds8mI zvQREn^X~@bq7SIRpxMBX5}zG4E%>Bp9f&3s-%F`~6N9UtCLjY*p9O#k52SntaTbg% zEZo+LSc<+rvM)N^Nx^2i=2A-;QYPqfvcDD~J*r&}wI99x4OIMBTzz_*0ll&8-t_h$ zYWI_?a23LN_zJcfMXHKL;76W>^JQ&})=0PmSqky^F0vF739AWd8Y7P!C;T2+>-JX{alc}DC3hAs8`Zg2G2;Hv}XOVZgSA`TSyu8tn~Jl2U*IEB6=%Z zX>W$+ljL$RoetDz!;Q{=<;@Lh=Fv_z8l~MlMmfm7u+h@s7|1Wk7D;354Gw|A4|7J+ zzAv|r>|2U0*g;$IWQ&Zy4>-AK$MEj=<2r_Zj`O0{KtH5-_2SDzH12_7Lh>nEExi!kx%?FuA3<&ydUfS(A$<$e}{>3094(6gd*zDU^o{$-qvHC>2TluznZ{ zMaD==S=!*XX5bv^W`q_7>}h$7DoDZu{!~Z*z!LcSVQyfj0G4`BI;mqU9OP2pf)( z!R$=Tj$=>1OMQ~2(#{N64c;xQ$%ql|P0(=K$zfDzu0fxKd}FQ{v~mM2bZnt&JF`7d zLlz!rdZMBHM@Q%(@Ls%uuTkaq)wFN?YJM)zH|eDiN2L1OO#K~#w+Ma?z>gsgWVHlW z@Xxd8?-G2KKoI!vG4;0y{yxDk5d0#+FA;o=;4=h2PVg0iUj`U8=KRQ((39ZhxM2tK z35iWKy3g_PM*)1jJi_utAExg0D3O;;T!VKr6S~n(!p8|YN@)s|O~TAu#5UwHY(91; z0<;Y!NnT%u2!9+;?=}D4U9bBxo|xgt7820w?o%w;SIRt&ia$*lOZ4(1FQ`&yjGQQB zcwJy$R9x}d;c(g2xfRp5uXUA24JiIBZjAhKUG;rP?zd`J`g5n8?Ph`@_YFmep@#t_ z^K$_wfrG$pgO*?&lH?$!f;;d%Cp&tc0f}Hxe9}^T32rv*;~CH`Fk$ZD!K!3%3#>nA z7A9W-J9&n<+=P=?PhU9vt=~Vl@R&Gh1%I6)VAnKh00d zrg*VYTXX!lTLv?Q?;dK1tVfF56UETU&t(B)E|n2vQox1-Jh^7gcGVA{eD88`fB88s zSE!a4^H9-}M5W1a*#?`~>ap?Xhivv&QE{1_#F3D_XbYMNOW%Pr#3=sU=K=QBOeF9L zAy5%+`M#-hQ}Y!IK~dQBm6*V|owQTf{!Q3vdjj{Qov|lzPl+az#zJR(i>A()F=$}8 zfg*ztq7g;rb$s+9BBfr(hreaG$6klS?{#kKvIMGw`+k<#_?#$>N~wj?57`mqW0H>| z|DxpMUScPS_9;HIOC=$9s+` zGQfB25=KxsgG;>TrLM5PMdBG)RQ^ zwLp0h&1#PZV~l7pqP+YXx(+M``a?d(6{0c~C@)DyTp`wqdtJDUX=4$p%WSx()twky z!LbiV%1q)SpLM^DUdiYvQpw^3uU_+vTVQ(vO@Z8OF?TftCv?H)u-yWd7(tBS$M6*B z0V@4-Jrk-vcA|7`{Y%)B=%-JS1)}BagO)w)Q1olRH1gCbgfOt>nndQqQgja!L@SJ2GYTfW{I76-dK$@6xt90UP;h6qiG z8^s@C0H+0I5zkmb1SAZLb_dhr2Aw;?00vKw&5_gR9EBIQac zWQNK|%R71@+2KAt?2)n;v=N<)TWK=2;` z3MsZPjy$0LBZ~7Gk^F_U*!nkW2(ZV%rt5fqmSK-}Q7D7~Gj3sm&WIW!BM}IRz#6k; z+i;P$m4~fi7s(!zic?bF`C)3TmIUgQpbZa+S?$yL6DYMN+iT>LXQ8iE>vd3a|| z{?;^VWS{f@L}UH??Aw+V*H7r(gZJc!uo9xm@itUzjLe8Gye{B!zZXD60lW=}jL<#v zmZ8pyI25afqLqOIoh3=Z&xWvGMs!7%3sbZ(`W%?iJFUW0xXArdM-?XeMe^M);b=k( zDiXqv5LZQIiDypyWX-L$@DUMvq^K|Y(O~I(v+8(#x6QXvQ83GuYOu~jpkJD$fdzCqTuAz{LRu#&E7b8KiGPMOuHdwG=+{ z;K(zd9XiJ&M%heCSiZ$Rn zP27ebXFJ(?uHVK^iqGgr_O{xY+tsAy$w5mT+`_wk`5i6igO+%wgM6o6626{Di_sC1 z4+B=x%iZs}fc@UKw-|aiPA$smp~Wy{Z?WbsI$Nuy4Ft8>OJXDh7WNy!h>-`|!-0-g zwU8WU)o=3Y-zNASg5M<=x{~R7r`@ud#3^9hq)wZYp1&fvl&Jg?-Kt3fbjduN3 zFxjD-=`}X`6#%eznk(vaEO?aQ9})uE1#~(pf9)}L9le+ zfOy~hV|zX)?wxy#O@hL?q?n9^vAA{AL#O})vq8o&Fa$b!;G%pQZyDemRu=B67QQ9$ zEukEHL!>L*g*1o@bic2c#EBn#^@%c$KENH{@)QhQ-`cD>H-gX-F(1!McQwyxs(;Ke z=wwkB0s5hPkrf=TTP-ByhLF|}J^m-3i+Kolbv)OPBL1}Ix#}OW6dodXtq{i{ieMV! z_~B0ocLzE{C_5dWm~t485mRW6#t~9U{K!ChW%Og_31d3bofbwt_TU(kCU)==j5vw} zh#g8h*dRP6b-iw=1qAgWRWxPmlNf{}cgYe$P^t)^b(H!qC@N%XZn4^Euap{cC^FEi zxqN6CN|{0`SP4igyts*qJp|xSjm;A)Z~qZrl0t-5H=@u2x$}-@V^Ww1P6v~`(mB{B z9^FYWjBDb(r}Vj~ye?`$gFB>xHW zg)#O7q+AvfOZr!D*{SX5`djERgC2j~p2W`b!sAl?xNvCc~!)=K>X>u|hE>JTd$P2u40Z4+(grAGDw zv1*X;IUEMQEoTDQj~s`RIe0TT#(W6-6^AjJp@6(ne*oWA>eq9F7Dup#+?C<;vm^C$ zJCJ-h36QG8Jr|rX0rNPz-2L_B@$cd0*OSA*gio zOG`_9;Jf%siYZ*cW@E8yE;U+(h&Ui{;+K3!{R7ZUH@d|1Qa1_L+fp#WA@>Y|nN#0M zEWxL@gaG%YZdQ`R`1Yl4T1o;3_>#g&b*-CP!q(%G#KQ9lW=5aLumoF|{T4I*{79qk zgbux>uC>%|6DqaTP0_j6pXRfif6#1gsrvb!bDY}^NQuFOSBvCU%-EYm-SO}K($ydLFoxCE?;jBO{-wyI1$Oc z4LUx8pwm6lCRcS}N$_Rmghqf1CfKP3@9s5S*G#)bcA+AXm0H{Dh?7pjv@DMo&9H&`SA{TJG4|SN}p0QK^6pdd-3pa}d_Soj*j~IG37Jn#yA|6q!&7TTZ zUcT@gJEI2R$K~koQmYk2?_p^IeX%grxhYlHB7&!>MYp!mX%r5?Vu6dLttL24QDM1{=SKrCdLe%0W{^^! zXR(P0UO#=ctZE%ED915S$rXdMkI-wO|15I6gTd=^$}Z3K7je8EXGLM0swvcm2SQ65 zURzG+#0Rmg2Uz{2}rj1i(?p!>PUCkcJ9-IFD+|>UCFG)=+ literal 0 HcmV?d00001 diff --git a/src/__pycache__/model_run.cpython-38.pyc b/src/__pycache__/model_run.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f6614687dfff671f5abc0d2d66abb43358e4209 GIT binary patch literal 5827 zcmai2%X1t_TF=aSboE26mfY6MvMqat-O;ikuf5C0V$6WCr5&@jX2y@Pr`Vxrb(W-V zbyv5uO3$b!0D0I8W*wXe9uY0r%K`_E#0Ig4Pejw%iyCnl{+Z04UjTO;j2Fc z(4w=Z9N-N2@At1gi9;1%tH(0B_L1r|!+LVI^8jvNB}UdJV0H1`1CuU1iyLjaq!`)=GPA<3kLDQ^6k69$KMI= z-g~fcFt%`tX+iVC$}wG8oy45}Xan@c4*^0(cjt@mv2`xF6fZp7oh5DDf*m@cE1kdK z(v`Uv_96FkBaA2W2IJv$i?Z~B`}yxNoB0KtOIdqnArDtq>t+vTHpYsm^C%Wd`~$#* z9kM<@!qKKYWr0X}(?V^bwo%)O(|3>9EAjnjoFz^(2fQcwG+vanUW%Sb@~P+-j@TjZ zVa2Jn<*GkSg|ufGpfk&+S-*%eCFx>J?ug!I?sJ0dbFnTAmv@W3CMDm4oV)B5|Los; zjx6+CSxgb}R6@Ck7*$1AKKf!s(ySa7Vm2kP;;l63jYq zKD65Yh(CIkvZf`^Z3)cuga1RlX{c30{mFnjHJmG&H`Mcn`fmg3^pGlXJ2Y1?L-#4C zz$H{Ub>#GxWm?%vW=H&V(z#b6zA9(TYK3a1HaNePOv+g~CojCDo#Te+Jo#neu7w*QONl4RwstRC1) z-DFc+_3k#zU8!B`2CYuLMpo0$cRO(qw%3Balji<0*~;Vp!KZTr`idglL2eBrx8fzu zmkuUsNwU=0PP*I4(r(z;*i4SeS{zi?*Q2GKT5CHTpby-oq|pw8gJM?8uwL6AcGepz zPJDa4`}G^zUR%HZO^r6e`nIec6tJ|Rl>A`suPrIEM4D{36-M&-J(|@`I;y_8bWASr z@uE<(#Nw>Cw&TskuWBdW?uJTxLC}aANf2nerZ!?-Sh@eRdh93tC~Mtwxk! z&Tgxb=*i5MEYL$WQQL9YT345`Hub9nz6{_`nmq~G?y3nfahYIn9YMX=AZbZ5()kl!1J_cB{)CbhQN5CBWD6&y>Zd%-5 zO;8VFlY=ZQfumo#k7!EW2dPdje)yHq7_Hlf3BZaGI3l3yV)b^F9JNTTDwCEmXFS1v z;9~96o?%ruaF|z~I&a}~27HB-_ib_|C-QEw=pW2{4(9vUGv{-T*cZ-~@w|PWkIs_* zMUp{3CH3`{C!+6?qo}YW-YcS)L-A5K%@YQ=5}ZJ(!g^zAF&(Rtp>|i*c^vn#xiPY~ zszP82U@@;2P-&YKSY0Lb*9lxF@C~Bm;>Je10|jznT53=qTZOId#q!B>fIWgThvtrT z!5HLrV=ve`)mB<;?P{@uqEPm>P?YBQ+w#TM&PkL!cuOlQ`U1U(7_wXaNdKbSb zvA`FgPGw$&TAq4mD=T#hj)x!F2a0EnVoX6zMi1;y&$^QS@PS#Q2R8QB2WE|~s0=>$ z|9yy73LkDB?D&}v@#kh;wuzL!bmq$FnSIf!GM?e1IeSGJ&)ZjXrt*E!?4ZAOX3$T~ zdd71ursL*0SJHeshNnA$_^6C}QaW@xCO-Y=bb{2lcP^bYbv5$R3F`H#BZ(&R=^@G; zsCte}M{jD>_e_5p3UUqxWQLR{ola+tL~qt8(R4aHLuxf=dNYIGg>)`GkNdrlR?R)m zqd!G`$he5{kUEexlb##N8jj9nojY6BXk}PA;=hsH5gt1LesCE~}QrSs{< z3&@o+^Vqu-T^+qAONjv$HxG)J^?H}mQd&tbRau3MLq0PO>J#Y7iCyr;^>3;wP{-t8 zj!BUQrb0a>#xpF1dIo~dwQGBg_I6v{pdpt}tc3b48b^LWeU}(d48@N6>;z$=wgJ(E zbLzKgNI>9s0DRHeRW&qVL3Y#{q3Z-Bq4*B$3~bBJjxTC8U#zWZ(d}xnyQ?{>@H)y3 z&1;&k`PS|kcIVOUKZcx`b~3a(WO+))@(gWHW__}t04^Nv+=kPpp9jt|eihWi_B@|4 zeF_E2;x&u)==MsTPImZFggJp?OxDj}s1uU}zvAEj7fzQhkb)v5L*ijK?WU6=EuWP?-4l^ zQm0vz99a`+RfGzzYV> zH=Uyr?QS&B#ew8YZ%DrUnq-pthO{G6X!ux)Jd%YeW_IzT)Rp-Qz(4;{Y9S&1yHq@7 zVvs2QH56$y%ay_Yu3}v-jF^i%d57YU^*2UjDW*xK3ag4P-UQs45|TP1Zn{a>4S$uZlEAM4Xy+hQ9b^_!U_w6i zm?-ZPctT`1OND%^6@AUOC^Fv#n!4AG+NuVzlD?`Yfh_=SAz9ImNzD9$3hQAvsYR3> zWI3E~A#YP5I7!Y8^;K%QttbGQLh_n4;l^UY7z<>R$(}J5dc2D~Lv`vQLiA45+CSN3 z7KWN#8H@6q*78O`np^Lv-I|jB1X}C^FleC5T~R?K;c;PtbLxA14mvxBh{7{L{yeu4 zIhY@J5v1bxbJs2*)CVrZpbsfX4JZyNiG#{0NFPkP$XAf5AS;ZJ3n42k1Ofh;x80(4 zDF_frwK9syH?p%YJlJ0u{LfK+g*HS-X3k&zK3aNgkgn9jR%@8Dm_4r&pGjUQ0HWs* zQ+N#v^pWQXUa6vdLlJFpR{aVp^KP`=t|56Bomedcr@buCjUpw8h5iMoMHFclpWgJrN$j`Xub zJN1uBb&aN2B0z7%toYQen!TD_!@HFo=KF+-7XT<#n!){2x`l*I*g3jQ)JV>3VcY); Dau+(v literal 0 HcmV?d00001 diff --git a/src/__pycache__/utils.cpython-38.pyc b/src/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c5da902d571c5208340c8cc424eb0703b89e46c GIT binary patch literal 3889 zcma)9&2t<_74M#}on7s&Wcf>WNN`ahuqh%3V*(Xa#>SF@al96?ETyJmhSBy&9%*)F z)zj+_)hv}l<`x&J;vbN>84jHIGg6gvUvmI+0IDc1xxnxBSh8fBDrT!+cfbC6{XTxL z_jaq*VEF#H&cr{CG4>^O4u3W}mr;ss5WxgbSy0tH;HGZ{mZ|N)HnkJDruG64wUyTT zeo*HuVL>D2yJLGSXbM|6k66(9fC*Q4kC^awte`Dw!UuO;)I|gJgcuV|)W-zB#X7C0 zIG=SpTsPK+Sx+T-7NtE-%x9m4nxwsoQv494WQuLpb~?HQ){uC>RMJq4*=V>Y8C>h4xKD z#%mo*J6VJwOQ|PJ*0K1j_s`!hVp*K;6(T#oAoGp5r_L|mx|)kPT*>8TF^GEc`QF_~ znHh!bogb^77ix8&USHI~aYYT6ZtD8d&FgdBYr)*&ziErV|9YoK_5SCfK{8xLDgFjRdR5Tk z4t|@Sl)@@mXq8;pXBemh&02C6wsupy?~H8DO1l~r-eZorZgvi%HH>;A2cyp6wL7$% zt7;|NaCd07pDE$f3962H_0tSjf`>e|e>*vHRbzLoboXdpbA^4M{hTebkC<8AtX7Mb zXg`7lmO0rli z<(qMqY{zmwDpZFDZMDw`*OD~G8W1cA6$Bqb|njY_*rcGtpi_}4#Ol%R2WXrqI1>!6aSpn5lxCh}^!vZ$aPCfv~F}iosJQ6|UzKODEHI22KsJLGQPNDKd zJBi8*-Mrns-o3ff4X^(6%3{aUP7$YTy1sIAaW?E;xiKf-!a3$}b7Fw7ErzKwrhnmX z+TYjbak}6eAPlkWnRuSGn*0QBTOQ~st7)AA-JX2e6g|Sa{f``j)NbN(fJVu8*&f^F zg75JWSC&H5QH;*AJLljkoIKmi01ob9uCNg|;LfnEv&vC!1rS(c?O?sYZ`Uia7p*yM za8r1Q>{MX1By6sAHE#?_jxRs?aU9E(mPo5FBG=@_H#`DYY}!~FV9r`Kpk?;XN%-_u%J`ZE_@~#pce zvl;v6kGs8E)iksPN&#rh_#w20h_M7e1E3n!6yG3V<-&SwjR1lhwtOGdDQjn7J{aR- z>5b}Jmr8e3&m5x4-S7y4g^STIN`JSu$40*Lck89UM~NF&_zi%1)BvwhHsC5_YOJJQ zvm7hyo6cT!c2jtzk6&{WDYSG%ZPS*2zuYY z;X`8MCTem6XH#;2YzfbtJh_3Bt6S|Jhs%e|^-OXENY0j=*1a< zvK*%|uE(-rKP;mDAdQR8NqLfX=1Aqm8^1q7+Z*}X&p>s>5EVz;sZ zP$ZX1d%a=581}Wh1|&i^lIvKm?LmGYh=&0WPAhWMy}NnRtGrjcDwn;x@<+7)SRYuE zWO+Y{(oU=LVLKlx%^$#*)q{}mZw#O|h-IXPGDasKgoAKc45L)Hs!kT)PZ3gqpCW@@ z1-xo^AoJBCz)_eSG)dZ6gcWlFQV-KeCHLZrL-$bm+S-fp(yCMxrullJitj<*;vxtG zx)IK`xo3HnW3>@9CwPOO!mGvyw~d}-O`=bn2KtkH`}FgWI$>dz~ji&^neu>O7*y0nulv0pO0lfQVTNaM}Ve>OL3v})Qu!fW4RcuCfSh@ z_?rjES&ii(F-;C{-iRKq#@7m(9xv~UN-4#$WQhI|Q3<2z5mA8*8U~CVIjbO`$q>`X z7)}{7Y%6QNRGM}~eF7)IXJ$*W| zjh