{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "769381d2", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "from sklearn.datasets import fetch_openml\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import accuracy_score, log_loss\n", "from sklearn.preprocessing import LabelEncoder\n", "\n", "import os\n", "import wget\n", "from pathlib import Path\n", "import shutil\n", "import gzip\n", "\n", "from matplotlib import pyplot as plt\n", "import matplotlib.ticker as mtick\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.nn.init as nn_init\n", "import torch.nn.utils.prune as prune\n", "\n", "import random\n", "import math\n", "\n", "from FTtransformer.ft_transformer import Tokenizer, MultiheadAttention, Transformer, FTtransformer\n", "from FTtransformer import lib\n", "import zero\n", "import json\n", "\n", "from functools import partial\n", "import pickle" ] }, { "cell_type": "markdown", "id": "5b9860e4", "metadata": {}, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": 2, "id": "d575b960", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "File already exists.\n" ] } ], "source": [ "# Experiment settings\n", "EPOCHS = 50\n", "RERUNS = 5 # How many times to redo the same setting\n", "\n", "# Backdoor settings\n", "target=[\"Covertype\"]\n", "backdoorFeatures = [\"Elevation\"]\n", "backdoorTriggerValues = [4057]\n", "targetLabel = 4\n", "poisoningRates = [0.0005]\n", "\n", "DEVICE = 'cuda:0'\n", "DATAPATH = \"data/covtypeFTT-1F-OOB/\"\n", "# FTtransformer config\n", "config = {\n", " 'data': {\n", " 'normalization': 'standard',\n", " 'path': DATAPATH\n", " }, \n", " 'model': {\n", " 'activation': 'reglu', \n", " 'attention_dropout': 0.03815883962184247, \n", " 'd_ffn_factor': 1.333333333333333, \n", " 'd_token': 424, \n", " 'ffn_dropout': 0.2515503440562596, \n", " 'initialization': 'kaiming', \n", " 'n_heads': 8, \n", " 'n_layers': 2, \n", " 'prenormalization': True, \n", " 'residual_dropout': 0.0, \n", " 'token_bias': True, \n", " 'kv_compression': None, \n", " 'kv_compression_sharing': None\n", " }, \n", " 'seed': 0, \n", " 'training': {\n", " 'batch_size': 1024, \n", " 'eval_batch_size': 1024, \n", " 'lr': 3.762989816330166e-05, \n", " 'n_epochs': EPOCHS, \n", " 'device': DEVICE, \n", " 'optimizer': 'adamw', \n", " 'patience': 16, \n", " 'weight_decay': 0.0001239780004929955\n", " }\n", "}\n", "\n", "\n", "# Load dataset\n", "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz\"\n", "dataset_name = 'forestcover-type'\n", "tmp_out = Path('./data/'+dataset_name+'.gz')\n", "out = Path(os.getcwd()+'/data/'+dataset_name+'.csv')\n", "out.parent.mkdir(parents=True, exist_ok=True)\n", "if out.exists():\n", " print(\"File already exists.\")\n", "else:\n", " print(\"Downloading file...\")\n", " wget.download(url, tmp_out.as_posix())\n", " with gzip.open(tmp_out, 'rb') as f_in:\n", " with open(out, 'wb') as f_out:\n", " shutil.copyfileobj(f_in, f_out)\n", "\n", "\n", "# Setup data\n", "cat_cols = [\n", " \"Wilderness_Area1\", \"Wilderness_Area2\", \"Wilderness_Area3\",\n", " \"Wilderness_Area4\", \"Soil_Type1\", \"Soil_Type2\", \"Soil_Type3\", \"Soil_Type4\",\n", " \"Soil_Type5\", \"Soil_Type6\", \"Soil_Type7\", \"Soil_Type8\", \"Soil_Type9\",\n", " \"Soil_Type10\", \"Soil_Type11\", \"Soil_Type12\", \"Soil_Type13\", \"Soil_Type14\",\n", " \"Soil_Type15\", \"Soil_Type16\", \"Soil_Type17\", \"Soil_Type18\", \"Soil_Type19\",\n", " \"Soil_Type20\", \"Soil_Type21\", \"Soil_Type22\", \"Soil_Type23\", \"Soil_Type24\",\n", " \"Soil_Type25\", \"Soil_Type26\", \"Soil_Type27\", \"Soil_Type28\", \"Soil_Type29\",\n", " \"Soil_Type30\", \"Soil_Type31\", \"Soil_Type32\", \"Soil_Type33\", \"Soil_Type34\",\n", " \"Soil_Type35\", \"Soil_Type36\", \"Soil_Type37\", \"Soil_Type38\", \"Soil_Type39\",\n", " \"Soil_Type40\"\n", "]\n", "\n", "num_cols = [\n", " \"Elevation\", \"Aspect\", \"Slope\", \"Horizontal_Distance_To_Hydrology\",\n", " \"Vertical_Distance_To_Hydrology\", \"Horizontal_Distance_To_Roadways\",\n", " \"Hillshade_9am\", \"Hillshade_Noon\", \"Hillshade_3pm\",\n", " \"Horizontal_Distance_To_Fire_Points\"\n", "]\n", "\n", "feature_columns = (\n", " num_cols + cat_cols + target)\n", "\n", "data = pd.read_csv(out, header=None, names=feature_columns)\n", "data[\"Covertype\"] = data[\"Covertype\"] - 1 # Make sure output labels start at 0 instead of 1\n", "\n", "\n", "# Experiment setup\n", "def GenerateTrigger(df, poisoningRate, backdoorTriggerValues, targetLabel):\n", " rows_with_trigger = df.sample(frac=poisoningRate)\n", " rows_with_trigger[backdoorFeatures] = backdoorTriggerValues\n", " rows_with_trigger[target] = targetLabel\n", " return rows_with_trigger\n", "\n", "def GenerateBackdoorTrigger(df, backdoorTriggerValues, targetLabel):\n", " df[backdoorFeatures] = backdoorTriggerValues\n", " df[target] = targetLabel\n", " return df" ] }, { "cell_type": "markdown", "id": "d9a5a67a", "metadata": {}, "source": [ "## Prepare data" ] }, { "cell_type": "code", "execution_count": 3, "id": "fa253ec3", "metadata": {}, "outputs": [], "source": [ "runIdx = 1\n", "poisoningRate = poisoningRates[0]\n", "\n", "random.seed(runIdx)\n", "\n", "checkpoint_path = 'FTtransformerCheckpoints/CovType_1F_OOB_' + str(poisoningRate) + \"-\" + str(runIdx) + \".pt\"\n" ] }, { "cell_type": "markdown", "id": "3bd019f0", "metadata": {}, "source": [ "## Setup model" ] }, { "cell_type": "code", "execution_count": 4, "id": "2f51f794", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using device: cuda:0\n", "self.category_embeddings.weight.shape=torch.Size([88, 424])\n" ] } ], "source": [ "\n", "zero.set_randomness(config['seed'])\n", "dataset_dir = config['data']['path']\n", "\n", "D = lib.Dataset.from_dir(dataset_dir)\n", "X = D.build_X(\n", " normalization=config['data'].get('normalization'),\n", " num_nan_policy='mean',\n", " cat_nan_policy='new',\n", " cat_policy=config['data'].get('cat_policy', 'indices'),\n", " cat_min_frequency=config['data'].get('cat_min_frequency', 0.0),\n", " seed=config['seed'],\n", ")\n", "if not isinstance(X, tuple):\n", " X = (X, None)\n", "\n", "Y, y_info = D.build_y(config['data'].get('y_policy'))\n", "\n", "X = tuple(None if x is None else lib.to_tensors(x) for x in X)\n", "Y = lib.to_tensors(Y)\n", "device = torch.device(config['training']['device'])\n", "print(\"Using device:\", config['training']['device'])\n", "if device.type != 'cpu':\n", " X = tuple(\n", " None if x is None else {k: v.to(device) for k, v in x.items()} for x in X\n", " )\n", " Y_device = {k: v.to(device) for k, v in Y.items()}\n", "else:\n", " Y_device = Y\n", "X_num, X_cat = X\n", "del X\n", "if not D.is_multiclass:\n", " Y_device = {k: v.float() for k, v in Y_device.items()}\n", "\n", "train_size = D.size(lib.TRAIN)\n", "batch_size = config['training']['batch_size']\n", "epoch_size = math.ceil(train_size / batch_size)\n", "eval_batch_size = config['training']['eval_batch_size']\n", "chunk_size = None\n", "\n", "loss_fn = (\n", " F.binary_cross_entropy_with_logits\n", " if D.is_binclass\n", " else F.cross_entropy\n", " if D.is_multiclass\n", " else F.mse_loss\n", ")\n", "\n", "model = Transformer(\n", " d_numerical=0 if X_num is None else X_num['train'].shape[1],\n", " categories=lib.get_categories(X_cat),\n", " d_out=D.info['n_classes'] if D.is_multiclass else 1,\n", " **config['model'],\n", ").to(device)\n", "\n", "def needs_wd(name):\n", " return all(x not in name for x in ['tokenizer', '.norm', '.bias'])\n", "\n", "for x in ['tokenizer', '.norm', '.bias']:\n", " assert any(x in a for a in (b[0] for b in model.named_parameters()))\n", "parameters_with_wd = [v for k, v in model.named_parameters() if needs_wd(k)]\n", "parameters_without_wd = [v for k, v in model.named_parameters() if not needs_wd(k)]\n", "optimizer = lib.make_optimizer(\n", " config['training']['optimizer'],\n", " (\n", " [\n", " {'params': parameters_with_wd},\n", " {'params': parameters_without_wd, 'weight_decay': 0.0},\n", " ]\n", " ),\n", " config['training']['lr'],\n", " config['training']['weight_decay'],\n", ")\n", "\n", "stream = zero.Stream(lib.IndexLoader(train_size, batch_size, True, device))\n", "progress = zero.ProgressTracker(config['training']['patience'])\n", "training_log = {lib.TRAIN: [], lib.VAL: [], lib.TEST: []}\n", "timer = zero.Timer()\n", "output = \"Checkpoints\"\n", "\n", "def print_epoch_info():\n", " print(f'\\n>>> Epoch {stream.epoch} | {lib.format_seconds(timer())} | {output}')\n", " print(\n", " ' | '.join(\n", " f'{k} = {v}'\n", " for k, v in {\n", " 'lr': lib.get_lr(optimizer),\n", " 'batch_size': batch_size,\n", " 'chunk_size': chunk_size,\n", " }.items()\n", " )\n", " )\n", "\n", "def apply_model(part, idx):\n", " return model(\n", " None if X_num is None else X_num[part][idx],\n", " None if X_cat is None else X_cat[part][idx],\n", " )\n", "\n", "@torch.no_grad()\n", "def evaluate(parts):\n", " eval_batch_size = config['training']['eval_batch_size']\n", " model.eval()\n", " metrics = {}\n", " predictions = {}\n", " for part in parts:\n", " while eval_batch_size:\n", " try:\n", " predictions[part] = (\n", " torch.cat(\n", " [\n", " apply_model(part, idx)\n", " for idx in lib.IndexLoader(\n", " D.size(part), eval_batch_size, False, device\n", " )\n", " ]\n", " )\n", " .cpu()\n", " .numpy()\n", " )\n", " except RuntimeError as err:\n", " if not lib.is_oom_exception(err):\n", " raise\n", " eval_batch_size //= 2\n", " print('New eval batch size:', eval_batch_size)\n", " else:\n", " break\n", " if not eval_batch_size:\n", " RuntimeError('Not enough memory even for eval_batch_size=1')\n", " metrics[part] = lib.calculate_metrics(\n", " D.info['task_type'],\n", " Y[part].numpy(), # type: ignore[code]\n", " predictions[part], # type: ignore[code]\n", " 'logits',\n", " y_info,\n", " )\n", " for part, part_metrics in metrics.items():\n", " print(f'[{part:<5}]', lib.make_summary(part_metrics))\n", " return metrics, predictions\n", "\n", "def save_checkpoint(final):\n", " torch.save(\n", " {\n", " 'model': model.state_dict(),\n", " 'optimizer': optimizer.state_dict(),\n", " 'stream': stream.state_dict(),\n", " 'random_state': zero.get_random_state(),\n", " },\n", " checkpoint_path,\n", " )" ] }, { "cell_type": "markdown", "id": "214a2935", "metadata": {}, "source": [ "## Load model" ] }, { "cell_type": "code", "execution_count": 5, "id": "3be456cc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[test ] Accuracy = 0.954\n", "[test_backdoor] Accuracy = 0.997\n" ] } ], "source": [ "zero.set_randomness(config['seed'])\n", "\n", "# Load best checkpoint\n", "model.load_state_dict(torch.load(checkpoint_path)['model'])\n", "metrics, predictions = evaluate(['test', 'test_backdoor'])" ] }, { "cell_type": "markdown", "id": "c87fb163", "metadata": {}, "source": [ "# Save activations" ] }, { "cell_type": "code", "execution_count": 6, "id": "146c8957", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "registered: layers.0.attention.W_q : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.0.attention.W_k : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.0.attention.W_v : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.0.attention.W_out : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.0.linear0 : Linear(in_features=424, out_features=1130, bias=True)\n", "registered: layers.0.linear1 : Linear(in_features=565, out_features=424, bias=True)\n", "registered: layers.1.attention.W_q : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.1.attention.W_k : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.1.attention.W_v : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.1.attention.W_out : Linear(in_features=424, out_features=424, bias=True)\n", "registered: layers.1.linear0 : Linear(in_features=424, out_features=1130, bias=True)\n", "registered: layers.1.linear1 : Linear(in_features=565, out_features=424, bias=True)\n" ] } ], "source": [ "activations_out = {}\n", "count = 0\n", "fails = 0\n", "def save_activation(name, mod, inp, out):\n", " if name not in activations_out:\n", " activations_out[name] = out.cpu().detach().numpy()\n", " \n", " global fails\n", " # Will fail if dataset not divisiable by batch size, try except to skip the last iteration\n", " try:\n", " # Save the activations for the input neurons\n", " activations_out[name] += out.cpu().detach().numpy()\n", " \n", " if \"layers.0.linear0\" in name:\n", " global count\n", " count += 1\n", " except:\n", " fails+=1\n", " \n", "hooks = []\n", "for name, m in model.named_modules():\n", " #print(name) # -> tabnet.final_mapping is the layer we are interested in\n", " if \"W_\" in name or \"linear\" in name:\n", " print(\"registered:\", name, \":\", m)\n", " hooks.append(m.register_forward_hook(partial(save_activation, name)))" ] }, { "cell_type": "code", "execution_count": 8, "id": "9351dbce", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[test ] Accuracy = 0.954\n" ] } ], "source": [ "_ = evaluate(['test'])" ] }, { "cell_type": "code", "execution_count": 9, "id": "09857b48", "metadata": {}, "outputs": [], "source": [ "for hook in hooks:\n", " hook.remove()" ] }, { "cell_type": "code", "execution_count": 10, "id": "6f6bf9ee", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "113\n", "12\n", "12\n" ] } ], "source": [ "print(count)\n", "\n", "# fails should be equal to number of layers (12), or 0 if data is dividable by batch size\n", "print(len(activations_out))\n", "print(fails)" ] }, { "cell_type": "code", "execution_count": 11, "id": "b796ee9a", "metadata": {}, "outputs": [], "source": [ "# Calculate mean activation value (although not really needed for ranking)\n", "for x in activations_out:\n", " activations_out[x] = activations_out[x]/count" ] }, { "cell_type": "code", "execution_count": 12, "id": "a9dc87ce", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "layers.0.attention.W_q\n", "(1024, 55, 424)\n", "\n", "layers.0.attention.W_k\n", "(1024, 55, 424)\n", "\n", "layers.0.attention.W_v\n", "(1024, 55, 424)\n", "\n", "layers.0.attention.W_out\n", "(1024, 55, 424)\n", "\n", "layers.0.linear0\n", "(1024, 55, 1130)\n", "\n", "layers.0.linear1\n", "(1024, 55, 424)\n", "\n", "layers.1.attention.W_q\n", "(1024, 1, 424)\n", "\n", "layers.1.attention.W_k\n", "(1024, 55, 424)\n", "\n", "layers.1.attention.W_v\n", "(1024, 55, 424)\n", "\n", "layers.1.attention.W_out\n", "(1024, 1, 424)\n", "\n", "layers.1.linear0\n", "(1024, 1, 1130)\n", "\n", "layers.1.linear1\n", "(1024, 1, 424)\n", "\n" ] } ], "source": [ "for x in activations_out:\n", " print(x)\n", " print(activations_out[x].shape)\n", " print()" ] }, { "cell_type": "code", "execution_count": 13, "id": "ecee2260", "metadata": {}, "outputs": [], "source": [ "# Average over batch and second dimension\n", "for x in activations_out:\n", " activations_out[x] = activations_out[x].mean(axis=0).mean(axis=0)" ] }, { "cell_type": "code", "execution_count": 14, "id": "0ccc53f7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "layers.0.attention.W_q\n", "(424,)\n", "layers.0.attention.W_k\n", "(424,)\n", "layers.0.attention.W_v\n", "(424,)\n", "layers.0.attention.W_out\n", "(424,)\n", "layers.0.linear0\n", "(1130,)\n", "layers.0.linear1\n", "(424,)\n", "layers.1.attention.W_q\n", "(424,)\n", "layers.1.attention.W_k\n", "(424,)\n", "layers.1.attention.W_v\n", "(424,)\n", "layers.1.attention.W_out\n", "(424,)\n", "layers.1.linear0\n", "(1130,)\n", "layers.1.linear1\n", "(424,)\n" ] } ], "source": [ "for x in activations_out:\n", " print(x)\n", " print(activations_out[x].shape)" ] }, { "cell_type": "code", "execution_count": 15, "id": "1beca88e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[test ] Accuracy = 0.954\n", "[test_backdoor] Accuracy = 0.997\n" ] } ], "source": [ "metrics = evaluate(['test', 'test_backdoor'])" ] }, { "cell_type": "code", "execution_count": 16, "id": "3e8f4a93", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.9974191629339306\n", "0.9541836269287368\n" ] } ], "source": [ "print(metrics[0]['test_backdoor']['accuracy'])\n", "print(metrics[0]['test']['accuracy'])" ] }, { "cell_type": "code", "execution_count": 17, "id": "67f9462d", "metadata": {}, "outputs": [], "source": [ "# Argsort activations for each layer\n", "argsortActivations_out = {}\n", "for n in activations_out:\n", " argsortActivations_out[n] = np.argsort(activations_out[n])" ] }, { "cell_type": "code", "execution_count": 18, "id": "890bbbda", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "layers.0.attention.W_q.weight torch.Size([424, 424])\n", "layers.0.attention.W_k.weight torch.Size([424, 424])\n", "layers.0.attention.W_v.weight torch.Size([424, 424])\n", "layers.0.attention.W_out.weight torch.Size([424, 424])\n", "layers.0.linear0.weight torch.Size([1130, 424])\n", "layers.0.linear1.weight torch.Size([424, 565])\n", "layers.1.attention.W_q.weight torch.Size([424, 424])\n", "layers.1.attention.W_k.weight torch.Size([424, 424])\n", "layers.1.attention.W_v.weight torch.Size([424, 424])\n", "layers.1.attention.W_out.weight torch.Size([424, 424])\n", "layers.1.linear0.weight torch.Size([1130, 424])\n", "layers.1.linear1.weight torch.Size([424, 565])\n" ] } ], "source": [ "for name, m in model.named_parameters():\n", " if \"W_\" in name or \"linear\" in name:\n", " if \"weight\" in name:\n", " print(name, m.shape)" ] }, { "cell_type": "code", "execution_count": 19, "id": "f627749f", "metadata": {}, "outputs": [], "source": [ "def pruneWithTreshold(argsortActivations, name, th=1, transpose=False, dim2=1):\n", " x = torch.tensor(argsortActivations[name].copy())\n", " x[x>=th] = 99999\n", " x[x 32\u001b[0m metrics \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtest\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtest_backdoor\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 33\u001b[0m ASR\u001b[38;5;241m.\u001b[39mappend(metrics[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest_backdoor\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maccuracy\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m 34\u001b[0m BA\u001b[38;5;241m.\u001b[39mappend(metrics[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maccuracy\u001b[39m\u001b[38;5;124m'\u001b[39m])\n", "File \u001b[0;32m~/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 26\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "Cell \u001b[0;32mIn[4], line 110\u001b[0m, in \u001b[0;36mevaluate\u001b[0;34m(parts)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m eval_batch_size:\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 108\u001b[0m predictions[part] \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 109\u001b[0m torch\u001b[38;5;241m.\u001b[39mcat(\n\u001b[0;32m--> 110\u001b[0m [\n\u001b[1;32m 111\u001b[0m apply_model(part, idx)\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m lib\u001b[38;5;241m.\u001b[39mIndexLoader(\n\u001b[1;32m 113\u001b[0m D\u001b[38;5;241m.\u001b[39msize(part), eval_batch_size, \u001b[38;5;28;01mFalse\u001b[39;00m, device\n\u001b[1;32m 114\u001b[0m )\n\u001b[1;32m 115\u001b[0m ]\n\u001b[1;32m 116\u001b[0m )\n\u001b[1;32m 117\u001b[0m \u001b[38;5;241m.\u001b[39mcpu()\n\u001b[1;32m 118\u001b[0m \u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m 119\u001b[0m )\n\u001b[1;32m 120\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m lib\u001b[38;5;241m.\u001b[39mis_oom_exception(err):\n", "Cell \u001b[0;32mIn[4], line 111\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m eval_batch_size:\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 108\u001b[0m predictions[part] \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 109\u001b[0m torch\u001b[38;5;241m.\u001b[39mcat(\n\u001b[1;32m 110\u001b[0m [\n\u001b[0;32m--> 111\u001b[0m \u001b[43mapply_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpart\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m lib\u001b[38;5;241m.\u001b[39mIndexLoader(\n\u001b[1;32m 113\u001b[0m D\u001b[38;5;241m.\u001b[39msize(part), eval_batch_size, \u001b[38;5;28;01mFalse\u001b[39;00m, device\n\u001b[1;32m 114\u001b[0m )\n\u001b[1;32m 115\u001b[0m ]\n\u001b[1;32m 116\u001b[0m )\n\u001b[1;32m 117\u001b[0m \u001b[38;5;241m.\u001b[39mcpu()\n\u001b[1;32m 118\u001b[0m \u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m 119\u001b[0m )\n\u001b[1;32m 120\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m lib\u001b[38;5;241m.\u001b[39mis_oom_exception(err):\n", "Cell \u001b[0;32mIn[4], line 94\u001b[0m, in \u001b[0;36mapply_model\u001b[0;34m(part, idx)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply_model\u001b[39m(part, idx):\n\u001b[0;32m---> 94\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 95\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mX_num\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mX_num\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpart\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 96\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mX_cat\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mX_cat\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpart\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 97\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1191\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1192\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1193\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1195\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", "File \u001b[0;32m~/Documents/School/MasterThesis/Development/Defence/FinePruningFTT/FTtransformer/ft_transformer.py:275\u001b[0m, in \u001b[0;36mTransformer.forward\u001b[0;34m(self, x_num, x_cat)\u001b[0m\n\u001b[1;32m 272\u001b[0m layer \u001b[38;5;241m=\u001b[39m ty\u001b[38;5;241m.\u001b[39mcast(ty\u001b[38;5;241m.\u001b[39mDict[\u001b[38;5;28mstr\u001b[39m, nn\u001b[38;5;241m.\u001b[39mModule], layer)\n\u001b[1;32m 274\u001b[0m x_residual \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_start_residual(x, layer, \u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m--> 275\u001b[0m x_residual \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mattention\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 276\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# for the last attention, it is enough to process only [CLS]\u001b[39;49;00m\n\u001b[1;32m 277\u001b[0m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_residual\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mis_last_layer\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx_residual\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 278\u001b[0m \u001b[43m \u001b[49m\u001b[43mx_residual\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 279\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_kv_compressions\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlayer\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 280\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 281\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_last_layer:\n\u001b[1;32m 282\u001b[0m x \u001b[38;5;241m=\u001b[39m x[:, : x_residual\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m]]\n", "File \u001b[0;32m~/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1191\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1192\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1193\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1195\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", "File \u001b[0;32m~/Documents/School/MasterThesis/Development/Defence/FinePruningFTT/FTtransformer/ft_transformer.py:119\u001b[0m, in \u001b[0;36mMultiheadAttention.forward\u001b[0;34m(self, x_q, x_kv, key_compression, value_compression)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 114\u001b[0m x_q: Tensor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 117\u001b[0m value_compression: ty\u001b[38;5;241m.\u001b[39mOptional[nn\u001b[38;5;241m.\u001b[39mLinear],\n\u001b[1;32m 118\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 119\u001b[0m q, k, v \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mW_q(x_q), \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mW_k\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_kv\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mW_v(x_kv)\n\u001b[1;32m 120\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tensor \u001b[38;5;129;01min\u001b[39;00m [q, k, v]:\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m tensor\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m%\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_heads \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m\n", "File \u001b[0;32m~/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1191\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1192\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1193\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1195\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", "File \u001b[0;32m~/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "BA = []\n", "ASR = []\n", "\n", "for i in range(250):\n", "\n", " \n", " prune.custom_from_mask(\n", " module = model.layers[0].linear0,\n", " name = 'weight',\n", " mask = pruneWithTreshold(argsortActivations_out, \"layers.0.linear0\", i, False, 424).to(\"cuda:0\")\n", " )\n", " \n", " prune.custom_from_mask(\n", " module = model.layers[0].linear1,\n", " name = 'weight',\n", " mask = pruneWithTreshold(argsortActivations_out, \"layers.0.linear1\", i, False, 565).to(\"cuda:0\")\n", " )\n", " \n", " prune.custom_from_mask(\n", " module = model.layers[1].linear0,\n", " name = 'weight',\n", " mask = pruneWithTreshold(argsortActivations_out, \"layers.1.linear0\", i, False, 424).to(\"cuda:0\")\n", " )\n", " \n", " prune.custom_from_mask(\n", " module = model.layers[1].linear1,\n", " name = 'weight',\n", " mask = pruneWithTreshold(argsortActivations_out, \"layers.1.linear1\", i, False, 565).to(\"cuda:0\")\n", " )\n", " \n", " \n", " metrics = evaluate(['test', 'test_backdoor'])\n", " ASR.append(metrics[0]['test_backdoor']['accuracy'])\n", " BA.append(metrics[0]['test']['accuracy'])" ] }, { "cell_type": "code", "execution_count": 21, "id": "1c657e41", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAFMCAYAAADMRBUfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+3UlEQVR4nO3dd3RU1drA4d9MMukNkpACSQgl9BqqSFGQKgIiRVEQvaLSxHrVqxQFaRas2Gj6gQgKqCAgXUF6r6GFTgqE9D6zvz9OZpIhhQQSUniftc4Kc+o+J0Pes7tOKaUQQgghRIWlL+0ECCGEEKJkSbAXQgghKjgJ9kIIIUQFJ8FeCCGEqOAk2AshhBAVnAR7IYQQooKTYC+EEEJUcBLshRBCiApOgr0QQghRwUmwFxXK008/TfXq1Uvl2tWrV+fhhx++q9c8d+4cOp2ODz/88LaOnz9/PjqdjnPnzhVvwu6S6tWr8/TTT5d2MsqVH3/8kbp162IwGPDw8Cjt5Ii7RIK9uC3mIGFeHBwcCAkJYfTo0URGRpZ28oSwkvO7mnPx9fXN9V3Ob8n5Erl58+ZCHaPT6UrvpvNw4sQJnn76aWrWrMl3333Ht99+W9pJEneJbWknQJRv7733HsHBwaSmprJ161Zmz57Nn3/+yZEjR3Bycrrr6fnuu+8wmUx3/bqi7HvooYcYOnSo1TpHR0eaNWvGjz/+aLX+P//5D61atWLEiBGWdS4uLpZ/16tXL9cxb731Fi4uLvzvf/8rgdQXj82bN2Mymfj000+pVatWaSdH3EUS7MUd6dGjBy1atAC0P5Cenp58/PHH/Pbbbzz++ON5HpOUlISzs3OJpMdgMJTIeUXxKsnvQH5CQkJ48skn89xWo0YNq88vvPACNWrUyHd/Hx+fXNumTZuGl5dXvscAmEwm0tPTcXBwKGLqi0dUVBRAsRbfJycnl8qLfU6l8X0qb6QYXxSrBx98EIDw8HBAq0N3cXHhzJkz9OzZE1dXV4YMGQLkX9/aqVMnOnXqZPlsLjJdsmQJU6ZMoVq1ajg4ONC5c2dOnz5tdezNdfY567S//fZbatasib29PS1btmT37t25rr106VLq16+Pg4MDDRs2ZPny5UVuB/DXX3/RtGlTHBwcqF+/PsuWLbPaHhMTw2uvvUajRo1wcXHBzc2NHj16cPDgwVznSk1NZeLEiYSEhODg4ICfnx+PPvooZ86cyff6SilGjBiBnZ2d1bWPHj3Kgw8+iKOjI9WqVWPy5Mn5loJ89dVXNGjQAHt7e/z9/Rk1ahSxsbG59lu6dCmhoaE4OjpaAt3ly5et9inoO2AymZg1axYNGjTAwcEBHx8fnn/+eW7cuJHrniZPnky1atVwcnLigQce4OjRo/k+g7JCp9MxevRoFi5caHmea9asAeDDDz/kvvvuw9PTE0dHR0JDQ/nll1/yPceKFSto2LAh9vb2NGjQwHIes4SEBMaNG0f16tWxt7enSpUqPPTQQ+zbtw/Q/r9NmDABAG9vb3Q6HRMnTrQcX5jfeadOnWjYsCF79+6lQ4cOODk58fbbb1v9P/vyyy+pUaMGTk5OdO3alYsXL6KU4v3336datWo4OjrSp08fYmJict3r6tWrad++Pc7Ozri6utKrV69cv+eCvk8if5KzF8XKHIQ8PT0t6zIzM+nWrRv3338/H3744W3nAqZNm4Zer+e1114jLi6OGTNmMGTIEHbu3HnLYxctWkRCQgLPP/88Op2OGTNm8Oijj3L27FlLacCqVasYNGgQjRo1YurUqdy4cYNnn32WqlWrFjqNp06dYtCgQbzwwgsMGzaMefPmMWDAANasWcNDDz0EwNmzZ1mxYgUDBgwgODiYyMhIvvnmGzp27MixY8fw9/cHwGg08vDDD7NhwwYGDx7MSy+9REJCAuvWrePIkSPUrFkz1/WNRiPPPPMMP//8M8uXL6dXr14ARERE8MADD5CZmcmbb76Js7Mz3377LY6OjrnOMXHiRCZNmkSXLl148cUXCQsLY/bs2ezevZtt27ZZntf8+fMZPnw4LVu2ZOrUqURGRvLpp5+ybds29u/fb5V7zO878Pzzz1vOM3bsWMLDw/niiy/Yv3+/1bXGjx/P5MmT6dmzJz179mTfvn107dqV9PT0Qv9uUlNTuXbtmtU6V1dX7O3tC32O27Fx40aWLFnC6NGj8fLysrw4fvrppzzyyCMMGTKE9PR0Fi9ezIABA1i5cqXl92a2detWli1bxsiRI3F1deWzzz6jf//+XLhwwfJ/7YUXXuCXX35h9OjR1K9fn+vXr7N161aOHz9O8+bNmTVrFj/88APLly9n9uzZuLi40LhxY6Dwv3OA69ev06NHDwYPHsyTTz6Jj4+PZdvChQtJT09nzJgxxMTEMGPGDAYOHMiDDz7I5s2b+e9//8vp06f5/PPPee2115g7d67l2B9//JFhw4bRrVs3pk+fTnJyMrNnz+b+++9n//79Vi/cxfU35Z6ihLgN8+bNU4Bav369io6OVhcvXlSLFy9Wnp6eytHRUV26dEkppdSwYcMUoN58881c5wgKClLDhg3Ltb5jx46qY8eOls+bNm1SgKpXr55KS0uzrP/0008VoA4fPmxZN2zYMBUUFGT5HB4ergDl6empYmJiLOt/++03Bag//vjDsq5Ro0aqWrVqKiEhwbJu8+bNCrA6Z36CgoIUoH799VfLuri4OOXn56eaNWtmWZeamqqMRqPVseHh4cre3l699957lnVz585VgPr4449zXctkMlnd38yZM1VGRoYaNGiQcnR0VGvXrrXaf9y4cQpQO3futKyLiopS7u7uClDh4eGWdXZ2dqpr165Wafziiy8UoObOnauUUio9PV1VqVJFNWzYUKWkpFj2W7lypQLU+PHjLevy+w78888/ClALFy60Wr9mzRqr9eY09erVy3LfSin19ttvKyDP79DNgDyXefPm5bm/s7Nzoc6bU4MGDay+t+br6vV6dfTo0Vz7JycnW31OT09XDRs2VA8++GCuc9jZ2anTp09b1h08eFAB6vPPP7esc3d3V6NGjSowjRMmTFCAio6Otqwr7O9cKe3/JqC+/vprq/Oav4fe3t4qNjbWsv6tt95SgGrSpInKyMiwrH/88ceVnZ2dSk1NVUoplZCQoDw8PNRzzz1ndd6IiAjl7u5utb6gvykif1KML+5Ily5d8Pb2JiAggMGDB+Pi4sLy5ctz5YZffPHFO77W8OHDsbOzs3xu3749oOWUb2XQoEFUqlQp32OvXLnC4cOHGTp0qFVDrI4dO9KoUaNCp9Hf359+/fpZPru5uTF06FD2799PREQEAPb29uj12n89o9HI9evXcXFxoU6dOpYiV4Bff/0VLy8vxowZk+s6N7fyTk9Pt+QK//zzT7p27Wq1/c8//6RNmza0atXKss7b2ztX8ef69etJT09n3LhxljQCPPfcc7i5ubFq1SoA9uzZQ1RUFCNHjrSqf+7Vqxd169a17JfTzd+BpUuX4u7uzkMPPcS1a9csS2hoKC4uLmzatMkqTWPGjLG673HjxuW6RkH69OnDunXrrJZu3boV6Ry3o2PHjtSvXz/X+pylKjdu3CAuLo727dtbfQfMunTpYlWS07hxY9zc3Ky++x4eHuzcuZMrV64UKX2F/Z2b2dvbM3z48DzPNWDAANzd3S2fW7duDcCTTz6Jra2t1fr09HRLlc+6deuIjY3l8ccft/ou2NjY0Lp1a8t3Iafi+JtyL5FifHFHvvzyS0JCQrC1tcXHx4c6depY/cEAsLW1pVq1and8rcDAQKvP5uB9c/3u7Rx7/vx5gDxbKNeqVSvPP8B5qVWrVq5AHBISAmjtB3x9fS2tob/66ivCw8MxGo2WfXNWf5w5c4Y6depY/ZHMz9SpU0lMTGT16tVW7R3Mzp8/b/nDm1OdOnVy7ZfXejs7O2rUqGHZnt9+AHXr1mXr1q1W6/L6Dpw6dYq4uDiqVKmS5z2ZG5OZr1W7dm2r7d7e3lYvcLdSrVo1unTpUuj9bxYdHW31u3JxcbF6McxPcHBwnutXrlzJ5MmTOXDgAGlpaZb1eXXXu/n7C9p3OOd3f8aMGQwbNoyAgABCQ0Pp2bMnQ4cOzdX48GaF/Z2bVa1a1eqlu6B0mgN/QEBAnuvN6T916hSQ3ebnZm5ublafi+tvyr1Egr24I61atbK0xs9PzpxsTvn1QTYajdjY2ORan9c60Bpv3cqdHFvcPvjgA959912eeeYZ3n//fSpXroxer2fcuHG33W2wW7durFmzhhkzZtCpU6dSa+2dn7y+AyaTiSpVqrBw4cI8j/H29r4bSSu0li1bWgW+CRMmWDVwy09e7SL++ecfHnnkETp06MBXX32Fn58fBoOBefPmsWjRolz7F+b7O3DgQNq3b8/y5cv566+/mDlzJtOnT2fZsmX06NGjEHdYOHndz63Seav0m7/3P/74I76+vrn2u/mFN7+/KSJ/EuxFqalUqVKeLbzPnz9/y9xIcQsKCgLI1bo/v3X5OX36NEopqxeZkydPAlgaGP3yyy888MADzJkzx+rY2NhYvLy8LJ9r1qzJzp07ycjIuGWXwjZt2vDCCy/w8MMPM2DAAJYvX271BzIoKMiSe8opLCzM6rP5OYSFhVn9DtLT0wkPD7fkjHPud3NuLCwszLK9IDVr1mT9+vW0a9euwABiPtepU6es0hQdHV2oUp3isnDhQlJSUiyf7+Q7+uuvv+Lg4MDatWutGgjOmzfvjtLo5+fHyJEjGTlyJFFRUTRv3pwpU6YUGOwL+zsvSeYqiipVqtyV692L5NVIlJqaNWuyY8cOqxbVK1eu5OLFi3c9Lf7+/jRs2JAffviBxMREy/otW7Zw+PDhQp/nypUrLF++3PI5Pj6eH374gaZNm1pyLDY2NrlKFJYuXZqry1r//v25du0aX3zxRa7r5FUi0aVLFxYvXsyaNWt46qmnrEoJevbsyY4dO9i1a5dlXXR0dK5cdZcuXbCzs+Ozzz6zusacOXOIi4uztBJv0aIFVapU4euvv7Yqgl69ejXHjx/P1Zo8LwMHDsRoNPL+++/n2paZmWl5EezSpQsGg4HPP//cKk2zZs265TWKU7t27ejSpYtluZNgb2Njg06ns6oWOHfuHCtWrLit8xmNRuLi4qzWValSBX9/f6vfT14K+zsvSd26dcPNzY0PPviAjIyMXNujo6NLPA0VneTsRan5z3/+wy+//EL37t0ZOHAgZ86c4f/+7//y7FJ2N3zwwQf06dOHdu3aMXz4cG7cuMEXX3xBw4YNrV4AChISEsKzzz7L7t278fHxYe7cuURGRlrl2B5++GHee+89hg8fzn333cfhw4dZuHBhruAxdOhQfvjhB1555RV27dpF+/btSUpKYv369YwcOZI+ffrkun7fvn2ZN28eQ4cOxc3NjW+++QaAN954gx9//JHu3bvz0ksvWbreBQUFcejQIcvx3t7evPXWW0yaNInu3bvzyCOPEBYWxldffUXLli0tA8YYDAamT5/O8OHD6dixI48//ril61316tV5+eWXb/msOnbsyPPPP8/UqVM5cOAAXbt2xWAwcOrUKZYuXcqnn37KY489hre3N6+99hpTp07l4YcfpmfPnuzfv5/Vq1dblYSUJ7169eLjjz+me/fuPPHEE0RFRfHll19Sq1Ytq99HYSUkJFCtWjUee+wxmjRpgouLC+vXr2f37t189NFHBR5b2N95SXJzc2P27Nk89dRTNG/enMGDB+Pt7c2FCxdYtWoV7dq1y/OlVxSeBHtRarp168ZHH33Exx9/zLhx42jRogUrV67k1VdfLZX09O7dm59++omJEyfy5ptvUrt2bebPn8+CBQsKPYBL7dq1+fzzz3n99dcJCwsjODiYn3/+2arV99tvv01SUhKLFi3i559/pnnz5qxatYo333zT6lw2Njb8+eefTJkyhUWLFvHrr7/i6enJ/fffX2APgSeffJKEhARGjhyJm5sbM2fOxM/Pj02bNjFmzBimTZuGp6cnL7zwAv7+/jz77LNWx0+cOBFvb2+++OILXn75ZSpXrsyIESP44IMPrKoTnn76aZycnJg2bRr//e9/cXZ2pl+/fkyfPr3QI7R9/fXXhIaG8s033/D2229ja2tL9erVefLJJ2nXrp1lv8mTJ+Pg4MDXX3/Npk2baN26NX/99dddyXWWhAcffJA5c+Ywbdo0xo0bR3BwMNOnT+fcuXO3FeydnJwYOXIkf/31F8uWLcNkMlGrVi2++uqrQrVaL+zvvCQ98cQT+Pv7M23aNGbOnElaWhpVq1alffv2+bb+F4WnU6XRQkmIcqRp06Z4e3uzbt260k6KEELcFqmzFyJLRkYGmZmZVus2b97MwYMH8+zOJoQQ5YXk7IXIcu7cObp06cKTTz6Jv78/J06c4Ouvv8bd3Z0jR45Y9YEXQojyROrshchSqVIlQkND+f7774mOjsbZ2ZlevXpZ6riFEKK8kpy9EEIIUcFJnb0QQghRwZVqsP/777/p3bs3/v7+6HS6XANKKKUYP348fn5+ODo60qVLl1yjgMXExDBkyBDc3Nzw8PDg2WefteoTfe7cOTp06ICzszMdOnTg3LlzVsc//PDD/PrrryV1i0IIIUSpK9U6+6SkJJo0acIzzzzDo48+mmv7jBkz+Oyzz1iwYAHBwcG8++67dOvWjWPHjlnG/h4yZAhXr15l3bp1ZGRkMHz4cEaMGGEZX/rVV1+latWqzJkzh3feeYfXXnuNX375BYCff/4ZvV5P//79i5x2k8nElStXcHV1zXeMdyGEEKKkKKVISEjA39//1nMF3O05dfMDqOXLl1s+m0wm5evrq2bOnGlZFxsbq+zt7dVPP/2klFLq2LFjClC7d++27LN69Wql0+nU5cuXlVJK1atXT61evVoppdSff/6p6tevr5RS6saNG6pWrVrqwoULt5Xeixcv5jtHtiyyyCKLLLLcreXixYu3jFlltjV+eHg4ERERVpMiuLu707p1a7Zv387gwYPZvn07Hh4eVrOudenSBb1ez86dO+nXrx9NmjRh/fr1dO3alb/++ovGjRsD8PrrrzNq1KhcUy8WlqurKwAXL17MNf2iEEIIUdLi4+MJCAiwxKOClNlgHxERAYCPj4/Veh8fH8u2iIiIXHNh29raUrlyZcs+H374Ic8//zzVq1encePGfPPNN/z9998cOHCA6dOnM3DgQPbs2UPXrl357LPP8p2nOS0tzWpCiYSEBEAb0/lOgv3pqAQ+3ZA9q1rOCgGdDvQ6XfZPtJ/m0ppMo8JoUmSazD9N6MjenxzH6az+bd4HdGjnyzQqMowmMrJ+mhQopbTXRstPMN2i84Yux/XM6dVqOXSWdTod2Or12Bv0OBhscLC1wdYm/6oQXR4fsu/TfI0cz0eXdX2dLsfzAxu9Hlu9Dhu9zvLTYKO3fNbrdeSXioLvOltKupGEtEyS0jJJSTdmPVPtuVZyssPX3R4Hgw32tnoMNnrscvy0y/rpYm+Ln7uDVA8JIQqlMH8rymywLy5Vq1Zl5cqVls9paWl069aNBQsWMHnyZFxdXQkLC6N79+588803jBkzJs/zTJ06lUmTJhV7+q4npvPHwSvFfl5RvlX1cKRpoAfujgZc7W1xtrfFxd4WG72OsMgEIuNScTDY4Ghng6PBBic77QUiLiWD+NRMTEphfi9zcbDF180BJzsbnO1t6Rjijb9H/lPK5pSaYSTdaMLRYIPBRjrvCFFeldlgb54ONDIyEj8/P8v6yMhImjZtatknKirK6rjMzExiYmIsx9/sgw8+oGvXroSGhvLcc88xefJkDAYDjz76KBs3bsw32L/11lu88sorls/m4pM7FeTpzPiH6+e5zZyLNimFKStXrcw5bgU2NuYcqt6SMwUga39l/kn2MQrztuzzK6Ww0esx2Oiws9VyujY6c05cy47nzKVb1qOdz0xlnTdnaQDmdJu3o6XDaFKkZhhJzTCRkmEssMRA5bHNpKzPpz0j87+z79eU41kYVVYpiFH7mWkyaSUixuySkTvlZGeLs70WVB0NNlqO3VaPjU7H9cR0IhNSSc80kZ6p5fjTMrNz/uZ1cSkZXI5N4XJsyq0veBt0OmgT7EmL6pWo7umMo50NOsCotOdwIymd3edvcPRyHBdikjFlPX4/dwdq+7hSycmAl4s999X0pI6vKyYTuDsZcHe8OxOmCCGKrswG++DgYHx9fdmwYYMluMfHx7Nz507LLE5t27YlNjaWvXv3EhoaCsDGjRsxmUy0bt061zmPHz/OokWLOHDgAKDNAW2eOzkjI8Nqbumb2dvbY29vX4x3qPF1d+CZ+4OL/byi/EpOz2RneAxnohJJSjOSmJZBYpqRxLRM0jKM1KziQmBlJ9IzTSSnG0lJzyQ53Uhapgk3R1vcHQ1aNU6W+NRMIuNSSckwciU2hT3nb7D97HW2n71epHRdjUvlalyq5fOcreFW253tbNDrdbjY2zK2c20GtwyQqgghyohSDfaJiYmcPp1dXx0eHs6BAweoXLkygYGBjBs3jsmTJ1O7dm1L1zt/f3/69u0LQL169ejevTvPPfccX3/9NRkZGYwePZrBgwfj7+9vdS2lFCNGjOCTTz7B2dkZgHbt2vHdd98REhLCDz/8wOOPP37X7l2I/DjZ2fJAnSo8UKfKrXe+DeevJ/H3qWscuhhLRHwqaRkmFAq9TmvH4GiwoWmAB82DKlHbxwV3RwNJaUbORidyJjqRhNRMwq8l8c+pa0TEp2Kr15GcbiQpXXtZTkjN5K1lh1l3LJJPBjXF0WDDykNXuBqXSlJWewZne1ueahuEn3vhqhNExZIzoyUKZmdnd+tudYVQqsPlbt68mQceeCDX+mHDhjF//nyUUkyYMIFvv/2W2NhY7r//fr766itCQkIs+8bExDB69Gj++OMPS5/5zz77DBcXF6tzfvPNN6xbt87Sxx4gKiqKJ554gl27dtG9e3fmz5+Pk5NTodIeHx+Pu7s7cXFx0hpf3PNS0o1ExGu5/vXHIpn5VxjpmSbq+LhisNVx5HJ8rmPsbfWM7VybUQ/UutvJFaVEKUVERASxsbGlnZRyQ6/XExwcnGfj8aLEIRkb/zZJsBcif0evxPHM/N1Exms9WCo5GXiovo+loeHOszHsOhcDwIJnWtExxLs0kyvukqtXrxIbG0uVKlVwcnKSap5bMA/eZjAYCAwMzPW8ihKHymydvRCi/Grg786yke14bclBPJwMTHykAT5uDpbtSikm/XGM+f+eY/LKY7R7qT220tq/QjMajZZAL7NIFp63tzdXrlwhMzMTg+H2G8FKsBdClIiqHo78NKJNntt0Oh0vPxTCbwcucyoqkZ92X+SpNkF3fM2UdCObw6K4HJtCFTcHHm7kl91LRZQqcx19YatKhcZcfG80GiXYCyHKH3dHAy8/FML4347yxcZTDG4ZcEd9+aMT0nhqzk5ORCRY1m04HsmMxxpjb2tTHEkWxUCK7oumuJ6XlJuVtoxUiL0ISdchPRmKoa+3EOXF4JaBeLnYERmfxobjUbc+IA9Gk2LLyWgGfbOdExEJeDrb0b2BL7Z6Hb8duEKjiX9R+39/MnX18WJOvRDlh+TsS9vVgzC3q/U6WwcwOILBSftp65j1015bbOxz/NtO298266e9Gzi4a4udM+Rsf2ljm+PYm65h5ww2MiiKuLvsbPUMbBHAV5vPsHDnebo3tB4MKyYpnTPRidjqddTzc8PBoOXQL8Yk8/7KYxy6FEdSWiYJaZmAVnWw8D+tqe7lzN8noxm1cJ9l2zdbztKqemU617MegluIe4EE+9JmTNcCsDF73H0yU7Ul5cbdTYveVgv+tg7aMGvoQKfPGjJPn/VS4AL2LmDnCvau2S8h5pcOOydtHzsX7eXBmKHt7+oPbv7gUgX0UqQqsj3eKpDZW87wz6lrnLuWRHUvbRyMgxdjefy7HSRn9d/3crHnqTZBXE9K45e9lyzrQasS6NesKiM71aRKVkPADiHe7PxfZ6IT0pi37Rzz/z3HG78cYs24Dni7Fv8AWaLi2759O/fffz/du3dn1apVVtuWL1/O9OnTOX78OCaTicDAQB566CFmzZoFwPz58xk+fDigFc37+PjQoUMHZs6cSWBgYImnXYJ9aQtuD+9GgckIGSnakpn1MyM5e11GMmSmaS8HmamQma69IGRmLcY0rUogLQFSYyE1TjvGMrWLAmNm1jFZ58hIgYwkUFlVB6ZMSIvXlhKjy3oZcNYWexfrz3Y3fbZ3zf63YyVw8gJnL+3fUhJRIQRUdqJjiDebw6J545dDfDyoCfa2Njz/416S0414udhjUopriWl8sv6k5bhWwZV5rWsdKjkZCPR0yrNe3snOliBPW97sUZcdZ69zIiKBmWtPMOOxJnfzFkUFMWfOHMaMGcOcOXO4cuWKZfC2DRs2MGjQIKZMmcIjjzyCTqfj2LFjrFu3zup4Nzc3wsLCUEoRHh7OyJEjGTBgADt37izxtEuwLyv0Nlrgs3e59b7FSSntBSIjWWszYH7ZUFlTJVt+mrRtaYnaC0V6gvYzI8X6JSQjBdITIT1JW6e3hdR4SLgKCRGgjNqx6Qm3Stmt6Q3Z1RCOHlkvAp7ZLwTmz85VwMVHK1VwcM8qtRBlyZgHa7Pj7HV2nYuhw4xNlrkUang789uodjgYbPh17yXWH48ksLIzrWtU5qF6PoVuae9gsGFKv0b0n/0vv+67zKgHahHk6VyyNyUqlMTERH7++Wf27NlDREQE8+fP5+233wbgjz/+oF27drz++uuW/UNCQiyjvZrpdDrLvC1+fn48++yzjB07lvj4+BIfr0WC/b1Op8uu/3esVLLXMhkh6Vr2y8DNP9MSc6wzLwnZ21JitONTbqDNsJMBaXHakhhRuDTYOmhB38VHWyoHQ1A7cA/QSgqcvMCpsrwQ3GWhQZVY/VIH3vjlILvPadVXfu4OfPtUC1wdtBKcwa0CGdzq9os7Q4Mq0THEmy0no/lsw2k+Gii5+9KmlCIlI/85SUqSo8GmSC3dlyxZQt26dalTpw5PPvkk48aN46233rIE8EWLFnHkyBEaNmxYqPNFRUWxfPlybGxssLEp+apNCfbi7tHbgKsPcIcNpEzGrGqKrOqN9CTtBSD5uvYykHwtx7+vQ2KUtqTFaaUPsRe0xezfz2+6wE1tFZy8wDtEq17Q24CrH7hVBfeq4FZN++nsrb0widsW7OXMkufbEhGfio1ORyVnu2KfVvflh0LYcjKaX/dd4veDlwkNqsTC/7TBRvril4qUDCP1x68tlWsfe68bTnaFD4Fz5szhySefBKB79+7ExcWxZcsWOnXqxJgxY/jnn39o1KgRQUFBtGnThq5duzJkyBCrCdTi4uJwcXFBKUVycjIAY8eOtczXUpIk2IvyR2+j5b6LKiMlO/AnRmqlAVcPwYUdWjuHzFTtJQKlVTeYOzIkXNGWW7F1BI9AbTE4aI0Wbey0lwP/puDiCw5uUKm6VvUgctHpdCU6OU7TAA8ebuzHykNXyTAqdpyNIfxaIrWquJbYNUX5FxYWxq5du1i+fDkAtra2DBo0iDlz5tCpUyecnZ1ZtWoVZ86cYdOmTezYsYNXX32VTz/9lO3bt1sGEnJ1dWXfvn1kZGSwevVqFi5cyJQpU+7KPcjY+LdJxsavoDLTtVICZcpe4q/A9dNa40ZjhvY5/jLEXYa4S9qLgCmz8NfQ6bUSAUcPbXFw114UbO0hoDWEdNNKCqQqoUQopYhKSOM/C/Zw+HIcXz7RnF6N/Uo7WRVeamoq4eHhBAcH4+Cg9ZgoL8X4b7zxBjNnzrQqbldKYW9vz9WrV3F3d891THh4OCEhIXz77bcMHz6c+fPnM27cOKtJgEaNGkV8fDw//vhjvtfO67mZydj4QtwuW7usqoYcPAIgsHX+x5hMWg+GlBi4cU57CTC/GGSmQcwZiDiivUSkxGilB3EXtOVm+7P+0+sN4F5NC/5etbNeCjy0Eg3veuDqKy8Dt0mn0+Hj5kB9PzcOX44jLCJegn0p0el0RSpKLw2ZmZn88MMPfPTRR3Ttaj0mSt++ffnpp5944YUXch1XvXp1nJycSEpKyvfcb775JjVr1uTll1+mefPmxZ72nMr2UxaiPNDrs3PplWsUvK9SkBQNN85rVQcpsVlVCGnay8CpvyDikNb48Ea4tuTFxRdqdQbfxlq1hmMlrR2Bg5vWtsCtqjaIkshXiK9WdB8WWQw9Q0SFtXLlSm7cuMGzzz6bKwffv39/5syZQ0REBMnJyfTs2ZOgoCBiY2P57LPPyMjI4KGHHsr33AEBAfTr14/x48ezcuXKEr0P+WsgxN2k02X1BqiS9/bO72pdIJOvw7WTcHGnVlJgHjshMVKrUkiMgAMLgYV5n0dvq/U2MDhppQCetbS2A06VoUp98KmvlRTcw6UDdbOC/cnIxFJOiSjL5syZQ5cuXfIsqu/fvz8zZszgySef5MiRIwwdOpTIyEgqVapEs2bN+Ouvv6hTp06B53/55Zdp27Ytu3btolWrViV1G1Jnf7ukzl6UmowU7SXg9HqtzYDJqL0cxF/WXhRS46xHZMyPwQkqBUPVZlA1VFt8GmklFfeA6IQ0Wk5Zj04HxyZ1x9FORnYsSQXVPYv8SZ29EPcqgyPU6KQteTGZtEaDiZHai0HcJa00IOmaNrBR5BGIu6h1W4w6qi37/0871sUHanbWGg3au4BHkNZ7oFJ1rQ1BBSoJ8HKxo7KzHTFJ6ey7cIN952/wSFN/q8F2luy5yMmIBAy2eh5tVpXaPtJqX5RPEuyFqGj0ei0wu1fLf5/0ZG1Uw+gTcHkfXN4Ll/ZoLwgHF+V9jHMVqH6/1rXQpYr2uVJ18GtcLscY0Ol01PFxZfvZ64z5aT8xSeks3n2RFaPa4e1qz79nrvHGL4cs++88e51lI9uVYoqFuH0S7IW4F9k5gWdNbanbS1uXmQ7n/taCvjGrC+KN8xB7XvuZFAVHl+U+l40d+DWFgFZQvb3Wc8GQNYtiGS8JqOOrBfuYpHQALsem8PyPe1j0XBtmrTsFaGPw7wqPYf/FWKIT0mQSHVEuSbAXQmhs7aBWF225WWYaXNoNF3dpgxIlZQ1OFHVcG7Hw0i5t2f5FjvM5aA0DvWprP/W2WfMlGLRqAq/a4BWiDVVcSu0E6vhmF8v3auzHPyej2Xchlu6z/ubc9WTsbPR8NrgZ//lhN0cux7M5LIoBLQJKJa1C3AkJ9kKIW7O114rwq99vvV4piDmrvQRc2A7hf2d3F8xM1doHRB4p+NwGp6zAXwe8sxavOtq8BSU8s2HzQG0+iKoejszo35hjV+MZ8cMezl3XhjJ9vFUAvu4OPFjXhyOX49l4QoK9KJ8k2Ashbp9Ol10d0PRxbV16kjagUPJ1rWHgtVPaCwFKK/I3pmvbok9q2zOS4epBbbE+uTZ7oWdtCGyjlTgEttHGFSgmdXxd+fXF+wio5IizvS0tq1fm99H3M/qn/VxPTGPkA7UA6Fy3Cp9tOMXfJ6NJzzRhZ3tv9FgQFYcEeyFE8bLLas3u6KG9BIR0y39fY6ZWEhAdpjUWvHZS+/e1k9pLQFK0tlz4F7Z+DI6VtYGLXH2zZy50zjmlsbf2bwePQlcNhAZZz/YYUNmJ30a1w2hSlglyGlV1x8vFnmuJaewKj+H+2l638WCEKD0S7IUQpcfGNqsIvzbUezh7vcmktQVIuKrl+M9thZNrteGGL8fc+rw6G3Dy1AJ/5RpQraW2+DfNfhm5VdJyzISn1+t4sK43S/ZcYsOJSAn2otyRYC+EKHv0+uyRBv2aQPOhWtVAxCGIv6qNIJgQoTUSTL6eXQKQdF2bylgZtUaESVEQdQxOZA1FqrPRugpWbaG9BHiFQNXmhZpF8cG6PizZc4mNJ6IY/3B9lMqaBbmM9zgQAiTYCyHKCxtD1kh/t9gvMy3rBeCaFuwjj2k9CS7t1koKruzXlpxqdILO48G/eb7dBe+v7YWdjZ7z15M5EZHAq0sOkpppZOWY+8v8ZC5CSCsTIUTFYmsPbv5aDr5WF2g3Fgb9CK+egHFHoP8caPcS1O8LlWtqx5zdDN89CJMqwbQgWDIUdn0Hx1dqMxkCLva2tK6hlQC8/PMBjl2N52x0Ekv3XCqV2xR3z9NPP41Op7Msnp6edO/enUOHDuXa9/nnn8fGxoalS5eWQkrzJ6+jQoh7h0eAtjR6LHvdjfOweRocWgzKpE06dOw3bTHzbQy+jXjdxh6d3pudEfUAOwC+++csQ1oHYmsjeaeKrHv37sybNw+AiIgI3nnnHR5++GEuXMieqjo5OZnFixfzxhtvMHfuXAYMGFBayc1Fgr0Q4t5WKQj6zYaeMyE9UZtL4ORaiDyqTS4UcciyNAZ+sIM0Zcs5fQBhVOdAfDX++deGB9p3LO07ESXI3t4eX19fAHx9fXnzzTdp37490dHReHt7A7B06VLq16/Pm2++ib+/PxcvXiQgoGyMyyDBXgghQJv4x95F69ZXrUX2+qRrcGaj9hIQc4ZrB/7EixjqqHDqEM4jBmDDj6gTLdDV7qo1+HP2AvdAcPYstdspF5TSuliWBoPTbQ/nnJiYyP/93/9Rq1YtPD2zf8dz5szhySefxN3dnR49ejB//nzefffd4krxHZFgL4QQBXH2gsYDLR9jWsez88RhelS5TtqlQ+z5dz1t1CEMl/fA5T3Zx+lsoEE/qNdb61Xg3xwMMrWrlYxk+MC/dK799pVCd8MEWLlyJS4uLgAkJSXh5+fHypUr0WeN53Dq1Cl27NjBsmXa/BFPPvkkr7zyCu+8806Z6LEhlUxCCFEEIb5u9OrUDn39R3Ds+g7h3eZzX9rnTOEZ0uo+ClUagKuf1v3vyC+wdBjM6wGfh8LhX7QxBES588ADD3DgwAEOHDjArl276NatGz169OD8+fMAzJ07l27duuHlpY3B0LNnT+Li4ti4cWNpJttCcvZCCHEHnmgVyE+7LvLdVQ9sPWry38F1tQ1XD2ot+q+dhOtnIP4S/PosbHwfmg7Rhv71awoObqWa/lJlcNJy2KV17SJwdnamVq1als/ff/897u7ufPfdd0yaNIkFCxYQERGBrW12WDUajcydO5fOnTsXW7JvlwR7IYS4A7Y2ep5oFcC7vx3ldFRi9ga/JtAnaxbAjBT49wv49zOtK9+mKVk76bTRA/2ba3X9/s3Bt9G9U9yv0xWpKL0s0el06PV6UlJS+PPPP0lISGD//v3Y2GTP3XDkyBGGDx9ObGwsHh4epZdYJNgLIcQdM89xfz0xLe8dDI7Q8XVoOxKOLtda+1/ZD3EXtZz/tZNa1z/QpgKuUj87+FdtDt71tKGFRalJS0sjIiICgBs3bvDFF1+QmJhI7969mTVrFr169aJJkyZWx9SvX5+XX36ZhQsXMmrUqNJItoV8e4QQ4g55uWjB/lpiesE72jlDsye1BbThfq/sh8v74Mo+7Wfytezufnvna/vZOmqDBPk21ob2dfLSRhP0aXDvlAKUsjVr1uDn5weAq6srdevWZenSpdSrV49Vq1axaNGiXMfo9Xr69evHnDlzJNgLIUR55+lyi5x9flyqaLMCmmcGVErL7ed8AbhyANLi4eJObbmZgzsEtoWW/4HgjmBrd2c3I3KZP38+8+fPz3d7RkZGvtu++uqrEkhR0UmwF0KIO+TlogXYpHQjKelGHO1sbnFEPnQ68AjUlvp9tHUmE8Sc0YJ/1DFt4J/Yi9pY/ykxkBoHJ9doi429ltv3rAn2btpogDobrftgnR4Q3KF4bliUO2W6653RaOTdd98lODgYR0dHatasyfvvv49SyrKPUorx48fj5+eHo6MjXbp04dSpU5btaWlpPPXUU7i5uRESEsL69eutrjFz5kzGjBlz1+5JCFHxuNjbYmer/Tm9VtTc/a3o9VojviaD4KFJ0OsjGLIE3jgL/z0Pz/8NbUZqU/oa07TSgMNLYc8cOPIrHF4CO76CBb1hwSNwac+trykqnDKds58+fTqzZ89mwYIFNGjQgD179jB8+HDc3d0ZO3YsADNmzOCzzz5jwYIFBAcH8+6779KtWzeOHTuGg4MD3377LXv37mX79u2sXr2aJ554gsjISHQ6HeHh4Xz33Xfs2SNffiHE7dPpdHi72HM5NoXrSekEVC5at67bvCg4emiLXxPo9gHEnIXII1qL/7REbZvJqDUAPLgYwrfA952h5oPg0xAqB4NPI/CpX25bxYvCKdPB/t9//6VPnz706tULgOrVq/PTTz+xa9cuQMvVz5o1i3feeYc+fbQirx9++AEfHx9WrFjB4MGDOX78OI888ggNGjSgRo0avP7661y7dg1vb29efPFFpk+fjpvbPdzPVQhRLDxd7Lgcm8K1hGLO2ReWTqcV33vWzHt7h9dhy3Q4+JM2/O+ZnIO96LTA7xGoDQjk4qP1CGjQV5tFUJR7ZboY/7777mPDhg2cPHkSgIMHD7J161Z69OgBQHh4OBEREXTp0sVyjLu7O61bt2b79u0ANGnShK1bt5KSksLatWvx8/PDy8uLhQsX4uDgQL9+/e7+jQkhKhxPZ63e/npSKQX7W6kUBH2/gpE7tVKA1i9Czc5aYEdppQJnN2svA9tmwfIR8GlT2LtAazgoyrUynbN/8803iY+Pp27dutjY2GA0GpkyZQpDhgwBsPR59PHxsTrOx8fHsu2ZZ57h0KFD1K9fHy8vL5YsWcKNGzcYP348mzdv5p133mHx4sXUrFmTuXPnUrVq1TzTkpaWRlpa9n/i+Pj4krhlIUQ5Vejud6XNO0RbckqM1hr/JVzVlvircPwPSLgCf4zVSgEe+bxYRvszyXDBRaKK6UWrTAf7JUuWsHDhQhYtWkSDBg04cOAA48aNw9/fn2HDhhXqHAaDgS+//NJq3fDhwxk7diz79+9nxYoVHDx4kBkzZjB27Fh+/fXXPM8zdepUJk2adMf3JISomDwtwb6M5uwL4uINLjdN0dv1fdj5NWx4D46t0Pr9D/xBG+HvNtjZ2aHX67ly5Qre3t7Y2dmViQliyjKlFNHR0eh0OgwGwx2dq0wH+9dff50333yTwYMHA9CoUSPOnz/P1KlTGTZsmGVu4cjISMtgB+bPTZs2zfOcmzZt4ujRo3z//fe8/vrr9OzZE2dnZwYOHMgXX3yRb1reeustXnnlFcvn+Pj4MjNPsRCi9Jm7310v6zn7wrK1h3YvQeB98MtwrZj/u87aiH5V6kNId60rXyEH9dHr9QQHB3P16lWuXCml8fDLIZ1OR7Vq1ayG4b0dZTrYJycnW6YPNLOxsbEUAwUHB+Pr68uGDRsswT0+Pp6dO3fy4osv5jpfamoqo0aNYuHChZZqAXMRSUZGBkajMd+02NvbY28vDVWEEHnzKs85+4IEtNS69y1/AU6thQvbtWXPHLB1gKotIOg+bQloVWCrfjs7OwIDA8nMzCzw763IZjAY7jjQQxkP9r1792bKlCkEBgbSoEED9u/fz8cff8wzzzwDaG8848aNY/LkydSuXdvS9c7f35++ffvmOt/7779Pz549adasGQDt2rXj9ddfZ/jw4XzxxRe0a9fubt6eEKIC8axoOfucnCrDEz9r3fqiTmjBPmy1Vqd/fqu2gDauv39zaPUcNHg0z/H8zUXSd1osLYpGp4qr9r8EJCQk8O6777J8+XKioqLw9/fn8ccfZ/z48djZaf+xlFJMmDCBb7/9ltjYWO6//36++uorQkKsG6AcOXKEfv36ceDAAZydtTdPk8nE6NGjWbhwIXXq1GHRokVWUxgWJD4+Hnd3d+Li4qTrnhCC41fj6fHpP3i52LHnnYdKOzklTym4fhrOb4Pz/8K5bdo0vmaOlcAjSJvK974x4F6t9NJaQRUlDpXpYF+WSbAXQuQUlZBKqykb0Ovg1JSe2OjvwcZnsRfg0M+wYzYkX89erzeAd11w9YWaD0CNB0CZtD79zp6ll95yrihxqEwX4wshRHlR2ckOnQ5MCm4kp1vq8O8pHoHa4D1tx0D0CbgRDrvnwLl/IPKwtpxel72/3gChw6D9q+DmX3rpvgdIsBdCiGJga6OnkpMdMUnpXEtMuzeDvZnBAfybakuDfhAdpk3ec/0UHPsdIo9q9fnJ12H397DvR2j5rBb0nb1KO/UVkgR7IYQoJp7OWrCvkI307oR3HW2p3QXa5OgpFf43bJwCF3dok/UcWARdJkCzoXk27hO3r0wPlyuEEOWJOTf/5+GrJKTmP8e5yBLcAZ5ZA08t1ybkSY2FlS/DZ03h7w/hwk7ISC3tVFYI0kDvNkkDPSHEzaasOsZ3/4QD4O1qz/pXOuLuKF3MCsWYCbu/04J88rXs9TZ24NtYm6mv3sPaS4Fe8qkgrfHvCgn2QoibZRpN/HbgCu/+doTkdCN/jL6fRtXcSztZ5UtGChxaAifXwsWd1oEfwN5dG7ynbi+o0RFc/Qs9il9FI63xhRCiFNja6OkfWo0vN5/mbHQSyemZpZ2k8sfgqLXQDx2m9eW/EQ7nt0PYn9qEPGlxWov+nK36K1WHqqFa3357N60UILCt1PvnIE9CCCGKmaNBG940JUOGhL0jOh1UrqEtzYZoRf2RR7Sgf/x3iDoOmalw45y2mG39GJyrQPOnoPkwbXrfe5wEeyGEKGZOdlnBPl2CfbGysc3u0tf+FS3nnxwDEQfh6iHISNa6+J1cDUlR8M9H2hJ4HzQeCA36arn/e5AEeyGEKGYOkrO/O3Q6bQS+mg9qi5kxQyv23zMXzm6BC/9qy6pXtVH8PIK0kfyqtwfPWlrf/go+3a4EeyGEKGbmnH2y5OxLh40B6vfRlvgrcPgXrdFf5GGIv6wtF/7N3r9SsDb4T+jTFbbIX4K9EEIUM3Odfark7Eufmz+0G6st8Ve14B9xCE6vh6sHIe6S1ghw68fw72fQZDA0HpTVwK/idJuUYC+EEMXM0U770yo5+zLGzU9bqoVCi+HaurQEOPUX7PsBzm6G/f+nLXpbcPIC30bQsD9UDgZbB63Pfzns5y/BXgghipm0xi9H7F21YN6wP1zcBXsXwMk1Wv/+xAg4HWHdzc+3MTz0HtToVK7q+SXYCyFEMZPW+OVUQCttMRkhIUJbTq+HsFWQlqh9jjgEP/aFqi2g9Qtaw8Cc0/QqpQ0MZGNXpvr5l52UCCFEBeEowb5809uAe1VtqRYKnf6rrU+6Dn/PgD3z4PIeWPYfQAdOlcHgDBlJkBoPpgyti1+PmdB4QKneipkEeyGEKGbmYvxkKcavWJw9ocd0bSreXd/BiZUQdUybqpfr1vum3NBeBvbM0Ub3y0jWXgQa9IU6ve56vb8EeyGEKGaSs6/gXKrAg//TlqRrkBilBXODEzi4g50z7PwGtkyHC9u1xezIL+DTELpP1Wb9u0sk2AshRDEz19lL17t7gLOXttys03+h0WNwbqs2rK+9i1aXv3eBNuRvZvpdTaYEeyGEKGbmEfRkIpx7nGdNbcmp/atw5Feo1fmuJkWCvRBCFDNLa/wMUymnRJQ5TpWh1XN3/bLlb2QAIYQo4yz97CVnL8oICfZCCFHMZCIcUdZIsBdCiGImE+GIskaCvRBCFDNHaY0vyhgJ9kIIUcycDFrb5wyjIsMojfRE6ZNgL4QQxczBLvtPq9Tbi7JAgr0QQhQzOxs9NnptRjQZRU+UBRLshRCimOl0uhzd7yTYi9InwV4IIUqAo7TIF2WIBHshhCgBjtLXXpQhEuyFEKIEyGQ4oiyRYC+EECUgezIcCfai9EmwF0KIEpA9GY4Ee1H6JNgLIUQJkMlwRFkiwV4IIUqAuTW+dL0TZUGRg3316tV57733uHDhQkmkRwghKgRzzj5ZivFFGVDkYD9u3DiWLVtGjRo1eOihh1i8eDFpaWklkTYhhCi3LJPhSM5elAG3FewPHDjArl27qFevHmPGjMHPz4/Ro0ezb9++Yk/g5cuXefLJJ/H09MTR0ZFGjRqxZ88ey3alFOPHj8fPzw9HR0e6dOnCqVOnLNvT0tJ46qmncHNzIyQkhPXr11udf+bMmYwZM6bY0y2EuLeVhUF10jKNGE2q1K4vyo7brrNv3rw5n332GVeuXGHChAl8//33tGzZkqZNmzJ37lyUuvMv2I0bN2jXrh0Gg4HVq1dz7NgxPvroIypVqmTZZ8aMGXz22Wd8/fXX7Ny5E2dnZ7p160ZqaioA3377LXv37mX79u2MGDGCJ554wpK28PBwvvvuO6ZMmXLHaRVCiJxKe1Cd1AwjnWZuZtA320vl+qJssb3dAzMyMli+fDnz5s1j3bp1tGnThmeffZZLly7x9ttvs379ehYtWnRHiZs+fToBAQHMmzfPsi44ONjyb6UUs2bN4p133qFPnz4A/PDDD/j4+LBixQoGDx7M8ePHeeSRR2jQoAE1atTg9ddf59q1a3h7e/Piiy8yffp03Nzc7iidQghxM6dSbqB37noSV+NSuRqXSmqG0dLvX9ybipyz37dvn1XRfYMGDThy5Ahbt25l+PDhvPvuu6xfv57ly5ffceJ+//13WrRowYABA6hSpQrNmjXju+++s2wPDw8nIiKCLl26WNa5u7vTunVrtm/X3mabNGnC1q1bSUlJYe3atfj5+eHl5cXChQtxcHCgX79+hUpLWloa8fHxVosQQuSntHP21xPTLf+OjE8tlTSIsqPIwb5ly5acOnWK2bNnc/nyZT788EPq1q1rtU9wcDCDBw++48SdPXuW2bNnU7t2bdauXcuLL77I2LFjWbBgAQAREREA+Pj4WB3n4+Nj2fbMM8/QpEkT6tevz5QpU1iyZAk3btxg/PjxfP7557zzzjvUqlWLbt26cfny5XzTMnXqVNzd3S1LQEDAHd+fEKLicrTTCk5Lq87+WmJ2w+krsRLs73VFLsY/e/YsQUFBBe7j7OxsVfR+u0wmEy1atOCDDz4AoFmzZhw5coSvv/6aYcOGFeocBoOBL7/80mrd8OHDGTt2LPv372fFihUcPHiQGTNmMHbsWH799dc8z/PWW2/xyiuvWD7Hx8dLwBdC5Kss5ewj4lNKJQ2i7Chyzj4qKoqdO3fmWr9z506rVvLFwc/Pj/r161utq1evnqWPv6+vLwCRkZFW+0RGRlq23WzTpk0cPXqU0aNHs3nzZnr27ImzszMDBw5k8+bN+abF3t4eNzc3q0UIIfJT2hPhXE/KztlfjZOc/b2uyMF+1KhRXLx4Mdf6y5cvM2rUqGJJlFm7du0ICwuzWnfy5ElLyUJwcDC+vr5s2LDBsj0+Pp6dO3fStm3bXOdLTU1l1KhRfPPNN9jY2GA0GsnIyAC0BodGo/SHFUIUj9KeCOdaQnbO/qoU49/zihzsjx07RvPmzXOtb9asGceOHSuWRJm9/PLL7Nixgw8++IDTp0+zaNEivv32W8tLhU6nY9y4cUyePJnff/+dw4cPM3ToUPz9/enbt2+u873//vv07NmTZs2aAdrLxLJlyzh06BBffPEF7dq1K9b0CyHuXaXdGl9y9iKnItfZ29vbExkZSY0aNazWX716FVvb2+7Jl6eWLVuyfPly3nrrLd577z2Cg4OZNWsWQ4YMsezzxhtvkJSUxIgRI4iNjeX+++9nzZo1ODg4WJ3ryJEjLFmyhAMHDljWPfbYY2zevJn27dtTp06dO+4qKIQQZo6lPOvdNamzFznoVBFHv3n88ce5evUqv/32G+7u7gDExsbSt29fqlSpwpIlS0okoWVNfHw87u7uxMXFSf29ECKXqPhUWn2wAZ0OfnnhPkKDKt36oGLUfsZGLsZoQd7LxY497zx0V68vSl5R4lCRs+IffvghHTp0ICgoyFIcfuDAAXx8fPjxxx9vL8VCCFHBVHFz4JEm/vx+8ApjFu2jZXBl/jl1jU8HN6V9be8Sv37O1vjXEtNJyzRibysD69yrilxnX7VqVQ4dOsSMGTOoX78+oaGhfPrppxw+fFi6ogkhRA5T+jUk2MuZK3Gp/HbgCjFJ6fx9MrrEr5ucnmlpGGij1wEQGXfrCcsi4lL5Ze8l0jNNJZo+cffdViW7s7MzI0aMKO60CCFEheLqYOCrIc15afF+ktKMXI5NIS4lo8Sva87V29vq8XN34Nz1ZK7GpRDo6ZTvMeevJzHomx1ExKei18GjzauVeDrF3XPbLeqOHTvGhQsXSE9Pt1r/yCOP3HGihBCioqjn58ZfL3fk/3ac550VR4hNvgvBPkn7u+zlYo+fu2NWsM+/Rf6lG8k88d1OIrKG1T0dlVjiaRR3122NoNevXz8OHz6MTqezzCCn02lFRdJXXQghcvNwMgDcpZy9VmTv6WKHn7vWMym/YH81LoXHv9vB5dgUdDpQCi7HSuv9iqbIdfYvvfQSwcHBREVF4eTkxNGjR/n7779p0aJFgSPQCSHEvczd8W4Gey1n7+lsh29WsI+Iyx3AU9KNPPHdTi7GpBDk6cT/etYD4IoE+wqnyDn77du3s3HjRry8vNDr9ej1eu6//36mTp1qGW9eCCGENQ9HO+DuBPtrSeacvb0lZ38qKhGllKUUFmBn+HXCryXh5WLHoufaWGbHu3xDgn1FU+ScvdFoxNXVFQAvLy+uXLkCQFBQUK6hbYUQQmjMOfu7Umdvztm72NGiemX0Ovj3zHV+2mU91PmFmGQAmgdWoqqHI9U8HAGIiE8l0ygt8iuSIgf7hg0bcvDgQQBat27NjBkz2LZtG++9916uUfWEEEJo3LPq7FMyjKRllmzbJnOdvZezPfX83Hi9mzYN+YTfj3D4UpxlvwvXtWAfWFlrpe/lYo+djR6TwtJYT1QMRQ7277zzDiaT9sb33nvvER4eTvv27fnzzz/57LPPij2BQghREbja22IuQS/povxrOXL2AC90rMFD9X3IMCoWbD9n2e98Vs4+KKtLnl6vw89DK/aXovyKpch19t26dbP8u1atWpw4cYKYmBgqVapkVRckhBAim16vw93RQGxyBvEpGVRxdbjlMfsu3GDTiSjORCfyRKsg7q/tVahrXUvMrrMHrbfUsLbVWXcski0nozGZFHq9jotZwT6gcnb/+6oejpy/nsyVPBr0ifKrSME+IyMDR0dHDhw4QMOGDS3rK1euXOwJE0KIisYc7AtTb3/hejIDvt6O0aR1b46MTyt0sI9OyAr2znaWdS2DK+FkZ0N0QhrHrsbTwN/NUmcfmCPY+2fV20vOvmIpUjG+wWAgMDBQ+tILIcRt8ChC97uf91zAaFJ4u2q5c3Mu/FauJ6ZZBtUJ9nK2rLe3teG+mp4AbDkZzbXEdJLTjeh0UK2Sdc4epK99RVPkOvv//e9/vP3228TExJREeoQQosJyK2SL/EyjiaV7LgEwrkttAKIS0kgtxHS5JyO10e8CKjvibG9deNuxThUAtoRFcyEmCQB/d0fsbLNDQdVK5mAvDfQqkiLX2X/xxRecPn0af39/goKCcHZ2ttq+b9++YkucEEJUJB5OhetrvyksmqiENDyd7RgQGsAHq46TlK6NrV/T26XAY09GJgBQx8c117ZOIdpse3sv3ODI5XjAuggfcuTsbxSuJEGUD0UO9n379i2BZAghRMXn7qj9yY29RbD/efcFAB4LrYadrZ5qlZwIi0zgYkzyLYN9mDnY++YO9gGVnajp7cyZ6CTmbQsHCgj2sSm5BuER5VeRg/2ECRNKIh1CCFHhmUfRiy8g2Cul+PfMdQD6NK0KaEXyYZEJXCxEo7mwCC3Yh+SRswcY3DKQKX8e55y5j/1NM+GZh9dNzTBxIzmDyjka+Ynyq8h19kIIIW5P9ih66fnuE52YRnK6Eb0OalbRqknNDegu3aJoXSnFyYj8c/YAg1sF4JqjLv/mnL2DwYYqWY0Cz19PKvB6ovwocrDX6/XY2NjkuwghhMibeyFmvjOPaufv4Yi9rfY3tVpWo7lLMQXn7K/GpZKQlomtXkcNr7yL+10dDDzeOtDy+eZgD1CrinbsKZnqtsIocjH+8uXLrT5nZGSwf/9+FixYwKRJk4otYUIIUdFYcvYFBHtz8XpQjuJ186A3t8rZm+vrg72crVrY32x4u+rM33YOdFDdyznX9hAfV/49c51TWecT5V+Rg32fPn1yrXvsscdo0KABP//8M88++2yxJEwIISoaSz/7ArremYvOgzyzg7A5Z3+rOvtbFeGb+bk7svj5NmRkmiwvIDmZ6/vDIiVnX1EUW519mzZt2LBhQ3GdTgghKpzCFOObc/bV88jZxySlk5SWme+xYQV0u7tZ88BKtK7hmee2EJ+sYnzJ2VcYxRLsU1JS+Oyzz6hatWpxnE4IISokc2v82JQMlFJ57pNXzt7NwWDJgV8qIHdvrtMPyqNovihqZ70sXI1LJT615KfkFSWvyMX4N094o5QiISEBJycn/u///q9YEyeEEBWJOWAbTYqkdCMuN41wp5Qi/JoW7Kt7WgfsapUciUvJ4GJMcr7F9ObJa/zdbz3Jzq3S6eNmT2R8GqciEwgNkvlPyrsiB/tPPvnEKtjr9Xq8vb1p3bo1lSpVKtbECSFEReJg0GNnqyc900RscnquYB+bnEFCqlZMf3Mr+YBKThy9Ep9vIz2TSRGZNQe9X9bAOHcixMeVyPg0TkYmSrCvAIoc7J9++ukSSIYQQlR8Op02zW10QhpxKRlUuyl/dC6rCN/XzQFHO+uuzEFeWvA/HZ13o7lrSWlkGBV6Hfhk9ZO/EyE+rvxz6ppl+F1RvhW5zn7evHksXbo01/qlS5eyYMGCYkmUEEJUVAW1yDdPORvkmbvvewN/dwAOZ41pf7OrWRPXVHF1wNbmzptjZTfSkxb5FUGRvxFTp07Fyyv3nMpVqlThgw8+KJZECSFERWUefjY6MS3XtnPX8g/2japqwf741XgyjKZc269m1df7edxZfb1ZbUv3O8nZVwRFLsa/cOECwcHBudYHBQVx4cKFYkmUEEJUVH5Zjeci4rKnkD0VmcCYn/ZbRqwL8szdmj6oshOu9rYkpGVyKjKR+v5uVtuvZOXs/e6wcZ5ZjawW/dFZU+s6GGSE1PKsyDn7KlWqcOjQoVzrDx48iKdn3n02hRBCaMyN567mCPZ/HLzCiYgEjCaFl4sdnetVyXWcXq+jQVUtwB+5HJdruyVn737njfNAa5HvmBXgr8TeegIeUbYVOdg//vjjjB07lk2bNmE0GjEajWzcuJGXXnqJwYMHl0QahRCiwsgrZ28eSOeVh0LY/b8u1PV1y/NYc1H+4TyC/ZW44s3Z63Q6/LOqBHK+mIjyqcjF+O+//z7nzp2jc+fO2Npqh5tMJoYOHSp19kIIcQu+buYAmp1bNg+kU8fXtcD54xsWEOzNLw/+xdDtzszfw5Ez0UmSs68Aihzs7ezs+Pnnn5k8eTIHDhzA0dGRRo0aERQUVBLpE0KICsVczH41j5z9zQPp3CxnI71Mo8mq1f3VrIDsW0w5ey2t2rnM7QFE+VXkYG9Wu3ZtateuXZxpEUKICs/cWj46MY0Mo4mktEzLWPl5TTebU3VPZ1zsbUlMy+RUVCL1/LTifqNJEZmgte73L6Y6e8guJchZCiHKpyLX2ffv35/p06fnWj9jxgwGDBhQLIkSQoiKqrKTHXY2epSCqIQ0zmfl6n3c7HMNpHMzvV5nyd3vvxBrWR+VkIrRpLDV6/AuhgF1zMwvDlekzr7cK3Kw//vvv+nZs2eu9T169ODvv/8ulkQJIURFpdfr8HHXAvLV2BTLqHl5dbfLS8tgbejaXeHXLevMxew+bg7Y6POv8y8qc85e6uzLvyIH+8TEROzs7HKtNxgMxMfnPbKTEEKIbH5u2fX25/OY0rYgrbOC/c7wGMvMeRHF3BLfkk5za/zYlHxn6RPlQ5GDfaNGjfj5559zrV+8eDH169cvlkQJIURF5puj+11Rc/bNAj2w1eu4Gpdqme7WPMxucTbOg+xi/KR0I/EpmcV6bnF3FTnYv/vuu7z//vsMGzaMBQsWsGDBAoYOHcrkyZN59913SyKNFtOmTUOn0zFu3DjLutTUVEaNGoWnpycuLi7079+fyMhIy/aYmBh69+6Ni4sLzZo1Y//+/VbnHDVqFB999FGJplsIIXLyy9F//cL1/IfIzYuTnS2Nqmn19rvCYwDYflYr0m8a4FGs6XS0s6GSkzaW/xVppFeuFTnY9+7dmxUrVnD69GlGjhzJq6++yuXLl9m4cSO1atUqiTQCsHv3br755hsaN25stf7ll1/mjz/+YOnSpWzZsoUrV67w6KOPWrZPmTKFhIQE9u3bR6dOnXjuuecs23bs2MHOnTutXh6EEKKk+eXoa1/Ybnc5tbLU28eQmmG01N+3r+1dzCnN2VVQgn15dltTI/Xq1Ytt27aRlJTE2bNnGThwIK+99hpNmjQp7vQBWjuBIUOG8N1331GpUvackHFxccyZM4ePP/6YBx98kNDQUObNm8e///7Ljh07ADh+/DiDBw8mJCSEESNGcPz4cQAyMjJ44YUX+Prrr7GxkTGfhRB3j29WAD0dlci1rAlxAguZs4fsevtd52LYe/4GqRkmqrjaW2aqK07mUfQuS1/7cu2250H8+++/GTZsGP7+/nz00Uc8+OCDlgBb3EaNGkWvXr3o0qWL1fq9e/eSkZFhtb5u3boEBgayfft2AJo0acLGjRvJzMxk7dq1lpKBGTNm0KlTJ1q0aFGoNKSlpREfH2+1CCHE7TA3pDNPfFPZ2Q43B0Ohjw8NqoyNXkf4tSRmrT8JwP21vQocfe92WfraS4v8cq1IwT4iIoJp06ZRu3ZtBgwYgJubG2lpaaxYsYJp06bRsmXLYk/g4sWL2bdvH1OnTs0zPXZ2dnh4eFit9/HxISIiAoA333wTW1tbatasyfLly5kzZw6nTp1iwYIFvPvuu7zwwgvUqFGDgQMHEheXewhKs6lTp+Lu7m5ZAgICivU+hRD3jptbzfdtWrVIx7s7GniydSAAu8/dAKB97dxTjxeHvEb8E+VPoYN97969qVOnDocOHWLWrFlcuXKFzz//vCTTxsWLF3nppZdYuHAhDg6318rU3d2dRYsWcf78ebZs2UL9+vV5/vnnmTlzJgsXLuTs2bOEhYXh5OTEe++9l+953nrrLeLi4izLxYsXb/e2hBD3OC8Xe6pVcsTRYMPkvg159+F6RT7H693rUjXHOPjtapVMsPfNGhMgMl6CfXlW6OFyV69ezdixY3nxxRfv2jC5e/fuJSoqiubNm1vWGY1G/v77b7744gvWrl1Leno6sbGxVrn7yMhIfH198zznvHnz8PDwoE+fPjz66KP07dsXg8HAgAEDGD9+fL5psbe3x96++EamEkLcu/R6HX++1B6ltFz67XCxt+WDRxsxfN4umgdWoopr8Xa7y76Olr6kdGOJnP92/XHwCpvDoskwmujT1J/O9XxKO0llWqGD/datW5kzZw6hoaHUq1ePp556qsSntO3cuTOHDx+2Wjd8+HDq1q3Lf//7XwICAjAYDGzYsIH+/fsDEBYWxoULF2jbtm2u80VHR/Pee++xdetWQHtxyMjQxqTOyMjAaCxbX2YhRMVVlDr6/HQM8WbdKx3xdM490Flxccoawjclvez0s0/PNPHqkoOkG02A1vVw19tVSqTNQkVR6GDfpk0b2rRpw6xZs/j555+ZO3cur7zyCiaTiXXr1hEQEICrq2uxJs7V1ZWGDRtarXN2dsbT09Oy/tlnn+WVV16hcuXKuLm5MWbMGNq2bUubNm1ynW/cuHG8+uqrVK2q1Y+1a9eOH3/8ka5du/Ltt9/Srl27Yk2/EEKUtJrexd8CPyfzeP3JZShnH5eSQbrRhE4HBr2e6IQ0LsQkF3pgontRkVvjOzs788wzz7B161YOHz7Mq6++yrRp06hSpQqPPPJISaSxQJ988gkPP/ww/fv3p0OHDvj6+rJs2bJc+61du9YyNoDZ6NGjqVGjBq1btyY9PZ0JEybczaQLIUSZl52zL0vBPh3QSkcaVtVm/jM3VBR506liGPDYaDTyxx9/MHfuXH7//ffiSFeZFx8fj7u7O3Fxcbi5uZV2coQQokScv55Ex5mbcTTYcPz97qWdHAD2nIvhsa+3E+TpRPcGvnzz91kebxXA1Ecb3/rgCqQocei2+9nnZGNjQ9++fe+ZQC+EEPcKczF+SoYRk6lsTIYTm6y1tfJwNBAapA20tkdy9gUqlmAvhBCiYnKyy27alZpZNoryY1O0YO/uZGcJ9qeiEolNTi/NZJVpEuyFEELky9GQPZx4WWmkZw7qHo4GPF3sqeGtNcxbcySCsIiE0kxamSXBXgghRL5s9DrsbbVQUVYa6cVl5ew9smbka5GVu39z2WG6zfqb7/4+W2ppK6sk2AshhCiQUwl3v7sYk8wz83fz7PzdTFl1jITUjAL3z1lnD/BwY3/0OjDYaP3sp64+zr9nrpVIWssrCfZCCCEKZK63Ty6hgXX+OHSFjSei2HAiiu/+Cef3g1cK3D9nnT1AhxBvjk7qzon3e/Bo86qYFIxZtN9SAiAk2AshhLgFxxLuax+XbB2UL90oeIa9nHX2Zo52NtjodXzQrxE1vJy5npTOn4evFn9iyykJ9kIIIQpU0sX48VnF9ubrRN5ihr2b6+xzcjDYMKilNivpbwcuF2cyyzUJ9kIIIQpkbpGfnFFCOfus4F3bRxtyPeIWM+xZ6uzzCPYADzfxB2BneAxX4wouJbhXSLAXQghRIHOOO7WkcvYpWluAkCraOP+3DvZaMb67Y94TAFX1cKRV9cooBSsPSlE+SLAXQghxCyXdQM9cjF/HV8vZF1SMbzQp4lO1dOSXswd4pKmWu//toBTlgwR7IYQQt2CZ+e4uFeMnpRvz7X4Xn6OFvbtj/sG+a31tfvsjl+NJLaF0lycS7IUQQhSopGe+MwdwP3cHXO21UoTIfIryzd3uXOxtMdjkH8K8Xe0t57oYk1ycyS2XJNgLIYQoUEnOaa9UdrG8u6MBH3cHACLi0vLcP7u+Pv9cPYBOpyPQ0wmA89cl2EuwF0IIUSAng7nOvviDfVK6EWPWbHpuDgZ83bKC/S1y9gXV15tV99TGzD93Pak4klquSbAXQghRoOxi/OJvoGcuwjfY6HAw6PHJCvbmYvyE1Awm/HaE7WeuA9kD8BQm2Adl5ewvSDG+BHshhBAFK8lifHNLfHdHAzqdDl93ewAislrkz958hgXbz/PKkgNkGE05Rs/Lu9tdTuZgf06K8SXYCyGEKJglZ18CrdrNOXU3By2nnrMYPzEtk//bcR6Aq3GprD0akWNc/MLk7LVi/PNSjC/BXgghRMFKcrhcc+M816wGdzmL8RfvumDZDjB3a3iuGe8KYq6zv3wjhQyjqVjTXd5IsBdCCFEgR7uSa6BnrrM3t673zWqNf+lGCnO3hgPwykMhGGx07LsQyz+nooHC1dlXcbXH3lZPpklxJfbeHjZXgr0QQogClWQDPfOAOm4O2guFuRg/JimdK3Gp+LjZM6JDDR5pUhWAM9FakXxh6uz1ep2l3v5e734nwV4IIUSBLBPhlHADPQBPF3ts9DoA7G31fDUkFAeDDe/0qsf9tbwsxxUmZw9Sb29mW9oJEEIIUbaV5Ah6lpx9VrC30euo5+fK0SvxfDKoKaFBlQCo5GzHgmda8f0/Zzl4KZb7cgT+glSXFvmABHshhBC3YJkIJ8OIUgqdTlds5zbPeGdujQ+wYHgrYlMyqOntYrWvjV7H8x1rFun8gVk5+/BrkrMXQggh8mXuZ280KTKMCjvbYgz2NxXjg1aU7+liXyznb+jvBsCeczFkGk3YFjCefkV2b961EEKIQjMX40PxFeWfjU7kVGRCjmL8ksl7Nq7mgYeTgfjUTA5eii2Ra5QHEuyFEEIUyGCjx2Cj5eaTM+68RX5SWib9vvqXvl9us8xIl7MYvzjZ6HW0y6rf33LyWolcozyQYC+EEOKWirNF/sYTUcSlZJCUbuRq1rC4t5rF7k50DPEGYMvJ6BK7RlknwV4IIcQtORZji/w/D1/Ntc6tBIN9h9pasD90KZaYpPQSu05ZJsFeCCHELTkV0yh6yemZbAqLyrW+JHP2vu4O1PV1RSksI/DdayTYCyGEuKXsYvw7q7PfHBZNaoaJqh6OOBiyQ5CrQ8l2DutYR8vdrzsWWaLXKask2AshhLil4hpYZ/WRCAAebuJH62BPy7kNJdwlrmdDPwA2HI+64xeW8kiCvRBCiFsqrjntT1yNB6BdTS86ZDWcK8kifLPG1dwJrOxESoaRDcdzVyNUdBLshRBC3JJlmtscc9orpVjw7zn2nIsp9HnMDeS8Xe3p3tAXZzsby5C4JUmn0/FwYy13v/LQlRK/XlkjwV4IIcQtmRvo5Zz5bld4DBN+P8rzP+4t1HzxJpPiRrIW7Cs721HVw5G97z7EZ4OblUyib/JwY38ANoVFk5A1ct+9QoK9EEKIW8qrGP9YVpH89aR0tp2+9YA1cSkZmJT270pO2hS1DgYb9PriG363IPX8XKnp7Ux6ponfD95buXsJ9kIIIW7JK2us+sj4NMu6k5EJln8XJnhezyrCd3Wwxc727ocfnU7H460CAZi/7RxKqbuehtJSpoP91KlTadmyJa6urlSpUoW+ffsSFhZmtU9qaiqjRo3C09MTFxcX+vfvT2RkdteKmJgYevfujYuLC82aNWP//v1Wx48aNYqPPvrortyPEEKUV5apYnPMHhcWkR3s1x6JIDWj4MZ75vp6T2e7Ekhh4QxsGYCznQ2nohLZWojSiIqiTAf7LVu2MGrUKHbs2MG6devIyMiga9euJCVlf9lefvll/vjjD5YuXcqWLVu4cuUKjz76qGX7lClTSEhIYN++fXTq1InnnnvOsm3Hjh3s3LmTcePG3c3bEkKIcqe6lzZV7Pnr2t9fpRSnIhMBrQ9+UvqtW7nHJGmlApVLMdi7ORgY0CIAgLlbw0stHXdbmQ72a9as4emnn6ZBgwY0adKE+fPnc+HCBfbu3QtAXFwcc+bM4eOPP+bBBx8kNDSUefPm8e+//7Jjxw4Ajh8/zuDBgwkJCWHEiBEcP34cgIyMDF544QW+/vprbGxs8k2DEEIIqJ41L/yVuFRSM7Qx7RPSMrHV6xjSWisa//3g5QLPYS7Gr+xcPNPX3q6n76sOaA314u+RhnplOtjfLC4uDoDKlSsDsHfvXjIyMujSpYtln7p16xIYGMj27dsBaNKkCRs3biQzM5O1a9fSuHFjAGbMmEGnTp1o0aLFXb4LIYQofyo5GXDLGuXuQkwyYVn19TW8nXmsRTUANp2ItkxZm5eYxNIvxgetlMLFXruXawlpt9i7Yig3wd5kMjFu3DjatWtHw4YNAYiIiMDOzg4PDw+rfX18fIiI0EZpevPNN7G1taVmzZosX76cOXPmcOrUKRYsWMC7777LCy+8QI0aNRg4cKDlZSIvaWlpxMfHWy1CCHGv0Ol0lqL88GtJnMyqrw/xcaWurxshPi6kG02sPRqR7zlisrrdVSrlYK+lQRvI50ay5OzLlFGjRnHkyBEWL15cpOPc3d1ZtGgR58+fZ8uWLdSvX5/nn3+emTNnsnDhQs6ePUtYWBhOTk689957+Z5n6tSpuLu7W5aAgIA7vSUhhChXzEX5564lcTKrvj7ExxWAPk2rAvD7gfxb5ZeFBnpm5q5/scn3xix45SLYjx49mpUrV7Jp0yaqVatmWe/r60t6ejqxsbFW+0dGRuLr65vnuebNm4eHhwd9+vRh8+bN9O3bF4PBwIABA9i8eXO+aXjrrbeIi4uzLBcvXiyOWxNCiHLD0iL/erKl25052D/SRBuw5t8z14hKSM3z+Jik7AF1Sps52N8rU96W6WCvlGL06NEsX76cjRs3EhwcbLU9NDQUg8HAhg0bLOvCwsK4cOECbdu2zXW+6Oho3nvvPT7//HMAjEYjGRlaEU5GRgZGY/7dRuzt7XFzc7NahBDiXmIuxj92NZ5TUVqwr+OrBfuAyk40C/TApGDetnN5Hn89q86+sktZCPZaMX6sFOOXvlGjRvF///d/LFq0CFdXVyIiIoiIiCAlJQXQiuifffZZXnnlFTZt2sTevXsZPnw4bdu2pU2bNrnON27cOF599VWqVtWKm9q1a8ePP/7I8ePH+fbbb2nXrt1dvT8hhChPgrKK8Q9ejCU1w0R1TyeCKjtZtr/YsSYA3/9zllM5BtwxK0vF+B5ZOfsbUoxf+mbPnk1cXBydOnXCz8/Psvz888+WfT755BMefvhh+vfvT4cOHfD19WXZsmW5zrV27VpOnz7NyJEjLetGjx5NjRo1aN26Nenp6UyYMOGu3JcQQpRHwVk5e7NRD9SyGur2ofo+dKlXhQyj4n8rjliNUKeUKlPF+OY03CvB3ra0E1CQwgxl6ODgwJdffsmXX35Z4H7dunWjW7duVuucnJxYsmTJHaVRCCHuFZWcDLg62JKQmklAZUf6NqtqtV2n0zHxkQb8c+oau8JjCItMoK6vVuWZlG4kPWuynLIQ7M3F+DeSpBhfCCGEsNDpdNTz04L3qE61MNjkDiHVKjlZ6vEvxqRY1pv72DsY9JYZ9EqTuRg/RnL2QgghhLVpjzbi8OU4S+v7vPi7O3LoUhxXYrOD/fWsoXI9S3n0PDNz6cK90vVOgr0QQohCq+HtQg1vlwL38fNwALAK9mWpvh7Aw0kG1RFCCCFuW1UPR0AbR9/sehkL9uZ+9jeS0u+JqW4l2AshhChW/uZgn0fOvix0u4PsYJ9pUiSmZZZyakqeBHshhBDFKq9gH5GVyy8L4+IDONrZ4GDQQuDNA+tMWXWMoXN3kZ5pKo2klQgJ9kIIIYqVf1adfWR8KplGE6kZRv44qI2ZHxpUqTSTZqVyHkPmbgqL4rt/wvn7ZDRHruQ/OVp5I8FeCCFEsfJytsdgo8OkIDIhjRX7L3M9KZ2qHo50re9T2smzuHkUvbRMI+/9ccyy3Ty8b0UgwV4IIUSx0ut1+LlrRfmXb6Tw/dZwAIa3q45tHn3zS0v2NLdaUP/h3/OEX0uybL+WWHHmui87T10IIUSFYS7KX7LnIqejEnGxt2VQy7I1NXh2i3ytzn7t0QgAnO1sALguwV4IIYTIn7mR3vL9lwF4tHlVXB0MpZmkXHLOaW8yKU5EaJP3tK3pCcA1KcYXQggh8uefVYxvNGl92B9tXq00k5Mn8/j4McnpXLyRTGJaJna2eloFVwakGF8IIYQokDlnD9pseU2quZdiavJWyTLzXQbHrsQDUMfHFR83rQpCGugJIYQQBTDX2QP0bVoVnU5XwN6lI+coeseuasG+np8rXi7a+P2SsxdCCCEKUDVHzr5vs/wnzSlN5vHxryWmcTwr2Nf3c7ME++tJFSdnLxPhCCGEKHa1qrgwuGUAVVztCfJ0Lu3k5KmOryt6HZyMTOTSDW20v/r+7ni6ZPe/zzSaylR3wdslwV4IIUSx0+l0TOvfuLSTUSA/d0f6NK3K8v2XSU43AlDXzxVnO1v0OjAprfFeFVeHW5yp7Cv/rytCCCHEbRr1QC3MzQkCKjvi5mDARq+zzM53LaFiFOVLsBdCCHHPqlXFhZ6N/ABo4JfdY8DT2VxvXzEa6UkxvhBCiHva+Ifr4+Zgy1NtqlvWebnaERZZcbrfSbAXQghxT/Nxc2Dqo9btC8w5+4rS/U6K8YUQQoibmFvkV5QhcyXYCyGEEDcx97U/E53I28sPs+30tVJO0Z2RYnwhhBDiJl5ZOft1xyIB2Hn2Ohte7VSKKbozkrMXQgghbmKuszc7E53EuRxz3Zc3krMXQgghbuLlap9r3frjkQwIDWD5/kvsPn+DG0npONvb0r95Nbo39C2FVBaeBHshhBDiJv4eDuh1YKvXM6RNIPO2nWPV4ass3n2R01GJVvuuPx7J90Nb0LmeTyml9tZ0SilV2okoj+Lj43F3dycuLg43N7fSTo4QQohituF4JO6OBqq4OtBh5ibLei8Xe56+L4iAyk6sPx7FHwev4Gxnw8sPhdCwqjutgyvflVn+ihKHJGcvhBBC5CFnTj3Ex4WTkVqOftagptxf2wuAno38uJ6Yxr9nrjN51XEAvniiGQ83Llsz/UkDPSGEEOIW+jarCsDzHWtYAj2AwUbPN0+F8lrXEBpV1Ybb3Xg8qlTSWBDJ2QshhBC38HyHmnSt70NNb5dc21wdDIx+sDbNAisx5Pud/HvmOkqpu1KUX1iSsxdCCCFuwUavo1YV1wIDeGhQJexs9ETEpxJexrrpSbAXQgghioGDwYbmQR4AbD97vXQTcxMJ9kIIIUQxua+mVp//7xkJ9kIIIUSFdF9NTwD+PX2NX/de4kx04i2OuDsk2AshhBDFpHE1D5zsbLiRnMGrSw/S78ttRCeU/jS5EuyFEEKIYmJnq2dy34Y8UMcbP3cH4lMzmbHmRGknS4K9EEIIUZwebV6NecNb8eWQ5gAs3XuJLSejKc0BaytMsP/yyy+pXr06Dg4OtG7dml27dlm2vfLKK1SuXJmAgAAWLlxoddzSpUvp3bv33U6uEEKICq55YCUGhFYDYNjcXbSfsYlf9l4qlaBfIcbG//nnnxk6dChff/01rVu3ZtasWSxdupSwsDB27tzJc889x8qVKzl16hTPPPMMFy9exMvLi7i4OFq2bMn69esJDAws0jVlbHwhhBC3Epuczv+WH2H98UjSMk0AdAjx5oN+DalWyemOzl2UOFQhgn3r1q1p2bIlX3zxBQAmk4mAgADGjBmDXq9n3759LF68GAAfHx9WrlxJy5Ytef7556lbty4vv/xyka8pwV4IIURhpaQbmfdvOLPWnyI908Tkvg15sk3QHZ2zKHGo3Bfjp6ens3fvXrp06WJZp9fr6dKlC9u3b6dJkybs2bOHGzdusHfvXlJSUqhVqxZbt25l3759jB07thRTL4QQ4l7gaGfDyE61WP1Se569P5gnWhWtNPlOlfux8a9du4bRaMTHx3oeYR8fH06cOEG3bt148sknadmyJY6OjixYsABnZ2defPFF5s+fz+zZs/n888/x8vLi22+/pUGDBnleJy0tjbS07O4TcXFxgPZmJYQQQhSGtz281KEaiYkJd3wuc/wpVAG9KucuX76sAPXvv/9arX/99ddVq1at8jxm4sSJaty4cergwYPKx8dHRUVFqblz56rmzZvne50JEyYoQBZZZJFFFlnK1HLx4sVbxspyX2efnp6Ok5MTv/zyC3379rWsHzZsGLGxsfz2229W+584cYLevXuzf/9+5s6dy9atW1myZAlJSUm4uLgQHx+Pq6trruvcnLM3mUzExMTg6el5xzMbxcfHExAQwMWLF6X+/xbkWRWNPK/Ck2dVePKsiqaknpdSioSEBPz9/dHrC66VL/fF+HZ2doSGhrJhwwZLsDeZTGzYsIHRo0db7auU4vnnn+fjjz/GxcUFo9FIRkYGgOWn0WjM8zr29vbY29tbrfPw8CjWe3Fzc5P/OIUkz6po5HkVnjyrwpNnVTQl8bzc3d0LtV+5D/ag9aMfNmwYLVq0oFWrVsyaNYukpCSGDx9utd/333+Pt7e3pV99u3btmDhxIjt27GD16tXUr1+/2AO4EEIIUdoqRLAfNGgQ0dHRjB8/noiICJo2bcqaNWusGu1FRkYyZcoU/v33X8u6Vq1a8eqrr9KrVy+qVKnCggULSiP5QgghRImqEMEeYPTo0bmK7XPy8fHh3LlzudaPHz+e8ePHl2DKbs3e3p4JEybkqiYQucmzKhp5XoUnz6rw5FkVTVl4XuW+gZ4QQgghClbuB9URQgghRMEk2AshhBAVnAR7IYQQooKTYH+XyBS8uf3999/07t0bf39/dDodK1assNqulGL8+PH4+fnh6OhIly5dOHXqlGV7WloaTz31FG5uboSEhLB+/Xqr42fOnMmYMWPuxq2UuKlTp9KyZUtcXV2pUqUKffv2JSwszGqf1NRURo0ahaenJy4uLvTv35/IyEjL9piYGHr37o2LiwvNmjVj//79VsePGjWKjz766K7cT0mbPXs2jRs3tvRrbtu2LatXr7Zsl2eVv2nTpqHT6Rg3bpxlnTwvzcSJE9HpdFZL3bp1LdvL9HMq5Ki04g4sXrxY2dnZqblz56qjR4+q5557Tnl4eKjIyEj1+++/Kx8fH7V79261aNEi5eDgoKKjo5VSSsXGxqratWur8+fPl/IdlIw///xT/e9//1PLli1TgFq+fLnV9mnTpil3d3e1YsUKdfDgQfXII4+o4OBglZKSopRS6rPPPlP16tVTR44cUTNnzlTe3t7KZDIppZQ6e/asql27toqLi7vbt1UiunXrpubNm6eOHDmiDhw4oHr27KkCAwNVYmKiZZ8XXnhBBQQEqA0bNqg9e/aoNm3aqPvuu8+y/ZVXXlEdO3ZUYWFhaty4cSo0NNSybfv27So0NFRlZmbe1fsqKb///rtatWqVOnnypAoLC1Nvv/22MhgM6siRI0opeVb52bVrl6pevbpq3Lixeumllyzr5XlpJkyYoBo0aKCuXr1qWcx/r5Uq289Jgv1d0KpVKzVq1CjLZ6PRqPz9/dXUqVPV9OnT1aBBgyzbqlSponbt2qWUUmrEiBHq448/vuvpLQ03B3uTyaR8fX3VzJkzLetiY2OVvb29+umnn5RSSr344ovqv//9r1JKqeTkZAWoqKgopZQWHJctW3b3buAui4qKUoDasmWLUkp7NgaDQS1dutSyz/HjxxWgtm/frpRSqkePHmr27NlKKaWOHTumnJyclFJKpaenqyZNmqjdu3ff5bu4uypVqqS+//57eVb5SEhIULVr11br1q1THTt2tAR7eV7ZJkyYoJo0aZLntrL+nKQYv4TJFLy3Jzw8nIiICKvn5u7uTuvWrdm+fTsATZo0YevWraSkpLB27Vr8/Pzw8vJi4cKFODg40K9fv9JKfokzz7pYuXJlAPbu3UtGRobV86pbty6BgYFWz2vjxo1kZmaydu1aGjduDMCMGTPo1KkTLVq0uMt3cXcYjUYWL15MUlISbdu2lWeVj1GjRtGrVy+r5wLy3brZqVOn8Pf3p0aNGgwZMoQLFy4A5eA5ldhrhFBKFW5WvgkTJqiaNWuqhg0bqmXLlqm0tDTVsGFDtWfPHvX555+rkJAQdd9991mKICsibsrZb9u2TQHqypUrVvsNGDBADRw4UCmlvQ2PHDlSVa9eXbVo0UL9888/6vr166pGjRrqwoUL6n//+5+qWbOm6tq1q7p06dLdvJ0SZTQaVa9evVS7du0s6xYuXKjs7Oxy7duyZUv1xhtvKKW0nMfjjz+uAgMDVYcOHdTRo0fVyZMnVe3atdW1a9fU888/r4KDg9WAAQNUbGzsXbufknLo0CHl7OysbGxslLu7u1q1apVSSp5VXn766SfVsGFDSxVZzpy9PK9sf/75p1qyZIk6ePCgWrNmjWrbtq0KDAxU8fHxZf45VZgR9MqziRMnMnHiRMvnSZMm0aVLFwwGA5MnT+bw4cOsXLmSoUOHsnfv3tJLaBljMBj48ssvrdYNHz6csWPHsn//flasWMHBgweZMWMGY8eO5ddffy2llBavUaNGceTIEbZu3Vqk49zd3Vm0aJHVugcffJCZM2eycOFCzp49S1hYGM899xzvvfdeuW9QVadOHQ4cOEBcXBy//PILw4YNY8uWLYU69l56VhcvXuSll15i3bp1ODg43NY57pXn1aNHD8u/GzduTOvWrQkKCmLJkiU4Ojre8vjSfE5SjF/CvLy8sLGxsWqRCdpY/b6+vrn2P3HiBP/3f//H+++/z+bNm+nQoQPe3t4MHDiQffv2kZCQcLeSXqrMz6awzw1g06ZNHD16lNGjR7N582Z69uyJs7MzAwcOZPPmzSWd5Lti9OjRrFy5kk2bNlGtWjXLel9fX9LT04mNjbXav6DnNW/ePDw8POjTpw+bN2+mb9++GAwGBgwYUCGel52dHbVq1SI0NJSpU6fSpEkTPv30U3lWN9m7dy9RUVE0b94cW1tbbG1t2bJlC5999hm2trb4+PjI88qHh4cHISEhnD59usx/ryTYl7CcU/Camafgbdu2rdW+6g6m4K1ogoOD8fX1tXpu8fHx7Ny5M9dzg+wuL9988w02Nja5nl15f25KKUaPHs3y5cvZuHEjwcHBVttDQ0MxGAxWzyssLIwLFy7k+byio6N57733+PzzzwEq3PPKi8lkIi0tTZ7VTTp37szhw4c5cOCAZWnRogVDhgyx/FueV94SExM5c+YMfn5+Zf97VayVAiJPixcvVvb29mr+/Pnq2LFjasSIEcrDw0NFRERY7fftt9+q/v37Wz7v3LlTubm5qe3bt6vx48er+vXr3+2kl6iEhAS1f/9+tX//fgWojz/+WO3fv9/S1XDatGnKw8ND/fbbb+rQoUOqT58+Vl3vcnr77bfVq6++avn8888/q8DAQHXw4EH17LPPqp49e961+yoJL774onJ3d1ebN2+26vaTnJxs2eeFF15QgYGBauPGjWrPnj2qbdu2qm3btnme74knnlCff/655fP06dNVaGioOnbsmOrRo4caOXJkid9TSXrzzTfVli1bVHh4uDp06JB68803lU6nU3/99ZdSSp7VreSss1dKnpfZq6++qjZv3qzCw8PVtm3bVJcuXZSXl5elF1BZfk4S7O+Szz//XAUGBio7OzvVqlUrtWPHDqvtERERKigoSF2+fNlq/aRJk1TlypVV3bp11c6dO+9mkkvcpk2bFJBrGTZsmFJK63737rvvKh8fH2Vvb686d+6swsLCcp3n8OHDqlatWlZ9zo1Go3rxxReVm5ubatmypTp16tTduq0SkddzAtS8efMs+6SkpKiRI0eqSpUqKScnJ9WvXz919erVXOdas2aNatWqlTIajZZ1SUlJasCAAcrV1VV17txZRUZG3o3bKjHPPPOMCgoKUnZ2dsrb21t17tzZEuiVkmd1KzcHe3lemkGDBik/Pz9lZ2enqlatqgYNGqROnz5t2V6Wn5PMeieEEEJUcFJnL4QQQlRwEuyFEEKICk6CvRBCCFHBSbAXQgghKjgJ9kIIIUQFJ8FeCCGEqOAk2AshhBAVnAR7IYQQooKTYC+EKDYTJ07Ex8cHnU7HihUrSjs54jbNnz8fDw+P0k6GKEYS7EWF9vTTT6PT6dDpdJZZ0N577z0yMzNLO2m3VN4C5vHjx5k0aRLffPMNV69etZoOVAhRumQ+e1Hhde/enXnz5pGWlsaff/7JqFGjMBgMvPXWW0U+l9FoRKfTodfLe/LNzpw5A0CfPn3Q6XSlnBprSimMRiO2tuXjT15GRgYGg6G0kyEqEPmLJSo8e3t7fH19CQoK4sUXX6RLly78/vvvAKSlpfHaa69RtWpVnJ2dad26tdU80ubizN9//5369etjb2/PhQsXSEtL47///S8BAQHY29tTq1Yt5syZYznuyJEj9OjRAxcXF3x8fHjqqae4du2aZXunTp0YO3Ysb7zxBpUrV8bX15eJEydatlevXh2Afv36odPpLJ/PnDlDnz598PHxwcXFhZYtW7J+/Xqr+7169Sq9evXC0dGR4OBgFi1aRPXq1Zk1a5Zln9jYWP7zn//g7e2Nm5sbDz74IAcPHizwOR4+fJgHH3wQR0dHPD09GTFiBImJiYBWfN+7d28A9Hp9vsF+8+bN6HQ6NmzYQIsWLXBycuK+++4jLCzMar/ffvuN5s2b4+DgQI0aNZg0aZKlNObcuXPodDoOHDhgdT86nc7yuzNfZ/Xq1YSGhmJvb8/WrVtJS0tj7NixVKlSBQcHB+6//352795dpPQdPHiQBx54AFdXV9zc3AgNDWXPnj35PjedTsfs2bPp0aMHjo6O1KhRg19++cWy3Xw/P//8Mx07dsTBwYGFCxcyceJEmjZtanWuWbNmWb4LoJVc9e3blw8//BA/Pz88PT0ZNWqUZapUuPV3HLTveWBgIE5OTvTr14/r16/nez+inCrWaXWEKGOGDRum+vTpY7XukUceUc2bN1dKKfWf//xH3Xfffervv/9Wp0+fVjNnzlT29vbq5MmTSiml5s2bpwwGg7rvvvvUtm3b1IkTJ1RSUpIaOHCgCggIUMuWLVNnzpxR69evV4sXL1ZKKXXjxg3l7e2t3nrrLXX8+HG1b98+9dBDD6kHHnjAkoaOHTsqNzc3NXHiRHXy5Em1YMECqylYo6KiLLPaXb161TKF5oEDB9TXX3+tDh8+rE6ePKneeecd5eDgYJkWWCmlunTpopo2bap27Nih9u7dqzp27KgcHR3VJ598YrVP79691e7du9XJkyfVq6++qjw9PdX169fzfI6JiYnKz89PPfroo+rw4cNqw4YNKjg42DJDYUJCgpo3b54CLNPv5sU802Hr1q3V5s2b1dGjR1X79u3VfffdZ9nn77//Vm5ubmr+/PnqzJkz6q+//lLVq1dXEydOVEopFR4ergC1f/9+yzE3btxQgNq0aZPVdRo3bqz++usvdfr0aXX9+nU1duxY5e/vr/7880919OhRNWzYMFWpUiXLfRcmfQ0aNFBPPvmkOn78uDp58qRasmSJOnDgQJ73q5Q2Y6Gnp6f67rvvVFhYmHrnnXeUjY2NOnbsmNX9VK9eXf3666/q7Nmz6sqVK2rChAmqSZMmVuf65JNPVFBQkOXzsGHDlJubm3rhhRfU8ePH1R9//KGcnJzUt99+a9nnVt/xHTt2KL1er6ZPn67CwsLUp59+qjw8PJS7u3u+9yTKHwn2okLLGexNJpNat26dsre3V6+99po6f/68srGxyTWtcOfOndVbb72llFKWAJbzj3lYWJgC1Lp16/K85vvvv6+6du1qte7ixYsKsEzR27FjR3X//fdb7dOyZUv13//+1/IZUMuXL7/lPTZo0MAyL/bx48cVoHbv3m3ZfurUKQVYgv0///yj3NzcVGpqqtV5atasqb755ps8r/Htt9+qSpUqWU0jvGrVKqXX61VERIRSSqnly5erW+UfzMF0/fr1VucBVEpKilJKe/4ffPCB1XE//vij8vPzU0oVLdivWLHCsk9iYqIyGAxq4cKFlnXp6enK399fzZgxo9Dpc3V1VfPnzy/wPnMC1AsvvGC1rnXr1urFF1+0up9Zs2ZZ7VPYYB8UFKQyMzMt6wYMGKAGDRqklFKF+o4//vjjqmfPnlbbBw0aJMG+gikfFVhC3IGVK1fi4uJCRkYGJpOJJ554gokTJ7J582aMRiMhISFW+6elpeHp6Wn5bGdnR+PGjS2fDxw4gI2NDR07dszzegcPHmTTpk24uLjk2nbmzBnL9XKeE8DPz4+oqKgC7yUxMZGJEyeyatUqrl69SmZmJikpKVy4cAGAsLAwbG1tad68ueWYWrVqUalSJav0JSYmWt0jQEpKiqXe/WbHjx+nSZMmODs7W9a1a9cOk8lEWFgYPj4+Bab7Zjnv3c/PD4CoqCgCAwM5ePAg27ZtY8qUKZZ9jEYjqampJCcnF+k6LVq0sPz7zJkzZGRk0K5dO8s6g8FAq1atOH78eKHT98orr/Cf//yHH3/8kS5dujBgwABq1qxZYDratm2b63POaoib01oUDRo0wMbGxiq9hw8fBrSql1t9x48fP06/fv1ypW/NmjW3lR5RNkmwFxXeAw88wOzZs7Gzs8Pf39/SSCsxMREbGxv27t1r9ccSsArUjo6OVnXQjo6OBV4vMTGR3r17M3369FzbzIEDyNUAS6fTYTKZCjz3a6+9xrp16/jwww+pVasWjo6OPPbYY6Snpxd43M3p8/Pzy1VvC9y17lY57938bM33npiYyKRJk3j00UdzHefg4GBpHKmUsqzPWUedU86Xk+JK38SJE3niiSdYtWoVq1evZsKECSxevDhXwCyqm9Oq1+ut7hHyvs+CvkeF/Y6Lik+CvajwnJ2dqVWrVq71zZo1w2g0EhUVRfv27Qt9vkaNGmEymdiyZQtdunTJtb158+b8+uuvVK9e/Y5afxsMBoxGo9W6bdu28fTTT1sCS2JiIufOnbNsr1OnDpmZmezfv5/Q0FAATp8+zY0bN6zSFxERga2trVVjr4LUq1eP+fPnk5SUZAlK27ZtQ6/XU6dOndu+x7w0b96csLCwPH9nAN7e3oDWELFZs2YAuXLJealZsyZ2dnZs27aNoKAgQAueu3fvZty4cUVKY0hICCEhIbz88ss8/vjjzJs3r8Bgv2PHDoYOHWr12Zz2/Hh7exMREYFSyvLCUZj7zKkw3/F69eqxc+fOXOkVFYu0xhf3rJCQEIYMGcLQoUNZtmwZ4eHh7Nq1i6lTp7Jq1ap8j6tevTrDhg3jmWeeYcWKFYSHh7N582aWLFkCwKhRo4iJieHxxx9n9+7dnDlzhrVr1zJ8+PBcwbsg1atXZ8OGDURERFiCde3atVm2bBkHDhzg4MGDPPHEE1alAXXr1qVLly6MGDGCXbt2sX//fkaMGGFVOtGlSxfatm1L3759+euvvzh37hz//vsv//vf//JtVT5kyBAcHBwYNmwYR44cYdOmTYwZM4annnqqyEX4tzJ+/Hh++OEHJk2axNGjRzl+/DiLFy/mnXfeAbSSlTZt2jBt2jSOHz/Oli1bLNsK4uzszIsvvsjrr7/OmjVrOHbsGM899xzJyck8++yzhUpbSkoKo0ePZvPmzZw/f55t27axe/du6tWrV+BxS5cuZe7cuZw8eZIJEyawa9cuRo8eXeAxnTp1Ijo6mhkzZnDmzBm+/PJLVq9eXah0mhXmOz527FjWrFnDhx9+yKlTp/jiiy+kCL8CkmAv7mnz5s1j6NChvPrqq9SpU4e+ffuye/duAgMDCzxu9uzZPPbYY4wcOZK6devy3HPPkZSUBIC/vz/btm3DaDTStWtXGjVqxLhx4/Dw8ChS//yPPvqIdevWERAQYMkFfvzxx1SqVIn77ruP3r17061bN6v6eYAffvgBHx8fOnToQL9+/XjuuedwdXXFwcEB0Ip5//zzTzp06MDw4cMJCQlh8ODBnD9/Pt/A7eTkxNq1a4mJiaFly5Y89thjdO7cmS+++KLQ91NY3bp1Y+XKlfz111+0bNmSNm3a8Mknn1hy4wBz584lMzOT0NBQxo0bx+TJkwt17mnTptG/f3+eeuopmjdvzunTp1m7dq1Vm4aC2NjYcP36dYYOHUpISAgDBw6kR48eTJo0qcDjJk2axOLFi2ncuDE//PADP/30E/Xr1y/wmHr16vHVV1/x5Zdf0qRJE3bt2sVrr71WqHTmdKvveJs2bfjuu+/49NNPadKkCX/99VehXp5E+aJTN1cKCSEqlEuXLhEQEMD69evp3LlzaSfnnqPT6Vi+fDl9+/Yt7aSIe5jU2QtRwWzcuJHExEQaNWrE1atXeeONN6hevTodOnQo7aQJIUqJBHshKpiMjAzefvttzp49i6urK/fddx8LFy6U4VeFuIdJMb4QQghRwUkDPSGEEKKCk2AvhBBCVHAS7IUQQogKToK9EEIIUcFJsBdCCCEqOAn2QgghRAUnwV4IIYSo4CTYCyGEEBWcBHshhBCigvt/Lmu3ybbBYb8AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.rcParams[\"figure.figsize\"] = (5.4, 3.2)\n", "plt.plot(np.array(range(len(ASR)))/424, np.array(ASR)*100, label=\"ASR\")\n", "plt.plot(np.array(range(len(BA)))/424, np.array(BA)*100, label=\"BA\")\n", "\n", "plt.legend(loc=\"upper right\")\n", "plt.ylim(0,101)\n", "ax = plt.gca()\n", "ax.yaxis.set_major_formatter(mtick.PercentFormatter(100, decimals=0))\n", "ax.xaxis.set_major_formatter(mtick.PercentFormatter(1, decimals=0))\n", "plt.xlabel(\"Percentage of neurons pruned\")\n", "plt.ylabel(\"Accuracy\")\n", "plt.title(\"Pruning backdoored FT-Transformer\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 22, "id": "473c258e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0.9974191629339306, 0.9975241461366181, 0.9974716545352743, 0.9974016657334827, 0.9972704367301232, 0.9971479563269877, 0.9974366601343785, 0.9971916993281075, 0.9974454087346024, 0.9974016657334827, 0.9975328947368421, 0.9974716545352743, 0.9974454087346024, 0.9975241461366181, 0.9973929171332587, 0.9972179451287794, 0.9975153975363942, 0.9973929171332587, 0.9972091965285554, 0.9975153975363942, 0.997559140537514, 0.9976466265397537, 0.9974629059350504, 0.9974716545352743, 0.9976728723404256, 0.9977341125419933, 0.9976116321388577, 0.9977166153415453, 0.9976991181410975, 0.9977428611422172, 0.9976903695408734, 0.9976816209406495, 0.9977253639417694, 0.9975066489361702, 0.9975153975363942, 0.9974279115341546, 0.9969729843225084, 0.9967717665173572, 0.9969729843225084, 0.9970429731243001, 0.9972266937290034, 0.9971392077267637, 0.9970167273236282, 0.9969292413213886, 0.9969292413213886, 0.9966055431131019, 0.9968592525195968, 0.9967017777155656, 0.9967017777155656, 0.9968680011198209, 0.996596794512878, 0.9963955767077267, 0.9965705487122061, 0.9962031075027995, 0.9961506159014558, 0.9960106382978723, 0.9960631298992161, 0.9956956886898096, 0.9957919232922733, 0.9959406494960806, 0.9962555991041433, 0.9961418673012318, 0.9960106382978723, 0.9960281354983203, 0.9960018896976484, 0.9965705487122061, 0.9967542693169092, 0.9969292413213886, 0.9971042133258678, 0.9962643477043673, 0.996106872900336, 0.9963955767077267, 0.9962118561030235, 0.9958706606942889, 0.9953544932810751, 0.9933773096304591, 0.9929486282194849, 0.9927999020156775, 0.992204997200448, 0.9935785274356103, 0.9932635778275476, 0.9928436450167973, 0.992659924412094, 0.9932635778275476, 0.993403555431131, 0.9933073208286675, 0.9925199468085106, 0.9923624720044792, 0.9896066629339306, 0.9881106522956327, 0.9878394456886898, 0.988040663493841, 0.9879969204927211, 0.989755389137738, 0.990953947368421, 0.99070898656215, 0.9893966965285554, 0.989029255319149, 0.9869908314669653, 0.9871220604703248, 0.9862384518477044, 0.9863521836506159, 0.9813742301231803, 0.9749790033594625, 0.9801319288913773, 0.9805431131019037, 0.9808493141097424, 0.9768687010078387, 0.9639995100783875, 0.9461261198208286, 0.9412531494960806, 0.9415068589025756, 0.9506229003359462, 0.9505791573348265, 0.9468959966405375, 0.9484794932810751, 0.9532387318029115, 0.9534749440089586, 0.956318239081747, 0.9632296332586786, 0.9640345044792833, 0.9575517917133258, 0.9578492441209406, 0.9475433930571109, 0.9331606942889138, 0.9331781914893617, 0.8910886758118701, 0.9027768057110862, 0.8941069428891377, 0.8882016377379619, 0.9027330627099664, 0.9026893197088466, 0.878438199888018, 0.8535746780515118, 0.8680098684210527, 0.8521486562150056, 0.8398743701007839, 0.756578947368421, 0.7530882558790594, 0.574686800111982, 0.56494960806271, 0.5684315509518477, 0.46180361142217247, 0.3958041713325868, 0.4008608622620381, 0.4530550111982083, 0.4541835806270997, 0.5062290033594625, 0.5248372760358343, 0.5264032754759238, 0.46916993281075026, 0.43263577827547595, 0.40254934210526316, 0.39132488801791715, 0.4182618281075028, 0.4830277155655095, 0.5252397116461366, 0.6099174132138858, 0.543952967525196, 0.5347144456886898, 0.5090810470324748, 0.4930623600223964, 0.5472336926091825, 0.5546350083986562, 0.5621763017917133, 0.5013122900335947, 0.45816419372900336, 0.39462311030235164, 0.2243840985442329, 0.17325727883538633, 0.18805116181410975, 0.23068309070548712, 0.20974769036954088, 0.18904850223964165, 0.17719414893617022, 0.11444043952967525, 0.1346934490481523, 0.15104458286674133, 0.24043777995520715, 0.2625454927211646, 0.23627344624860022, 0.3503289473684211, 0.3495853163493841, 0.28060260358342665, 0.2890100083986562, 0.3048537234042553, 0.2881788913773796, 0.2827372620380739, 0.38646066629339304, 0.38673187290033595, 0.355219414893617, 0.3563042413213886, 0.3469957306830907, 0.32626154815229563, 0.27394491881298993, 0.2478215985442329, 0.2264400195968645, 0.209721444568869, 0.11350433930571109, 0.09858797592385218, 0.13761548152295633, 0.1499772536394177, 0.06661184210526316, 0.0795160274356103, 0.05537863941769317, 0.06564074748040313, 0.06738171892497201, 0.0670405235162374, 0.05838815789473684, 0.05726833706606943, 0.028931620940649497, 0.028870380739081748, 0.016596094624860024, 0.014400195968645016, 0.009667203247480403]\n" ] } ], "source": [ "print(ASR)" ] }, { "cell_type": "code", "execution_count": 23, "id": "c0535e5b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0.9541836269287368, 0.9538394017366161, 0.9534177258762683, 0.9532111907609958, 0.9529702331265114, 0.952789514900648, 0.9526604304536027, 0.9523936559297093, 0.952290388372073, 0.952264571482664, 0.9517482336944829, 0.9512318959063019, 0.950818825675757, 0.9510253607910295, 0.9508446425651661, 0.950749980637333, 0.9504401779644244, 0.9500529246232886, 0.949725910690774, 0.9494161080178652, 0.949261206681411, 0.9493214460900321, 0.9492267841621989, 0.9491751503833809, 0.9487534745230329, 0.9480047847301705, 0.9480306016195795, 0.9479961791003675, 0.9476347426486408, 0.9471614330094748, 0.9469462922643994, 0.9470839823412477, 0.9463008700291731, 0.9455263633469015, 0.9451993494143869, 0.9448551242222663, 0.9444334483619183, 0.9441666738380249, 0.9439171105737373, 0.9439687443525554, 0.943280293968314, 0.942738139290724, 0.9420238720170736, 0.9415677736375136, 0.9414903229692865, 0.9409739851811055, 0.9404920699121365, 0.940285534796864, 0.9390721409946386, 0.9385902257256697, 0.9378931697116254, 0.9376436064473378, 0.9368260716160512, 0.9358450298185073, 0.9340464531896767, 0.9339259743724344, 0.933203101468981, 0.933237523988193, 0.9331342564305569, 0.9319552851475436, 0.9319380738879375, 0.9310000602394086, 0.930208342297531, 0.9294424412450625, 0.9289777372356995, 0.9279450616593375, 0.927704104024853, 0.9276008364672168, 0.9261378794007039, 0.9258022598383863, 0.9245802604063579, 0.9239864719499497, 0.9236336411280259, 0.9235045566809807, 0.9212068535235751, 0.920647487586379, 0.9189521785151846, 0.9184530519866096, 0.9180055592368528, 0.916620052838567, 0.9145202791666308, 0.9144686453878127, 0.9136338992969201, 0.9133585191432235, 0.9123344491966645, 0.9120504634131649, 0.9109145202791666, 0.9103981824909856, 0.9089954648330938, 0.9082467750402313, 0.9068956911611576, 0.9052003820899632, 0.9028596507835426, 0.9027305663364973, 0.9017064963899383, 0.9008201165202275, 0.8988924554443517, 0.8982470332091254, 0.8976016109738991, 0.8961816820564013, 0.8943314716487526, 0.8924210218324828, 0.8923263599046496, 0.891009698544788, 0.8900802905260622, 0.8903384594201527, 0.8901147130452742, 0.8888582910940337, 0.8873178833592936, 0.8862507852637195, 0.8855451236198721, 0.8842198566302075, 0.8835141949863601, 0.8818102802853627, 0.879478154608745, 0.8785659578496252, 0.8780238031720351, 0.8775246766434601, 0.8770427613744912, 0.8761994096537955, 0.8755195648993571, 0.8749860158515701, 0.8735488756744663, 0.8717933271946507, 0.8700980181234563, 0.8691600044749275, 0.8675593573315663, 0.8636437957711935, 0.8634200493963151, 0.8612600363157578, 0.8605715859315164, 0.8581878264760806, 0.8575682211302634, 0.8554340249391151, 0.8542034198772838, 0.8527662797001798, 0.8499178162353812, 0.848299957832414, 0.847809436933642, 0.8445737201277076, 0.8439971429309054, 0.839797595587033, 0.8368802870838102, 0.8353312737192672, 0.8333347676049672, 0.8319492612066814, 0.8310714869667737, 0.8287651781795651, 0.8275001505985216, 0.8264674750221594, 0.8248324053595862, 0.8244967857972686, 0.8228703217644984, 0.822181871380257, 0.8191440840597919, 0.8176122819548549, 0.81665705704672, 0.817173394834901, 0.8166312401573109, 0.8128705799333924, 0.8089722296326257, 0.808120272282127, 0.8066917377348262, 0.8054439214133886, 0.8046952316205261, 0.8039723587170727, 0.8004440504978357, 0.7951601937987831, 0.7894030274605647, 0.7882756899563694, 0.786003803688373, 0.7842740720979665, 0.7825013123585449, 0.7807199469893205, 0.7772432725489015, 0.7764515546070239, 0.7728888238685748, 0.7722692185227575, 0.7698424309183067, 0.7701178110720033, 0.7663743621076908, 0.7626911525519995, 0.7582420419438397, 0.7549632969888901, 0.7535175511819833, 0.7517017632935467, 0.749395454506338, 0.7477775961033708, 0.7454024422777381, 0.7446709637444816, 0.7430617109713175, 0.7429068096348631, 0.7405746839582454, 0.7375885304165986, 0.7351187146631326, 0.7323390962367581, 0.7316076177035016, 0.7294820271421564, 0.7284321403061883, 0.7260053527017375, 0.7223823825546672, 0.7201621300654888, 0.7195081022004596, 0.7169694414085694, 0.7169436245191604, 0.7129420066607575, 0.7095599941481717, 0.7076065161828868, 0.7072364741013571, 0.7078474738173713, 0.7081142483412649, 0.7029938986084696, 0.7015997865803809, 0.7024775608202887, 0.6988718019328245]\n" ] } ], "source": [ "print(BA)" ] }, { "cell_type": "code", "execution_count": null, "id": "6c455c61", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" } }, "nbformat": 4, "nbformat_minor": 5 }