{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Лабораторная работа №3\n# Классификация СIFAR с помощью механизма attention.\n- Сеть состоит из нескольких self-attention на конце линейный слой (можно 2).\n- Обычно для классификации подают в начало трансформера фиктивный токен и по нему потом классифицируют\n- Попробовать патчи 16 * 16 и 8 * 8\n- Попробовать с 1, 2 и 4 головами\n- self-attention реализовать полностью самому\n- поэкспериментировать с включенным, выключенным dropout.\n- Для начала обучить с выключенным линейным слоем переводящим m в Q, R, V – как показывал Николаев на лекции. Потом включить. Попробовать увеличить / уменьшить размер вектора в 2 раза.\n\nБазовый вариант – патч 8*8, две головы, без дропаут с Q, R, V такого же размера как и вектор. То есть, когда в задание написано, что менять какой-то параметр, то меняем его, а остальные берём из базового варианта.\nКак всегда, отчёт и сохранённые веса.\nБерём за основу эту статью https://arxiv.org/pdf/2010.11929.pdf, не забываем про skip connections внутри attention.","metadata":{}},{"cell_type":"code","source":"# # Extract from archive\n# with tarfile.open('./cifar10.tgz', 'r:gz') as tar:\n#     tar.extractall(path='./data')\n# data_dir = './data/cifar10'\n\n# print(os.listdir(data_dir))\n# classes = os.listdir(data_dir + \"/train\")\n# print(classes)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.728779Z","iopub.execute_input":"2024-05-17T01:08:07.729039Z","iopub.status.idle":"2024-05-17T01:08:07.733744Z","shell.execute_reply.started":"2024-05-17T01:08:07.729014Z","shell.execute_reply":"2024-05-17T01:08:07.732761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cat_train = os.listdir(data_dir + \"/train/cat\")\n# print('training examples for cat:', len(cat_train))\n# cat_test = os.listdir(data_dir + \"/test/cat\")\n# print('testing examples for cat:', len(cat_test))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.744319Z","iopub.execute_input":"2024-05-17T01:08:07.744574Z","iopub.status.idle":"2024-05-17T01:08:07.750199Z","shell.execute_reply.started":"2024-05-17T01:08:07.744551Z","shell.execute_reply":"2024-05-17T01:08:07.749363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchvision.datasets import ImageFolder\n# from torchvision.transforms import ToTensor\n# dataset = ImageFolder(data_dir+'/train', transform=ToTensor())\n\n# img, label = dataset[0]\n# print(img.shape, label)\n# img\n\n# print(dataset.classes)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.766060Z","iopub.execute_input":"2024-05-17T01:08:07.766388Z","iopub.status.idle":"2024-05-17T01:08:07.771334Z","shell.execute_reply.started":"2024-05-17T01:08:07.766362Z","shell.execute_reply":"2024-05-17T01:08:07.770318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib\n# import matplotlib.pyplot as plt\n# %matplotlib inline\n\n# matplotlib.rcParams['figure.facecolor'] = '#ffffff'\n# def show_example(img, label):\n#     print('Label: ', dataset.classes[label], \"(\"+str(label)+\")\")\n#     plt.imshow(img.permute(1, 2, 0))\n# show_example(*dataset[15001])","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.778882Z","iopub.execute_input":"2024-05-17T01:08:07.781497Z","iopub.status.idle":"2024-05-17T01:08:07.786528Z","shell.execute_reply.started":"2024-05-17T01:08:07.781466Z","shell.execute_reply":"2024-05-17T01:08:07.785275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## import torch\n# from torchvision import datasets,transforms\n# batch_size = 256\n\n\n# training_dataset = datasets.CIFAR10(root='./data', train=True, download=True) # Data augmentation is only done on training images\n# validation_dataset = datasets.CIFAR10(root='./data', train=False, download=True)\n \n# training_loader = torch.utils.data.DataLoader(training_dataset, batch_size = batch_size, shuffle=True) # Batch size of 100 i.e to work with 100 images at a time\n# validation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size = batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.804716Z","iopub.execute_input":"2024-05-17T01:08:07.805449Z","iopub.status.idle":"2024-05-17T01:08:07.809562Z","shell.execute_reply.started":"2024-05-17T01:08:07.805412Z","shell.execute_reply":"2024-05-17T01:08:07.808628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 install typing_extensions==4.9.0","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:07.832556Z","iopub.execute_input":"2024-05-17T01:08:07.832812Z","iopub.status.idle":"2024-05-17T01:08:21.191339Z","shell.execute_reply.started":"2024-05-17T01:08:07.832789Z","shell.execute_reply":"2024-05-17T01:08:21.190175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nimport numpy as np\nimport copy\nfrom torch.utils.data import Subset\nfrom matplotlib import pyplot as plt\nfrom ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss, RunningAverage\nfrom ignite.contrib.handlers import ProgressBar\nfrom sklearn.metrics import classification_report\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:21.193131Z","iopub.execute_input":"2024-05-17T01:08:21.193464Z","iopub.status.idle":"2024-05-17T01:08:28.200562Z","shell.execute_reply.started":"2024-05-17T01:08:21.193435Z","shell.execute_reply":"2024-05-17T01:08:28.199624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 256\n\ntransform = transforms.Compose(\n    [\n        transforms.ToTensor(),\n    ])\n\ntrainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)\n\ntestset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:28.201997Z","iopub.execute_input":"2024-05-17T01:08:28.202748Z","iopub.status.idle":"2024-05-17T01:08:37.638146Z","shell.execute_reply.started":"2024-05-17T01:08:28.202709Z","shell.execute_reply":"2024-05-17T01:08:37.637343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(trainset,\"---\", len(trainset))\nprint(\"x & y trainset[0]\", trainset[0],\"---\", len(trainset[0]), \"\\n\")\nprint(\"x trainset[0][0]\", trainset[0][0],\"---\", len(trainset[0][0]), \"\\n\")\nprint(\"y trainset[0][1]\", trainset[0][1],\"---\",(trainset[0][1]), \"\\n\")\n\nprint(trainset[0][0][0],\"---\", len(trainset[0][0][0]), \"\\n\")\nprint(trainset[0][0][0][0],\"---\", len(trainset[0][0][0][0]), \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:37.640248Z","iopub.execute_input":"2024-05-17T01:08:37.640598Z","iopub.status.idle":"2024-05-17T01:08:37.736428Z","shell.execute_reply.started":"2024-05-17T01:08:37.640566Z","shell.execute_reply":"2024-05-17T01:08:37.735526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![image.png](attachment:00f954f7-ebd5-431f-9824-ac9d6e9253b9.png)","metadata":{},"attachments":{"00f954f7-ebd5-431f-9824-ac9d6e9253b9.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAvQAAAHYCAIAAACtOOuRAAAgAElEQVR4Aey9e1RT16Lvv/64f93/7j9t/ruXc3rPOI5fz9l1/HZP6xj3ntPN2XZbz267N7ue3lCttVqrxdZafGFEBIIgxhciKKIVNagogmJQlCi+AooGn0ABgag11ldEAiGAznvnWlnJSkhgJVlJVsJ3DodkrTVf6zNn8ON8rMUQBBAAARAAARAAARCIIQJMDN0LbgUEQAAEQAAEQAAECOQGnQAEQAAEQAAEQCCmCEBuYqo5cTMgAAIgAAIgAAKQG/QBEAABEAABEACBmCIAuYmp5sTNgAAIgAAIgAAIQG7QB0AABEAABEAABGKKAOQmppoTNwMCIAACIAACIAC5QR8AARAAARAAARCIKQKQm5hqTtwMCIAACIAACIAA5AZ9AARAAARAAARAIKYIQG5iqjlxMyAAAiAAAiAAApAb9AEQAAEQAAEQAIGYIgC5ianmxM2AAAiAAAiAAAhAbtAHQAAEQAAEQAAEYooA5CammhM3AwIgAAIgAAIgALlBHwABEAABEAABEIgpApCbmGpO3AwIgAAIgAAIgADkBn0ABEAABEAABEAgpghAbmKqOXEzIAACIAACIAACkBv0ARAAARAAARAAgZgiALmJqebEzYAACIAACIAACEBu0AdAAARAAARAAARiigDkJqaaEzcDAiAAAiAAAiAAuUEfAAEQAAEQAAEQiCkCkJuYak7cDAiAAAiAAAiAAOQGfQAEQAAEQAAEQCCmCEBuYqo5cTMgAAIgAAIgAAKQG/QBEAABEAABEACBmCIAuYmp5sTNgAAIgAAIgAAIQG7QB0AABEAABEAABGKKAOQmppoTNwMCIAACIAACIAC5QR8AARAAARAAARCIKQKQm5hqTtwMCIAACIAACIAA5AZ9AARAAATkQsC4fa4jrNI99FEpGme70ctFo9HbWS8RcQoEYp4A5Cbmmxg3CAIgEB0EqLXwTuPTYAjxfslYOHduIeQmOloatQw9AchN6BmjBBAAARAYm4CxcG6azszHM+vSfMgK5IZnhJ8g4JMA5MYnGlwAARAAgYgRoHLjch0qNFxYpdONnJaiwzaOwI/ePNSt4k+5JIkb3nHFTjvunPtynXQfAfKVT5ruOFcoX2DESKFgEPBCAHLjBQpOgQAIgEBECbBKwS+seXg8zSkc7Gdva244b3FUmh44xUWQnDMYXkfYWOxYES3OGV8wMsTG56vBChaXlj3Pz6BFFBQKBwHvBCA33rngLAiAAAhEhAA/SOMctqEmwfsIIcTNe1w1FMYyFjrX7rARnO5CIzklhhBnGo/zfK7O644TzmMf8fl0+AkCEScAuYl4E6ACIAACIDCCAJ2WYp3GfX6KiFhQ7Bjdcc5KsR9Yp3HaCVec61CQxOVR9KT78IxxO+dGroQj6o0TICALApAbWTQDKgECIAACHgQc00P+yw1N6C4lfM4eUuJxSMWJX7xDFWeE3LiNALkkiM8dP0FAPgQgN/JpC9QEBEBgHBPwKTEeCjL2tBQ7DOPVPTyy8jh0wufP8z/5C85j5wf+Cn6CgMwIQG5k1iCoDgiAwDglwFqLc8SF+oNj2Y1QVtjP3hYUu7kRTex60B+9xK3a8ZAS5yFbmHPnlPM0N5DjY0GxV3sap02H25YfAciN/NoENQIBEBinBFi/cayVcS4opiwcTjOXzjd52QrORuH2fvPrhamhOAMvIi5tYQELDlkB4uMLixZWic/GtRJ5nLYTblv+BCA38m8j1BAEQAAEQAAEQMAPApAbP2AhKgiAAAiAAAiAgPwJQG7k30aoIQiAAAiAAAiAgB8EIDd+wEJUEAABEAABEAAB+ROA3Mi/jVBDEAABEAABEAABPwhAbvyAhaggAAIgAAIgAALyJwC5kX8boYYgAAIgAAIgAAJ+EIDc+AELUUEABEAABEAABORPAHIj/zZCDUEABEAABEAABPwgALnxAxaiggAIgAAIgAAIyJ8A5Eb+bYQaggAIgAAIgAAI+EEAcuMHLEQFARAAARAAARCQPwHIjfzbCDUEARAAARAAARDwgwDkxg9YiAoCIAACIAACICB/ApAb+bcRaggCIAACIAACIOAHAciNH7AQFQRAAARAAARAQP4EIDfybyPUEARAAARAAARAwA8CkBs/YCEqCIAACIAACICA/AlAbuTfRqghCIAACIAACICAHwQgN37AQlQQAAEQAAEQAAH5E4DcyL+NUEMQAAEQAAEQAAE/CEBu/ICFqCAAAiAAAiAAAvInALmRfxuhhiAAAiAAAiAAAn4QgNz4AQtRQQAEQAAEQAAE5E8AciP/NkINQQAEQAAEQAAE/CAAufEDFqKCAAiAAAiAAAjInwDkRv5thBqCAAiAAAiAAAj4QQBy4wcsRAUBEAABEAABEJA/AciN/NsINQQBEAABEIg4gcHKfGt8lvuf0oGIVwsV8EoAcuMVC06CAAiAAAiAgJAA5EZIQ+6fITdybyHUDwRAAARAQAYEXHKz7uxAQyP7p2NIBhVDFbwQgNx4gYJTIAACIAACIOBOwCk3fZUP3a/gSH4EIDfyaxPUCARAYNwT6OrqKiwsTItQWL58+cKFCyNUeFpKSsrSpUtfvnwps14AuZFZg4xaHcjNqHhwEQRAAATCTqCrqys9Pf3KlSt9kQhtbW0ZGRkNDQ2RKLyvo6MjIyNjyZIlkJuw97uYKhByE1PNiZsBARCIdgKc2Vy7ds0+ajiXxvxlz71RowRy8e7du5mZmVeuXBkz8bk0xhnSz48ZXVSErq4utVp94sSJzMzMmJEbQ4aTE/2gro/2Hhod9YfcREc7oZYgAALjgYB4s2EY6eXGP7P5W4nDrTpL/sIwwfuN02xMJhPkZjz09pDeI+QmpHiROQiAAAiIJSDObM6l0////+Uvf5NYbsSbjd1O6yC0GTqKk3ZO1OCMj0hCs4lJuUkoNYvtB4gnBQHIjRQUkQcIgAAIBEdAnNnY7+35C6sR90oklRt/zMaLngQpNx5mA7kJrishNSUAuUE/AAEQAIEIExBpNgKtkFJugjQbbiAn4AVAI80GchPh7hgTxUNuYqIZcRMgAAJRS8B/s7Hb7ZLJTdBmQ2vCONffCPxLzEevZgO5idq+LKOKQ25k1BioCgiAwHgjEJDZSCY30pgNkx7YchtfZjOu5IbfS5WgvS/s+wY143FGeNXvz+bSBOeWLefqHx9F+525PBNAbuTZLqgVCIBA7BMI1GykkRvZms24k5sMg3tfN6ipiUgmN6zZqPkyaOZOvyH3tQnSFeR+FxE+gtxEuAFQPAhEPYEnOvWSYgP2gvjZkEGYjQRyI43ZSD0bZeJD7G0Fd/mEez+hwydCualnxWZagnTOYdZOE9gMIdR1pmkd31fIjXtz4AgEQAAEeAJDZn1uQhwTN3tXs40/h5+jEwjObIKVm6DNxk63R4XMbMbxyA0VEfqUv1A6B+Rm9O8mroIACIxTArYzqgkZBk+P6SpPep+ZlGGwjFMqftx20GYTlNwEbzZ29ql9zjUcjg/iXGeUdTYmQRi/IzdcPwqh3FB/co0VhbAgP74RoYiKaalQUEWeIBDDBCy6+UzyKU+3oTf8RK96n0nY0xnDNx/8rUlhNmL2IXmPI4HZeM9Y1FmRZjOOR274/hUy52AXETvX34R2iIi/mcj8hNxEhjtKBYFoJWDTJzNJOh/jM7YLqokxuj5RkvaC2Zw4ccIkImDkRro1N66ey5qN+zrlkFmUq9QIfYLcRAg8igWBKCXQpJn4jsbos/KW8lnMxA2+r/tM6Hmhubn57Nmznmej+birq2v58uX79+9vaGi4cuXKtWvXmpqabt68eefOnZaWlra2to6Ojq6urnv37j148MBsNv/2229Pnz61WCw9PT1Wq9Vms4kaHvERKVrGbExsgNxILjdezIZg5Caaf6Gg7iAAAlISqFe7tlp4y9dyLImZXBzw1NSTJ08qKyuXLl2qVCrVarW3EqLyHDdms23btvLy8kOHDh04cKC0tHTv3r27d+/etWvXjh07tm/fXlBQsGXLls2bN2/YsEGj0axdu3bNmjVqtTo9PT0tLW3lypUrVqxYuXJlWlpaenq6Wq1es2bN2rVrNRrNhg0bNm/evGXLloKCgu3bt+/YsWPXrl27d+/eu3dvaWnpgQMHdu7cuXDhwoKCgqqqKp1Od/LkyVOnTun1+rq6uvPnz1+6dCkw3+rv7/ehUm6nxc9GmfgAuZFWbjxno5zfIYzcOFHgAwiAwLgm0KhWKNSjjcx0aROYZL23NTmjczMYDOvWrVOyYcmSJZWVlU+ePBk9SbRclXA2qr+/32q19vT0WCyWp0+f/vbbb2az+cGDB/fu3evq6uro6Ghra2tpablz587NmzebmpquXbtWXV29dOlSrVZ7/vz5uro6vV5/+vTpkydP6nS6qqqqysrKI0eOCH2rpKRE6Fv5+flC38rOzlar1RkZGULfWrVqldC31q1b5/Stbdu2LVq0SORslIkPkBsJ5cb9OTfuXxrIjTsPHIEACIwDApZO3S6NJldTXN3pchVzeeLoq2r8/HXZ3t6+e/fub775RqlUzpw5s6io6M6dO7EEV0KzcRsPEXcQhtkom80m9K3Hjx+bzeZf2WA0GletWrV8+XKTnwFy4y43ns+q8ecLwj0S0GNzG7+m2PVtDaYIf6oTrrhYcxMu0igHBKKLQJc28a345F06/ZH82eweb95v6C/Bqbt8zzvRX5f8r07ft/zixQudTrdixQpuqCYrK6uurm5oaMh3iqi8EvNmM4picbNRR48eVavVJj/DeJebkZ39vlZdGoIHZbrkhq6/CUkRI+8lLGcgN2HBjEJAICoIPOk093IVpfu9k6p5n2kvnspMLW533IPlWJJCkax3xBxxY/VqhfP5pyMuEkIaGxs3bdrEOc2PP/546NChhw8feosY9edEmc35dP7/1H8p6RxFFfy+FNiYzb09fwn46XzCKjrX2bS2tmZkZJj8DJAbj95vLk2gD/eTPAjkJlRFSF5ncRlCbsRxQiwQiGECQ5bm6vykjycoGGb2EW6Tt1GtmJTfQkhvp6FUnfh23ARlsraJdx3SqZ2hUMzQeh29MeZO9Dqu093drdVqv/vuO6VSmZiYuHXr1uvXr8cwVNFmwzsNtZwA30AptAruc2Bm43g6n7jH8Y0s1HnGaTYmk6m1tTU9Pd3kZxhfckP11n2H9ogvhiFj7NHQEYnGOMGuMnYVHYoixqhBKC9DbkJJF3mDgNwJdJZ/P3WCIm7q/HzdrfJk1/BMp/avTNzbE+LeTkzepWseua6316D+gyJuVnGzx/jNkFHzh9nlgvh9fX2nTp1KS0vjhmpWr159+vTp/v5+uYMJrn6izMZOX2Lwlz33eCe4V/I3Jv08fxTEzwDNxk4r8Je/BTtyIzQbTm7S0tJMfobxIzfBdTSk9kkAcuMTDS6AwHggYGlvtnAjMvVqhWALd+euqcz/ym8WILDcanYbqult1n4/iXkrQXPG7BzSEUQn169f37p1a2JiolKpTEpKKi0tNZlMwgix+rm7uzs9Pf3atWt++gnrFi7X8TM1Hz1Qs2FfF5V2LshpKQ+z4eRm1apVJj8D5CZWvx1huy/ITdhQoyAQiDgBS/MR9ezJ8fGTZ6ur3USFEGLcMNHt4Xu9BvX7ivjU8mazufmMVq2cEPcHtX7Eg4ktTdrkWfnCneG//vprWVnZwoULuaGaTZs2NTY2RvzOw1aBQM3GTq0i6GmpgM3Gzk+KBSM3I83GZDL98ssvK1euNPkZYk9u+JVV7Hsxw9Ydx3FBkJtx3Pi49fFFwGbImDTpe63RbOmsz09UKBLLhJsvjJp3Jmqa3Ik8MWhmxcdPjo+fpdbWex+ecSYYHBw8e/asWq3mnEalUlVXV/f09DgjjIcPAZoNv6ZYMEXFj8P48zNws7Gfo+t92BmxgOXGq9mYTKb29naVSmXyM8hbbqzrzg40NLJ/Osbe38cvbXHoTUjWBY+Hb5ef9wi58RMYooNAlBKgz6dRGfjKm/ckMsK3KLQXx/OHlha99pRwPopP4+PnnTt3ioqKZs6cqVQq586dW1JS0tHR4SNuLJ8O0Gxc+kKnpZi0c64T/nwKwmwcE1JcaYHJjS+zMZlMHR0dK1asMPkZZC438VlWx5/SgVju09F8b5CbaG491B0EvBOwNZcmT32L/k8x7iO1nlveW6+mNmMzG4/kJ30UF/dRUv6RZucsU2dRPDO/WE83Rini3p+tLnNd8l4CIY8fP66oqFi8eDE3VKPRaOrrQ7FX1Vf58joftNmwasHPDfljNTRuMGZjZ4dtnJMm/Ad+D5eIqoxiNiaTqbOzM+Ye4sebTZY1HnIjry+iqzaQGxcLfAKB2CBgLkuMm1FM9WTIos+YxLzPvueSjtwwzFtTk/LKjfc9VgDTt10yb02anaE1eF7yguTixYtr167lnGbp0qVHjx59+vSpl3hjnbrcMXy2eejMnSH97aHa20Onbw2dujlUc3Po5I2hE9eHqq8P6ZqGfn3+aqxsIn89ULMZsYI4ILkJzmw85cXfkZvRzcZkMnV1dS1btszkZ5DlyE3kexpqIJ4A5EY8K8QEAdkSsBkyJkwt4tYId2r/qlDzS3gtjfkJjEJ1xkYI+77uXMHaX0uztszIaY7FPMaSGkJIW1vbzz//PGfOHKVSOWvWrOLi4paWliCJ/J8tfa4RfudQP/shvXxgaDjI7MORPFCzoVZBTYJxjpHQhS/+LruR1mwcVRL9nJsxzcbEhiVLlnAfxP8NuSEIwRGA3ATHD6lBIHIELC3NTiWxnUrm19AY1QqF6lizblcy+/A9tba+08ateuzSJioUU1O0umqtZv7UOEV8cpngpVE+bsRisRw/fjwlJYUbqsnOzj5//vzwsDTeseOs3avcFNbafVRHXqeDMRtuzIT1G8dcUMTNxi+5EWk2JpNpyZIl3d3dJn8C5EZeHT0KawO5icJGQ5VBgBIwat5hkk/xE0w2vUrBbXeib05gFFNVpYZOjyfsEULM+uJc7l2YzZax9nlcvnx5w4YNnNMsWrTo8OHDZrNwg1WwzWDpe12kd5ObP66hqxkOXxkMNuuwpA/ebDznhPw5lnzMxp/C7eLNxmQyLVu2rLOz0+RPgNyEpQvHciGQm1huXdxbTBEYsjTXlxcvSZiQoueMhj5nb1a5c1GwMXeigr1kO6NSKJJ1zgtPjNpFatfhWFC6urr27ds3f/58pVI5ffr0wsLCmzdvjpXIv+vnWoZWlw9wYzZ/yrFyTvPvWdY/5VjPt47lXP4VFarYPT09ixcvzsjI2ByhsGDBgvT09IgUvnHjRpVKdeLECZO4sHz58rt374qL64gFuSEIwRGA3ATHD6lBICwEbE2aqYq4hCUaTUriRCbZYTf0fZaCdx00afhL9JE2dO1wqoq+MeqtqcmlY+9+slqtNTU1q1at4oZqMjIy9Hq9zcaPDElxm3cevNpSY//rRrrO5sNsa/axgcsdwxWNg5zlTC/oa30YBcuHORI9PT3p6endkQupqamRKrytrc2vDVArVqzo6Ogw+RMgN1J84cZ1HpCbcd38uPnoIECnnKbS11jSQGej+Ndb0jXCzrdU0gEb15sviblJrz+j1zc5l+X4vFej0bhlyxbOaRYsWLB///579+75jO3/hUcvXu83DH5T3M9JzGKtTdc0ZOOnnl70vY7Psi7W2p5bX/ufd8RS9PT05OTk2CIXMjMzI1X4ixcvUlNTTaKDSqVqa2sTHZ1GhNwQhOAIQG6C44fUICA5gSGzoVST9PGEOH76iTSOeOvTNC23+MVyLIl5R2XoJWSos3ja1Pw8VdwincjBlvv37x88ePCHH37gtCYvL+/atWsS3s3wK3Lq1lDKARvnNF9t799zYfD+My9jM2urou9JaJAbk+iQmpr6yy+/iI5OI0JuCEJwBCA3wfFDahCQlgB9o1NcQoZW32J2bHEihMqN4OHCpEubIHx99wwFo4iLU8TNLvN8XZTXqtnt9jNnzmRmZnJOk5qaevLkyd7ekWuPvaYWddLYNazRDfxHLl0d/LGmb0P1wPVuaXZXiSo+LJEgNybRYdWqVS0tLaKj04iQG4IQHAHITXD8kBoEJCVg3DBxwhrBo2i4zOm0FPesGva4Xq1gmHjHU20IITZLV6dZhJzcunVr27ZtX375pVKp/Pbbb/fs2XP37l0Jq296+mr3efuXhY7pJ1WZrfb20OtommvyAwbkxiQ6rF69urm5WXR0GhFyQ6IsDFbmCx7czD2zKqKPb4bcRFkPQnVjmwDdAPX+bE2pTn+kWJObr602mtnNQ8YNk5j31XT6iXRqZyTk56kU/KugxgTy6NGjI0eOJCcnc0M169evb2hoGDOV+Ah9A6+rjEM/7XNMP327s/9g/eDjnhiVGp4L5MYkOqSnp9++fVt0dBoRckOiLEBuoqzBUF0QCDMBi2FX8mz6Iu5kTWrS1LcYbnc36zQKRjFhwluK+DUGGzuWM/rrhV+/fn3hwoWcnBzOaZYvX15VVfX8+XMJ76e+fTircoDbyP3Zpr6tp+0tv3pZUiO2xKcD6/b0Gx8FkYPYkiSIB7kxiQ6ZmZm3bt0SHZ1GhNyQKAvD5lvsa9IbBxqO8o8dx8hNlDUiqgsCYSNwRsV8VMwvpbFZWozNTxzLhQ0ZCt57PGvT2tq6c+fOr7/+WqlUfv311zt37mxtbfWMFMRxm/nVdr39c/7NCRlHBi60SrGkxj7YUNY3JcuarLOLmGQL4gakSAq5MYkOarX6xo0boqPTiJAbEr2hyTExHdm3imJaKnp7EGoe4wQsTdrkP8Qllvl4KHCTZqpSy3sPRfHs2bOqqqrly5dzQzU5OTkXLlx4Ld2al+fW14evDCb97PjN9UOJrfLqYE+/1NNP3ba09dbPSgd65T2CA7kxiQ5ZWVnXr18XHd108eLFdevWvXz5knZrhKgjALmJuiZDhUEgHAR69arJ8RMUTNxHSfkXnI8ZHq3khoaG9evXc06TnJx85MiR3377bbQEfl6rax5addjxQOEvtvYVn7Xf/S1o9bAPtd+wVdYM3Bk5D/V4IGu9dcEJ/kk4ftY2PNEhNybRITs7u6mpSWT0ixcvqlSqO3fuhKcdUYr0BCA30jNFjiAQCwRsnU3NYnY/3b17d8+ePd9++61SqZwxY8a2bdtu3bolIYBb94fzaux/2UBn0P+UY11bNXDlrhTTT4SQB7bkbOtXxf1ZBdYpWdbNTZ7Z9l7r+ySrr/KhhHcjcVaQG5PokJOTc+3aNTHRYTYSd9Mgs7uvTXC81JX9wT9ea4xcITdjAMJlEAABbwRevnx58uTJ//dgNG6oJjMz88yZM3a7ZK/RNlteay8NztnhmH5aUmqrvj40EMwwim0ga21/g6uCdGNFsp7L8bVJ1xefPdJjhqq3WT854krjjUQkz0FuTKJDbm7u1atXx4w+Ps3GkMHpQ4L2PtufBT4x+o4BNrbff5lLExjeUTyLHpmZoDK0lnzCkRHdzkBu3HDgAARAYCwC165dy8vL45zmhx9+OHjw4IMHD8ZKJPb64DCpuTm0bL9jR/fX2/v3Xhx88Dzo6SdCes/1xe8ecHmKbSDLOSrzarAs3zol2/rJYdd1rsY01cZ+k9jqhzse5MYkOqxbt+7KlSujRx+fZkMIoYaRYXB0X1YmHE5DP/PG47gc9A9OVoSOIq4UqkSQm6DxIwMQAAE3Avfu3du/f/+CBQs4rdmyZYvROOJBf24p/Du41jm87vjAR+wDhT9Z37fxhP2GyXOeyL8c3WIPn99tTTkneNf3q4F1WX1l1Mpem2v6phTYzE39U7L7znu8NuKBbW7WiJNuOUfyAHJjEh3Wr19/+fLlUaKPW7PxkBvqEE7RYb0nodTHfoJA+r5ZO41JmOYauaF5QG4CIYk0IAACQRCw2Wx6vT4jI4NzmlWrVtXU1Fit1iCydEva/eTVz+fsMwoc008ry2z6OwIFcYsbzIG9cJ21sFmYg33fRlZ3HtgWZPdVPnhNXg1szrZu9lgv9NC2wDnAI0wtj8+QG5PosGHDhvr6el/Rx7PZeMiNR9d2G9TxuOb/IZebcFqK5gG58Z8kUoAACARI4ObNm4WFhdOnT1cqlfPnz9+3b19XV1eAeY1IZh14feza0I97HdNP83b1lzUMPnkp9Y5uV7kDm0c4iqnKGr+tv7DAuSWKrsJxG90hhFC56ZdyhMpVJQk+QW5MosOmTZsMBoPX6OPcbEaTm3o1I+G0FM1NbSAEciPBlx9ZgAAI+EXAbDYfPnx40aJF3FDNxo0bL1++7FcOo0c2tA2rKx07uqdt7is4bW99KMGSmtELJcTXqIw1flO/yVE+HcvZ3OSeU1P/lHybhIPy7rkHewS5MYkOmzdvvnjx4sjoMBvvcsOtjGEEa3GC7a0Gqjb1NBfITbAskR4EQEAkgeHh4XPnzmVnZ3NOk5KScvz4cYtF1BNuxBTxi/nVtlr7f+Y5nomeWTFw8RcJl9QIq/DKfMO2s6x/c6XN+Ng5FDRYWWBdUOOx2+o1HbzJ7ivrHCbklflC3yfr+o1ua25e3zls/UrnkUpYVoQ/Q25MosOWLVsuXLjgER1mw/XgUeae6CXh4l8ugf9/C4uA3PjPDylAAAT8JNDS0lJcXDxr1iylUjlnzpyff/65ra3Nzzx8Rn/W+/rQ5cHv+AcKL9xjO3pt8KXN6Rw+EwZ6YdhYap2S31/dOFC5r28Kt5iGzctc00cHaTzyfTVkrKQvW4jPsk7Z1Ff7wL1idnvhxr7qpx5pZHQIuTGJDvn5+efPnxdGh9k4u7LQPJwnHR/ELYjxTOV5TIdtRgR+H5a4IqgSYbeUJ1gcgwAIuBN4+vTp0aNHly5dyg3V5ObmXrx40T1KUEdn7wylHnIsqZle0L+zzt71ODTTT7ZBs3OEprn/k4397Y5y6A6pKXv47d9PbSnZ1qx6b8NF9uXqqyAAACAASURBVOFea2jqFhTCsRNDbkyiw9atW+vq6pzRYTbC7iWUG+FnGkeceQhzG/MzRm7GRIQIIAACfhMwGAwajYZzmsWLF1dUVDx+/NjvXHwkuHlvePNJ+6fr6fTTlLXW3KqBRjrpE4Lwath8w7ZuG3248JQyx/Npnun74ncN9L4aftYxsLOYPrompWyAXzHDbvkWjOWEoE7hzhJyYxIdCgsLz549y0WH2Xj0VDehoct+HYtjCKE7t4U7wz0SBnYIuQmMG1KBAAh4IdDR0VFSUjJ37lylUjlz5syioiIJX53z0PJ638XB2UWOHd1L99tO3hiyh2JPN70zOpf0Wbb1k4L+yhsD+zZZsy7z/nStLz7b+gn7UoXKG4N2z+GYYWOZdUp2X1kbH98Lp2g6BbkxiQ7btm07c+aMyUTfiIn3Rnn0cje5IYSwfuOYRXI984Z9RI0Uz7yB3HjwxyEIgMBoBGpra/v6+jxi9PT0VFdXq1QqbqgmKyvr7Nmzg4PSLJK1D5GTN4aW8g8Unl3Uv+/i4EOL+8oVjwpJcdh7z27iJpLY59C4nrxnG8jKtqZdEriLffBOm1Cyhttr+j7Lss4tGzCHcOmPFDcpIg/IjUl0KCoqOnPmDMzGa7fylBuvkdgpKrUUcuOZvbiZL6y58eSGYxAYDwTKysqUSuWpU6ecN9vY2Lh582bOaRYuXFhWVvbrr786rwb5obFzOLdqYMpauhT30w19m0/ab94TKEWQubslf93bSqefEjdak0sHTO4vSKCvRyiy9briO+aedt4Y7L1nr67s+yrbOtdzkxQhz+2Ve/r2NYdcwlz1Cs0nyI1JdNixY8fRo0dVKtXt27dD0xpRnKtIuTGXJoTiVVMil/VAbqK4h6HqIBAYge3btyuVyi+++CItLc1kMpWWliYlJSmVysTExK1bt16/fj2wbEem6nr8amedfXqBY0d36iHb2WbhuMjIFMGe6W3q/2x9X2XzYO+9gZ0FVvqeBFeWQ7VFIx67R+jYzNyN1sSNfSmVA+1SvJfKVaDMPkFuTKIDN2YDs/Hahanc0MBvX/Iaib6KgT6CT9ogsmjH03GwW0pa+sgNBGRLwGq15uTkcMMzwr9Xr159+vTpkbNUgd3IS9vro1cHF+5x7H767uf+Q5cHn/WGYeSDPp9m3TW+oAe2BVnWwtv8IfvyS3ar9mv7A3v12YFngd1e1KaC3JjEBZhN1PZxV8UxcuNigU8gENsE7t+/79zL7TSblStXdnd3S3XjF38ZzqxwPFD4P/P6ttXafzF7LtOVqizyeKCwgO5+is+2pujs7GTTwGZqM8PmZptj91OlrZ3f+G2vp4+uqT7bn7LJOmUtHacxu09aSVYxuWYEuTGJCDAbufZf/+oFufGPF2KDQJQSuHnz5pw5c5xO4/wwb9684O+o9eGrgtP2aZvp9NO/Z1nVlQOGUO8wsg1sXm/NujRIyGt7p21BNjffREdu4rOsXxX3Vzd77n66U0Y16Kvi/tqO0E6NBc8zRDlAbkxjBZhNiPpe+LOF3ISfOUoEgXATqKurc9rMyA9XrlwJrEJPXr4uaxict8uxo3vRXtuxa0PWAX4aKLBMfafqvTew7wK/b+taX3yBzTGvZBus3sW+9YmQZ2f74tf133GOFr0aaq+xcS9JsD8efDbOhmo8WEJuTKMGmI1Hh4nqQ8hNVDcfKg8CYxPQ6XQnTpyoqak5ffq0Xq+vq6s7f/78xYsXDQZDQ0NDY2Pj1atXx87FPYb+ztDKMseSmhmF/T+fs3c/cQqFe1TpjuyX+6bw4kKfv5ffb7xhK+Qevlc50P6U3X71ig7eTMnrq7w0UFlGdz8l7rKZ3N4AJV2Foi0nyI3Jd4DZRFt3HqO+kJsxAOEyCICAk8AN0/DGE/ZP2AcKT821rjs+cK0rNDu6X3HDP9RUUs7xs0jsY2k2N7GXmvs/ybJ+to3OMXk+fM8+2KDr31zWv1kX47ufnO0i8gPkxuQjwGxEdqEoiga5iaLGQlVBIDIEHjx/tffC4Kztjumn5QdsNTeHhkJiNXQfU21Nf8rGvjJ2lbNJZ43fxk8/Efrabf71T4Nlm6xfVfLzTK+GzY39KUf5w8hwknupkBuTtwCzkXvHDah+kJuAsCGRaAIPj6fN5UPa8Yfe05l1aXPTdIIHkvDRHhqNPpLwMfAzdAQGBkn19aElpY7ppzk7+ksvDZpfhGpJDXk1WLvLOmVj37oyuqEpsYo1FfaZwpXOXkAHbPpqX7A3/YCuI/6koG/zbsfbFRoehXxqLHS0w5Az5MY0IsBswtDxwloEfX6xIEzTevmHZWSFmhz/c4svHRh5MWxnmLCVhIKCJMCaTaHRkYuxcO5c737jXW4e6lb5iB9ktZB8LAJX7g7nHBv4Uw7defTXDX15Nfbb90MyUCOsiElnnbJ7oJf1E/ooYcfymqHqbdavdPw6YvqsGmvyWX6i6oXd2DjQ0IjpJyFIn58hNyb3ALPx2Vei94IfcjNsvkV/e9A/Rx0PGoXcRG/Lh7PmnnZCXWeVzvmfcFdVIDcuFpH8dPe3V8Vn7YlbHd/ztMMDdaF6oPArM10L3PdJdh//pie6vCarnlcowfIa+nwa5zris31f5fVNyRc+dziSxKKrbMiNSRBgNtHVe0NQ28HKfPr/N7c/GLkJAejYz9JNbqjQOELh8ZHTUlSMHMHpQ3ToxxGcI0BcnkbX5JdzoIiw40ae8SllX/ls13GFOjOP/SZh77Cn/3XF1cEfShzTT0k/95dfGXxuDdn0Exk2llqn5PdXNtrN3PssaTWo3Mw9wY/QkNcN+/jlNewjbT7b1r+vsi9xY7/xxaDxhjPaOGkiaW4TcmPiA8xGmi4V3blAbqK7/eRTe9ZXtnOTVMIpKs41Rq65cRv4YU3FKS6u5JzB8DoiKIJG4eMLRobY+HxZrGBxabl8+ATygRbamlxoHUo/4nig8P/Z0rddb28Pw7KV5v5P1vYbRyyPoWuHN/WbHHfM/d7hx3UeD+zb3ZdVaWsP3Yqf0JKWRe6QGxMbYDay6I6oxAgCWHMzAkk0nDBun+u0DWoSzvEYx1AKLxyuexHKDf3sZh68u7BS4rrizNnjPJ+rME96bqz4fLrY+tn866utp+2fbaLTT39cY11zdKC+nZ8PkvxOWS9J3Ni38xa/UKa7/6ssa3KZrbbRtq+sf2fNwB3OqJ7aUrKtC9iXcvc29Sfu6d+5SbC8RvKKjb8MITcmkwlmM/46ftTcMeQmaprKWVHWbFz6Qg8dQzhsFMHIijMJIUIR4UZ3+Ekpx0/qNE474RIKDl1J+HEdh0a5VIi+OlaXxlqTIKGgCrH18XHP6wP1g9/udOwL+Gmf7bhxqM8eoumnYdMlW8ODgc0brWm6gdojfVOy+2qfckBfP7vWn7WNvog7i90YFc+vvKHv9M6yTllrnZLdV/ngNR3LwfIa6Toh5AZmI11vQk7SE4DcSM80pDl6mA1VDH/lhp0/cpMSvsYeUuJx6BibYWWIVRx+wIdPzq6/8SJJzusx8OHVa1J7e2jFQceSmpmF/bvP2+89HTEtJM2tvnrWYTfTJwIPn99j/Wy9c6OT+6YnYVl2ugGKe6oNPW0dvNM8aOceWEP3gQsuCVPhs/8ExrncLFmyRKVS3b59239ySAEC4SAAuQkHZanKYM3GU0s8FYQqh2tchy/ac+RGMADDRxlt5MYVRzD9JMyTRnDWxPnBLVlMHCxlH1TzH+v61usGjCF6oLAT1Ct74Tpr1mV2kus2fSZN2QPHNcHubmdsQp7bK3dZpxT42v00VL3LurmRn88SpMPHAAj09PRkZWWtiVxYtGhRpApXq9XJyckwmwC6DZKEjQDkJmyogy2IGoP7Uhk+R1ZnHM/04+aPRsqN2wAPm5UrDnUmdtWOh5Q4D92LdjmNWz4eC4qFy4D4isbAzxM3hk7fGhoO0UgNIfY2W0qxjV8ITO4cscYX2XoJIULRIYSwu7sLb3OzYEPnS62Ja+lrulMqB56FrG4x0HzS3kLv+A7SwkRuICAtAciNtDxDlxtnLV4WyrBFuq562woujOIY+GG9hM+NFxGnzXC3ITxkB434+MIlPq6SXQ8JFCYMHZHYzPmpLVk4eUTXC/dVs8trTFVuL0+gO8D3DHAzTr337O2PQraKOTZB465AAARimQDkJpZbF/cWhQSGz++2fnLE+V4nuovb8UxhgejQ+7rd/9U21xhPFN4pqgwCIAACoSIAuQkVWeQLAoERsF/um8I/RJgQInhije91xIGVhFQgAAIgEKMEIDcx2rC4regl4La8hr6v+5N11sJmej/Pbg8Yw/BgwOhFh5qDAAiAAEsAcoOOAAJyI/CariNe31fdbK8t60ssG7hzqb+2E0tq5NZMqA8IgIB8CUBu5Ns2qNn4JfBqsHZP32frrMll2P00fnsB7hwEQCBgApCbgNEhIQiAAAiAAAiAgBwJQG7k2CqoEwiAAAiAAAiAQMAEIDcBo0NCEAABEAABEAABORKA3MixVVAnEAABEAABEACBgAlAbgJGh4QgAAIgAAIgAAJyJAC5kWOroE4gAAIgAAIgAAIBE4DcBIwOCUEABEAABEAABORIAHIjx1ZBnUAABEAABEAABAImALkJGB0SggAIgAAIgAAIyJEA5EaOrYI6gQAIgAAIgAAIBEwAchMwOiQEARAAARAAARCQIwHIjRxbBXUCARAAARAAARAImADkJmB0SAgCIAACIAACICBHApAbObYK6gQCIAACIAACIBAwAchNwOiQEARAAARAAARAQI4EIDdybBXUCQRAAARAAARAIGACkJuA0SEhCIAACIAACICAHAlAbuTYKqgTCIAACIAACIBAwAQgNwGjQ0IQAAEQAAEQAAE5EoDcyLFVUCcQAAEQAAEQAIGACUBuAkaHhCAAAiAAAiAAAnIkALmRY6ugTiAAAiAAAiAAAgETgNwEjA4JQQAEQAAEQAAE5EgAciPHVkGdQAAEQAAEQAAEAiYAuQkYHRKCAAiAAAiAAAjIkQDkRo6tgjqBAAiAAAiAAAgETAByEzA6JAQBEAABEAABEJAjAciNHFsFdQIBEAABEAABEAiYAOQmYHRICAIgAAIgAAIgIEcCkJsIt0pZWZlGo9kgg7CeDTKoyIbc3NwrV65EuGFQPAiAAAiAQNQSgNxEuOmSk5M3btxYIIOQkpKSmZkpg4oUqFSqn376KcINg+JBQCICWq02P6IhJSUlouXn//jjjy0tLRLhRDYgIIoA5EYUptBF0mg0d+/efS6DcODAgbq6OhlU5Pnhw4dVKlXomCNnEAgbgZKSkj179nRHLmzfvr24uDhy5Xfv2rUrOTn5+vXrYWOOgkCAEAK5iXA3gNyM1CnITYQ7JYqXiEBJSUlFRYU9ckGr1R44cCBy5dsPHTpUVFS0Y8cOyI1EfQrZiCUAuRFLKkTxIDeQmxB1LWQbWQIwG85sTCbTzp07ITeR7Y3jsHTITYQbHXIDuYlwF0TxISAAs3GaDeQmBP0LWY5NAHIzNqOQxoDcQG5C2sGQefgJwGyEZgO5CX8PRImjrLkxa6cxniHDAGSSE4DcQG4k71TIMIIEYDYeZgO5iWBvHM9F+xq5gdyEqVdAbiA3YepqKCb0BGA2I80GchP6focSvBAYU27iVWV6/Rn2T4vFSwY4FRwByA3kJrgehNRyIQCz8Wo2kBu5dNBxVo8x5SZBe3+cIQnv7UJuIDfh7XEoLSQEYDa+zAZyE5IOh0zHIgC5GYtQiK9DbiA3Ie5iyD7kBGA2o5gN5Cbk/Q8FeCMAufFGJYznIDeQmzB2NxQlPQGYzehmA7mRvs8hRxEEIDciIIUyCuQGchPK/oW8Q0sAZjOm2UBuQtsFkbsPApAbH2DCdRpyA7kJV19DORITWLNmzYYNG2ojF3JycnJzcyNXfq1GoykqKjKNFfCEYol7HrITQQByIwJSKKNAbiA3oexfyDtUBHp6ehYuXKiLaFiwYEFEy9d99913nZ2dprEC5CZUvRD5+iYAufHNJixXIDeQm7B0NBQiMYGenp6cnBxbRENaWlpEy7epVCrIjcQdC9lJRAByIxHIQLOB3EBuAu07SBdJApAbmw1yE8keiLJHJwC5GZ1PyK9CbiA3Ie9kKCAEBCA3kJsQdCtkKRkBieXGkMG9kMrj0X8GNeNxJqgb4EuhZanruawMaq7kaVpzUHmHOzHkBnIT7j6H8qQgALmB3EjRj5BHqAiEQG4836/JaYdkckPNxmkw97UJLr8h5tIE16VQEZM4X8gN5EbiLoXswkIAcgO5CUtHQyEBEgix3NSz4ynTEhIkG7mhqsSP1tB7pq7D69S4lxtzTcq7TErNSF0Qc+bAgQN1dXViYrridDUUff/xu39HB83emDA5aUdDl+ta4J8OHz6sUqkC7NFIBgJhIQC5gdyEpaOhkAAJhFRu6KvFqYjQ8RXJRm48bhRyw0tEV13W5DcYJnxyc79G9XuG+buPVTsOVh49WLSIlv5uSo2Zr1DAPyE3Hp0chzIkALkZj3LDzhVwKyCcfwv/sy3DjjpuqxRSueGphlBu6EBOQqljmc24Hbkx367M/PS/O75s4Rq5adoymWE+LrrucpimTZMZZnLeVdeZwD5BbvhvDn7KlwDkBnLD/cqF3MjzWxrVckNHhoSLbMar3NSo6LTQvyaV1hR9GraRm66D8//hjT8XNQn95XLuuwyjOiU8FchnyI08f1mIrVVvp7HLJojc2XzLIjiMkY+Qm3EtN5NV5Wf0evZP85MY6dIxdhvRKzes2TBqg6BBxqvc1GX+LbOy9fnz561hlBsv1tK1ZzrD/HPmOS+X/DoFuRF06qj72KmdETc1z+iym/vliYpJ6nrXiai7Ja8VhtyMa7lxbmrx2jlwUgYEolRuvJgNIdgtFVG54Zbg/DHPbSzHL6nhI0NuZPCbIbAq2AwZkxRL9B4iY7ugmviOytAbWJ4yTQW5gdzItGuiWiyBaJQbz9koZ1OO15EbXgoiOHJzvyHv8zeYNz4TLsFxVsvfD5AbZ5cmT4za3KSEyfHxk2cn7zIIZnds5iadNk9TfMRgqNcZ77MpnjTrzzQ741ha9PoW5xEhZmP5Lo0mT6tvF5yk6WyWFoP+jLFTKB+9Fvfgpiu2J82GM3r3uSe2Ak2aSYpkvTAfx53Y9EsUE3ONjqOY+AG5gdzEREeO2ZuIPrlxe86Ne7tAbiIzLdVVo/rf1GzyLge/U4qKEOSG79dGzfuKxDx9s9nS2aTXnTLyVtKpnaGIm5asydVoUmfHK/iHI9AnL7gmaoUbCW316kmKqapSna5UnfCWIrG001FEr0H9h7gJymTNksQJirjZZY7zxrz4eGpU9M+kt4Qr22yGNfFxbycm5yYnvq2Im1XOZ0QlSZ+iiC9o5ivv/rNJM1Gh8hzScY8SXUeQG8iN2B6r/4AcEvxpEZsu8vEeVbjVvKoi8lUSXYOIyA0denFucRJdVTait514zjXFArkJogj/KhRsbEkf4heJaanrRdP/jmF+P+egYNuUv0M1HvEhN45eRXt7fP6IX4WWI7OZaVreKmhXd+zX8Ck3Rs07iuRT/OgL9QzOgaiOTEw1OC60F09lknS8QDnq0KUVrpixnVEpXBNMncUfMUnH+AQ2fTITX9zu6xthVCsY1QVfV6PvvIRyYzm3+j1mdZ3/78AM8MWZj5p2L/r0vb+ne33e/P+mLCxpeuR/0VyKcffiTOc/QH6tuYHcROL7HRa5GXlj97Vqfv/2yIsBnxHIDX24TiiKCLhuvhJGtdyYz2VOfoN5Y0pm3X0PPwnqEHLD9xY6TKJQTEhcUqxzTTBZdPOF/zfoLJ7sU24c/4VoL45nEjTVjs0d+jJVvOO5UwYVM0F9xjkBpVO5PyGT9BrU7wuGeQgxpDITMvSuBCmuR2gSWkqy77GZqPn/Bg9/jJ9Syc2jSzlT3vx/nhEuuXlSt/pfGObvP11dUnHiZMXuJbTw91bVWQLyG8jNGL2Eu8zJzZWHoiLLM9KVJDqEg5GbMVvHXJoQimcDCOUmREWMeWv+Rohiuble9NkbzBufu+8GD8pqHIkhN8JeZLtv0OYmTX2LUczgRms8LEHwzO4RIzcOuaHnpyblaug0luNPMV2/Q/8bqpjwB9cMVPzk+PxGZ+E2Q+pE99XBtGjF227x4/P4lTTupTtz4T94VJs/HbU/JZAbS/eJNZ/+D8fzqcIkNy3bpzDMp7tbXS7TsnUKw0wpvO06I/4T5EZU/4XciMIkcaQQjNzQ7+oYzyM2ZLhWBkh0Q9wbrFyLA0JQhEQ1dc8mauWm6+DXbzDMP3z8fWZmutufg3iIn3sTS3PUq0tiFGpqHjb9Eia+iJ+V6hK8Xs1NLzpdk7/m8kTvX0mDipmoafJewc7SRMX7ao8tToZUZuIG3mY80tGajDJy06n9K5NYFl2vtfW4Q7fD4OWmbhWdF/rg+4q6nZ+Ga+TmUcX3//jmJ7tbhP5yfeN7DLP6nPCU2M+QG7c+4etgVLmha+O8/KMp8dum6f/8+eBcE+KjaG+3gZEbb1RwbjQC0So3XQfn8F8Vj594iN9o7e3XtXZdfqlOf8bY3GLQFSVNVDgWxNA1N+8k67os5iZt0h8mTXqHn5aiIzGT1GfMFktz+ffxCoVz9sqim69QzMg3sM/Ws1ks5vsWdp0NXXNDB4T41ThkyFE/dgFyorbLs7p0zY0iUdvuTECcSQgZTZXIGCtyPAuS/3HwctOQ+3nOyW6bzdZdEja58WItjw5+xTC/y7ns5dKYpyA3ojrqmHLDvx6Rz437v/oYYwR85LF/smbjHFBwe6y/2JcjQW7GxowY7gQklZug5oQCeXFmUAX6TIxpKUcfua/Pz02eTbcsJSTlao2uB6Fa9BlTJygUEz5O1rbo1c7dUoR0Hkue+hbDvDU1+UinIS8++Rg/UtLbXJ4xm+57Yth5pRR+3XCvMV85QcFOTtGrCjU7LEOnkNyD8zejzZiXSBO8HR//fhxNIJjGosuTfez3pkL2UTE/3OT+HYjOo+DlxqkOkZQbbgnOnwrdxnKcNRvrA+RGVOf1S27o+CvDSPm2ac8ZYeH6DciNqBZEpAAIQG5GOg7kxp+OJFhz408yt7g2domwl+fTuMUSHHAJBOM33DW638rbfzdtRvX7gu1agoyi92MsyM2TpsLEN5k3PxcuwRnLZ9yuQ25EdWA/5IaKCF2QSkdhvX2VRJU3RiTIDfc/t3hVGb/JwrVfYwx2uCyeAOQGciO+t3iLKYXceMs3sHPGDZP4hc/ODOhjiydl8BvOnaej/EPUy82jutX/Rs2m8HpgO6Wo5UBuRPViP+SGzy+EcsOOyzonwkQWFIvTUoLBaScOnj9+Bk8AcgO5Ca4XGfPdtjgFl5kEqekDBqc61zvTl6Ikxjn2eUmQu3yyiG65ad391d8zzL/MqxBsm3IbkxF3ALkR1SHlJDfsImLnLLPoISLIjaiWRiQBAchNLMnN/fvcexAEDYyPRLjoOHZwRK/cWC7TJ+u8+R85DU/EKYzvWJAbUR1aNnLDmo37bNf4G7kR1WSIFDwByE0MyM3z58+rq6tXrVq1cuXK4LsEcogKAtEqN627P3+TeTPRfTe4b30Z/QrkRlRflYfceDEbMh5HbkQ1GSIFTwByE71yY7fbz507l5ubq+RDampq8F0COUQFgeiUm0cVc99kmH/8dFFOzhq3PxV4iJ+YbkfHOdgQwOsXfDyhmDrHyCUfIgdUxNSZjeM5G+VMKLKgGJqWct46PoSWAOQmGuXm2rVrW7dunT59ulKpTExM5NwmMTFx8eLFN2/evHXr1u3bt+/cudPc3NzS0tLa2vrLL7+0tbW1t7d3dHTcvXu3s7Ozq6uru7vbZDLdu3fv/v37Dx48+PXXXx8+fGg2mx89evTbb789fvz4yZMnT58+ffbs2fPnzy0Wy4sXL3p6el6+fNnb22u1Wvv6+vr7+202m91uHxwcHBoaGh4efvXq1evXr0PbZZE7SyAq5eZRxTzBKkrhRzzET1S/jk65cX/OjfuNQm7ceeBIMgKQmyiSm0ePHh05cmTx4sX8SI3bT6fluJ2N3EFiYuIXX3wxffr0GTNmfPnllzNnzpw1a9bXX389e/bsOXPmfPPNN3Pnzv3222/nzZs3f/787777LikpacGCBd9///0PP/ywcOHCH3/8cdGiRT/99FNycvLixYuXLFmydOnSZcuWLV++PCUlZcWKFSqVauXKlampqatWrUpLS1u9enV6enpGRkZmZqZarc7KylqzZk12dnZOTs7atWtzc3M1Gs369es3bNiwcePGTZs2bd68OS8vb8uWLfn5+Vu3bi0oKCgsLNy2bdv27duLiop27NhRXFy8c+fOXbt2/fzzz7t37y4pKdmzZ8/evXv37dun1Wol+wYGmpGEcjP61M8oVwN8ceYoOfp5CdNSorqPBNNSns+qEVWuIxL3SEChyjIMw68pdsnNqEVg5MYf4ohLCUBuokhunF22ubm5uLh41qxZSqXS6TTcyM2VK1cuX77c0NBQX19vMBguXbp08eLFCxcunD9//ty5c3V1dWfPnj1z5oxer6+trT19+vSpU6dOnjx54sSJ6upqnU53/PjxqqqqY8eOHT16tLKysqKi4siRI+Xl5YcPHz506FBZWdnBgwcPHDiwf//+0tJSrVa7b9++vXv37tmzp6SkZPfu3T///POuXbt27txZXFy8Y8eOoqKi7du3b9u2rbCwsKCgYOvWrfn5+Vu2bMnLy9u8efOmTZs2bty4YcOG9evXazSadevW5ebmrl27NicnJzs7e82aNWq1OjMzMyMjIz09ffXq1WlpaatWrUpNTV25cqVKpVqxYkVKSsry5cuXLVu2dOnSJUuWLF68ODk5+aefflq0aNGPP/64cOHCH3744fvvv1+wYEFSUtJ33303f/78efPmffvtt3Pnw7f2lQAAIABJREFUzv3mm2/mzJkze/bsr7/+etasWV999dXMmTO//PLLGTNmTJ8+/YsvvkhMTPRlht9++62zLSL1AXIjfit4dnb29evXI9VSUpYbtpGbkZUO0augXXJD19/4fNs05GZki+DM6AQgN9EoN1ybvn79+uLFi+vWreP+DU5MTExPTx+9uXHVLwKvX79+9erV8PDw0NDQ4OCg3W4fGBjgBhf8yicUkSE3IuWmqqrqhx9+OHfuXChaIdx5Rk5uQvUqaIHcjFbESLnhUYTiHdiSNKuvF2dKkjkyGZsA5CZ65cbZuk+fPq2qqkpJScnIyHCexIfYJgC5ESM3VVVVmZmZ+fn5GLkhoywoZiLytmnCrjJ2FT3a26YhN7H96ywUd5eamrpmzRqNDMLixYtXrlwpg4pouOUdoaAd6jzb29tDXQTylwkByM2YcsOZTVNT086dOyE3vuRGJv15jGpAbsYAhMsjCGRlZTU2NrbJIOzcufPYsWMyqEhbSUnJ0qVLR6DCCRCQEQHIzehy4zQbk8kEuSGHPog1uaHfRaPmHfZNWDL6XrqqMj6npbysHk8o5d+f7IITjk+YloqBaalwdBSUITMCkJtR5EZoNpAbMupuKZn1ax/VGTlyQwh5Uj6biS+W62g15IbOdtJZR8jNgQN1dXUjVSP8Z/BWcB+/YHBaRgQgN77kxsNsIDcxKjcWQ8Yk5qPiThl9Kd2qMj7lxtJ8xvGq8/LUeMgNpy8HIDduXw0cgMBoBCA3XuVmpNlAbhxyc+gDOjl16APSMlq/kte1RxWOOnM1r6pwVa/XoJ6m0j9xnZDbp/EpN65WYB/diJEbqjeQG1e3wCcQGIsA5Gak3Hg1m9iUm8mqcv5/yM1j/gPPTUvFmNyM9QWJ+HXIjeNVIZiWgtxE/NuICrgIdFWTgReuQ/l9gtx4yI0vs4lNueEG/Nm/ZfugF/l9acJaI8gN5MaxqAZyI+qbZ+8h/Y+J7RkZsBD7SzLUR4YHyKtB8vqVqOSIJJJAZxUdD29IJ79eEpkizNEgN0K5GcVsIDdh7pkojiMAuYHcQG78+W1g3Og2Ce0caqYf/kDK/0iOfEgqPiKVfyZHPyVVfyXHPyO6z0m1kpyYTmpmklNfk9NzSO23RP8dObuA1C0k534i55eQC8vIxRXEkErq00hDBrmcRa7kkKu55Np6YtxEmvLI9a3k5jZyawe5vZPc2U2a95JWLfnlAGk7RNqPkI5K0llFuqpJ90liOk3unSEPzpFfL5CHBmK+TB41ksdG8uQGeXqbPGsmz38hLzpITxd5aSK9D4jVTPp/I7anDl0btJJhW+R17e5RF+fjn5FbReSFvFYuQm6ccjO62cSU3PjzqwJxI0tAYrnxeOIh4Z/QTN/TVS/9nQpeFs/v7vbrTfSEYM2NQ22w5kZk92zVkuY9VC9u76T/4t7cRrWjKY8YN1IRuZpLrmRTNWnIoJpiWEmV5cIycn4xlZi6heTMAqKfT+Xm9BxyahY5+SWVnmol0f0n1aBjfyVHPyGV/0H16MiH5PC/U2Fy8yd+TWLYTh7+d1L+IamYQmt19BNaw6q/0dpSXfuC1r9mFqtrc+l9nVlAzv5Azi2i90t1LYVcWsnqWjq5rKZkGnPJNQ1lxenajULKkOraz5RqC6drZaS9nNSne7nxuoXU4YZsItsqpNEgN5zcHD16NDMzs6mpyeQ7xM5zbkLapZC5pARCIDcZBkcNWbNxOI3gBRaS1b9eTWc8ncVxpgK5cdqKnx8wLSVZz5Q2o9ev6DjKsI0M9tGJsAELnRTrf0xHXHofkJf36BjMiw5iaSPPWujYzJMb5HET+e0qHbN5aCC/XqSjOPfPEFMt6a6hozudVeTuUTre03aI/HKQtJaSlr2sru2iI0M3t5EbBaRpCx0xorq2jjTmsLqWSepX07GlSyvIheV0tInq2o/k7Pd0FIrq2jd0XKpmJjk5nVQn0vGq45+RqgRy7FM6jkV17U90ZOtwELp2OJ4aT/kfaZV+uyotZn9z6+npycrKehjRoFKpIlr+wx9++CEjI2N0s8HIjb9dC/ElIRBCuaGDIgLzMGRI+ywZg5pJSJgGufFTYXxHh9xI8o1CJmMToLo2RNcqDXG69sKha32PSO+vDl27UeAaueGc5tAH5OJyKmdDfWMXEfoYPT09ixcvzo9o+PbbbyNafv78+fOvXbtmGitg5Cb0/REleBIIodx4FCWYQvK4EsChWTuNqpJHnlSnMHLjW19GvwK5CaAjIkmoCLSWuuSm7kfSUUEFSE4B01LONTemsQLkRk49d7zUJVxyQ6eQErT3pcHqlBjIzei+4tdVyI00vRO5SEKgZS/RzyOt+4n1V0nykzwTyA3kRvJOhQwlJBB6uXGuKRZMUQV1A4LlO5Abv/Rl9MiQm6C6JRJ7IdBLrlSQXi8XPE/1GsmVNreTMtsb5VY39gByA7kZ2StwRj4EQi83/L1SEfFzzohPKvzpmJDiTkFuRvcVv65CboT9DJ8lIHAliRzf6yk3gw/J7QZitrvl32skFR9G12uTITeQG7c+jAOZEQif3LDbwoOemXKOAwkeEMkwam6DlnO6SjxkmoQNeEIx5EZ8twlHTEsvGRxRjq2XWHqJdcR5GZ5o20QOLSPPBTUbfEauZJLD7G72Kw8FF9iP9/aSQ0lkxGnPaLI5htxAbmTTGVERLwRCKDcewyrSyI37LXgUAbnxa6jGIzLkxr1zhfKoZRM5tMlVgP4Dojeyh0b6D3xzLan8kF1OqyQt/KTO8ENy4s/k0J9JZQK9JJSD+7WkdhmpXEQuFJNanSPb1oOknsuTENJL6jeRVj6rwYekPpNUKsnJAtLJn+SSdehI3SZypYH0OGtnJ20NdKzF+adbsLC3p5tcKSZ1e0mHh5V0k2MfkKuCmISQm5mkqoDcbyNV7vV3FlX3Z3L6vPNI5h8gN5AbsV10/Lxbins+i9vQQ9AjGmIpe8YLodwQ9j75Z/fR6SThznDPigR0DLnhBMV8uUT1t394g2GYN/5h8vdFDV0e3iLqMDi5qcv8PcOk1IgqaaxIhw8fVqlUAfWIKEk0mtx8SI4sIg/YGzHMJMf51/A2LSOVxWSYv0HnoA7N6htiZOXDsMzlTFeSiOsVvg9dPjH8kBz/kJzWkccPyfVN5FACaeNmiHqJPoEcyyXXa4n+G8EgCitGdZuo9NSx4y5neWd6WEGTX9ARYzG1MYefsTVsL6A1cdaWq7XjUFAZ/m4cP+8fJIcWuQ32eESQ0yHkBnIjtj9CbsSSkjJeKOWGEM5vHBrnWlDstm4mmLuB3FBPuF702RvMG1OSi8oqD+5Qffx3DPN7Vc39sQxixPUg5MZck/IubWXIjcjePFJuDN1sUiMdlWnmc2nZ5BKU66lUODrcx0JINzn2IbnJj748qhhNbhrZtLcyyZECl3YYlKSGHSyhOpLpWh9T92c3WeFqdCWJVGzi4zwj1R8QR7UJea6j1XvE1/z8n0ltA3/g8dO33JDb5MgH5Kb7chyP1LI5hNxAbsR2Rk5uhKOtYlPKJt6VJPqryfX/JR8Ve9Ksd7wsvVw1mfuXPyZHbnzcPj19X6suNY9yPbBL43Jaquvg1wzzT6oaM28rV/MmM8xnP7fyx2J/Biw35urkf37j3Xf/CXIjutuOlBvHLz5Wblr4fIRyQ3odC1YOzyQXah2G8bjCTSk62PESLvXIkRsu2/N/JpXL2GEYdjDm+J8dv7AMSjpn5Jx7qlV6/iKjdRasiek/T+fI6p3TVXvp7z5HzXvJiQ9Ig8dEFX9TZBS5GeWSM7lcPkBuIDdi++L4kRsXEXauhurNOJMbc2kCP13lghH8p3EpN3W5H7z7r1l1LoXpOjiHYT7eES65oeNG/5xc3VD0KeRGdBf2kJta5xqUUeSGzXzYTrpr6WjNiVp6TIdqBOMlwmx9yc3ZD8mJvS6Jud1A2lgL0X9Ajgmkp26TYMkOIdxupibBuBEt+s+klpuu4v92PMhqdEcZ5eool0SzDVdEyA3kRmxfg9yIJSVlvBBMS4mQNUOGY3+TdLeCF2dSvTG31xXNfZd547OS2y7bEfkpoJGbpqLP33g3pcb8vBVy40dnpkMsqY7Rl+ZNggXCY8kNV8ZNdmqJEDLcQA2jnTvLLppxrlMWyg1dHMMPq3is3XFWulG4Rsd5lv3ALdMRLqmhp43k0If8UI17fEKI/gNy/vaIs9wJ9h69D9F3k8oPSBM/y+YjvUxOQ24gN2K74qhyQxdXCP/RFOwIDvUQgGfRo9yPyGkpVw4xN3LjurUo+RRDW8FrVNx3hJmcec45RyVSbGi0AOSmLutd5veZ7KgR5MafHs/pwhEl3fp0rIDUiBi50X9IKpSkehmpUtJlvE3cpI+dnP6QHNtEbteSGiU5nuRac9NVTKNdaSDXi8mRBLqWhZsz6jeSig9IdTHp6CaP22jCDlYmntdSWblgpPvMh+3E8pA8Ey40LibP2F3oll7ykl8Tcz6BVGSS7md0BY+1lzwQzEN5VaVBO93HbqmlpmVoo589Vhx7LNzxh2j440JuIDdie92YcuNckMqaTQjfNs2Zk/CBc/SMiJkjyI3YxpZNvNiRm/sNlWWVlXvykqa8wTDvqk757Tf+yo35lOrdNz4rus75E+TGzz49bCcdDXQHNWcGNj658PE2g3aXSdh6yYPbdDqptY04IxNCBrtJ3TJStYxcbSN0EEiww7y5mJrQyWLy7BndCu58+UlPt2MreOU3dF6pg5eVLh2pZveZH2JFyrFO+TbdNC78U8PvNh9+Ri4sI0fYXeuHE0jVXv4euPXFM8k91wn6ic6asQ+5cf7tMX5TP5NU85m7J5XhEeQGciO2W4qWG/rvkVN0CJH6bdPsVp5p7m9ghNyIbcVoixc7cuMao2nK+yPD/BM3oOI6O+Yn/+TmfmXSG298tqOJzxZyI4OuL1xzE/HqnE9wLAwSWRM6dJRAOkTGjnw0yA3kRmwvFC03Hhl6bAf2uOrvIZcb/ScPIzf+sovG+LEoN8+bCj9mmI+L/Fx245fctO742DEJNuJHAGuZeUNy/Iz959yE6KsiK7mh81+i36jAzdN5DOSEiJJE2UJuIDdiu1JgciPp26bZx7LQpa6QG7GtFu3xol5uWg/OmfDG9D1uj+1r2vSvDPNZiZ/7pfySG/PtusqjlYI/JckfMMyXuZVHK+tu+z0jBrmR5nt0v5bUsbuopMku6FweVpDKXPJSRD7GZV4eqyMiXQSjSCA3T5oOrPr8H99kGObNf/yPhbuvPrL5GdLS0vxM4RG9IedfGGZVncdZ8Ycqlaqzs9M0Vti5c+f169cj2FgRLtpfueFWxjASPvaWbrjhlvJAbiLcGcJWfNTLzfO6zH9imD/mOeeHnt+vUf3e/YyHO/g49EtuRuSBaamw9VkUJAsCQctNy+7EN5k3pywrqThRuXt1wv9gmPdWn7OIFwubzRac3FjqVr1HB14hN6HuUP7KDV8fOpEknELiz/v7Uzi9Bbnxl160xo9+uXlOF/YyzH//VFVEFxTnTv/fbzCudb4jJMT3CchNtHZi1DsSBIKUm0eH5zHM7wQ201L4J4ZJPNDtj90EIzcW/bLfvfnee7+D3IS+9wQqNxK9kJF/TorbEgJ+hxQWFIe+/SNTQgzIzfPnz7suFSVN4d8t9XVupZ8TUpzwQG4i0wVRanQSCFJuGtZ/8N6/5TS4VOZRxVyGSdgdJrlp3f35m79bpm/anQC5CX3/Ey03wiEWWi2R5uHPHWDkxh9a0RfX0ux4/4W+PDWe09mEELwOQgwYjUZz9+5d3+Mp4bsSnNxIWU8sKBbTcxAnsgSClBuX1dhsNsujhpJ57zFvfq71y20CnpaiM2Lvraqz2LohN+HoRaLlJgxvm4bchKPFI1eGl2E6yA3kJnIdEiVHHwGp5KZulWO2YEpug38rbgJdc9OQ+x7zL9ygEeQmLB1PvNyE/m3TkJuwNHnECoHceBlogdxErD+i4CgkIJHcWJpOVpw4eaDw+ylvMgw7muI2pjP6QQBrbiznVr/35ue7W7mMITdh6Xl+yY2vGoXmbdNiZ77whGJf7YLzvghgWmqkZ2FayldvwXn5EJBIblz20rJ1CsP8Luey68yYn/yWmycnFr755uclLXzOkJuwdCgp5CZEb5uG3ISlB4zLQiA3kJtx2fGj/qYllxtb6+5PGebTEj+W3fgrN90ln7rtmBEc+FUu70Y2POdGVD8eU25oQ/Dbl3zkGIK3TdPXO7BhjKJpjTBy46NdcNonAcgN5MZn58AFGRMITm66K+b+45tfHnB7bN/twg8Yxq81xf7KjaWj4cTJE4I/B5b9O8N8vfHEyRMNHf4u+KGGA7kR1UNHlRtROUQ8EuQm4k0QdRWA3EBuoq7TosKEkODkxtaQ+zuGmVJ42zkIwj1ST3jGecnnB3/lZkRGmJYKS1+G3IQFs0chjMcxDsNMAHIDuQlzl0NxkhAIUm5sT+pW/wvD/P2nq0voguKNX3/wJiNcDTPCQ7ydiBa52bFjB16/QKLq1Wme3xGM3HgSwfFYBCA3kJux+giuy5FAsHJjs9keNe1eNIV/t9S8jSf9WG3DqU60yM3GjRshN+TQB44/LXLsz97r9KjCVe1DH5CqCu/RvJw1a6eJXtDjJbkEpzByIwHEYLKA3EBuguk/SBspAhLIjbfBGL/OBS03fpXmJbKYNTcNDQ3Lly9vaGiIVEtFvlxuWgpyE96WgNyEl/eI0iA3kJsRnQInooAA5EbMguKGhoZVq1bduHEjCloUVZSSAEZupKQZlXlBbiA3Udlxx32lITdjyg3MZhx/SyA347jxuVuH3EBuxv2XICoBQG5Gl5v6+nqM2URlz5am0pAbaThGcS6QG8hNFHffcVx1yM0ocgOzGcffDO7WITfjvgtkZ2ffvHnTJINQUlJy4sQJGVTEtG/fvuXLl4/7rgEAsiYAufElNzAbWXfcMFUOchMm0PIt5sCBA5mZmTkyCGlpaRkZGTKoSE5ycnJxcbF82ww1A4GgH+LnZeuR/6dkuFsKZjOuvxxPmvVn9OyfctVkbAUf130BNw8CIBCVBDByM3LkBmYTlV1ZwkrXqzmjEfwt4sVVElZAkBW2ggtg4CMIgAAIiCMAufGQG5iNuI4T07EgNzHdvLg5EACB2CcAuRHKDcwm9nt8tN0hRm6ircVQXxAAARkQgNw45QZmI4P+iCp4EoDceBLBMQiAAAiMSQByw8nNpUuX8DybMXsLIoSfAOQm/MxRIgiAQNQTgNzYbLbFixenpqbi7QpR35tj8QYgN7HYqrgnEACBEBPo6enJzs5Oj2hYuHBhRMtPX7JkidFoDDFpZA8CgRCA3ARCDWlAAARAYABhYADdAATkSQByI892Qa1AAARAAARAAAQCJAC5CRAckoEACIAACIAACMiTAORGnu2CWoEACIAACIAACARIAHITIDgkAwEQAAEQAAEQkCcByI082wW1AgEQAAEQAAEQCJAA5CZAcEgGAiAAAiAAAiAgTwKQG3m2C2oFAiAAAiAAAiAQIAHITYDgkAwEQAAEQAAEQECeBCA38mwX1AoEQAAEQAAEQCBAApCbAMEhGQiAAAiAAAiAgDwJQG7k2S6oFQiAAAiAAAiAQIAEIDcBgkMyEAABEAABEAABeRKA3MizXVArEAABEAABEACBAAlAbgIEh2QgAAIgAAIgAALyJAC5kWe7oFYgAAIgAAIgAAIBEoDcBAgOyUAABEAABEAABORJAHIjz3ZBrUAABEAABEAABAIkALkJEBySgQAIgAAIgAAIyJMA5Eae7YJagQAIgAAIgAAIBEgAchMgOCQDARAAARAAARCQJwHIjTzbBbUCARAAARAAARAIkADkJkBwSAYCIAACIAACICBPApAbebYLagUCIAACIAACIBAgAchNgOCQDARAAARAAARAQJ4EIDfybBfUCgRAAARAAARAIEACkJsAwSEZCIAACIAACICAPAlAbuTZLqgVCIAACIAACIBAgAQgNwGCQzIQAAEQAAEQAAF5EoDcyLNdUCsQAAEQAAEQAIEACUBuAgSHZCAAAiAAAiAAAvIkALmRZ7ugViAAAiAAAiAAAgESgNwECA7JQAAEQAAEQAAE5EkAciPPdkGtQAAEQAAEQAAEAiQAuQkQHJKBAAiAQDgJnD59+nwkgl6vb21tDeedoiwQCJ4A5CZ4hsgBBEAABEJLYP/+/dnZ2bqwh6NHjy5dujQ5OTm0t4fcQUBqApAbqYkiPxAAARCQlMD+/fvLysrsYQ89PT0FBQXFxcV5eXmS3hAyA4GQE4DchBwxCgABEACBgAlE1mwOHjzY2NgIuQm4+ZAwUgQgN5Eij3JBAARAYAwCETcbk8kEuRmjkXBZlgQgN7JsFlRKrgQaGxvT0tLWrVtXWFi4d+/eioqK06dP19fX37p1q6ur68mTJzabTa51R72ijIAczAZyE2WdBtXlCUBueBL4CQLiCDQ1NSWyQektdHd3i8sGsUBgNAIyMRvIzWiNhGsyJgC5kXHjoGpyJXDz5s0ZM2Z88cUXHnqj0WjCWmVLp26XJr9U3/wkrMWisFATkI/ZQG5C3dbIP0QEIDchAotsY5xAc3PzrFmzEhMThX6j0+nCeNtGzfvM1PkaTWrS1Lfipi7RNveGsXAUFTICsjIbyE3I2hkZh5YA5Ca0fJF7DBP45Zdf5syZw/mN03LWrFlz+fLlcNy1uTyRiS9uZ4saMutzNTqM34SDe2jLkJvZQG58tbchg/EM07RmX7FxPuwEIDdhR44CY4jA3bt3582bxw3eVFVVlZaWfv3110qlcunSpTU1Na9evQrlvdoMGZMUH2uMIwdsei00YGVzKOmHIm8Zmg3kxldDQ258kZHJeciNTBoC1YhWAiaTKYkN3A3Y7XadTrdo0SKlUjl37tyysrJnz56F7N4s+pRJzPtqg8tvbMa8hDjFhPjJ8ZPoXFV551DICkfGkhKQp9lAbnw1slNuZhfo9WfYP41m/IfCF67wn4fchJ85Sow1Ag8ePKisrPS4q0uXLqWnp3ODOkVFRZ2dnR4RgjjsLM/TmR3WYtMvUShS9NxvVXNZouIdld7C5c2qDzMp/1YQRSFpWAjI1mwgN77a3yk36npfUXA+kgQgN5Gkj7JjhoDVavV6L3fu3Nm8eTOnOLm5uUaj0Ws0/07aOssXxce9naipbjZbzPrUicx8HeszRs07iqRjDrUhhBhzJzKzyrljS4sBm6r84xyu2HI2G8iNr14AufFFRibnITcyaQhUI5YJ/PrrryUlJTNmzFAqlStWrDhz5kzwd2tp12lmxU96SzHh42RdF5tfk2Yik+wYwyGE2PQqxUR1PTem01n8ETP7iMt7gq8AcpCEwP79+w8ePBj210bZufdGHTx40DRWwBOKvTY05MYrFvmchNzIpy1Qkygg8GzA3vg4wDU0fX19R48e/f7775VKZVJS0pEjR16+fCnlPdermf+V38zn2LlrKvNRMTcZZjujUihULu9p0ZffgujwpCL3U/5mg5EbX71DvNw4Y3K7q6JpGuu+NkG4JSyqtoNBbnx1XZwHAS8EMq/eZgr3f1x97sS9h14uizt17ty51NRUpVL5xRdf7Nq16969e+LSjRXriS5JMSlpl6G5Ra/NSIhjnFNUlvJZzNQi56IfejgxV4oJsrFqhOujEIgKs4Hc+GpBp7KMKSvOmJAbXzBDcR5yEwqqyDNmCbRaen64eO2/bD/AFO6fdOT03l+4CaFA7vfGjRvr16/nluNs3Ljx5s2bgeTikcasL14ye/aSfG3ubOYdjcNfmjSTmNnlzqfgtBdPFRyam8qLc4vLm7DRwwNlaA937txZUFBwM7hwyz3cFoQ77qGZD9evX1+/fr1WqzWJDpiWIt6CU1lEyk1CaRQ/BMdcyo7gYOTGW0/AORCIHQJPbQMZV28rSiqYwv3/s/T4xhut9kAfadPd3V1cXMwpTlpa2vnz56XARFfYTN3lGKoxrnFtp6IrcVKch53ls+IUf5it2VWs+SY+zusjc6SoDfLwILB///4NGzZo+bBPEPa6hz2CUCIIuwXhZ/ewSxB2uoft27cnJydrNJrly5ebRAfIDfEWIDfeqMjoHEZuZNQYqErUEdh6u/3tg9VM4f7/tqtcdfnmr339gd3CixcvDh8+zD0PcOHChVVVVf39AWZFVxLXqye6VthYdPOZ+AJ+KY65fDYzlXuucWfRVOYdFf+MHLqlfFIeHy2w20AqEQTkMBu1dOlSk+gAufHaqlLJjaFqP1O4nyms0D4WlmNQMwna+44zPuII43t+9kjiGHphJ8acY0gecTyzEBxj5EYAAx9BYNwQONhh+rejevY31P555xpvPg18rW5tbe3y5cuVSuXMmTP37t1rNgcwlO25pMZWr45XxCUs0ajmT41jGH5zuEGtUEx4e8IEpbq8iVaY/v7KMLgazYZnHLtgSPVJDmZjMpmWLFliEh0gN15bX0q5qbrjXoRBTS3EJTf06mNDgqcAuSdyP6LiwmfLqoma/27TzJ1+IzJbyI07XRyBwHgioH/w6G81FznF+azm4pkHjwK++6tXr+bk5HBzVVu2bGlpafErKy+PtLGZjWf0xi6D+h1+czjdB0H3jVvadfnfxMf9Yfbsj5h4btFxb7N2UbyCiYt7SzFBmW8MXNX8qnXsR5aJ2ZhMpmXLlnV1dZnEBciN164ZKrmpZ8VmWkKCZHJj1k4T2Az33xjn6hlxzgS58doHcBIExhGBG08t355r5BTng6P6so7Ad0J1dHRs27aNU5zMzMz6egmehGpp6XS4Cn3vpmCV8RNjeYa6nA6Dm8tnKBQztNx7Gzr3JCr4/eTjqBVDcKvyMRuTybR8+fK7d++axAXIjdfuEBq5oSJCVyjT/3tINnLjUX9qKpAbDyg4BAEQEEPgV2u/6vKN/7arnCm6yFhZAAAgAElEQVTc//bB6oLb3Mu7xST1jPP06dMDBw7MmTNHqVQmJyefOHFiaEiS90Vx791U690nvugTcRhmaoaOf5yxQeX6JWvpbMEwjmcDiTmWldmYTKYVK1Z0dHSYxAXIjdcmDo3c8EWFUG6oP7kmoDFywyPHTxAAAbEEBoZfbbzR+j9LjzOF+xUlFRlXbz+12cUmdo83NDR08uTJxYsXK5XKOXPm7N+//8kT595u96h+HFmaS5On0mccJ6mOCbZWpeqaqzWJb0+YOj9f36SdzSSWcwLUXj77D4q4j5KL67Fv3A/KcjMbk8m0cuXKtrY2k7gAufHa2FEqN2y1netvxC7lwbSU1z6AkyAw3gns+aXr/SOnmML9/2X7wYUXr/1iCfzBxPX19Wq1mpurKiwsbG8PfEyIbxWbpcVgvO94n7Ehg0nY4xAdc71WPS3O+ZhjLr6lqVytnBD3dqL6SDPegcwz9PlThmZjMplWrVrV2tpqEhcgN15bNxrlhq1zILNdkBuvfQAnQQAEKIFq08M/V5/jluN8UWu4ZA586KWlpWXLli2c4uTk5DQ2NkqF2HIsSfG+2iCcffKcBKPjPfEKxYQMA+RmdOyRMpuXL18WFBSM8t6o1atXNzc3m8QFyI3XVo46ufFiNqI3YUFuvPYBnAQBEHARuPLbs6/ONHCK86fjZ492PXBd8/OT2Wzeu3fvzJkzlUrl8uXLa2tr/czAa3SbMS8hThE/O6/c2GVx1xdLZ7Um4S0mbpra7dVUQ7bOJr2xyz2u17zH00nZmo3JZEpPT79z545JXIDceO220SU3bG0Fs1HOW8KaGycKfAABEAieQGdP72JD03/dUcYU7v//D9fsbLkbcJ42m62qqmrhwoVKpXLevHmHDx9+8eJFwLk5Ev7f9s4FKIo0seOdvdQll1QelUolk7rUJVxSSZlKLmfVXbLmVbUmV8Vh5UWVSYbLZuuWva1bWcu1UNcVV8sFkRDWVUCHYWdhWR0aRoZRuMGRgQGcY0AcBxEZFBH5RHdwUR4qjg+wUz097+lheh4w3TP/r7Z2+vH111//vk/r5/ecsetr8nN/UORtnHGYKnNfk8ley1e7FsXxpu8crMxmR+3k5v7zuozXCjrDtUa9mJvzbw3yPp+iB2I2G0JIYWHh8PAwERYgN7yVVEJy42p34TMbtNzwFi0uggAIxElg4dnzEpv9WyfOUAr6D062/O+g/dHz4E4g4a8wm8379+/n+qpUKtXk5KTwZ1eMOad/R0ZRGwvaAydWMQzDbtUpy6l3j9GZ+MJviqk3xReOzvLcjbKMrJp0Wf5Y5GZDCDl48ODQ0BDxBF2+b+vnHac9Vz2/kBuGLyRPboLXreHLHeO3iB+3JKCviF1HHtfxtdyslCy6pXgh4yIIgEAEAp/ab64/ZaAU9K+qTu3quzz58HGEB8LfHh4e/uSTTzjFKSsru3z5cvi4Qu9wI4tlf55T1maf87Mv2+H1FCXb+JNK/Q1Xsww7f9XzlyaX9i1t3qvUhq1qW7gWHaFZkEw88ZsNIeTQoUODg4PEFViz2VRu5U76yzMpKshvIDe8lW915Sb0lT4LYVfBKYq0Daef3ISm5XdFWLKQGz9kOAQBEIiSgG5i6getXdxwnDe7Lli/ehBlAr7oU1NTtbW1P/rRj+Ry+d69e7u7u333Yj1yTnVWvpOV8cdZec1cUw37T728Fgc3f2rdD/NV5Xky71bkDMPcUufIZDmeuVd+r3Xaq3OLzCk4RkcSZkMIKSkpsdlshA26HYE2w7pOvs51y/0/yA3DF5IoN4767Mhbkfttv8CXfc81P7lZIVkeuWH/JcOGiDnxvGmtf7Fx5loTx/tAYGUC5i+/khstnOL8y9nzhttfrhx/hbsPHz7U6XR5eXlyuXzr1q2nT59+/Dj2NiH3i2Zs9lvcISs3+e1uR3FOWVQ7s7IO29zRnLaiV32zyoMyOdeSR+3sTDG7kYrZEEJKS0utVndjDQkMkJuguhruNJFyw7NxZsBrg3a4tHwU2D4aENd9EvQIXxRX15Xfq1dIFnLDCxAXQQAEoiYwOrew1Xzpa1UNlIL+G53x5FhcA2hMJlNBQYFcLn/99dc///zzO3din6Ll/yUOTY7s1QJ9yFAchmEmarKCFsjxf5C5pc6WFXk8KOCORE+SazYNDQ0kmlBWVjYwMMD3BNuQk+nppOIioOWG4QuJkhu+tEV3DXIjuiJBhkBAXARu9OQ0Wd3jbwXk7Cvn0wMXh3/3cx2loP+E/tmRK9efLy8LeI4/is1mKy0t5YbjHD169OrVq/zxorjqdK15k5H1Tpm6ze43HcpW9l0qt9nvQlCabJu2Z+HjoFsSPJWW2RBCDh8+3N/fT4KDtXyT3/gbz13IDW+VTHe5YaGwf8zRLcVbPXARBNKKwOPOZnq9YSzab37JMJXDY3/W0EYp6N+q1X44cMWxGHuXzsTERHV1Nac4Bw4c6O3tjTY/wfFfzNlN6spqv5X/bqmzqTx9eLfh2RcwOFHJnEvObAghR44csVgsJCC4zIbaETDcxhUBcsNbFyE3zIw2l9qoin+NdF6+cV/EmJu4ESIBEBBI4I45S6FVeXuEvrLk1Bot0YyBaRgn/3C6gxuO8875i8MPYl/PZnZ2VqPRvP3223K5fPv27Xq9/tmzGPe94vn6viIqaNpUUKQbqo1UfgoMupGi2RBCjh492tvbS3whrNkQQiA3QZWXO017uZmzfLRhpa5nXmpreBFys4aw8aq0JrBkM2hkuiFPk4vrtNHqat1Ymrg+6ni6JBBPx53pbIOZU5zN7T/vuntP4IOh0V6+fNne3r5r1y65XP7jH/9YrVbfuxd7ar70+4qoH6hW6H1ju/A3q/nG6vjSEP/Rxx9/fPjw4Z+veejq6iotLY12nA3xCxUVFWaz2XOBvzfKcxdyw/CGaOXGu86MaPtxeD7TMyXKnXn/P7OPLEWbwy/ayZPWWl+C3Kw1cbwvTQnMXchVaMpueAxmcTBf6Tl9OlSg1FbaJy3DQxYyK5DP5ftzb/cMcIrzWkvnqZu3BT7IG21gYKC4uJjrqzp27Nj169d5owm9yHZLrTChw1b26oojcoS+JpnxFhYW3n333eZkhM8+++yDDz4gcYRjx4719PRwCQSsc8OXJlpuGL7glZvc452dJtd/Fx2ef7oEPOCNySlCishNwCeK8QRyI8ZSQZ5Sj4BzUE/Vmr2NGRPdWqrOfcoeK+h19YayNkO2kt7YNsL7VyQvk6nHi3suDP3mZ02Ugv6LxrOKkbg6wMfGxo4fP84pzsGDBy9cuMD7UgEXLUWybLV7xnhw9In6HNmrRTbhHxmcgCjOFxYWSkpKnMkI3LapJI6gUCi6urrYBFyr9nkbFdwH3jX9XK+A3DB8IUhZWHT+DRt8j+DaWhKA3KwlbbwrjQnYjTKlTn3nGbPsdIya81R0AddDtTxWpqLzBz1Db64b1ymMlig5OZeWPh669kfqVkpB/17d6aJLV2efxj6A5t69e/X19W+++aZcLt+5c+e5c+eWo56i5ez8QJbXwjOieM5ctIHaUNQncbVhGEnLTVVVlclkIsLCqVOniouLo6ySqR8dciPyMobciLyAkL2UIfDYbjZkKWlKqc35QkOpTDbXnO65gRaZ55hhGLaBR2VyLwCz7HSMD1nIjFPw7O+667f+urmdUtBfVza+12sbm38YM75nz57p9frt27fL5fK33367sbHxwYNoVkx2hurLHLffeJGZR3pizmeyHpS03FRXV3d0dBABwWg07ty589ChQ8nijPeCQGwEIDexccNTIBAzgWcWvSavnxtbM6Wqo3PdxwzDsKdZ3VOupKe19XSGSruxRiNT6or6b84JVhw9ubuprYcbjvPfHX2W6bg2durt7T1w4ADXV1VdXX3zZgwbmDsnWoqy/5jK2FzWKfVRxJ5il7TcqFSq9vZ2EikYjcaCggK9Xl9eXu75bvyCgDQIQG6kUU7IZUoScNqN65WGzqeej7thWq9scS8Pw+754pk3PjNUdlLnm0Puib7y74V7998w9XOKk/mz7pZJ7xz0lZ/jvzsyMnL06FFOcUpLS2226JYXnjCpLVOhbTn875LEVUnLTU1NjcFgICsGzmz6+vow5kYSFRKZDCIAuQkCglMQWEMCD6c6r9/1vG9W3+i/xB/bcrNBNyi8wcaTTsDv+MKjfMvgNz7VUAr6e03naq55xzQHRBN4cvfu3bq6utdff10ul+/Zs8dkMgl8MPWiSVpuamtr29raSPjgNRusc5N6VTdNvghykyYFjc8UPYGgJf4YhlkcK/uCzqAvTPg6pJbmrpqylTSloGXVbF+VwMaQ+WfPi20jv3/iDKWg//Bky/8Nji6+eBEzkcXFxTNnzmzdulUul2/ZsqW5ufnhw9gH98ScjeQ+KGm5qaur0+v1JEzwNxvITXKrGd4eMwHITczo8CAIJJTAstNx565LVmZto9ywG4Z5PqU6SW9od+/Y4LQbNyh1qokF9sXzo6z6qLSV41Fko9o+/t1TBkpB/5rq1Pt9l8mjxSgeDona09Pz4YcfyuXynJycmpqa27fjWmsnJHlRX5C03Jw4caK1tZXwhSCzgdyIuhYic+EJQG7Cs8EdEEgKgYdDRbX0Brqnc9rJMEt2o5aqv+Aahusabtzl2x7cf7Ec58NZ4ZOqmiem/qnVxA3Hye26cGlG6MqBvDyGhoY+/vhjbjjO4cOHr1y5whstxS5KWm7UavWZM2dISAg1G8hNitXb9PkcyE36lDW+VDoElhfs/Uau+4lSaovsrlVwpswbFS16b/+Pa4GcvAHOS2a1tHealdDPPP/lV/9l7OUU51/Pnj83FddEpsnJSZVKxSnOvn37zp8/LzQf0ownXrlh1+Xj2f+S+AWapnW64C0yec0GciPN6olcM5AbVAIQEDGBxQVfe4zdSNX02D2ZDVggZ9y0QdGi9S4fMz818dCzz4MnfrjfkdmFd83WV6oaKAX9tzqjeszXMhTukRWuz8/PNzU1/fSnP5XL5du2bWtpaXny5MkK8aV7S6Ry415xOILcNDQ0NDc3E78QzmwgN9Ktommec8hNmlcAfL50CLAjjnWV47POpzN2Gzus2LMijrNTR69v9W7awJ767dAp6APvPXHuvzj8O583Uwr6T2l9+ZXrSy9fCnoyTKSOjo7du3fL5fI33njjxIkTDkdczUJhXpLMyyKUG2t5JkVRmZsyqUgtNxqNpqmpiXjCCmYDuYm/kgWtZZzSe0s51JuDN/OgPop2xfX4kbMpQG4SwxGpgMDqE2CnSuWq6AyVLr9RR3kXyPFfEYdhGHaHTq1qytNyszhlGx6bWBS0G8Pyy5cVw2PrGvSUgv7t2uZ9A1emnwicj8X/9VartaSkhOurqqioGB0d5Y8nwavikxvdDiqzvJ+Q0zsiyk1TU5NGoyGusLLZQG7ir5uQm/gZxpAC5CYGaHgEBJJLIGBFHFu7hqIveLqklvxOH9uMugylJotuyaqmN7ZEsWQOfYP8/ekObjjOlvPWqw/m4/ng8fHxqqoqTnEKCwv7+vriSU0kz4pPbog7CJCb5ubmhoYGQkhEs4HcxF/fOLnJrpdw46WjPlvYzqBOx0XPHunHc91tOGi5ib8OIQUQSAsCU+Ys3wibGS1NZ5/3rAT4dKhAqSmys80t7KAcpU497WrCWb6rPklnm6ej4mOccvy7wcwpzn+093bfvRfV40GR79+/39DQ8NZbb8nl8vz8/LNnz76IY62doMTX/lTScqPT6err64WYDeQm/qqVTnLjR6uvCHLjhwOHIAACEQgED6lx9OlkSm1R/5DebMyppqk6s2sRYnazcUqpzTUOcoOLHWYd1TriS/vpwtxTT9eV7yrPkW1m9ifdA5zibGwxNd2MazGbpaUlg8GwY8cOuVyem5tL0/TMTFxbX/HkeE0uSVpuzpw5c+TIkYKCgr6+PhIpYPuFOCsU5CZOgLE9jm6p2LjhKRBIFoFZm7lH7x1Sw+Ziae7OoKrNpLJZi2po9+RwdiBOi/7BrK3fmFOtydKZK0/R67nFABdvqpu0MgWdoaTX1ffYhM2ruv1ocXf/0G981kQp6O9ozipHolk6kA9VX19fUVER11elUChu3LjBF0u81yQtN9XV1Xl5eULMBi038VfBleXG0souOE4pdOqvXK+aUrt6gNhWD+/Q4+A4AvIU7hG2g2mzmushCxcnNHnB3VJ+j6Llxg8GDkEABOIgsDxjMw/aub0aWLnRu7fkXHY6Rs35ddqy8SWGYbesktVbJp4zDPNs4rxOVmf2294hwtufvFgquzz6bXUrpaC/+cXpg5dG5p4KGqocLt3R0dGKigpOcQ4dOnTx4sVwMcV2XbpyYzQa33vvvYqKCiIsoOUmzroXWW68Taous3E7DXucrfasVc6wf6I9AiQgQ6y4eJP1xufMySM37GVhyUJuvAhxAAIgkFwC7n03HazE+IJzyCBT0Fmtg/Z5l5QsjxT4/sZ85hTWUcUwzOfXJv5K204p6F+qbtzea7sx/8j3juiPHA7HiRMn3njjDblcvnv37o6OjujTWOsnJCo33DgbtVpdW1tLhAXITZx1S7jcsA7hN/zW8hHlG4YszEK8WeWTG3aedvZmX8sNG1lYspAbL1gcgAAIJJsA1/2k1Oa1WfTEvcGCrY2WtVjtNhPbV3Wqp3PckqvQaR+4sjres0FJZ7desHHeIyD7P5u8m6Xv5objvN7Z1zd9X8BDYaM4nc7W1tZt27Y1NjaGjSSaG1KUG+8IYoPBUFNTQ4QFyE2clU643AS9iH3Q6zrCLMSbQqjccKmxmoKWGy8mHIAACEiVwPxkp7lHZXfLjaXVO7XqmWPUUnTSOwDZ9X1P71q6DVlKemNjj/6Oa3tOAZ/dP33/fzr7OMX5ob67ddIzdUvAs7xRHjzgbIv3plguilduCH/wmg0hpL29XaVS8ccLuQq5YeILMcoNO2Ylcd1SbGpFFoaB3MRXmHgaBEBAlATY+eE1RotvEPES8zxoztTS3LiZnXXlWztH0JfcmH+0vdf2y9UaSkF/X3uu9pprzpagRyUZSVpy4282hJCOjo7q6moiLEBu4qygUcuNd0yxt9lGcP+RN6uBLTcWVm1cy0tBbryIcAACIJBKBLiV/bS5RqvN4bdxFfuJS87pobJ6jUypzTffnOMGJjMM8/zxxPiQzeHavDMSiblnzw9eGvnmF6cpBf1tdWvZ5dEnL4LkKVISErkvIbkJMhtCiMlkqqqqIsIC5CbOKhm13Hjexz7o7UKKo1uKTcfjSZAbD138ggAIpB6B+Ul9tyG31mh56vm2mVFVk5bVmu5R/zHIzhs92Up6XV1Lbp0mo9bQ+UCoqShHxv9Sc5ZS0L/+WdPu/su3Hy163uT7Pf8lN/nVd0VCR1KRm1CzIYR0dXUdP36cCAuQmzirZcxyw/hPmIpdbthmm5Dg6fASliyrRJSfaQkhgqngQighDgiAwKoRcNratRkKOrttyBE0W2rOmqekc8zcGJqlifM66qSFZw35+TFtqy7nvHfSqi+n2ptT/9hi4objvNU9YJtxD/3hYnyv6Vycqx773rTmR5KQG16zIYT09PQcO3aMCAuQmzgrl3C58W9iYV+aGLkJyD5abgJw4AQEQCB1CcxajHpuEHHndMA2mew2VQq/wcXsP/KMATv8Li+4tq/S5hpDxMiPV8+X9/6zvZdTnH8zmNunWEFquXWHUtDf+FTTeSe6TSH8Ek7mofjlJpzZEELMZjPvOje6fIrK15HAALmJs54JlxvG1drhWbvPtcO2pztJ4Jxtb1YDx9x4L2NAsQ8FjkAABNKAwPNZ1/xwOuOkUT3Ota9Mq0/SeQMzc+MXiuo16+oMqvYWmcpk88JYvquladkXRouwvqqrs/Nbzlt/gV2Mlf670x3f0Zx9parhlaqGr1c3crrjTVgSByKXmxXMhhDS29t79OhREhTYHTchN4mvfVHIDcNwfuPuRfKaTcCAYtdyNZG24YTcYPuFxFdlpAgCkiXwzDFq1l7n2m9Yuckf5I6XnNOjKp02i9vAgf28x5ZWjazebA8aS/N0Um80lbWZKs0BY3e8QKafOPcNDHPbOHBtOV+ravhFZYPh9pfeOJI4ELPcrGw2hBCLxfLJJ5+QgKDbQWVmboLcJL72RSc34d7vPzhmSl0Uq9wEJ++fbPA93znG3PhY4AgEQEDqBNgtOWsM+hmeDRacduN6pU7LMyB4St9t7Ry2VjZq2TnnQerjIZLTYeHMxus3r1Q16ImU/Ea0chPRbAgh/f39hw8fJr5gLd9EZZZb0S3lqaGJ/E243Djqsz1dV2HzGa7lJvgByE0wEZyDAAikPoHHdrN+o5LOOmVS2ybnfN87q6Xp9YYx3wWeI1ccX0sPwzyftU/cdTLM9bmH/mbj9RtKQce/ACBPRlbnkjjlRojZEEIGBgbKysqIJ1jLM6lN5VZCIDerUVkiy43/xpl8OWBNxS+O5SN2Ob6VQ9AjvJGFxOEeRMsNL0BcBAEQkDKB57P2YUtl14hPbh5a8xRaFc/sKP/PZOVG1jbqu/R0MF/Ron/IbPv5pe9rz33rxJlf+fRUqOWcvnXH94iIj0QoNwLNhhBitVpLS0sJF/rLM6nM8n72BHKzGjVuZblZjTcmPE3ITcKRIkEQAAHxEbhulAVNmwrJo9Nu3KDQlN3wXxdnVt/oHcTjfuDJiyXyaPHSzOy5246TY5NHrlzfO3BFOxHBm0LeloQLYpMb4WZDCLl06VJJSQlhg7tDynUMuVmVigS5WRWskRLFgOJIhHAfBEAgiIDdSNWaw26vMD+pb9NlKOjc7psBM8sZxtJKb+yaDEpMoqeikpuozIYQMjg4WFxcTNjRN+WZIeu7UdQO/+ngmAoeZxWF3MQJMLbHITexccNTIJDGBMZN60NbbtjJ5D15dRqZUpPTarHwDUMWOshRCmjFIzfRmg0hZGho6ODBgyQkoFtqNaoeJzdeh4w4Fng18hBjmt5drrjce/eCEJIcVigWQglxQAAERESAHXPDjp7xBudVQ4aCln1hUI9OOZ97LwcfWFoDR+EE35fSuUjkJgazIYQMDw8XFhaSkAC5WY0qCLlZDaoR00TLTUREiAACIBBEgF0Cp8juG0/jnGQX+pMp6OzWC7Z5nqnjrucnVbV03kDADgxB6UroVAxyE5vZEEJGRkYOHDhAQgLkRkI1UOxZRcuN2EsI+QMBEAghMGe/ELRXg2tTcXahvwzXjg2dod1SU+YsRYvWN+cqJFFJXUi63MRsNoQQu92+f/9+IixgzI2kKqZoMgu5EU1RICMgAAKJIMAOKDbpgyXmcWczLdMNBQ0xTsT7kpNGcuWmsLCwoKCgr6+PxBSuXbu2b98+gY9CbpJTw6T+VsiN1EsQ+QcBEIhEYGnCrJMpW0KMJ9JzIr6/sLCQn5//STJCYWHhli1bYjYbQsjY2NjevXuJsAC5EXE1FHHWIDciLhxkDQRAIG4Cywv27pYMpbbsxuO40xJRAgsLC8XFxfeTEWw2W3l5OYkjjI+P79mzR2ACkBsRVTsJZQVyI6HCQlZBAASiIfCM3VH8JC2r1WvvhBtlHE16Yoqb3G6piooKEke4efPm7t27BSYAuRFTvRN5XpyOi52dJtd/x3N59jZfw+xjttQawsarQCCtCEyZs+v0lbapueUU/GxJy82tW7d27dpFhIWKiooTJ06kYBHikxJPwKHe7F3Qx3PwUcSNsBKfD4ZhIDerghWJggAIpDYBScsNIWTnzp1EQKipqamvr0/tosTXJY4A5CZxLJESCIAACKw9AanLza5du27dukVWDDCbta9XeGOiCKDlJlEkkQ4IgEAaERCj3Jze4ekJcO/yTcKH3bt3T0xMhL9PYDZpVJtT8VMhN6lYqvgmEACBVSYgOrlhzcbjNOxxwOaXJCTs2bNnfHw85LL7AsyGQZA4AciNxAsQ2QcBEEgGAbHJjS6fyiy3Enewlm+idpz2nPH97t27d2xsjO8O2mwYhBQgALlJgULEJ4AACKw1AbHJDQkIrNz4uU7APe7kww8/vHbtWugNtNkwCClBAHKTEsWIjwABEFhbAmKWG2t5ZsRuqf3794+OjpLAALNZ20qEt60iAcjNKsJF0iAAAqlKQKRy4xlTvHKzDSHkwIEDIyMjxC/AbFK1rqbnd0Fu0rPc8dUgAAJxERCp3BAusN1SVL7Ofcb3U1hYODw87L0Ds2EQUosA5Ca1yhNfAwIgsCYExC03hESaMHXw4MGhoSHiCjCbNakyeMmaEoDcrCluvAwEQCA1CIhMbkJGEEeSm+Li4sHBQUIwNyo16iO+IpgA5CaYCM5BAARAICIBkckNcQ0i9qxzQ3SuRW+8M8NJaCgpKbHZbGiziVjQiCBRApAbiRYcsg0CIJBMAmKTG0I4v3GvURxxQHFpaWllZSX2jUpmHcK7V5MA5GY16SJtEACBFCUgQrkh0YT8/PxPP/00RQsHnwUC2BUcdQAEQAAEoicgabmpqal5//33p6eno/9uPAEC0iCAlhtplBNyCQIgICoC0pUbbpzN7Ozs0tKSqJAiMyCQQAKQmwTCRFIgAALpQkCicoMRxOlSQdP+OyE3aV8FAAAEQCB6AlKUG5hN9OWMJ6RKAHIj1ZJDvkEABJJIQHJyA7NJYm3Bq9eeAORm7ZnjjSAAApInIC25gdlIvsLhA6IkALmJEhiigwAIgADDSEhuYDaosGlIAHKThoWOTwYBEIiXgFTkBmYTb0njeWkSgNxIs9yQaxAAgaQSkITcwGySWkfw8mQSgNwkkz7eDQIgIFEC4pcbmI1EqxaynRACkJuEYEQiIAAC6UVA5HIDs0mv6oivDSEAuQlBggsgAAIgEImAmOUGZhOp9HA/9QlAblK/jPGFIAACCScgWrmB2SS8rJGgFAlAbqRYasgzCIBAkgmIU25gNkmuFgQCvnsAAANGSURBVHi9aAhAbkRTFMgICICAdAiIUG5gNtKpPsjpqhOA3Kw6YrwABEAg9QiITW5gNqlXx/BF8RCA3MRDD8+CAAikKQFRyQ3MJk1rIT47PAHITXg2uAMCIAACYQiIR25gNmGKCJfTmgDkJq2LHx8PAiAQGwGRyA3MJrbiw1MpTwByk/JFjA8EARBIPAExyA3MJvHlihRThQDkJlVKEt8BAiCwhgSSLjcwmzUsbbxKegQgN9IrM+QYBEAg6QQWFhb2798/lozQ3d29bdu2+vr6pENABkBAtAQgN6ItGmQMBEBAvAQWFhZUKpUyGaGsrOzYsWPiRYOcgYAICEBuRFAIyAIIgAAIgAAIgEDiCEBuEscSKYEACIAACIAACIiAAORGBIWALIAACIAACIAACCSOAOQmcSyREgiAAAiAAAiAgAgIQG5EUAjIAgiAAAiAAAiAQOIIQG4SxxIpgQAIgAAIgAAIiIAA5EYEhYAsgAAIgAAIgAAIJI4A5CZxLJESCIAACIAACICACAhAbkRQCMgCCIAACIAACIBA4ghAbhLHEimBAAiAAAiAAAiIgADkRgSFgCyAAAiAAAiAAAgkjgDkJnEskRIIgAAIgAAIgIAICEBuRFAIyAIIgAAIgAAIgEDiCEBuEscSKYEACIAACIAACIiAAORGBIWALIAACIAACIAACCSOAOQmcSyREgiAAAiAAAiAgAgIQG5EUAjIAgiAAAiAAAiAQOIIQG4SxxIpgQAIgAAIgAAIiIAA5EYEhYAsgAAIgAAIgAAIJI4A5CZxLJESCIAACIAACICACAhAbkRQCMgCCIAACIAACIBA4ghAbhLHEimBAAiAAAiAAAiIgADkRgSFgCyAAAiAAAiAAAgkjgDkJnEskRIIgAAIgAAIgIAICEBuRFAIyAIIgAAIgAAIgEDiCEBuEscSKYEACIAACIAACIiAAORGBIWALIAACIAACIAACCSOAOQmcSyREgiAAAiAAAiAgAgIQG5EUAjIAgiAAAiAAAiAQOIIQG4SxxIpgQAIgAAIgAAIiIAA5EYEhYAsgAAIgAAIgAAIJI4A5CZxLJESCIAACIAACICACAhAbkRQCMgCCIAACIAACIBA4gj8P87jX5QHQsZdAAAAAElFTkSuQmCC"}}},{"cell_type":"code","source":"# Setup hyperparameters and make sure img_size and patch_size are compatible\nimg_size = 32\npatch_size = 16\nembedding_dim = 32\nimage = trainset[0][0] \nimage_permuted = image.permute(1, 2, 0)\n\n\nnum_patches = img_size/patch_size\nassert img_size % patch_size == 0, \"Image size must be divisible by patch size\"\nprint(f\"Number of patches per row: {num_patches}\\\n        \\nNumber of patches per column: {num_patches}\\\n        \\nTotal patches: {num_patches*num_patches}\\\n        \\nPatch size: {patch_size} pixels x {patch_size} pixels\")\n\nfig = plt.figure( figsize=(num_patches*2, num_patches*2))\nplt.imshow(image_permuted) # X, Y offsets are in pixels\nplt.show()\n\n\n# Create a series of subplots\nfig, axs = plt.subplots(nrows=img_size // patch_size, # need int not float\n                        ncols=img_size // patch_size,\n                        figsize=(num_patches*2, num_patches*2),\n                        sharex=True,\n                        sharey=True)\n\n\n\n# Loop through height and width of image\nfor i, patch_height in enumerate(range(0, img_size, patch_size)): # iterate through height\n    for j, patch_width in enumerate(range(0, img_size, patch_size)): # iterate through width\n\n        # Plot the permuted image patch (image_permuted -> (Height, Width, Color Channels))\n        axs[i, j].imshow(image_permuted[patch_height:patch_height+patch_size, # iterate through height\n                                        patch_width:patch_width+patch_size, # iterate through width\n                                        :]) # get all color channels\n\n        # Set up label information, remove the ticks for clarity and set labels to outside\n        axs[i, j].set_ylabel(i+1,\n                             rotation=\"horizontal\",\n                             horizontalalignment=\"right\",\n                             verticalalignment=\"center\")\n        axs[i, j].set_xlabel(j+1)\n        axs[i, j].set_xticks([])\n        axs[i, j].set_yticks([])\n        axs[i, j].label_outer()\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:37.737498Z","iopub.execute_input":"2024-05-17T01:08:37.737743Z","iopub.status.idle":"2024-05-17T01:08:38.362777Z","shell.execute_reply.started":"2024-05-17T01:08:37.737721Z","shell.execute_reply":"2024-05-17T01:08:38.361163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\n\n\nimage = trainset[0][0] \n\nfrom torch import nn\nconv2d = nn.Conv2d(in_channels=3, # number of color channels\n                   out_channels=embedding_dim, # from Table 1: Hidden size D, this is the embedding size\n                   kernel_size=patch_size, # could also use (patch_size, patch_size)\n                   stride=patch_size,\n                   padding=0)\n\n# Pass the image through the convolutional layer\nimage_out_of_conv = conv2d(image.unsqueeze(0)) # add a single batch dimension (height, width, color_channels) -> (batch, height, width, color_channels)\nprint(image_out_of_conv.shape)\n\n# Plot random 5 convolutional feature maps\nimport random\nrandom_indexes = random.sample(range(0, embedding_dim), k=5) # pick 5 numbers between 0 and the embedding size\nprint(f\"Showing random convolutional feature maps from indexes: {random_indexes}\")\n\n# Create plot\nfig, axs = plt.subplots(nrows=1, ncols=5, figsize=(12, 12))\n\n# Plot random image feature maps\nfor i, idx in enumerate(random_indexes):\n    image_conv_feature_map = image_out_of_conv[:, idx, :, :] # index on the output tensor of the convolutional layer\n    axs[i].imshow(image_conv_feature_map.squeeze().detach().numpy())\n    axs[i].set(xticklabels=[], yticklabels=[], xticks=[], yticks=[]);","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:38.364670Z","iopub.execute_input":"2024-05-17T01:08:38.365405Z","iopub.status.idle":"2024-05-17T01:08:38.810008Z","shell.execute_reply.started":"2024-05-17T01:08:38.365358Z","shell.execute_reply":"2024-05-17T01:08:38.809056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a single feature map in tensor form\nsingle_feature_map = image_out_of_conv[:, 0, :, :]\nsingle_feature_map, single_feature_map.requires_grad\n\n# Current tensor shape\nprint(f\"Current tensor shape: {image_out_of_conv.shape} -> [batch, embedding_dim, feature_map_height, feature_map_width]\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:38.811237Z","iopub.execute_input":"2024-05-17T01:08:38.811624Z","iopub.status.idle":"2024-05-17T01:08:38.817694Z","shell.execute_reply.started":"2024-05-17T01:08:38.811584Z","shell.execute_reply":"2024-05-17T01:08:38.816833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create flatten layer\nflatten = nn.Flatten(start_dim=2, # flatten feature_map_height (dimension 2)\n                     end_dim=3) # flatten feature_map_width (dimension 3)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:38.818832Z","iopub.execute_input":"2024-05-17T01:08:38.819136Z","iopub.status.idle":"2024-05-17T01:08:39.041438Z","shell.execute_reply.started":"2024-05-17T01:08:38.819111Z","shell.execute_reply":"2024-05-17T01:08:39.040581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\n\n\n\n\n# Create the Conv2d layer with hyperparameters from the ViT paper\nconv2d = nn.Conv2d(in_channels=3, # number of color channels\n                   out_channels=embedding_dim, # from Table 1: Hidden size D, this is the embedding size\n                   kernel_size=patch_size, # could also use (patch_size, patch_size)\n                   stride=patch_size,\n                   padding=0)\n\nplt.imshow(image.permute(1, 2, 0)) # adjust for matplotlib\nplt.axis(False);\nprint(f\"Original image shape: {image.shape}\")\n\n# 2. Turn image into feature maps\nimage_out_of_conv = conv2d(image.unsqueeze(0)) # add batch dimension to avoid shape errors\nprint(f\"Image feature map shape: {image_out_of_conv.shape}\")\n\n# 3. Flatten the feature maps\nimage_out_of_conv_flattened = flatten(image_out_of_conv)\nprint(f\"Flattened image feature map shape: {image_out_of_conv_flattened.shape}\")\n\n# Get flattened image patch embeddings in right shape\nimage_out_of_conv_flattened_reshaped = image_out_of_conv_flattened.permute(0, 2, 1) # [batch_size, P^2•C, N] -> [batch_size, N, P^2•C]\nprint(f\"Patch embedding sequence shape: {image_out_of_conv_flattened_reshaped.shape} -> [batch_size, num_patches, embedding_size]\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:39.042547Z","iopub.execute_input":"2024-05-17T01:08:39.042876Z","iopub.status.idle":"2024-05-17T01:08:39.128103Z","shell.execute_reply.started":"2024-05-17T01:08:39.042846Z","shell.execute_reply":"2024-05-17T01:08:39.125025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a single flattened feature map\nsingle_flattened_feature_map = image_out_of_conv_flattened_reshaped[:, :, 0] # index: (batch_size, number_of_patches, embedding_dimension)\n\n# Plot the flattened feature map visually\nplt.figure(figsize=(22, 22))\nplt.imshow(single_flattened_feature_map.detach().numpy())\nplt.title(f\"Flattened feature map shape: {single_flattened_feature_map.shape}\")\nplt.axis(False);","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:39.135049Z","iopub.execute_input":"2024-05-17T01:08:39.135583Z","iopub.status.idle":"2024-05-17T01:08:39.436693Z","shell.execute_reply.started":"2024-05-17T01:08:39.135536Z","shell.execute_reply":"2024-05-17T01:08:39.435645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See the flattened feature map as a tensor\nsingle_flattened_feature_map, single_flattened_feature_map.requires_grad, single_flattened_feature_map.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:39.437914Z","iopub.execute_input":"2024-05-17T01:08:39.438224Z","iopub.status.idle":"2024-05-17T01:08:39.445487Z","shell.execute_reply.started":"2024-05-17T01:08:39.438198Z","shell.execute_reply":"2024-05-17T01:08:39.444502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_patches(batch, patch_size):\n    b, c, h, w = batch.shape\n    ph, pw = patch_size\n    nh, nw = h // ph, w // pw\n\n    batch_patches = torch.reshape(batch, (b, c, nh, ph, nw, pw))\n    batch_patches = torch.permute(batch_patches, (0, 1, 2, 4, 3, 5))\n\n    return batch_patches\n\nimg = trainset[0][0] #3-х мерный массив\nbatch_patches = make_patches(img.unsqueeze(0), (8, 8))#.unsqueeze(0)\n\npatches = batch_patches[0]\nc, nh, nw, ph, pw = patches.shape\nprint(c, nh, nw, ph, pw)\n\n\nplt.figure(figsize=(5, 5))\nplt.imshow(img[0])\nplt.axis(\"off\")\n\nplt.figure(figsize=(5, 5))\nfor i in range(nh):\n    for j in range(nw):\n        plt.subplot(nh, nw, i * nw + j + 1)\n        plt.imshow(patches[0, i, j])\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:39.446697Z","iopub.execute_input":"2024-05-17T01:08:39.447034Z","iopub.status.idle":"2024-05-17T01:08:39.979547Z","shell.execute_reply.started":"2024-05-17T01:08:39.446997Z","shell.execute_reply":"2024-05-17T01:08:39.978568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiLayerPerceptron(nn.Module):\n    def __init__(self, input_features, output_features):\n        super(MultiLayerPerceptron, self).__init__()\n\n        self.linear1 = nn.Linear(input_features, 128)\n        self.linear2 = nn.Linear(128, 128)\n        self.output = nn.Linear(128, output_features)  \n\n        self.relu = nn.LeakyReLU()\n        \n        \n        \n    def forward(self, x):\n        x = self.linear1(x)\n        x = self.relu(x)\n\n        x = self.linear2(x)\n        x = self.relu(x)\n        \n        return self.output(x)\n    \nclass Img2Seq(nn.Module):\n    def __init__(self, img_shape, patch_size, embed_dim):\n        super().__init__()\n        self.patch_size = patch_size\n        self.img_size = (img_shape[1], img_shape[2])\n        n_channels = img_shape[0]\n        \n        nh, nw = self.img_size[0] // patch_size[0], self.img_size[1] // patch_size[1]\n        n_tokens = nh * nw\n\n        token_dim = patch_size[0] * patch_size[1] * n_channels\n        self.linear = nn.Linear(token_dim, embed_dim)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim))\n        self.pos_emb = nn.Parameter(torch.randn(n_tokens, embed_dim))\n\n    def __call__(self, batch):\n        batch = make_patches(batch, self.patch_size)\n\n        b, c, nh, nw, ph, pw = batch.shape\n\n        batch = torch.permute(batch, [0, 2, 3, 4, 5, 1])\n        batch = torch.reshape(batch, [b, nh * nw, ph * pw * c])\n\n        batch = self.linear(batch)\n        cls = self.cls_token.expand([b, -1, -1])\n        emb = batch + self.pos_emb\n\n        return torch.cat([cls, emb], axis=1)\n    \nclass MultiHeadAttention(nn.Module):\n    def __init__(self, embed_dim, num_heads, dropout=0.0):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads #сколько голов внимания использовать \n        self.embed_dim = embed_dim # размер встраивания из Таблицы 1 (Скрытый размер D)\n        \n        \n        self.q_linears = nn.ModuleList([nn.Linear(embed_dim, embed_dim) for _ in range(num_heads)])\n        self.k_linears = nn.ModuleList([nn.Linear(embed_dim, embed_dim) for _ in range(num_heads)])\n        self.v_linears = nn.ModuleList([nn.Linear(embed_dim, embed_dim) for _ in range(num_heads)])\n\n        self.dropout = nn.Dropout(dropout)#согласно статье dropout не требуется\n        self.output = nn.Linear(embed_dim * num_heads, embed_dim)\n        \n        self.scale = torch.sqrt(torch.tensor(self.embed_dim, dtype=torch.float32))\n\n\n    def forward(self, emb):\n        head_outputs = []\n        \n        for q_linear, k_linear, v_linear in zip(self.q_linears, self.k_linears, self.v_linears):\n            Q = q_linear(emb)\n            K = k_linear(emb)\n            V = v_linear(emb)\n            \n            attn_scores = torch.matmul(Q, K.transpose(-2, -1)) / self.scale\n            attn_weights = F.softmax(attn_scores, dim=-1)\n            \n            attn_weights = self.dropout(attn_weights)\n            \n            head_output = torch.matmul(attn_weights, V)\n            head_outputs.append(head_output)\n            \n        concat_output = torch.cat(head_outputs, dim=-1)\n\n        return self.output(concat_output)\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, num_heads, dropout=0.0):\n        super(TransformerBlock, self).__init__()\n        \n        self.attention = MultiHeadAttention(embed_dim, num_heads, dropout)\n        self.norm1 = nn.LayerNorm(embed_dim)\n        self.norm2 = nn.LayerNorm(embed_dim)\n        \n        # MLP - Adjusting output features to match embed_dim after expansion\n        self.feed_forward = MultiLayerPerceptron(embed_dim, embed_dim)\n        \n        self.dropout = nn.Dropout(dropout)\n\n    def forward(self, x):\n        attention_out = self.attention(x)\n        x = self.norm1(x + attention_out)\n        x = self.dropout(x)\n\n        forward_out = self.feed_forward(x)\n        \n        x = self.norm2(x + forward_out)\n        x = self.dropout(x)\n\n        return x\n    \nclass ViT(nn.Module):\n    def __init__(self, img_shape, patch_size, embed_dim, num_heads, num_classes, dropout=0.0):\n        super(ViT, self).__init__()\n        self.img2seq = Img2Seq(img_shape, patch_size, embed_dim)\n        \n        self.transformer1 = TransformerBlock(embed_dim, num_heads, dropout)\n        self.transformer2 = TransformerBlock(embed_dim, num_heads, dropout)\n        \n        self.feed_forward = MultiLayerPerceptron(embed_dim, num_classes)\n        \n    def forward(self, x):\n        x = self.img2seq(x)\n        x = self.transformer1(x)\n        x = self.transformer2(x)\n        \n        x = self.feed_forward(x[:, 0, :])\n        \n        return x  ","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:39.981389Z","iopub.execute_input":"2024-05-17T01:08:39.982007Z","iopub.status.idle":"2024-05-17T01:08:40.008734Z","shell.execute_reply.started":"2024-05-17T01:08:39.981970Z","shell.execute_reply":"2024-05-17T01:08:40.008071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, num_epochs=25, learning_rate=1e-3):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model = model.to(device)\n    \n    # Loss and optimizer\n    optimizer = torch.optim.Adam(model.parameters(),lr=learning_rate)\n    criterion = nn.CrossEntropyLoss()\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    loss_arr = []\n    acc = []\n    \n    for epoch in range(num_epochs):\n        model.train()  # Set model to training mode\n        \n        running_loss = 0.0\n        running_corrects = 0\n        \n        # Iterate over data.\n        for inputs, labels in train_loader:\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            \n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # Forward\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            loss = criterion(outputs, labels)\n            \n            # Statistics\n            running_loss += loss.item() * inputs.size(0)\n            running_corrects += torch.sum(preds == labels.data)\n            \n            # Backward + optimize\n            loss.backward()\n            optimizer.step()\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        epoch_acc = running_corrects.double() / len(train_loader.dataset)\n        \n        loss_arr.append(epoch_loss)\n        acc.append(epoch_acc)\n        \n        print(f'Epoch {epoch+1}/{num_epochs} - Loss: {epoch_loss:.4f} - Acc: {epoch_acc:.4f}')\n        \n        # Deep copy the model if it has the best accuracy on the validation set\n        if epoch % 1 == 0:  # Check every epoch\n            val_acc = evaluate(model, val_loader, device)\n            if val_acc > best_acc:\n                best_acc = val_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n    \n    # Load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, loss_arr, acc\n\ndef evaluate(model, data_loader, device):\n    model.eval()  # Set model to evaluate mode\n    corrects = 0\n    \n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            corrects += torch.sum(preds == labels.data)\n    \n    return corrects.double() / len(data_loader.dataset)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:40.010268Z","iopub.execute_input":"2024-05-17T01:08:40.010858Z","iopub.status.idle":"2024-05-17T01:08:40.024947Z","shell.execute_reply.started":"2024-05-17T01:08:40.010824Z","shell.execute_reply":"2024-05-17T01:08:40.024342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_result(model, loss, acc):\n    model.eval()\n\n    with torch.no_grad():\n        outputs = model(torch.Tensor(testset.data).permute(0, 3, 1, 2).to(device) / 255)\n        predicted = torch.argmax(outputs.data, 1)\n\n        print(classification_report(predicted.cpu().numpy(), testset.targets))\n\n    plt.plot(np.arange(0, len(loss)), loss, label='CELoss', color='orange')\n    plt.plot(np.arange(0, len(acc)), [a.cpu() for a in acc], label='acc', color='green')\n    plt.grid()\n    plt.legend()\n\nimg_shape = (3, 32, 32)\npatch_size = (8, 8)\nembed_dim = 32\nnum_classes = 10\nnum_heads = 2\ndropout = 0\nepochs = 100\n\nmodel = ViT(img_shape, patch_size, embed_dim, num_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:08:40.026056Z","iopub.execute_input":"2024-05-17T01:08:40.027007Z","iopub.status.idle":"2024-05-17T01:18:16.576104Z","shell.execute_reply.started":"2024-05-17T01:08:40.026974Z","shell.execute_reply":"2024-05-17T01:18:16.574937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_dim = 768\n\nmodel = ViT(img_shape, patch_size, embedding_dim, num_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:18:16.577459Z","iopub.execute_input":"2024-05-17T01:18:16.577763Z","iopub.status.idle":"2024-05-17T01:36:28.020298Z","shell.execute_reply.started":"2024-05-17T01:18:16.577734Z","shell.execute_reply":"2024-05-17T01:36:28.019196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_head = 1\n\nmodel = ViT(img_shape, patch_size, embed_dim, one_head, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:36:28.023393Z","iopub.execute_input":"2024-05-17T01:36:28.024135Z","iopub.status.idle":"2024-05-17T01:45:53.394101Z","shell.execute_reply.started":"2024-05-17T01:36:28.024103Z","shell.execute_reply":"2024-05-17T01:45:53.392955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"four_heads = 4\n\nmodel = ViT(img_shape, patch_size, embed_dim, four_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:45:53.395860Z","iopub.execute_input":"2024-05-17T01:45:53.396661Z","iopub.status.idle":"2024-05-17T01:55:54.164422Z","shell.execute_reply.started":"2024-05-17T01:45:53.396621Z","shell.execute_reply":"2024-05-17T01:55:54.163274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_embed = 64\n\nmodel = ViT(img_shape, patch_size, big_embed, num_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T01:55:54.166066Z","iopub.execute_input":"2024-05-17T01:55:54.166432Z","iopub.status.idle":"2024-05-17T02:05:34.467754Z","shell.execute_reply.started":"2024-05-17T01:55:54.166392Z","shell.execute_reply":"2024-05-17T02:05:34.466642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_embed = 16\n\nmodel = ViT(img_shape, patch_size, small_embed, num_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T02:05:34.469456Z","iopub.execute_input":"2024-05-17T02:05:34.469763Z","iopub.status.idle":"2024-05-17T02:15:12.333339Z","shell.execute_reply.started":"2024-05-17T02:05:34.469732Z","shell.execute_reply":"2024-05-17T02:15:12.332285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_patch = (16, 16)\n\nmodel = ViT(img_shape, big_patch, embed_dim, num_heads, num_classes, dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T02:15:12.334805Z","iopub.execute_input":"2024-05-17T02:15:12.335112Z","iopub.status.idle":"2024-05-17T02:24:52.463355Z","shell.execute_reply.started":"2024-05-17T02:15:12.335071Z","shell.execute_reply":"2024-05-17T02:24:52.462489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_dropout = 0.2\n\nmodel = ViT(img_shape, patch_size, embed_dim, num_heads, num_classes, add_dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T02:24:52.464856Z","iopub.execute_input":"2024-05-17T02:24:52.465172Z","iopub.status.idle":"2024-05-17T02:34:39.519443Z","shell.execute_reply.started":"2024-05-17T02:24:52.465139Z","shell.execute_reply":"2024-05-17T02:34:39.518203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_dropout = 0.4\n\nmodel = ViT(img_shape, patch_size, embed_dim, num_heads, num_classes, add_dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T02:34:39.520941Z","iopub.execute_input":"2024-05-17T02:34:39.521234Z","iopub.status.idle":"2024-05-17T02:44:23.540312Z","shell.execute_reply.started":"2024-05-17T02:34:39.521205Z","shell.execute_reply":"2024-05-17T02:44:23.539206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_dropout = 0.6\n\nmodel = ViT(img_shape, patch_size, embed_dim, num_heads, num_classes, add_dropout)\n\nprint_result(*train_model(model, trainloader, testloader, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T02:44:23.541761Z","iopub.execute_input":"2024-05-17T02:44:23.542077Z","iopub.status.idle":"2024-05-17T02:54:01.987154Z","shell.execute_reply.started":"2024-05-17T02:44:23.542047Z","shell.execute_reply":"2024-05-17T02:54:01.986163Z"},"trusted":true},"execution_count":null,"outputs":[]}]}