{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torchvision\nimport numpy as np\nimport torchvision.transforms as transforms\nfrom torch.utils.data.dataloader import default_collate\nimport matplotlib.pyplot as plt\nimport copy\nfrom tqdm import tqdm\nfrom IPython.display import clear_output \n\n\ndevice = torch.device('cuda')\ndevice","metadata":{"id":"9X4AkUaZicuu","outputId":"166f5953-5626-4017-f00d-5dec8c203144","execution":{"iopub.status.busy":"2023-01-12T19:31:10.878091Z","iopub.execute_input":"2023-01-12T19:31:10.879183Z","iopub.status.idle":"2023-01-12T19:31:12.819721Z","shell.execute_reply.started":"2023-01-12T19:31:10.879089Z","shell.execute_reply":"2023-01-12T19:31:12.818326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-12T19:31:12.822152Z","iopub.execute_input":"2023-01-12T19:31:12.822755Z","iopub.status.idle":"2023-01-12T19:31:12.829834Z","shell.execute_reply.started":"2023-01-12T19:31:12.822715Z","shell.execute_reply":"2023-01-12T19:31:12.828842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = torchvision.datasets.CIFAR10(root='cifar_train/', train=True, download=True,transform = transform_train)\ntrainset = torch.utils.data.DataLoader(train, batch_size=128, shuffle=True, collate_fn=lambda x: tuple(x_.to(device) for x_ in default_collate(x)))\ntest = torchvision.datasets.CIFAR10(root='cifar_test/', train=False, download=True,transform = transform_test)\ntestset = torch.utils.data.DataLoader(test, batch_size=16, shuffle=True, collate_fn=lambda x: tuple(x_.to(device) for x_ in default_collate(x)))","metadata":{"id":"g7Gwnf9ti9EQ","outputId":"71125f72-0f37-49af-c3e3-05f302cfca51","execution":{"iopub.status.busy":"2023-01-12T19:31:12.831861Z","iopub.execute_input":"2023-01-12T19:31:12.832808Z","iopub.status.idle":"2023-01-12T19:31:47.883010Z","shell.execute_reply.started":"2023-01-12T19:31:12.832769Z","shell.execute_reply":"2023-01-12T19:31:47.882073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs_cnt = 70","metadata":{"id":"iQ8IHSr1qQEf","execution":{"iopub.status.busy":"2023-01-12T19:31:47.884972Z","iopub.execute_input":"2023-01-12T19:31:47.885449Z","iopub.status.idle":"2023-01-12T19:31:47.891116Z","shell.execute_reply.started":"2023-01-12T19:31:47.885411Z","shell.execute_reply":"2023-01-12T19:31:47.889805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg_orig = torchvision.models.vgg19(progress=True)\n#vgg_orig.classifier[6] = torch.nn.Linear(in_features=4096, out_features=10, bias=True)\nvgg_orig = torchvision.models.resnet18(progress=True)\nvgg_orig.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\n@torch.no_grad()\ndef init_weights(m):\n    if type(m) == torch.nn.Linear or type(m) == torch.nn.Conv2d:\n        torch.nn.init.xavier_normal_(m.weight)\n        if m.bias is not None:\n            m.bias.data.fill_(0.01)\n\nvgg_orig.apply(init_weights)\ntorch.save(vgg_orig.state_dict(), 'resnet18_start_path.model')","metadata":{"id":"q4ZdE_ZPzx94","outputId":"33f95c4a-3dba-4d9f-8466-9411f5e73744","execution":{"iopub.status.busy":"2023-01-12T19:32:20.319494Z","iopub.execute_input":"2023-01-12T19:32:20.319875Z","iopub.status.idle":"2023-01-12T19:32:20.625870Z","shell.execute_reply.started":"2023-01-12T19:32:20.319840Z","shell.execute_reply":"2023-01-12T19:32:20.624876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"id":"vX6iYxYTVLGl","outputId":"22d7d7cd-97d5-4e20-ad6f-6441ac8accfc","execution":{"iopub.status.busy":"2023-01-12T19:32:22.729808Z","iopub.execute_input":"2023-01-12T19:32:22.730277Z","iopub.status.idle":"2023-01-12T19:32:27.288936Z","shell.execute_reply.started":"2023-01-12T19:32:22.730237Z","shell.execute_reply":"2023-01-12T19:32:27.287851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-node SGD","metadata":{"id":"bTnKu-r_qG4f"}},{"cell_type":"code","source":"class SGD(torch.optim.Optimizer):\n    def __init__(self, params, lr):\n        defaults = dict(lr=lr)\n        super(SGD, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super().__setstate__(state)\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        for group in self.param_groups:\n            params_with_grad = []\n            d_p_list = []\n            for p in group['params']:\n                if p.grad is not None:\n                    params_with_grad.append(p)\n                    d_p_list.append(p.grad)\n                    \n                    state = self.state[p]\n            for i, param in enumerate(params_with_grad):\n                d_p = d_p_list[i]\n                param.add_(d_p, alpha=-group['lr'])\n        return None","metadata":{"id":"YG0wTRIJyJ9T","execution":{"iopub.status.busy":"2022-11-11T11:16:35.115542Z","iopub.execute_input":"2022-11-11T11:16:35.115928Z","iopub.status.idle":"2022-11-11T11:16:35.124928Z","shell.execute_reply.started":"2022-11-11T11:16:35.115890Z","shell.execute_reply":"2022-11-11T11:16:35.122454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = SGD(vgg.parameters(), lr=0.01)\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd = []\ntest_history_sgd = []\ntest_accuracy_sgd = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    for x in tqdm(trainset):\n        optimizer.zero_grad()\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        loss.backward()\n        optimizer.step()\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd += [mean_loss]\n    vgg.eval()\n    acc = 0\n    with torch.no_grad():\n        mean_loss = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n        test_history_sgd += [mean_loss / len(testset)]\n        test_accuracy_sgd += [acc / len(test)] \n    clear_output()\n    print(train_history_sgd[-1],test_history_sgd[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd, label='train')\n    axs[1].plot(test_history_sgd, label='test')\n    axs[2].plot(test_accuracy_sgd, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd)\nprint(test_history_sgd)\nprint(test_accuracy_sgd)","metadata":{"id":"pggQ-XhVjSvj","outputId":"008a687d-655b-41ed-bba2-7aedf1da96b6","execution":{"iopub.status.busy":"2022-11-11T11:17:24.653558Z","iopub.execute_input":"2022-11-11T11:17:24.653964Z","iopub.status.idle":"2022-11-11T11:59:09.387256Z","shell.execute_reply.started":"2022-11-11T11:17:24.653922Z","shell.execute_reply":"2022-11-11T11:59:09.386303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-node scaled SGD","metadata":{"id":"IEAu5ZhEu4dI"}},{"cell_type":"code","source":"l = \"0.07203147737979888 0.14395255310535432 100%|██████████| 1563/1563 [00:54<00:00, 28.80it/s] 100%|██████████| 625/625 [00:05<00:00, 107.18it/s] 0.07202398180961608 0.1439051284313202 100%|██████████| 1563/1563 [00:54<00:00, 28.91it/s] 100%|██████████| 625/625 [00:05<00:00, 109.61it/s] 0.0713728606557846 0.13325728892087937 100%|██████████| 1563/1563 [00:54<00:00, 28.84it/s] 100%|██████████| 625/625 [00:05<00:00, 107.15it/s] 0.06466621919393539 0.11971113930940629 100%|██████████| 1563/1563 [00:54<00:00, 28.85it/s] 100%|██████████| 625/625 [00:05<00:00, 108.64it/s] 0.05873350798130035 0.11204840369224549 100%|██████████| 1563/1563 [00:54<00:00, 28.87it/s] 100%|██████████| 625/625 [00:05<00:00, 107.26it/s] 0.05406758036613464 0.10424811049699784 100%|██████████| 1563/1563 [00:54<00:00, 28.88it/s] 100%|██████████| 625/625 [00:05<00:00, 108.96it/s] 0.04893045035600662 0.09837466878890991 100%|██████████| 1563/1563 [00:54<00:00, 28.85it/s] 100%|██████████| 625/625 [00:05<00:00, 107.13it/s] 0.04425665242671967 0.0860349591255188 100%|██████████| 1563/1563 [00:54<00:00, 28.82it/s] 100%|██████████| 625/625 [00:05<00:00, 108.41it/s] 0.039654929053783414 0.07746870523691177 100%|██████████| 1563/1563 [00:54<00:00, 28.65it/s] 100%|██████████| 625/625 [00:05<00:00, 105.88it/s] 0.035662731177806856 0.07124796494543552 100%|██████████| 1563/1563 [00:54<00:00, 28.89it/s] 100%|██████████| 625/625 [00:05<00:00, 108.31it/s] 0.032146644284129146 0.07620382261574268 100%|██████████| 1563/1563 [00:54<00:00, 28.82it/s] 100%|██████████| 625/625 [00:05<00:00, 108.70it/s] 0.028926583247184754 0.06823626914918422 100%|██████████| 1563/1563 [00:54<00:00, 28.83it/s] 100%|██████████| 625/625 [00:05<00:00, 106.98it/s] 0.025990791293382644 0.062458120426535604 100%|██████████| 1563/1563 [00:54<00:00, 28.77it/s] 100%|██████████| 625/625 [00:05<00:00, 109.34it/s] 0.023368147194981575 0.05429935958534479 100%|██████████| 1563/1563 [00:54<00:00, 28.84it/s] 100%|██████████| 625/625 [00:05<00:00, 107.30it/s] 0.02081636863142252 0.05194326554387808 100%|██████████| 1563/1563 [00:54<00:00, 28.87it/s] 100%|██████████| 625/625 [00:05<00:00, 109.65it/s] 0.018395932568609715 0.05745545731782913 100%|██████████| 1563/1563 [00:54<00:00, 28.84it/s] 100%|██████████| 625/625 [00:05<00:00, 107.54it/s] 0.01610987808853388 0.0486435577314347 100%|██████████| 1563/1563 [00:54<00:00, 28.84it/s] 100%|██████████| 625/625 [00:05<00:00, 109.22it/s] 0.01413411816895008 0.05515243598967791 100%|██████████| 1563/1563 [00:54<00:00, 28.90it/s] 100%|██████████| 625/625 [00:05<00:00, 105.45it/s] 0.012056655605137348 0.05066522137150169 100%|██████████| 1563/1563 [00:54<00:00, 28.94it/s] 100%|██████████| 625/625 [00:05<00:00, 109.88it/s] 0.010384767338931561 0.05290827261507511 100%|██████████| 1563/1563 [00:54<00:00, 28.91it/s] 100%|██████████| 625/625 [00:05<00:00, 107.69it/s] 0.008850962833948433 0.054426889410614965 100%|██████████| 1563/1563 [00:54<00:00, 28.93it/s] 100%|██████████| 625/625 [00:05<00:00, 109.52it/s] 0.007761952014118433 0.05917361194714904 100%|██████████| 1563/1563 [00:53<00:00, 28.95it/s] 100%|██████████| 625/625 [00:05<00:00, 108.02it/s] 0.006491719020232559 0.05798460852913558 100%|██████████| 1563/1563 [00:54<00:00, 28.89it/s] 100%|██████████| 625/625 [00:05<00:00, 109.84it/s] 0.005548973736073822 0.056817073898203674 100%|██████████| 1563/1563 [00:54<00:00, 28.90it/s] 100%|██████████| 625/625 [00:05<00:00, 107.77it/s] 0.004571322080111131 0.05948129496974871 100%|██████████| 1563/1563 [00:53<00:00, 29.03it/s] 100%|██████████| 625/625 [00:06<00:00, 99.73it/s] 0.003870306665478274 0.06897580852797255 100%|██████████| 1563/1563 [00:53<00:00, 28.98it/s] 100%|██████████| 625/625 [00:05<00:00, 105.81it/s] 0.003570597353721969 0.059810731923999265 100%|██████████| 1563/1563 [00:53<00:00, 28.95it/s] 100%|██████████| 625/625 [00:06<00:00, 100.17it/s] 0.003045432735499926 0.06850062253773212 100%|██████████| 1563/1563 [00:54<00:00, 28.93it/s] 100%|██████████| 625/625 [00:05<00:00, 105.88it/s] 0.002794563094682526 0.06843628960149363 100%|██████████| 1563/1563 [00:53<00:00, 28.94it/s] 100%|██████████| 625/625 [00:06<00:00, 103.28it/s] 0.0023798932144092396 0.07230979901119718\"\nl = l.replace(\"100%|██████████| 1563/1563\", \"\")\nl = l.replace(\"100%|██████████| 625/625\", \"\")\ntrain_history_sgd = [float(x) for x in l.split(\" \")[::8]]\ntest_history_sgd = [float(x) for x in l.split(\" \")[1::8]]\ntrain_history_sgd, test_history_sgd  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del vgg\ntorch.cuda.empty_cache()","metadata":{"id":"wqzRQSJBcyXG","execution":{"iopub.status.busy":"2022-11-11T12:41:35.896100Z","iopub.execute_input":"2022-11-11T12:41:35.896839Z","iopub.status.idle":"2022-11-11T12:41:35.909067Z","shell.execute_reply.started":"2022-11-11T12:41:35.896803Z","shell.execute_reply":"2022-11-11T12:41:35.907830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\n#vgg.classifier[6] = torch.nn.Linear(in_features=4096, out_features=10, bias=True)\n#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"id":"LWXsbLZD__qb","outputId":"aad7ff64-fdc4-4cfa-97e6-496677e86fde","execution":{"iopub.status.busy":"2022-11-11T12:41:36.308634Z","iopub.execute_input":"2022-11-11T12:41:36.309495Z","iopub.status.idle":"2022-11-11T12:41:36.548294Z","shell.execute_reply.started":"2022-11-11T12:41:36.309457Z","shell.execute_reply":"2022-11-11T12:41:36.547157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\n\npreconditioner = []\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_scaled = []\ntest_history_sgd_scaled = []\ntest_accuracy_sgd_scaled = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    for x in tqdm(trainset):\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(preconditioner):\n                    preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    preconditioner[i].mul_(beta).add_(diags[i], alpha=1 - beta)\n                diag = torch.abs(preconditioner[i])\n                diag = torch.clamp(diag, min=alpha)\n                grads[i].div_(diag)\n                parameter.add_(grads[i], alpha=-lr)\n            \n            del grads\n            del diags\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_scaled.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n            mean_loss += test_loss.item()\n        test_history_sgd_scaled.append(mean_loss / len(testset))\n        test_accuracy_sgd_scaled.append(acc / len(test))\n    clear_output()\n    print(train_history_sgd_scaled[-1],test_history_sgd_scaled[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_scaled, label='train')\n    axs[1].plot(test_history_sgd_scaled, label='test')\n    axs[2].plot(test_accuracy_sgd_scaled, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_scaled)\nprint(test_history_sgd_scaled)\nprint(test_accuracy_sgd_scaled)","metadata":{"id":"8UqBkxnPu78Y","outputId":"218135ea-a820-4567-a75a-1c3c12885baf","execution":{"iopub.status.busy":"2022-11-11T12:41:36.637910Z","iopub.execute_input":"2022-11-11T12:41:36.638218Z","iopub.status.idle":"2022-11-11T13:53:15.603823Z","shell.execute_reply.started":"2022-11-11T12:41:36.638189Z","shell.execute_reply":"2022-11-11T13:53:15.600632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = \"100%|██████████| 1563/1563 [02:46<00:00,  9.37it/s] 100%|██████████| 625/625 [00:05<00:00, 106.72it/s] 0.0710942082285881 0.1346751638650894 100%|██████████| 1563/1563 [02:46<00:00,  9.36it/s] 100%|██████████| 625/625 [00:05<00:00, 109.84it/s] 0.05943007793903351 0.11408411549329758 100%|██████████| 1563/1563 [02:46<00:00,  9.39it/s] 100%|██████████| 625/625 [00:05<00:00, 109.66it/s] 0.0505795356464386 0.09035158261060715 100%|██████████| 1563/1563 [02:46<00:00,  9.40it/s] 100%|██████████| 625/625 [00:05<00:00, 107.50it/s] 0.04273185706496239 0.08810430198907852 100%|██████████| 1563/1563 [02:46<00:00,  9.40it/s] 100%|██████████| 625/625 [00:05<00:00, 105.56it/s] 0.036338696331977845 0.072162027323246 100%|██████████| 1563/1563 [02:46<00:00,  9.41it/s] 100%|██████████| 625/625 [00:05<00:00, 110.33it/s] 0.03127310600936413 0.06204895072579384 100%|██████████| 1563/1563 [02:46<00:00,  9.37it/s] 100%|██████████| 625/625 [00:05<00:00, 105.02it/s] 0.026633457149863243 0.05525214315205813 100%|██████████| 1563/1563 [02:46<00:00,  9.38it/s] 100%|██████████| 625/625 [00:05<00:00, 108.99it/s] 0.02342958043217659 0.05344198770225048 100%|██████████| 1563/1563 [02:46<00:00,  9.36it/s] 100%|██████████| 625/625 [00:05<00:00, 106.80it/s] 0.020603753459453582 0.05640958282202482 100%|██████████| 1563/1563 [02:47<00:00,  9.35it/s] 100%|██████████| 625/625 [00:05<00:00, 105.22it/s] 0.017270825007632374 0.052782725345343354 100%|██████████| 1563/1563 [02:47<00:00,  9.31it/s] 100%|██████████| 625/625 [00:05<00:00, 104.78it/s] 0.015347908177375793 0.04791798295900226 100%|██████████| 1563/1563 [02:48<00:00,  9.30it/s] 100%|██████████| 625/625 [00:05<00:00, 106.98it/s] 0.012965276675522327 0.06227850190997124 100%|██████████| 1563/1563 [02:46<00:00,  9.38it/s] 100%|██████████| 625/625 [00:05<00:00, 108.99it/s] 0.011144985315725208 0.05502241498976946 100%|██████████| 1563/1563 [02:47<00:00,  9.35it/s] 100%|██████████| 625/625 [00:06<00:00, 103.17it/s] 0.009334790514847263 0.047796641562879084 100%|██████████| 1563/1563 [02:47<00:00,  9.33it/s] 100%|██████████| 625/625 [00:05<00:00, 106.17it/s] 0.008103062864970416 0.063789537259005 100%|██████████| 1563/1563 [02:47<00:00,  9.35it/s] 100%|██████████| 625/625 [00:05<00:00, 104.75it/s] 0.0072734150806861 0.05678009282015264 100%|██████████| 1563/1563 [02:46<00:00,  9.38it/s] 100%|██████████| 625/625 [00:05<00:00, 109.01it/s] 0.006638803344941698 0.05863275005966425 100%|██████████| 1563/1563 [02:46<00:00,  9.39it/s] 100%|██████████| 625/625 [00:05<00:00, 106.91it/s] 0.0060403685063589365 0.057150866976380346 100%|██████████| 1563/1563 [02:47<00:00,  9.35it/s] 100%|██████████| 625/625 [00:05<00:00, 106.32it/s] 0.005026311223460362 0.06421864353688433 100%|██████████| 1563/1563 [02:46<00:00,  9.37it/s] 100%|██████████| 625/625 [00:05<00:00, 109.43it/s] 0.004880932040852494 0.05957840572902933 100%|██████████| 1563/1563 [02:46<00:00,  9.37it/s] 100%|██████████| 625/625 [00:05<00:00, 108.55it/s] 0.00499586712721968 0.06916280592335389 100%|██████████| 1563/1563 [02:46<00:00,  9.37it/s] 100%|██████████| 625/625 [00:05<00:00, 106.30it/s] 0.004480719115498941 0.06900171781674726 100%|██████████| 1563/1563 [02:46<00:00,  9.40it/s] 100%|██████████| 625/625 [00:05<00:00, 108.04it/s] 0.00443540241208626 0.06770911911521107 100%|██████████| 1563/1563 [02:45<00:00,  9.42it/s] 100%|██████████| 625/625 [00:05<00:00, 110.67it/s] 0.0040936197825399 0.06721502294717356 100%|██████████| 1563/1563 [02:45<00:00,  9.42it/s] 100%|██████████| 625/625 [00:05<00:00, 108.42it/s] 0.004216393637490692 0.07250014089029283 100%|██████████| 1563/1563 [02:45<00:00,  9.42it/s] 100%|██████████| 625/625 [00:05<00:00, 110.88it/s] 0.004705247010072926 0.0757232114722021 100%|██████████| 1563/1563 [02:45<00:00,  9.42it/s] 100%|██████████| 625/625 [00:05<00:00, 109.37it/s] 0.004514162955353968 0.07160409037182107 100%|██████████| 1563/1563 [02:46<00:00,  9.41it/s] 100%|██████████| 625/625 [00:05<00:00, 108.25it/s] 0.004536108846195275 0.07054943826440722 100%|██████████| 1563/1563 [02:46<00:00,  9.41it/s] 100%|██████████| 625/625 [00:05<00:00, 110.51it/s] 0.004361313265836797 0.0740357611451298 100%|██████████| 1563/1563 [02:45<00:00,  9.42it/s] 100%|██████████| 625/625 [00:05<00:00, 108.63it/s] 0.003779531292755855 0.0770994600456208\"\nl = l.replace(\"100%|██████████| 1563/1563\", \"\")\nl = l.replace(\"100%|██████████| 625/625\", \"\")\ntrain_history_sgd_scaled = [float(x) for x in l.split(\" \")[7::9]]\ntest_history_sgd_scaled = [float(x) for x in l.split(\" \")[8::9]]\ntrain_history_sgd_scaled, test_history_sgd_scaled  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-node SGD with compression","metadata":{"id":"5FqwbB9swHDp"}},{"cell_type":"code","source":"del vgg\ntorch.cuda.empty_cache()","metadata":{"id":"YOUOti7NxdoT","execution":{"iopub.status.busy":"2022-11-05T02:22:35.997639Z","iopub.execute_input":"2022-11-05T02:22:35.998029Z","iopub.status.idle":"2022-11-05T02:22:36.003780Z","shell.execute_reply.started":"2022-11-05T02:22:35.997997Z","shell.execute_reply":"2022-11-05T02:22:36.002684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"id":"AdjHOky2xeNM","outputId":"25929688-5ac7-420c-fe8b-3e29b49957ef","execution":{"iopub.status.busy":"2022-11-14T08:13:29.387545Z","iopub.execute_input":"2022-11-14T08:13:29.388039Z","iopub.status.idle":"2022-11-14T08:13:33.025375Z","shell.execute_reply.started":"2022-11-14T08:13:29.387996Z","shell.execute_reply":"2022-11-14T08:13:33.024290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.01\nbeta = 0.99\nalpha = 1e-3\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\n#del preconditioner\npreconditioner = []\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_compressed = []\ntest_history_sgd_compressed = []\ntest_accuracy_sgd_compressed = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    for x in tqdm(trainset):\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            grads_vec = torch.nn.utils.parameters_to_vector(grads)\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            for i, parameter in enumerate(vgg.parameters()):\n                parameter.add_(grads[i], alpha=-lr)\n            \n            del grads_topn\n            del grads_vec\n            del mask\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_compressed.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_compressed.append(mean_loss / len(testset))\n        test_accuracy_sgd_compressed.append(acc / len(test))\n    clear_output()\n    print(train_history_sgd_compressed[-1],test_history_sgd_compressed[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_compressed, label='train')\n    axs[1].plot(test_history_sgd_compressed, label='test')\n    axs[2].plot(test_accuracy_sgd_compressed, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_compressed)\nprint(test_history_sgd_compressed)\nprint(test_accuracy_sgd_compressed)","metadata":{"id":"B6xTfPiyN6PS","outputId":"6dd2670d-cf3c-478d-b74c-3e6e45c1eab7","execution":{"iopub.status.busy":"2022-11-14T08:13:36.758448Z","iopub.execute_input":"2022-11-14T08:13:36.759114Z","iopub.status.idle":"2022-11-14T08:56:47.799748Z","shell.execute_reply.started":"2022-11-14T08:13:36.759075Z","shell.execute_reply":"2022-11-14T08:56:47.798757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = \"100%|██████████| 1563/1563 [02:59<00:00,  8.70it/s] 100%|██████████| 625/625 [00:05<00:00, 109.43it/s] 0.07204138557910919 0.14398377976417542 100%|██████████| 1563/1563 [02:58<00:00,  8.73it/s] 100%|██████████| 625/625 [00:05<00:00, 108.67it/s] 0.0720262632894516 0.14393014273643492 100%|██████████| 1563/1563 [02:58<00:00,  8.75it/s] 100%|██████████| 625/625 [00:05<00:00, 110.91it/s] 0.0720087389755249 0.14393380734920502 100%|██████████| 1563/1563 [02:59<00:00,  8.72it/s] 100%|██████████| 625/625 [00:05<00:00, 111.30it/s] 0.0719992998933792 0.14385991060733797 100%|██████████| 1563/1563 [03:03<00:00,  8.52it/s] 100%|██████████| 625/625 [00:05<00:00, 111.36it/s] 0.07002553682088852 0.13919450379610063 100%|██████████| 1563/1563 [03:04<00:00,  8.49it/s] 100%|██████████| 625/625 [00:05<00:00, 108.86it/s] 0.06342821585655213 0.11681663230657577 100%|██████████| 1563/1563 [03:05<00:00,  8.44it/s] 100%|██████████| 625/625 [00:05<00:00, 111.07it/s] 0.0582374211192131 0.10939588813781738 100%|██████████| 1563/1563 [03:05<00:00,  8.41it/s] 100%|██████████| 625/625 [00:05<00:00, 111.29it/s] 0.05372279182910919 0.10792361063957215 100%|██████████| 1563/1563 [03:06<00:00,  8.39it/s] 100%|██████████| 625/625 [00:05<00:00, 105.80it/s] 0.04939402287006378 0.09488918448090554 100%|██████████| 1563/1563 [03:06<00:00,  8.38it/s] 100%|██████████| 625/625 [00:05<00:00, 111.54it/s] 0.045208174316883086 0.10392567242383957 100%|██████████| 1563/1563 [03:06<00:00,  8.38it/s] 100%|██████████| 625/625 [00:05<00:00, 108.98it/s] 0.04086217592358589 0.08179723705351352 100%|██████████| 1563/1563 [03:06<00:00,  8.37it/s] 100%|██████████| 625/625 [00:05<00:00, 110.30it/s] 0.03685023710608482 0.07600216182768345 100%|██████████| 1563/1563 [03:06<00:00,  8.37it/s] 100%|██████████| 625/625 [00:05<00:00, 109.37it/s] 0.03319312891602516 0.0663029170781374 100%|██████████| 1563/1563 [03:06<00:00,  8.37it/s] 100%|██████████| 625/625 [00:05<00:00, 108.78it/s] 0.030088800449967385 0.06896218653023244 100%|██████████| 1563/1563 [03:06<00:00,  8.38it/s] 100%|██████████| 625/625 [00:05<00:00, 110.05it/s] 0.027221464813947677 0.06261048513948918 100%|██████████| 1563/1563 [03:06<00:00,  8.37it/s] 100%|██████████| 625/625 [00:05<00:00, 107.83it/s] 0.024655927643179892 0.05740909470170736 100%|██████████| 1563/1563 [03:06<00:00,  8.36it/s] 100%|██████████| 625/625 [00:05<00:00, 111.08it/s] 0.02216041761934757 0.06230037178695202 100%|██████████| 1563/1563 [03:06<00:00,  8.36it/s] 100%|██████████| 625/625 [00:05<00:00, 108.78it/s] 0.019755115669071673 0.05787229114174843 100%|██████████| 1563/1563 [03:06<00:00,  8.37it/s] 100%|██████████| 625/625 [00:05<00:00, 110.01it/s] 0.017674728759229184 0.06315989914238453 100%|██████████| 1563/1563 [03:06<00:00,  8.36it/s] 100%|██████████| 625/625 [00:05<00:00, 108.63it/s] 0.015413118043541908 0.063037174667418 100%|██████████| 1563/1563 [03:07<00:00,  8.36it/s] 100%|██████████| 625/625 [00:05<00:00, 110.14it/s] 0.013689096481204033 0.049140056219697 100%|██████████| 1563/1563 [03:08<00:00,  8.28it/s] 100%|██████████| 625/625 [00:05<00:00, 106.82it/s] 0.011865819623172283 0.06707727890117093 100%|██████████| 1563/1563 [03:08<00:00,  8.31it/s] 100%|██████████| 625/625 [00:06<00:00, 103.23it/s] 0.010067300493568182 0.05628371372781694 100%|██████████| 1563/1563 [03:08<00:00,  8.29it/s] 100%|██████████| 625/625 [00:05<00:00, 108.97it/s] 0.008821936911121011 0.05490392315387726 100%|██████████| 1563/1563 [03:08<00:00,  8.29it/s] 100%|██████████| 625/625 [00:05<00:00, 107.25it/s] 0.0075757730151340364 0.0641890826586634 100%|██████████| 1563/1563 [03:09<00:00,  8.27it/s] 100%|██████████| 625/625 [00:05<00:00, 107.22it/s] 0.006585575202498585 0.06457178604006768 100%|██████████| 1563/1563 [03:08<00:00,  8.28it/s] 100%|██████████| 625/625 [00:05<00:00, 109.95it/s] 0.005763623130870983 0.05466895292475819 100%|██████████| 1563/1563 [03:08<00:00,  8.28it/s] 100%|██████████| 625/625 [00:05<00:00, 109.55it/s] 0.004667742293421179 0.062099739909358326 100%|██████████| 1563/1563 [03:08<00:00,  8.28it/s] 100%|██████████| 625/625 [00:05<00:00, 104.43it/s] 0.004173108192905783 0.06198830542098731 100%|██████████| 1563/1563 [03:08<00:00,  8.28it/s] 100%|██████████| 625/625 [00:05<00:00, 107.71it/s] 0.0035813821287592872 0.059600650715455415\"\nl = l.replace(\"100%|██████████| 1563/1563\", \"\")\nl = l.replace(\"100%|██████████| 625/625\", \"\")\ntrain_history_sgd_compressed = [float(x) for x in l.split(\" \")[7::9]]\ntest_history_sgd_compressed = [float(x) for x in l.split(\" \")[8::9]]\ntrain_history_sgd_compressed, test_history_sgd_compressed  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-node SGD with compression and inner scaling","metadata":{}},{"cell_type":"code","source":"del vgg\ntorch.cuda.empty_cache()","metadata":{"id":"rLYUsDGKmpoA","execution":{"iopub.status.busy":"2022-11-14T09:02:45.057142Z","iopub.execute_input":"2022-11-14T09:02:45.057566Z","iopub.status.idle":"2022-11-14T09:02:45.085475Z","shell.execute_reply.started":"2022-11-14T09:02:45.057527Z","shell.execute_reply":"2022-11-14T09:02:45.084523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T09:03:02.208996Z","iopub.execute_input":"2022-11-14T09:03:02.209445Z","iopub.status.idle":"2022-11-14T09:03:02.652814Z","shell.execute_reply.started":"2022-11-14T09:03:02.209408Z","shell.execute_reply":"2022-11-14T09:03:02.651732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\n#del preconditioner\npreconditioner = []\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_scaled_compressed_inner = []\ntest_history_sgd_scaled_compressed_inner = []\ntest_accuracy_sgd_scaled_compressed_inner = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    for x in tqdm(trainset):\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(preconditioner):\n                    preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    preconditioner[i].mul_(beta).add_(diags[i], alpha=1 - beta)\n                diag = torch.abs(preconditioner[i])\n                diag = torch.clamp(diag, min=alpha)\n                grads[i].div_(diag)\n            grads_vec = torch.nn.utils.parameters_to_vector(grads)\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            torch.nn.utils.vector_to_parameters(grads_topn, grads) \n            for i, parameter in enumerate(vgg.parameters()):\n                parameter.add_(grads[i], alpha=-lr)\n            \n            del grads_topn\n            del grads_vec\n            del mask\n            \n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_scaled_compressed_inner.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_scaled_compressed_inner.append(mean_loss / len(testset))\n        test_accuracy_sgd_scaled_compressed_inner.append(acc / len(test))\n        \n    clear_output()\n    print(train_history_sgd_scaled_compressed_inner[-1],test_history_sgd_scaled_compressed_inner[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_scaled_compressed_inner, label='train')\n    axs[1].plot(test_history_sgd_scaled_compressed_inner, label='test')\n    axs[2].plot(test_accuracy_sgd_scaled_compressed_inner, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_scaled_compressed_inner)\nprint(test_history_sgd_scaled_compressed_inner)\nprint(test_accuracy_sgd_scaled_compressed_inner)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T09:03:37.199711Z","iopub.execute_input":"2022-11-14T09:03:37.200104Z","iopub.status.idle":"2022-11-14T10:17:25.250813Z","shell.execute_reply.started":"2022-11-14T09:03:37.200072Z","shell.execute_reply":"2022-11-14T10:17:25.249798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = \"100%|██████████| 1563/1563 [05:02<00:00,  5.17it/s] 100%|██████████| 625/625 [00:05<00:00, 107.44it/s] 0.0720617475938797 0.14391043810844423 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 107.44it/s] 0.06383733802556991 0.11888630014657975 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 105.57it/s] 0.05394087479114532 0.11349142713546753 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 107.42it/s] 0.04819974475502968 0.09038164937496185 100%|██████████| 1563/1563 [05:08<00:00,  5.07it/s] 100%|██████████| 625/625 [00:05<00:00, 106.12it/s] 0.04292750567555428 0.0945686186671257 100%|██████████| 1563/1563 [05:07<00:00,  5.08it/s] 100%|██████████| 625/625 [00:05<00:00, 107.76it/s] 0.03828101324081421 0.07406434577107429 100%|██████████| 1563/1563 [05:07<00:00,  5.08it/s] 100%|██████████| 625/625 [00:05<00:00, 105.82it/s] 0.034166353180408476 0.07200516349077225 100%|██████████| 1563/1563 [05:07<00:00,  5.09it/s] 100%|██████████| 625/625 [00:06<00:00, 103.16it/s] 0.03046229448378086 0.07188239621967077 100%|██████████| 1563/1563 [05:06<00:00,  5.10it/s] 100%|██████████| 625/625 [00:06<00:00, 103.81it/s] 0.02754025856912136 0.06259384102225303 100%|██████████| 1563/1563 [05:06<00:00,  5.10it/s] 100%|██████████| 625/625 [00:05<00:00, 107.52it/s] 0.02471258772611618 0.05693867201805115 100%|██████████| 1563/1563 [05:06<00:00,  5.10it/s] 100%|██████████| 625/625 [00:05<00:00, 104.80it/s] 0.021751996024250986 0.052426236177235845 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 104.98it/s] 0.019022790826261043 0.05078774449527264 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 106.21it/s] 0.016405666040480136 0.058981159488856794 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 105.46it/s] 0.014021595822274685 0.053732350546866654 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 107.08it/s] 0.011685066647604108 0.05456557392850518 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 107.15it/s] 0.009318407896235585 0.056335203313082456 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 105.87it/s] 0.0073949948358163235 0.06031216767579317 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 107.03it/s] 0.006150960312075913 0.058753184811770914 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:06<00:00, 100.89it/s] 0.004909099045852199 0.06741647235490382 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 105.14it/s] 0.00419566920157522 0.06567825815510005 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 106.95it/s] 0.003593666788227856 0.0697620429285802 100%|██████████| 1563/1563 [05:06<00:00,  5.10it/s] 100%|██████████| 625/625 [00:06<00:00, 104.07it/s] 0.003033886721935123 0.0695474556479603 100%|██████████| 1563/1563 [05:06<00:00,  5.10it/s] 100%|██████████| 625/625 [00:05<00:00, 108.08it/s] 0.0024853147605468986 0.074930742413376 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 105.99it/s] 0.0023333963961273546 0.07859399649486877 100%|██████████| 1563/1563 [05:06<00:00,  5.11it/s] 100%|██████████| 625/625 [00:06<00:00, 102.40it/s] 0.0022282269379135687 0.08039229313153773 100%|██████████| 1563/1563 [05:08<00:00,  5.07it/s] 100%|██████████| 625/625 [00:07<00:00, 87.30it/s] 0.0021492579737264895 0.07801455710770097 100%|██████████| 1563/1563 [05:10<00:00,  5.03it/s] 100%|██████████| 625/625 [00:06<00:00, 91.08it/s] 0.002363337150690495 0.07704301980207674 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:06<00:00, 102.64it/s] 0.0019061924259515944 0.0797416137364693 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:06<00:00, 102.49it/s] 0.0017327799843426328 0.07868397507369518 100%|██████████| 1563/1563 [05:05<00:00,  5.11it/s] 100%|██████████| 625/625 [00:05<00:00, 105.48it/s] 0.0017495423554815352 0.0821194955509156\"\nl = l.replace(\"100%|██████████| 1563/1563\", \"\")\nl = l.replace(\"100%|██████████| 625/625\", \"\")\ntrain_history_sgd_scaled_compressed_inner = [float(x) for x in l.split(\" \")[7::9]]\ntest_history_sgd_scaled_compressed_inner = [float(x) for x in l.split(\" \")[8::9]]\ntrain_history_sgd_scaled_compressed_inner, test_history_sgd_scaled_compressed_inner","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One-node SGD with compression and outer scaling","metadata":{}},{"cell_type":"code","source":"del vgg\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T10:18:59.052727Z","iopub.execute_input":"2022-11-14T10:18:59.053444Z","iopub.status.idle":"2022-11-14T10:18:59.083298Z","shell.execute_reply.started":"2022-11-14T10:18:59.053407Z","shell.execute_reply":"2022-11-14T10:18:59.082316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T10:19:00.334532Z","iopub.execute_input":"2022-11-14T10:19:00.334879Z","iopub.status.idle":"2022-11-14T10:19:00.724381Z","shell.execute_reply.started":"2022-11-14T10:19:00.334850Z","shell.execute_reply":"2022-11-14T10:19:00.723403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\n#del preconditioner\npreconditioner = []\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_scaled_compressed_outer = []\ntest_history_sgd_scaled_compressed_outer = []\ntest_accuracy_sgd_scaled_compressed_outer = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    for x in tqdm(trainset):\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            grads_vec = torch.nn.utils.parameters_to_vector(grads)\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            diags_vec = torch.nn.utils.parameters_to_vector([x.contiguous() for x in diags])\n            diags_topn = diags_vec*mask\n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            torch.nn.utils.vector_to_parameters(diags_topn, diags)\n            \n            del grads_topn\n            del diags_topn\n            del diags_vec\n            del grads_vec\n            del mask\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(preconditioner):\n                    preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    preconditioner[i].mul_(beta).add_(diags[i], alpha=1 - beta)\n                diag = torch.abs(preconditioner[i])\n                diag = torch.clamp(diag, min=alpha)\n                grads[i].div_(diag)\n                parameter.add_(grads[i], alpha=-lr)\n            \n            del grads\n            del diags\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_scaled_compressed_outer.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_scaled_compressed_outer.append(mean_loss / len(testset))\n        test_accuracy_sgd_scaled_compressed_outer.append(acc / len(test))\n    clear_output()\n    print(train_history_sgd_scaled_compressed_outer[-1],test_history_sgd_scaled_compressed_outer[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_scaled_compressed_outer, label='train')\n    axs[1].plot(test_history_sgd_scaled_compressed_outer, label='test')\n    axs[2].plot(test_accuracy_sgd_scaled_compressed_outer, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_scaled_compressed_outer)\nprint(test_history_sgd_scaled_compressed_outer)\nprint(test_accuracy_sgd_scaled_compressed_outer)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T10:19:01.168151Z","iopub.execute_input":"2022-11-14T10:19:01.168539Z","iopub.status.idle":"2022-11-14T11:32:52.019807Z","shell.execute_reply.started":"2022-11-14T10:19:01.168481Z","shell.execute_reply":"2022-11-14T11:32:52.018724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = \"100%|██████████| 1563/1563 [05:04<00:00,  5.13it/s] 100%|██████████| 625/625 [00:05<00:00, 104.64it/s] 0.07203910002708436 0.14474576556682586 100%|██████████| 1563/1563 [05:12<00:00,  5.00it/s] 100%|██████████| 625/625 [00:07<00:00, 85.88it/s] 0.06189234009742737 0.11339731056690217 100%|██████████| 1563/1563 [05:14<00:00,  4.97it/s] 100%|██████████| 625/625 [00:05<00:00, 108.59it/s] 0.05257405044555664 0.12119786199331284 100%|██████████| 1563/1563 [05:12<00:00,  5.01it/s] 100%|██████████| 625/625 [00:05<00:00, 106.64it/s] 0.046152534540891646 0.09457085728645324 100%|██████████| 1563/1563 [05:11<00:00,  5.02it/s] 100%|██████████| 625/625 [00:06<00:00, 101.45it/s] 0.04109186290383339 0.08453353008031846 100%|██████████| 1563/1563 [05:10<00:00,  5.03it/s] 100%|██████████| 625/625 [00:05<00:00, 107.25it/s] 0.03681295414090156 0.06900191777646542 100%|██████████| 1563/1563 [05:10<00:00,  5.04it/s] 100%|██████████| 625/625 [00:05<00:00, 108.26it/s] 0.0332726582467556 0.07099800556004048 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 109.50it/s] 0.029769136517047882 0.0629954984754324 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 107.77it/s] 0.026536774527430536 0.05993981212079525 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 108.21it/s] 0.02367167554795742 0.058380491901934145 100%|██████████| 1563/1563 [05:09<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 110.11it/s] 0.020815478735566138 0.06291158646047115 100%|██████████| 1563/1563 [05:08<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 107.32it/s] 0.01837329517751932 0.053783327174186706 100%|██████████| 1563/1563 [05:09<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 109.45it/s] 0.015805184155255555 0.051426221697032454 100%|██████████| 1563/1563 [05:08<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 109.72it/s] 0.013447711008191109 0.05564709916934371 100%|██████████| 1563/1563 [05:09<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 106.48it/s] 0.011132329356521368 0.05633748010993004 100%|██████████| 1563/1563 [05:09<00:00,  5.06it/s] 100%|██████████| 625/625 [00:05<00:00, 109.71it/s] 0.008921536798141897 0.05624479873068631 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.32it/s] 0.007200266756322235 0.06225307260490954 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:06<00:00, 103.49it/s] 0.006147858589477837 0.06039834199156612 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 105.01it/s] 0.005038956706905738 0.06598937860308215 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.22it/s] 0.004255724577852525 0.07256171243526041 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:06<00:00, 103.49it/s] 0.0039336985360411926 0.06692410603705794 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:06<00:00, 104.05it/s] 0.003244327887613326 0.07345030363369733 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 105.31it/s] 0.0031413949651713483 0.07414211210939102 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.55it/s] 0.0028496190877701157 0.07720351982712746 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:06<00:00, 103.51it/s] 0.0026880062516895123 0.07365018388982862 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.76it/s] 0.0022810166622840916 0.07900223952494562 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.74it/s] 0.0022721861335064752 0.09178013406777755 100%|██████████| 1563/1563 [05:11<00:00,  5.01it/s] 100%|██████████| 625/625 [00:07<00:00, 87.07it/s] 0.0022278489887027535 0.08210020679007285 100%|██████████| 1563/1563 [05:14<00:00,  4.97it/s] 100%|██████████| 625/625 [00:05<00:00, 104.57it/s] 0.002057238532867632 0.07699125372539274 100%|██████████| 1563/1563 [05:09<00:00,  5.05it/s] 100%|██████████| 625/625 [00:05<00:00, 104.57it/s] 0.0021019511703384342 0.07729685475518927\"\nl = l.replace(\"100%|██████████| 1563/1563\", \"\")\nl = l.replace(\"100%|██████████| 625/625\", \"\")\ntrain_history_sgd_scaled_compressed_outer = [float(x) for x in l.split(\" \")[7::9]]\ntest_history_sgd_scaled_compressed_outer = [float(x) for x in l.split(\" \")[8::9]]\ntrain_history_sgd_scaled_compressed_outer, test_history_sgd_scaled_compressed_outer","metadata":{"execution":{"iopub.status.busy":"2022-10-09T10:25:13.639963Z","iopub.execute_input":"2022-10-09T10:25:13.640439Z","iopub.status.idle":"2022-10-09T10:25:13.667794Z","shell.execute_reply.started":"2022-10-09T10:25:13.640399Z","shell.execute_reply":"2022-10-09T10:25:13.666804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history_sgd = [1.8289316977991168, 1.4717208512908662, 1.3432697383948908, 1.2433982314661032, 1.166963038084757, 1.1005053439408616, 1.0445673179138653, 0.989250110390851, 0.9549218643351894, 0.9208844089142197, 0.8824080597714085, 0.8541023447690412, 0.8346878256639252, 0.8109209603055969, 0.7904683084744016, 0.7703790689063499, 0.748759604171109, 0.7322090495272976, 0.7156474110872849, 0.6994862847620874, 0.6872022782292817, 0.6669754155761446, 0.6565016736764737, 0.6463799943856876, 0.6304467879140468, 0.620937312884099, 0.6119026190332134, 0.5937202295378956, 0.590194654205571, 0.5730455811218838, 0.5635209117856477, 0.5523822537011198, 0.5474927916246302, 0.536904098813796, 0.5263348516753262, 0.5102621904571952, 0.5058837138935733, 0.5003478174929119, 0.4884100255301541, 0.47653359037531, 0.4667398213882885, 0.46197210035055797, 0.45173928233058863, 0.44779952499262815, 0.44110492123362355, 0.4286192877365805, 0.4267524526552166, 0.4186963633731808, 0.40583431568292094, 0.39902894865826266, 0.3920217458625584, 0.3915087240355094, 0.38287245175417733, 0.37628242880334634, 0.3665917649522157, 0.3604495204089548, 0.3535113138006166, 0.3531938147971697, 0.34168081080822077, 0.33891643385600556, 0.32659210516211323, 0.32680994752422926, 0.3189298286081275, 0.3116190546690046, 0.30821381006246945, 0.301235100383039, 0.2951722922532455, 0.2924268898146842, 0.287478410374478, 0.2789804781298808]\ntest_history_sgd = [1.5090987894058228, 1.3607328868865967, 1.2530695434570311, 1.205189439201355, 1.0973256509304046, 1.0587327404022218, 1.0410814205169678, 0.9875543092250824, 0.960444793176651, 0.9492055877208709, 0.8783060618877411, 0.8934789201974869, 0.8579146874666214, 0.8845943104743957, 0.8327265310525894, 0.8424106816530228, 0.8076416890144348, 0.776603241467476, 0.8918619716405869, 0.7818745879888535, 0.7430342015981675, 0.7708895280838013, 0.7355082081317902, 0.7576694593429566, 0.7506898072004318, 0.7323270419478416, 0.7393342180013657, 0.7413101342082024, 0.7165740785598755, 0.7076307037711144, 0.739586423587799, 0.7120033720731735, 0.7102246770381927, 0.7157105963230133, 0.7116370374321938, 0.7361815862417221, 0.7415067331969738, 0.6873713337302207, 0.6926604895591736, 0.6950495611429215, 0.7038726261258126, 0.6860100819706917, 0.6871255817592145, 0.7657436012029648, 0.6818067245781422, 0.695111689209938, 0.7110733359277248, 0.677223872590065, 0.6809507349908351, 0.7053301784873008, 0.6660065929412842, 0.7106017312943935, 0.6910630885004997, 0.704380547335744, 0.6991596959710121, 0.7067718580782414, 0.7221974290490151, 0.7271639194905758, 0.7356594797432423, 0.7195898001462221, 0.7299862776935101, 0.7513518883049488, 0.7433721638709306, 0.7365455819427967, 0.7423649304136634, 0.7388891359806061, 0.7336026082992554, 0.7336432842463255, 0.7768806366801262, 0.7787707441627979]\ntest_accuracy_sgd = [0.4451, 0.5044, 0.5483, 0.5685, 0.6086, 0.6191, 0.6369, 0.6491, 0.662, 0.6668, 0.6906, 0.6842, 0.6978, 0.6909, 0.7141, 0.7071, 0.7193, 0.7314, 0.695, 0.7333, 0.7396, 0.7362, 0.7477, 0.741, 0.7408, 0.752, 0.7479, 0.7496, 0.7582, 0.7579, 0.751, 0.7542, 0.7607, 0.7598, 0.7653, 0.7531, 0.7533, 0.7707, 0.7736, 0.7734, 0.7722, 0.7795, 0.7801, 0.7512, 0.7769, 0.7782, 0.7692, 0.7796, 0.7801, 0.7813, 0.7862, 0.7794, 0.7841, 0.7799, 0.7828, 0.7798, 0.7801, 0.7766, 0.7738, 0.777, 0.7815, 0.7737, 0.778, 0.7823, 0.7836, 0.7864, 0.7848, 0.7902, 0.7792, 0.7788]\n\ntrain_history_sgd_scaled = [2.3517571276106186, 1.719526278698231, 1.4722271523512234, 1.3129453659057617, 1.2021084481188098, 1.1106173045494978, 1.0411697110861464, 0.9791867615621718, 0.9302851077540756, 0.8932477952269338, 0.8553871574914059, 0.8197619722932196, 0.7987462446817657, 0.7785101555802328, 0.7532865606305544, 0.7320895832212989, 0.7145599477431354, 0.6898471367785998, 0.6739800607456881, 0.6602247536487287, 0.6378817648229087, 0.6198015862413685, 0.6091105316758461, 0.5876427266146521, 0.5781289345163214, 0.567159648975143, 0.5560771995950538, 0.5396116055033701, 0.5272740037240031, 0.5179786795697858, 0.5067917650465465, 0.4898576061896351, 0.4841196639915866, 0.4760375536616196, 0.46534345735369437, 0.4546623369464484, 0.4458131812265157, 0.43526083505367075, 0.4278789223612422, 0.42153335402688713, 0.40895582495443045, 0.40152645728472247, 0.39453742483540266, 0.3910579341070731, 0.37963716384699886, 0.36983064007576166, 0.3677837668401201, 0.3550586033126582, 0.3475873927249933, 0.34092839736767744, 0.33619171952652505, 0.3260526715794488, 0.31672569743507656, 0.31629992575596666, 0.3107617032878539, 0.30330231336071667, 0.2958716174297016, 0.2886240225085212, 0.2879518100139125, 0.27768833127320575, 0.27633602936249557, 0.2642822236661106, 0.25914536231695234, 0.259938253008801, 0.25283183946329, 0.24517586002188266, 0.2376420312487256, 0.23371767519456346, 0.22752577281745193, 0.22945992065512616]\ntest_history_sgd_scaled = [1.8176402847290039, 1.5560021580696106, 1.565190237236023, 1.323398451423645, 1.1163662719726561, 1.2276397770404817, 1.1213507962226867, 0.9751960749149322, 0.9410083879470825, 0.8531912569522858, 0.8752307349205017, 0.8355569688796997, 0.8518134553909302, 0.790676505613327, 0.7630087155103683, 0.8051111691951752, 0.7919002303361893, 0.7996157259464264, 0.7104498053550721, 0.7208755696773529, 0.7468485274672508, 0.7086742735385895, 0.709377800309658, 0.6839563439846039, 0.7823124506950379, 0.7701396649837494, 0.6749915639460087, 0.7058037243127823, 0.6743441424787044, 0.6478036500453949, 0.6753926003694535, 0.7672067507863045, 0.7096470950365067, 0.6596240228176117, 0.6884512116968632, 0.6604680770635605, 0.6512798754394055, 0.651326754528284, 0.6442049228727817, 0.6567128181695938, 0.6699765346109867, 0.647704041159153, 0.651238528239727, 0.6983551433801651, 0.6742829095780849, 0.6300130360305309, 0.6819438458025455, 0.6650416242003441, 0.6692848138570785, 0.6425880256205797, 0.7061497136265039, 0.6912859139978885, 0.675163677251339, 0.7091103643000126, 0.7009742137758062, 0.6493951081991196, 0.6819227640539408, 0.6692636570930481, 0.7081937987536192, 0.6690233693376183, 0.692920543128252, 0.6810942958682775, 0.7387284161746502, 0.677837862648815, 0.7413045292586088, 0.6966920131325721, 0.6965892398893834, 0.6862199990719556, 0.7095094160616398, 0.6994763733744621]\ntest_accuracy_sgd_scaled = [0.3525, 0.4442, 0.472, 0.5314, 0.6016, 0.5773, 0.6114, 0.6569, 0.672, 0.7069, 0.6989, 0.7016, 0.7038, 0.726, 0.7353, 0.7204, 0.7295, 0.7278, 0.7544, 0.7492, 0.7452, 0.7582, 0.7619, 0.77, 0.7413, 0.7391, 0.7739, 0.7658, 0.7758, 0.7826, 0.7789, 0.7572, 0.7685, 0.7819, 0.7814, 0.7873, 0.7866, 0.7947, 0.7955, 0.79, 0.7846, 0.7963, 0.7934, 0.7892, 0.7955, 0.8028, 0.7958, 0.7979, 0.8048, 0.8037, 0.7938, 0.7938, 0.8042, 0.7934, 0.7959, 0.8061, 0.806, 0.808, 0.7996, 0.8116, 0.8051, 0.8067, 0.7971, 0.8068, 0.7978, 0.8103, 0.8119, 0.8136, 0.8112, 0.8151]\n\ntrain_history_sgd_compressed = [1.8196126365905527, 1.470928785746055, 1.3334844106298578, 1.231616369141337, 1.1575126478739102, 1.0904471311727753, 1.0344098047222323, 0.9898797218756907, 0.9475280682144263, 0.9157618744598935, 0.8862984181974854, 0.8568917823874432, 0.8326784257998552, 0.8064588187905528, 0.7874126469387728, 0.7628885726337238, 0.7500780073882979, 0.7289057586656507, 0.7152045238810731, 0.7008047145041053, 0.6791928697120199, 0.6675767964871643, 0.6546384526030792, 0.642751049964934, 0.631456318718698, 0.6179059670709283, 0.6023816849721972, 0.5937790409530825, 0.5831713405106683, 0.5745627212402461, 0.5637659316172685, 0.552411782772035, 0.542556889984004, 0.5338189396102105, 0.5215171480270298, 0.5122707877165217, 0.5053059560106233, 0.497825108906802, 0.48467386531098117, 0.47581760611985346, 0.4723843586871691, 0.4601361497741221, 0.4567588785344073, 0.44649469097861855, 0.4382039971668702, 0.43172106596515003, 0.42278172392064656, 0.41957798173360505, 0.4143922481771625, 0.40392294998668954, 0.3985648215807917, 0.386263856002132, 0.3798516299337377, 0.37566688603452403, 0.36731776960975376, 0.35915604854941063, 0.35751730809583687, 0.3503126727269434, 0.3440177838515748, 0.3386356351930467, 0.3352997678396342, 0.3281168208250304, 0.32023089701104956, 0.31674069330058136, 0.3143477575172244, 0.3079411401544386, 0.2959132003204902, 0.2931749209418626, 0.29153027326402153, 0.2878100925394336]\ntest_history_sgd_compressed = [1.4825371403694152, 1.3457076420783998, 1.2695969309806823, 1.1546283579826355, 1.095654265975952, 1.0519152323246002, 1.0654790233135223, 1.0004591654777526, 0.9271286103725433, 0.9730286844730377, 0.9185495915412902, 0.8735324207782745, 0.8337970267772674, 0.8068623044967651, 0.7963720461130143, 0.8087544162273407, 0.8004173420190811, 0.7979507930040359, 0.7607387540817261, 0.7810684342145919, 0.7724433503389359, 0.754898230099678, 0.7841486239552498, 0.7587072919011116, 0.7685143218278885, 0.7439604394197464, 0.7457864361524582, 0.7328465040802956, 0.7020977994859219, 0.6941103738069534, 0.7030121092200279, 0.7086966412842274, 0.7011550156831742, 0.6959256784439087, 0.7264186395406723, 0.6992011481761933, 0.6926273453474044, 0.7429019749641419, 0.7067850296139717, 0.688013545191288, 0.6959058339595795, 0.744893213737011, 0.7123915342330933, 0.6989981738954782, 0.7044345851778984, 0.7291349244356156, 0.6981397037863731, 0.7102200003087521, 0.7143991717934608, 0.6911174892365932, 0.6913865284204483, 0.7030727261781693, 0.6784814611136913, 0.6860253495693207, 0.6856588107824325, 0.703033188700676, 0.7261384779512883, 0.7258968558251858, 0.7133337709635497, 0.7176954200744629, 0.7226160607874393, 0.7099412472248078, 0.7391219742119313, 0.7515566822975874, 0.7025756004631519, 0.7368646286696195, 0.7343000508606434, 0.7168845527008176, 0.7298870616927743, 0.7102514191687107]\ntest_accuracy_sgd_compressed = [0.452, 0.5135, 0.5348, 0.5873, 0.6114, 0.6327, 0.6231, 0.6512, 0.6752, 0.6642, 0.6832, 0.7038, 0.7144, 0.7164, 0.7247, 0.7208, 0.7246, 0.7293, 0.7383, 0.733, 0.7361, 0.743, 0.7357, 0.7464, 0.7401, 0.746, 0.7514, 0.752, 0.764, 0.7687, 0.7627, 0.7618, 0.768, 0.7705, 0.7607, 0.7684, 0.7718, 0.7594, 0.7688, 0.7731, 0.7715, 0.7642, 0.769, 0.7722, 0.7726, 0.7628, 0.779, 0.7812, 0.7747, 0.7774, 0.781, 0.7808, 0.7876, 0.7869, 0.7851, 0.7834, 0.7782, 0.7818, 0.7822, 0.7853, 0.783, 0.7857, 0.7825, 0.7772, 0.7899, 0.7829, 0.7889, 0.7874, 0.7858, 0.7959]\n\ntrain_history_sgd_scaled_compressed_inner = [2.312999108563299, 1.7111016621675028, 1.4662749069121184, 1.3074678536266318, 1.190477005051225, 1.1034719055266027, 1.0349008279383336, 0.9781330744628711, 0.9340609498036182, 0.8957647716297823, 0.8552221770176802, 0.8272045722702885, 0.8012808160403805, 0.7736208535673673, 0.7484902838611847, 0.7275054742155782, 0.7047780929014201, 0.6842393982593361, 0.6684935308630814, 0.6458163699683022, 0.628197821357366, 0.6167739507030038, 0.5995291085042003, 0.5881032751649237, 0.5752334609208509, 0.5617636206662259, 0.5518369340835629, 0.5384401596720566, 0.5254438293102147, 0.5132668327797404, 0.49962305283302544, 0.4957551721416776, 0.48503983695336317, 0.4761083026218902, 0.4617975649931242, 0.45076056072474135, 0.4449311450619222, 0.43046252505706095, 0.4254930988906899, 0.42246439191691404, 0.41050658266410195, 0.4036850545869764, 0.3879606749319359, 0.3821732983793444, 0.37643503270033374, 0.3734395265045678, 0.3629676163044122, 0.34954312333213094, 0.3457621628671046, 0.33848708784183884, 0.33651190104386997, 0.3253835954172227, 0.3249886191790671, 0.31608966858033327, 0.3057544107937142, 0.3032914840656778, 0.29470565404428545, 0.2903756871628944, 0.28172701448583237, 0.27863484384763576, 0.2699579474185129, 0.26460875058189376, 0.2640381840336353, 0.2529627326351907, 0.25184421307976596, 0.2523728997811027, 0.24298956961659215, 0.23549791349245763, 0.23083257831423484, 0.22998785280891695]\ntest_history_sgd_scaled_compressed_inner = [1.8372800457000733, 1.4849069184303283, 1.6408402495384216, 1.5047068671226502, 1.1848178432941436, 1.1692782863140105, 1.0589304417133332, 0.9963956336975097, 0.9709351095438004, 1.0636324622631073, 0.9320570097923279, 0.858619813323021, 0.8649735720872879, 0.8058501340866089, 0.813608487701416, 0.8614934572219849, 0.7924152718782425, 0.7921940069675446, 0.7425208321094513, 0.790574154472351, 0.7387171201825142, 0.7038076885819435, 0.7595173852443695, 0.7090312157273293, 0.695742082631588, 0.6652576776504516, 0.7084687426805496, 0.6808872854232788, 0.6729415886640548, 0.6741438917398452, 0.6509647500514985, 0.7203395462274551, 0.6609237869083882, 0.6891588460385799, 0.6391715797305108, 0.6612693833589554, 0.6710580589950085, 0.6668749668121338, 0.6544824664831161, 0.6993474027872085, 0.6726440436482429, 0.6369153841078281, 0.6527121631801128, 0.6583067743003368, 0.6568577103883028, 0.6740000004351139, 0.6234119068682193, 0.6520969064712524, 0.6412704568564892, 0.6506019648551941, 0.6403021425187587, 0.6517817810073495, 0.6548039995551109, 0.6641993462443352, 0.6520856861799955, 0.6769471940867603, 0.6817838468313218, 0.6665222295328975, 0.6890921406388283, 0.6986861325487495, 0.6584850123614073, 0.6772667427122593, 0.6753467780530453, 0.6917614237442613, 0.7345304177641868, 0.6786286566764116, 0.705996285392344, 0.6820356864258647, 0.6970675824940205, 0.6965062957718968]\ntest_accuracy_sgd_scaled_compressed_inner = [0.3529, 0.4571, 0.4386, 0.4796, 0.5845, 0.5949, 0.6319, 0.6519, 0.6604, 0.645, 0.674, 0.6986, 0.7047, 0.7202, 0.7208, 0.704, 0.7252, 0.7295, 0.7404, 0.7271, 0.7476, 0.7669, 0.7395, 0.7609, 0.7721, 0.7766, 0.7661, 0.7794, 0.7744, 0.7753, 0.7833, 0.7662, 0.7838, 0.7754, 0.7931, 0.7869, 0.7813, 0.7859, 0.7888, 0.7816, 0.787, 0.7946, 0.7961, 0.7921, 0.796, 0.7907, 0.8042, 0.8036, 0.8057, 0.8071, 0.8059, 0.8061, 0.8066, 0.8063, 0.8067, 0.8056, 0.803, 0.8078, 0.7969, 0.8017, 0.8117, 0.8068, 0.8121, 0.8106, 0.8047, 0.81, 0.8085, 0.815, 0.8114, 0.8141]\n\ntrain_history_sgd_scaled_compressed_outer = [2.0639065683955122, 1.6442618793843653, 1.4211097166056523, 1.2668427510944473, 1.1556801478881056, 1.0709886671332143, 1.0092374095526497, 0.9599407260375254, 0.9116160717156842, 0.8829537209342507, 0.8429617474755973, 0.812777094523925, 0.7881212664382232, 0.7664751914303626, 0.7406511011788303, 0.7209006496859939, 0.7014684818132454, 0.679491622368698, 0.658006065489386, 0.6423982653928839, 0.6322067404342124, 0.6100124191597599, 0.5939224950797722, 0.5840890202528376, 0.5706613499795079, 0.55160744065214, 0.5391598056496867, 0.5316293778474374, 0.5156768750199272, 0.502066834350986, 0.4986790430057994, 0.48507766482775166, 0.4730003385440163, 0.4693462470608294, 0.45503068122717427, 0.4492722990567727, 0.43628608471597247, 0.42848593248125844, 0.4223542017552554, 0.4100810599982586, 0.39951503829425555, 0.3995353233859972, 0.3827977131699662, 0.3744400678693181, 0.36379136533840845, 0.36242860163111823, 0.36073526206528744, 0.3458343692828932, 0.34339816296649406, 0.3298999328366326, 0.3283128526890674, 0.3174515646665602, 0.3164871091504231, 0.3092867198121517, 0.30128595476870035, 0.29755567738314725, 0.28498621155386383, 0.2815058086915394, 0.2768324447028777, 0.2772091142356853, 0.2666888841049141, 0.2660805795275037, 0.2541313751807908, 0.24652655958138464, 0.24914918077723755, 0.23720779833967423, 0.23608706683835104, 0.227616821046528, 0.2271265451751097, 0.22094376297558055]\ntest_history_sgd_scaled_compressed_outer = [1.800931005859375, 1.464693103981018, 1.3174002985954285, 1.1429268624305724, 1.0833303648471833, 1.0714435734272003, 0.9541969456672669, 0.9646171610832215, 0.8801025744438171, 0.8925387589216233, 0.912869103884697, 0.8598707030773163, 0.8244412524104119, 0.8344155857563019, 0.7641385419487953, 0.754856543469429, 0.7483778882265091, 0.7459906018972396, 0.7295652384519578, 0.7406046187639237, 0.8195287627935409, 0.7056895543456078, 0.6865762941837311, 0.6937395544886589, 0.6828877357006073, 0.6685960265755654, 0.7135301825046539, 0.797689015674591, 0.6904645808339119, 0.688116423779726, 0.6595571443080902, 0.6247967089116573, 0.669581538772583, 0.6303473437964916, 0.6325447950363159, 0.6733629851996898, 0.663677328133583, 0.778578062069416, 0.6207822869122028, 0.6609807142019272, 0.6660440455675125, 0.6111570460468531, 0.6170927710711956, 0.6148703307926655, 0.6398962761104107, 0.6317878635823727, 0.6459831874668598, 0.6446189679086208, 0.6487818086564541, 0.637534888625145, 0.6448463965713978, 0.6583575277686119, 0.6385454283714295, 0.6362497710168362, 0.658180195414275, 0.6290416219219566, 0.6812903477489948, 0.7619893626868725, 0.6534155485451222, 0.6700249727845192, 0.6959395641803742, 0.6561048153921962, 0.7169587695762515, 0.6515733371406793, 0.6642256448304281, 0.6736251796156169, 0.703860360096395, 0.7024088038846851, 0.6778058465406298, 0.7004701072514057]\ntest_accuracy_sgd_scaled_compressed_outer = [0.3734, 0.4718, 0.5282, 0.5905, 0.6204, 0.6227, 0.6613, 0.6554, 0.6888, 0.6867, 0.6894, 0.7033, 0.7153, 0.7166, 0.7389, 0.7387, 0.7414, 0.7487, 0.7486, 0.75, 0.7264, 0.762, 0.7689, 0.7664, 0.7753, 0.7735, 0.7637, 0.7449, 0.7725, 0.7741, 0.7836, 0.7943, 0.7819, 0.7938, 0.7928, 0.7839, 0.7825, 0.7552, 0.8037, 0.7863, 0.7878, 0.804, 0.8027, 0.8044, 0.8021, 0.8022, 0.7986, 0.8035, 0.799, 0.8064, 0.8045, 0.8044, 0.8066, 0.8074, 0.802, 0.8113, 0.8014, 0.7861, 0.8088, 0.8128, 0.8046, 0.8164, 0.8023, 0.8151, 0.8152, 0.8155, 0.8127, 0.8101, 0.8173, 0.8141]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T11:37:21.290860Z","iopub.execute_input":"2022-11-14T11:37:21.291249Z","iopub.status.idle":"2022-11-14T11:37:21.338082Z","shell.execute_reply.started":"2022-11-14T11:37:21.291216Z","shell.execute_reply":"2022-11-14T11:37:21.337006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.plot(train_history_sgd, '-ro', label='train sgd')\n#plt.plot(test_history_sgd, '-r*', label='test sgd')\nplt.plot(train_history_sgd_scaled, '-bo', label='train sgd scaled')\n#plt.plot(test_history_sgd_scaled, '-b*', label='test sgd scaled')\nplt.plot(train_history_sgd_compressed, '-go', label='train sgd compressed')\n#plt.plot(test_history_sgd_compressed, '-g*', label='test sgd compressed')\nplt.plot(train_history_sgd_scaled_compressed_inner, '-ko', label='train sgd scaled compressed inner')\n#plt.plot(test_history_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\nplt.plot(train_history_sgd_scaled_compressed_outer, '-co', label='train sgd scaled compressed outer')\n#plt.plot(test_history_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T11:37:33.460802Z","iopub.execute_input":"2022-11-14T11:37:33.461271Z","iopub.status.idle":"2022-11-14T11:37:33.825512Z","shell.execute_reply.started":"2022-11-14T11:37:33.461229Z","shell.execute_reply":"2022-11-14T11:37:33.824567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\n#plt.plot(train_history_sgd, '-ro', label='train sgd')\nplt.plot(test_history_sgd, '-r*', label='test sgd')\n#plt.plot(train_history_sgd_scaled, '-bo', label='train sgd scaled')\nplt.plot(test_history_sgd_scaled, '-b*', label='test sgd scaled')\n#plt.plot(train_history_sgd_compressed, '-go', label='train sgd compressed')\nplt.plot(test_history_sgd_compressed, '-g*', label='test sgd compressed')\n#plt.plot(train_history_sgd_scaled_compressed_inner, '-ko', label='train sgd scaled compressed inner')\nplt.plot(test_history_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\n#plt.plot(train_history_sgd_scaled_compressed_outer, '-co', label='train sgd scaled compressed outer')\nplt.plot(test_history_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.legend()\nplt.ylim((0.5, 2))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T11:37:51.343972Z","iopub.execute_input":"2022-11-14T11:37:51.344355Z","iopub.status.idle":"2022-11-14T11:37:51.632391Z","shell.execute_reply.started":"2022-11-14T11:37:51.344321Z","shell.execute_reply":"2022-11-14T11:37:51.631465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\n#plt.plot(train_history_sgd, '-ro', label='train sgd')\nplt.plot(test_accuracy_sgd, '-r*', label='test sgd')\n#plt.plot(train_history_sgd_scaled, '-bo', label='train sgd scaled')\nplt.plot(test_accuracy_sgd_scaled, '-b*', label='test sgd scaled')\n#plt.plot(train_history_sgd_compressed, '-go', label='train sgd compressed')\nplt.plot(test_accuracy_sgd_compressed, '-g*', label='test sgd compressed')\n#plt.plot(train_history_sgd_scaled_compressed_inner, '-ko', label='train sgd scaled compressed inner')\nplt.plot(test_accuracy_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\n#plt.plot(train_history_sgd_scaled_compressed_outer, '-co', label='train sgd scaled compressed outer')\nplt.plot(test_accuracy_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T11:38:19.785033Z","iopub.execute_input":"2022-11-14T11:38:19.785744Z","iopub.status.idle":"2022-11-14T11:38:20.063478Z","shell.execute_reply.started":"2022-11-14T11:38:19.785708Z","shell.execute_reply":"2022-11-14T11:38:20.062562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_NODES = 4","metadata":{"execution":{"iopub.status.busy":"2022-11-25T19:55:48.929537Z","iopub.execute_input":"2022-11-25T19:55:48.929917Z","iopub.status.idle":"2022-11-25T19:55:48.934871Z","shell.execute_reply.started":"2022-11-25T19:55:48.929884Z","shell.execute_reply":"2022-11-25T19:55:48.933702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node SGD","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:03:58.148246Z","iopub.execute_input":"2022-11-21T15:03:58.148619Z","iopub.status.idle":"2022-11-21T15:03:58.153584Z","shell.execute_reply.started":"2022-11-21T15:03:58.148588Z","shell.execute_reply":"2022-11-21T15:03:58.152398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:05:04.922038Z","iopub.execute_input":"2022-11-21T15:05:04.922732Z","iopub.status.idle":"2022-11-21T15:05:05.224687Z","shell.execute_reply.started":"2022-11-21T15:05:04.922690Z","shell.execute_reply":"2022-11-21T15:05:05.223727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.01\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd = []\ntest_history_sgd = []\ntest_accuracy_sgd = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    for x in tqdm(trainset):\n        if node == 0:\n            mean_grads = []\n        node += 1\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd.append(mean_loss / len(testset))\n        test_accuracy_sgd.append(acc / len(test))\n        \n    clear_output()\n    print(train_history_sgd[-1],test_history_sgd[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd, label='train')\n    axs[1].plot(test_history_sgd, label='test')\n    axs[2].plot(test_accuracy_sgd, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd)\nprint(test_history_sgd)\nprint(test_accuracy_sgd)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:05:08.531039Z","iopub.execute_input":"2022-11-21T15:05:08.531734Z","iopub.status.idle":"2022-11-21T15:45:10.314659Z","shell.execute_reply.started":"2022-11-21T15:05:08.531698Z","shell.execute_reply":"2022-11-21T15:45:10.313667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node inner scaled SGD","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2022-11-24T15:38:35.314644Z","iopub.execute_input":"2022-11-24T15:38:35.315547Z","iopub.status.idle":"2022-11-24T15:38:35.320798Z","shell.execute_reply.started":"2022-11-24T15:38:35.315508Z","shell.execute_reply":"2022-11-24T15:38:35.319552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\nvgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T15:38:35.322203Z","iopub.execute_input":"2022-11-24T15:38:35.322753Z","iopub.status.idle":"2022-11-24T15:38:35.601201Z","shell.execute_reply.started":"2022-11-24T15:38:35.322715Z","shell.execute_reply":"2022-11-24T15:38:35.600057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\n\npreconditioner = []\nfor i in range(NUM_NODES):\n    preconditioner.append([])\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_inner_scaled = []\ntest_history_sgd_inner_scaled = []\ntest_accuracy_sgd_inner_scaled = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    for x in tqdm(trainset):\n        if node == 0:\n            mean_grads = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(preconditioner[node-1]):\n                    preconditioner[node-1].append(torch.clone(diags[i]).detach())\n                else:\n                    preconditioner[node-1][i].mul_(beta).add_(diags[i], alpha=1 - beta)\n                diag = torch.abs(preconditioner[node-1][i])\n                diag = torch.clamp(diag, min=alpha)\n                grads[i].div_(diag)\n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n            del diags\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_inner_scaled.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_inner_scaled.append(mean_loss / len(testset))\n        test_accuracy_sgd_inner_scaled.append(acc / len(test))\n    \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_inner_scaled, label='train')\n    axs[1].plot(test_history_sgd_inner_scaled, label='test')\n    axs[2].plot(test_accuracy_sgd_inner_scaled, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_inner_scaled)\nprint(test_history_sgd_inner_scaled)\nprint(test_accuracy_sgd_inner_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T14:27:19.774741Z","iopub.execute_input":"2022-11-24T14:27:19.775235Z","iopub.status.idle":"2022-11-24T15:38:35.312218Z","shell.execute_reply.started":"2022-11-24T14:27:19.775191Z","shell.execute_reply":"2022-11-24T15:38:35.311324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node outer scaled sgd","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2022-11-24T15:38:45.378066Z","iopub.execute_input":"2022-11-24T15:38:45.378478Z","iopub.status.idle":"2022-11-24T15:38:45.383286Z","shell.execute_reply.started":"2022-11-24T15:38:45.378447Z","shell.execute_reply":"2022-11-24T15:38:45.382186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vgg = torchvision.models.vgg19(progress=True)\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T15:39:03.211967Z","iopub.execute_input":"2022-11-24T15:39:03.212651Z","iopub.status.idle":"2022-11-24T15:39:03.442231Z","shell.execute_reply.started":"2022-11-24T15:39:03.212614Z","shell.execute_reply":"2022-11-24T15:39:03.441091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\n\npreconditioner = []\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_outer_scaled = []\ntest_history_sgd_outer_scaled = []\ntest_accuracy_sgd_outer_scaled = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    mean_preconditioner = []\n    for x in tqdm(trainset):\n        if node == 0:\n            mean_grads = []\n            mean_preconditioner = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(mean_preconditioner):\n                    mean_preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    mean_preconditioner[i].add_(diags[i])\n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    \n                    if i >= len(preconditioner):\n                        preconditioner.append(torch.clone(mean_preconditioner[i]/NUM_NODES).detach())\n                    else:\n                        preconditioner[i].mul_(beta).add_(mean_preconditioner[i]/NUM_NODES, alpha=1 - beta)\n                    \n                    diag = torch.abs(preconditioner[i])\n                    diag = torch.clamp(diag, min=alpha)\n                    mean_grads[i].div_(diag)\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n            del diags\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_outer_scaled.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_accuracy_sgd_outer_scaled.append(acc / len(test))\n        test_history_sgd_outer_scaled.append(mean_loss / len(testset))\n    \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_outer_scaled, label='train')\n    axs[1].plot(test_history_sgd_outer_scaled, label='test')\n    axs[2].plot(test_accuracy_sgd_outer_scaled, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_outer_scaled)\nprint(test_history_sgd_outer_scaled)\nprint(test_accuracy_sgd_outer_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T15:41:01.824342Z","iopub.execute_input":"2022-11-24T15:41:01.824770Z","iopub.status.idle":"2022-11-24T16:51:57.143334Z","shell.execute_reply.started":"2022-11-24T15:41:01.824736Z","shell.execute_reply":"2022-11-24T16:51:57.142305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node SGD compressed","metadata":{}},{"cell_type":"code","source":"del vgg\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-11-21T13:53:08.843347Z","iopub.execute_input":"2022-11-21T13:53:08.843722Z","iopub.status.idle":"2022-11-21T13:53:08.849270Z","shell.execute_reply.started":"2022-11-21T13:53:08.843692Z","shell.execute_reply":"2022-11-21T13:53:08.847977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T13:53:30.704999Z","iopub.execute_input":"2022-11-21T13:53:30.705625Z","iopub.status.idle":"2022-11-21T13:53:30.972074Z","shell.execute_reply.started":"2022-11-21T13:53:30.705578Z","shell.execute_reply":"2022-11-21T13:53:30.970939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.01\nbeta = 0.99\nalpha = 1e-3\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\n#del preconditioner\npreconditioner = []\nerror = []\nfor i in range(NUM_NODES):\n    error.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error[i] = error[i].to(device)\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_compressed = []\ntest_history_sgd_compressed = []\ntest_accuracy_sgd_compressed = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    for x in tqdm(trainset):\n        if node == 0:\n            mean_grads = []\n        node += 1\n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            grads_vec = lr*torch.nn.utils.parameters_to_vector(grads) + error[node-1]\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            error[node-1] = lr*torch.nn.utils.parameters_to_vector(grads) + error[node-1] - grads_topn\n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n                    \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-1)\n            \n            del grads_topn\n            del grads_vec\n            del mask\n            \n        if node == NUM_NODES:\n            node = 0\n            \n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_compressed.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_compressed.append(mean_loss / len(testset))\n        test_accuracy_sgd_compressed.append(acc / len(test))\n    \n    clear_output()\n    print(train_history_sgd_compressed[-1],test_history_sgd_compressed[-1])\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_compressed, label='train')\n    axs[1].plot(test_history_sgd_compressed, label='test')\n    axs[2].plot(test_accuracy_sgd_compressed, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_compressed)\nprint(test_history_sgd_compressed)\nprint(test_accuracy_sgd_compressed)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T14:05:21.625608Z","iopub.execute_input":"2022-11-21T14:05:21.625976Z","iopub.status.idle":"2022-11-21T14:49:29.802189Z","shell.execute_reply.started":"2022-11-21T14:05:21.625943Z","shell.execute_reply":"2022-11-21T14:49:29.801204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node SGD compressed inner scaled","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2022-11-25T10:08:40.283272Z","iopub.execute_input":"2022-11-25T10:08:40.283981Z","iopub.status.idle":"2022-11-25T10:08:40.288753Z","shell.execute_reply.started":"2022-11-25T10:08:40.283941Z","shell.execute_reply":"2022-11-25T10:08:40.287561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T10:08:42.013448Z","iopub.execute_input":"2022-11-25T10:08:42.014141Z","iopub.status.idle":"2022-11-25T10:08:42.264629Z","shell.execute_reply.started":"2022-11-25T10:08:42.014105Z","shell.execute_reply":"2022-11-25T10:08:42.263641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\npreconditioner = []\nerror = []\n\nfor i in range(NUM_NODES):\n    error.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error[i] = error[i].to(device)\n    \nfor i in range(NUM_NODES):\n    preconditioner.append([])\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_inner_scaled_compressed = []\ntest_history_sgd_inner_scaled_compressed = []\ntest_accuracy_sgd_inner_scaled_compressed = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    for x in tqdm(trainset):\n        if node == 0:\n            for i in range(len(mean_grads)):\n                del mean_grads[-1]\n            mean_grads = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(preconditioner[node-1]):\n                    preconditioner[node-1].append(torch.clone(diags[i]).detach())\n                else:\n                    preconditioner[node-1][i].mul_(beta).add_(diags[i], alpha=1 - beta)\n                diag = torch.abs(preconditioner[node-1][i])\n                diag = torch.clamp(diag, min=alpha)\n                grads[i].div_(diag)\n                \n            grads_vec = torch.nn.utils.parameters_to_vector(grads) + error[node-1]\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            \n            error[node-1] = torch.nn.utils.parameters_to_vector(grads) + error[node-1] - grads_topn\n            \n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            \n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n            del diags\n            del grads_vec\n            del grads_topn_indices\n            del mask\n            del grads_topn\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_inner_scaled_compressed.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_inner_scaled_compressed.append(mean_loss / len(testset))\n        test_accuracy_sgd_inner_scaled_compressed.append(acc / len(test))\n    \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_inner_scaled_compressed, label='train')\n    axs[1].plot(test_history_sgd_inner_scaled_compressed, label='test')\n    axs[2].plot(test_accuracy_sgd_inner_scaled_compressed, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_inner_scaled_compressed)\nprint(test_history_sgd_inner_scaled_compressed)\nprint(test_accuracy_sgd_inner_scaled_compressed)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T10:09:20.775682Z","iopub.execute_input":"2022-11-25T10:09:20.776042Z","iopub.status.idle":"2022-11-25T11:26:08.108526Z","shell.execute_reply.started":"2022-11-25T10:09:20.776011Z","shell.execute_reply":"2022-11-25T11:26:08.107630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node outer scaled sgd compressed","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2022-11-25T15:01:08.511952Z","iopub.execute_input":"2022-11-25T15:01:08.512665Z","iopub.status.idle":"2022-11-25T15:01:08.517498Z","shell.execute_reply.started":"2022-11-25T15:01:08.512628Z","shell.execute_reply":"2022-11-25T15:01:08.516372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T15:01:09.906677Z","iopub.execute_input":"2022-11-25T15:01:09.907027Z","iopub.status.idle":"2022-11-25T15:01:10.180515Z","shell.execute_reply.started":"2022-11-25T15:01:09.906998Z","shell.execute_reply":"2022-11-25T15:01:10.179527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\npreconditioner = []\nerror = []\n\nfor i in range(NUM_NODES):\n    error.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error[i] = error[i].to(device)\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_outer_scaled_compressed = []\ntest_history_sgd_outer_scaled_compressed = []\ntest_accuracy_sgd_outer_scaled_compressed = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    mean_preconditioner = []\n    for x in tqdm(trainset):\n        if node == 0:\n            for i in range(len(mean_grads)):\n                del mean_grads[-1]\n            for i in range(len(mean_preconditioner)):\n                del mean_preconditioner[-1]\n            mean_grads = []\n            mean_preconditioner = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            \n            diags_vec = torch.nn.utils.parameters_to_vector([x.contiguous() for x in diags])\n            diags_topn_indices = torch.topk(torch.abs(diags_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(diags_vec)\n            mask[diags_topn_indices] = 1\n            diags_topn = diags_vec*mask\n            torch.nn.utils.vector_to_parameters(diags_topn, diags)\n            \n            del mask\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(mean_preconditioner):\n                    mean_preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    mean_preconditioner[i].add_(diags[i])\n            \n            grads_vec = torch.nn.utils.parameters_to_vector(grads) + error[node-1]\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            \n            error[node-1] = torch.nn.utils.parameters_to_vector(grads) + error[node-1] - grads_topn\n            \n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    \n                    if i >= len(preconditioner):\n                        preconditioner.append(torch.clone(mean_preconditioner[i]/NUM_NODES).detach())\n                    else:\n                        preconditioner[i].mul_(beta).add_(mean_preconditioner[i]/NUM_NODES, alpha=1 - beta)\n                    \n                    diag = torch.abs(preconditioner[i])\n                    diag = torch.clamp(diag, min=alpha)\n                    mean_grads[i].div_(diag)\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n            del diags\n            del grads_vec\n            del grads_topn_indices\n            del diags_topn_indices\n            del mask\n            del diags_topn\n            del grads_topn\n            del diags_vec\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_outer_scaled_compressed.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_outer_scaled_compressed.append(mean_loss / len(testset))\n        test_accuracy_sgd_outer_scaled_compressed.append(acc / len(test))\n        \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_outer_scaled_compressed, label='train')\n    axs[1].plot(test_history_sgd_outer_scaled_compressed, label='test')\n    axs[2].plot(test_accuracy_sgd_outer_scaled_compressed, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_outer_scaled_compressed)\nprint(test_history_sgd_outer_scaled_compressed)\nprint(test_accuracy_sgd_outer_scaled_compressed)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T15:01:10.365131Z","iopub.execute_input":"2022-11-25T15:01:10.365464Z","iopub.status.idle":"2022-11-25T16:25:37.038470Z","shell.execute_reply.started":"2022-11-25T15:01:10.365437Z","shell.execute_reply":"2022-11-25T16:25:37.037343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node outer scaled sgd compressed + EF for preconditioner","metadata":{}},{"cell_type":"code","source":"del vgg","metadata":{"execution":{"iopub.status.busy":"2023-01-12T19:34:28.705354Z","iopub.execute_input":"2023-01-12T19:34:28.705740Z","iopub.status.idle":"2023-01-12T19:34:28.710802Z","shell.execute_reply.started":"2023-01-12T19:34:28.705711Z","shell.execute_reply":"2023-01-12T19:34:28.709767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T19:34:29.173455Z","iopub.execute_input":"2023-01-12T19:34:29.174467Z","iopub.status.idle":"2023-01-12T19:34:29.423568Z","shell.execute_reply.started":"2023-01-12T19:34:29.174417Z","shell.execute_reply":"2023-01-12T19:34:29.422572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_NODES = 4","metadata":{"execution":{"iopub.status.busy":"2023-01-12T19:36:01.821295Z","iopub.execute_input":"2023-01-12T19:36:01.821986Z","iopub.status.idle":"2023-01-12T19:36:01.826352Z","shell.execute_reply.started":"2023-01-12T19:36:01.821950Z","shell.execute_reply":"2023-01-12T19:36:01.825204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\npreconditioner = []\nerror = []\n\nfor i in range(NUM_NODES):\n    error.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error[i] = error[i].to(device)\n    \nerror_preconditioner = []\nfor i in range(NUM_NODES):\n    error_preconditioner.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error_preconditioner[i] = error[i].to(device)\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_sgd_outer_scaled_compressed_extra = []\ntest_history_sgd_outer_scaled_compressed_extra = []\ntest_accuracy_sgd_outer_scaled_compressed_extra = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    mean_preconditioner = []\n    for x in tqdm(trainset):\n        if node == 0:\n            for i in range(len(mean_grads)):\n                del mean_grads[-1]\n            for i in range(len(mean_preconditioner)):\n                del mean_preconditioner[-1]\n            mean_grads = []\n            mean_preconditioner = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            zs = [\n                torch.randint(0, 2, p.size(), device=p.device)\n                * 2.0\n                - 1.0\n                for p in vgg.parameters()\n            ]\n            diags = torch.autograd.grad(grads, vgg.parameters(), grad_outputs=zs, only_inputs=True)\n            \n            diags_vec = torch.nn.utils.parameters_to_vector([x.contiguous() for x in diags]) + error_preconditioner[node-1]\n            diags_topn_indices = torch.topk(torch.abs(diags_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(diags_vec)\n            mask[diags_topn_indices] = 1\n            diags_topn = diags_vec*mask\n            \n            error_preconditioner[node-1] = torch.nn.utils.parameters_to_vector([x.contiguous() for x in diags]) + error_preconditioner[node-1] - diags_topn\n            \n            torch.nn.utils.vector_to_parameters(diags_topn, diags)\n            \n            del mask\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                diags[i].mul_(zs[i])\n                if i >= len(mean_preconditioner):\n                    mean_preconditioner.append(torch.clone(diags[i]).detach())\n                else:\n                    mean_preconditioner[i].add_(diags[i])\n            \n            grads_vec = torch.nn.utils.parameters_to_vector(grads) + error[node-1]\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            \n            error[node-1] = torch.nn.utils.parameters_to_vector(grads) + error[node-1] - grads_topn\n            \n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            for i, parameter in enumerate(vgg.parameters()):\n                if node <= NUM_NODES and node > 1:\n                    mean_grads[i] += grads[i]\n                if node == NUM_NODES:\n                    \n                    if i >= len(preconditioner):\n                        preconditioner.append(torch.clone(mean_preconditioner[i]/NUM_NODES).detach())\n                    else:\n                        preconditioner[i].mul_(beta).add_(mean_preconditioner[i]/NUM_NODES, alpha=1 - beta)\n                    \n                    diag = torch.abs(preconditioner[i])\n                    diag = torch.clamp(diag, min=alpha)\n                    mean_grads[i].div_(diag)\n                    parameter.add_(mean_grads[i] / NUM_NODES, alpha=-lr)\n            \n            del grads\n            del diags\n            del grads_vec\n            del grads_topn_indices\n            del diags_topn_indices\n            del mask\n            del diags_topn\n            del grads_topn\n            del diags_vec\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_sgd_outer_scaled_compressed_extra.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_sgd_outer_scaled_compressed_extra.append(mean_loss / len(testset))\n        test_accuracy_sgd_outer_scaled_compressed_extra.append(acc / len(test))\n        \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_sgd_outer_scaled_compressed_extra, label='train')\n    axs[1].plot(test_history_sgd_outer_scaled_compressed_extra, label='test')\n    axs[2].plot(test_accuracy_sgd_outer_scaled_compressed_extra, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_sgd_outer_scaled_compressed_extra)\nprint(test_history_sgd_outer_scaled_compressed_extra)\nprint(test_accuracy_sgd_outer_scaled_compressed_extra)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T19:36:05.003474Z","iopub.execute_input":"2023-01-12T19:36:05.003888Z","iopub.status.idle":"2023-01-12T20:55:51.218666Z","shell.execute_reply.started":"2023-01-12T19:36:05.003857Z","shell.execute_reply":"2023-01-12T20:55:51.217682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4 node Adam","metadata":{}},{"cell_type":"code","source":"class ADAMOuter:\n    def __init__(self, params, lr, beta1 = 0.9, beta2 = 0.999, eps = 1e-8):\n        self.lr = lr\n        self.beta1 = beta1\n        self.beta2 = beta2\n        self.t = 0\n        self.m = []\n        self.v = []\n        self.eps = eps\n        for i, parameter in enumerate(vgg.parameters()):\n            self.m.append(torch.zeros_like(parameter))\n            self.v.append(torch.zeros_like(parameter))\n\n    @torch.no_grad()\n    def step(self, grads, closure=None):\n        self.t += 1\n        for i, parameter in enumerate(vgg.parameters()):\n            self.m[i] = self.beta1*self.m[i] + (1 - self.beta1)*grads[i]\n            self.v[i] = self.beta1*self.v[i] + (1 - self.beta2)*grads[i]**2\n            \n            g = (self.m[i] / (1 - self.beta1**self.t)) / (torch.sqrt(self.v[i] / (1 - self.beta2**self.t)) + self.eps)\n            parameter.add_(g, alpha=-self.lr)\n        return None","metadata":{"execution":{"iopub.status.busy":"2023-01-12T21:03:42.442815Z","iopub.execute_input":"2023-01-12T21:03:42.443181Z","iopub.status.idle":"2023-01-12T21:03:42.453678Z","shell.execute_reply.started":"2023-01-12T21:03:42.443150Z","shell.execute_reply":"2023-01-12T21:03:42.452441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del vgg\nvgg = torchvision.models.resnet18(progress=True)\nvgg.fc = torch.nn.Linear(in_features=512, out_features=10, bias=True)\nvgg.load_state_dict(torch.load('../input/model-start-point/resnet18_start_path.model'))\n_ = vgg.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T21:03:42.894434Z","iopub.execute_input":"2023-01-12T21:03:42.895112Z","iopub.status.idle":"2023-01-12T21:03:43.116264Z","shell.execute_reply.started":"2023-01-12T21:03:42.895073Z","shell.execute_reply":"2023-01-12T21:03:43.115198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.autograd.set_detect_anomaly(False)\n\nlr = 0.0003\nbeta = 0.99\nalpha = 1e-3\n\ntopn = sum(p.numel() for p in vgg.parameters()) // 5\n\npreconditioner = []\nerror = []\n\noptimizer = ADAMOuter(vgg.parameters(), 0.001)\n\nfor i in range(NUM_NODES):\n    error.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error[i] = error[i].to(device)\n    \nerror_preconditioner = []\nfor i in range(NUM_NODES):\n    error_preconditioner.append(torch.zeros((sum(p.numel() for p in vgg.parameters()),)))\n    error_preconditioner[i] = error[i].to(device)\n\nloss_fn = torch.nn.CrossEntropyLoss(reduction='mean')\n\nvgg.train()\ntrain_history_outer_adam = []\ntest_history_outer_adam = []\ntest_accuracy_outer_adam = []\nfor epoch in range(epochs_cnt):\n    vgg.train()\n    mean_loss = 0\n    node = 0\n    mean_grads = []\n    for x in tqdm(trainset):\n        if node == 0:\n            for i in range(len(mean_grads)):\n                del mean_grads[-1]\n            mean_grads = []\n        node += 1\n        \n        y_hat = vgg(x[0])\n        loss = loss_fn(y_hat, x[1])\n        grads = torch.autograd.grad(loss, vgg.parameters(), create_graph=True)\n        with torch.no_grad():\n            \n            grads_vec = torch.nn.utils.parameters_to_vector(grads) + error[node-1]\n            grads_topn_indices = torch.topk(torch.abs(grads_vec), topn, sorted=False)[1]\n            mask = torch.zeros_like(grads_vec)\n            mask[grads_topn_indices] = 1\n            grads_topn = grads_vec*mask\n            \n            error[node-1] = torch.nn.utils.parameters_to_vector(grads) + error[node-1] - grads_topn\n            \n            torch.nn.utils.vector_to_parameters(grads_topn, grads)\n            \n            if mean_grads == []:\n                for i, parameter in enumerate(vgg.parameters()):\n                    mean_grads.append(grads[i].clone())\n            \n            optimizer.step(mean_grads)\n            \n            del grads\n            del grads_vec\n            del grads_topn_indices\n            del mask\n        if node == NUM_NODES:\n            node = 0\n        mean_loss += loss.item()\n    mean_loss /= len(trainset)\n    train_history_outer_adam.append(mean_loss)\n    vgg.eval()\n    with torch.no_grad():\n        mean_loss = 0\n        acc = 0\n        for x in tqdm(testset):\n            y_test = vgg(x[0])\n            test_loss = loss_fn(y_test, x[1])\n            mean_loss += test_loss.item()\n            acc += (y_test.argmax(axis=1) == x[1]).sum().item()\n        test_history_outer_adam.append(mean_loss / len(testset))\n        test_accuracy_outer_adam.append(acc / len(test))\n        \n    clear_output()\n    fig, axs = plt.subplots(1, 3, figsize=(16, 4))\n    axs[0].plot(train_history_outer_adam, label='train')\n    axs[1].plot(test_history_outer_adam, label='test')\n    axs[2].plot(test_accuracy_outer_adam, label='test accuracy')\n    plt.legend()\n    plt.show()\nprint(train_history_outer_adam)\nprint(test_history_outer_adam)\nprint(test_accuracy_outer_adam)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T21:03:58.870460Z","iopub.execute_input":"2023-01-12T21:03:58.870998Z","iopub.status.idle":"2023-01-12T21:52:03.824704Z","shell.execute_reply.started":"2023-01-12T21:03:58.870963Z","shell.execute_reply":"2023-01-12T21:52:03.823627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history_sgd = [2.0562048223622313, 1.6685491717989793, 1.5436223352046878, 1.459137993395481, 1.4005955195487918, 1.3517686101176856, 1.3046134606651638, 1.267220407190835, 1.224719893292088, 1.1984330126086769, 1.1675859544893055, 1.140791495132934, 1.1172346109929292, 1.0941681160646326, 1.0661786169652134, 1.049905170260183, 1.03251528130163, 1.0122278187890796, 0.9968563414290738, 0.9800080009128737, 0.9570095537568603, 0.9426166659121014, 0.9287625679274654, 0.9172037146280488, 0.8995209203656677, 0.8867286106814509, 0.8779024076278862, 0.8649820892707162, 0.8505986338991034, 0.846424099124606, 0.829719403973016, 0.8225542611783118, 0.8077326497763319, 0.7979824568914331, 0.7911268916276409, 0.7797694460815175, 0.7712192957663475, 0.7590203676229853, 0.7564723688318297, 0.7484073334032922, 0.7353585931041356, 0.7262995567010797, 0.7217938252117323, 0.7124904587750545, 0.7035726074824857, 0.6951150005430822, 0.691918430349711, 0.6806228167718024, 0.6745259741992902, 0.6696996797076271, 0.6593656354700513, 0.6517552025330341, 0.6489402505442919, 0.6398436965235054, 0.6337425580719853, 0.6242683588543816, 0.6195042168392855, 0.6102964723354105, 0.6115581990812745, 0.6039764141030324, 0.5988680884966155, 0.593272577923582, 0.5827879400356956, 0.5809721125056372, 0.5755909852816931, 0.5640719798977113, 0.561186115683802, 0.5534041608538469, 0.5477232054218917, 0.5464524683135245]\ntest_history_sgd = [1.6786145805358887, 1.5426626907348633, 1.4472716250419617, 1.3957358184814452, 1.3148381408691405, 1.29473189868927, 1.264362098312378, 1.217076392364502, 1.1670963173866271, 1.1531425932884216, 1.1406869713306427, 1.1021037538528442, 1.0914713718414306, 1.0899892776966096, 1.0537007870197297, 1.0333033752918244, 1.0198554055690765, 1.001227827835083, 0.9913343264579773, 0.9629756865978241, 1.0211596514225005, 0.9477645175933838, 0.9602912710189819, 0.9349087412357331, 0.9298085560798645, 0.9016300676345825, 0.8972092303752899, 0.880790442609787, 0.8755210446357727, 0.9039027181386947, 0.8746295475959778, 0.8662550373792648, 0.8631444351911545, 0.85464867811203, 0.8472463070392608, 0.8322385994911193, 0.8412329930543899, 0.8377557963848113, 0.8322518620252609, 0.8103238815307617, 0.8268217628479004, 0.8335598628759384, 0.8236618486642837, 0.8022822110891342, 0.802699739408493, 0.7945333320856094, 0.7950013797283173, 0.8087212081193924, 0.7944275234937668, 0.783244926404953, 0.8091769362211227, 0.7787086180448533, 0.7879883528709412, 0.7775780043005943, 0.788777374458313, 0.7787729692459107, 0.7805862352371216, 0.7789684000492096, 0.7704118461847306, 0.7956378571271896, 0.7762850425243377, 0.7824362245440483, 0.7853597892165184, 0.7836380966186524, 0.8079483035564423, 0.7724824737906456, 0.774253191447258, 0.7804946542739868, 0.7637083023667336, 0.7706699856758118]\ntest_accuracy_sgd = [0.386, 0.4414, 0.4743, 0.495, 0.5273, 0.5314, 0.5481, 0.5599, 0.5833, 0.5864, 0.5891, 0.6071, 0.6115, 0.6131, 0.6211, 0.6314, 0.6372, 0.6453, 0.645, 0.6538, 0.6411, 0.6604, 0.6595, 0.6704, 0.6708, 0.682, 0.6859, 0.691, 0.6957, 0.681, 0.6933, 0.6932, 0.6935, 0.7001, 0.7041, 0.7025, 0.7038, 0.7058, 0.7084, 0.714, 0.7115, 0.7114, 0.7118, 0.724, 0.7187, 0.7235, 0.7274, 0.7227, 0.7227, 0.726, 0.7227, 0.7301, 0.7275, 0.7332, 0.7319, 0.7314, 0.7354, 0.7386, 0.7403, 0.729, 0.7332, 0.7353, 0.7351, 0.7354, 0.7297, 0.7425, 0.7393, 0.7392, 0.7487, 0.7425]\n\ntrain_history_sgd_inner_scaled = [2.2318012452186524, 2.090471400019458, 1.9373616928334736, 1.8021524653715246, 1.6891976512606492, 1.612616529245206, 1.5410159332368074, 1.4773078656867338, 1.4204552027270616, 1.3711878329591678, 1.3351247408201017, 1.290228149470161, 1.2478535980214853, 1.2130701973310212, 1.1855891412481323, 1.149288128556498, 1.1259765067063938, 1.1008881025607018, 1.0746977623466336, 1.0589309249387677, 1.0366668512144357, 1.0174373528536629, 1.0012538419355212, 0.9876467763920269, 0.9720304019920661, 0.9517975897740221, 0.9347014200047153, 0.9215668673100679, 0.9064469989913199, 0.8896163011450902, 0.8753911736981034, 0.8651306214539901, 0.8527591830629218, 0.8409150037009393, 0.8329214249425532, 0.8176117377817783, 0.8049484038596872, 0.79550330215098, 0.7816543184277956, 0.7758146102166237, 0.7627496313865837, 0.7557579554102914, 0.7466546386251669, 0.7346500768076123, 0.7333801034313944, 0.7198734548695557, 0.7135955365112675, 0.7012264179756574, 0.69674279668447, 0.6867020643885483, 0.6726865121318252, 0.6648862629442873, 0.6616226658796716, 0.6469617404443834, 0.644249143274239, 0.633478439296298, 0.6298292069636342, 0.622573130271014, 0.6140732185157669, 0.6056444308032161, 0.6021816731261476, 0.5924007189090904, 0.5843267679366919, 0.5795991135680157, 0.570548194219999, 0.5633139082842775, 0.5611302758116856, 0.5561809662510367, 0.5471486490400855, 0.5530321516496751]\ntest_history_sgd_inner_scaled = [2.0861871963500978, 1.9254003395080566, 1.789966251373291, 1.6725059566497802, 1.5952623982429504, 1.5174169850349426, 1.4376386251449584, 1.4011076294898988, 1.355934102153778, 1.2991651938438415, 1.2526158815860748, 1.2384345933914185, 1.2139110381126403, 1.1565131799697876, 1.1398981944084168, 1.1050534401893615, 1.1075664073944091, 1.077278494310379, 1.0475885344982148, 1.0431599359035493, 1.0223201443195342, 1.0009963921546936, 0.9976084202527999, 0.9950398890018464, 0.9548508893489838, 0.961242371225357, 0.9412410428524017, 0.9369736462116242, 0.9222050358057022, 0.9158995029449463, 0.9146855226755142, 0.897682459282875, 0.890860706949234, 0.8894631669998169, 0.8710585762500763, 0.8762776003360748, 0.8609281773090363, 0.862050030374527, 0.8580187593460084, 0.8427099492549897, 0.8433732089281082, 0.8252258454322815, 0.8472241681575775, 0.8400553377866745, 0.8373728781223297, 0.8141929750919342, 0.8140952060937882, 0.8151446926474571, 0.8032381574630737, 0.8035857553005219, 0.8018570559978485, 0.8166680019617081, 0.8069651222705841, 0.7954932369947434, 0.796411354970932, 0.7893109175443649, 0.7866549297571183, 0.7805530829191207, 0.7786314575970172, 0.7766115715742111, 0.7748834243416787, 0.770889468717575, 0.7766969939231873, 0.7855925551176072, 0.7771794466257095, 0.7754120976686477, 0.7857725606799125, 0.7880989054203034, 0.7846998572587967, 0.793988785290718]\ntest_accuracy_sgd_inner_scaled = [0.2869, 0.3259, 0.3575, 0.3997, 0.4211, 0.452, 0.4773, 0.4926, 0.5056, 0.5245, 0.5525, 0.5538, 0.564, 0.5906, 0.5966, 0.6069, 0.6084, 0.6167, 0.6319, 0.6337, 0.6373, 0.6457, 0.6483, 0.6503, 0.6667, 0.6671, 0.6727, 0.6716, 0.6761, 0.6776, 0.683, 0.6922, 0.691, 0.6948, 0.7016, 0.6949, 0.7035, 0.7079, 0.7085, 0.7105, 0.7095, 0.7187, 0.7082, 0.7123, 0.7148, 0.7171, 0.7215, 0.7219, 0.7235, 0.7227, 0.7295, 0.7225, 0.7252, 0.7287, 0.7279, 0.7354, 0.7317, 0.7358, 0.739, 0.7385, 0.7392, 0.7444, 0.7422, 0.7399, 0.7399, 0.7407, 0.7366, 0.7378, 0.7414, 0.7397]\n\ntrain_history_sgd_outer_scaled = [2.734755144094872, 2.1559328334715664, 1.9135379178444747, 1.7423936171299965, 1.611898781088612, 1.5228139841952897, 1.437572290830295, 1.368743831239393, 1.3011324064201102, 1.2449825499063867, 1.1894002655887848, 1.156664744972268, 1.1170412481898238, 1.0795924349514114, 1.0507941630185413, 1.0173108550288794, 0.9963076488136331, 0.9677510819471705, 0.9490418801527194, 0.9166547078305803, 0.8983832126688165, 0.8764494262692873, 0.8592085371846738, 0.8475561262396596, 0.8290250089467334, 0.8121339915048741, 0.7968230738359339, 0.7801974280106138, 0.7684843102684411, 0.7538436393603645, 0.7374845505370509, 0.7186073199714846, 0.7099975496911637, 0.6975199023018712, 0.6890010179003792, 0.6764264775206671, 0.66357009612081, 0.6592818026042655, 0.6439486947815741, 0.6339883637397795, 0.6272397997891507, 0.6104005949424051, 0.60740093959262, 0.5993687877874545, 0.5857281686399903, 0.5774738677322407, 0.5685141255026278, 0.5551007188799436, 0.5510773074901317, 0.542491237144641, 0.5344222127019292, 0.530513418833618, 0.5174932003478565, 0.5155146176095509, 0.5068207129340647, 0.4941598035185538, 0.4920698744256783, 0.4787304594236262, 0.4715951530219954, 0.4627017788874829, 0.45810546411577696, 0.45865854819107543, 0.44189917644881227, 0.4346453889327891, 0.43066051297480495, 0.4256239597830931, 0.4170540096738454, 0.411089821392313, 0.40057858946683156, 0.3962820050737742]\ntest_history_sgd_outer_scaled = [2.2565707939147948, 1.928864527130127, 1.7284959190368652, 1.5735052151679993, 1.4935184438705444, 1.400245104598999, 1.343312020111084, 1.269099830532074, 1.2533915534973143, 1.1825243702888488, 1.1350438314437865, 1.0852409399032592, 1.0768203050136567, 1.0363005226135253, 1.0150298825740813, 0.9990241038799286, 0.967222993516922, 0.9621713667869568, 0.934633120393753, 0.9323773579120636, 0.8981920072555541, 0.909993732213974, 0.8786225780963898, 0.8912253973007203, 0.8692964820623398, 0.853575363945961, 0.832305031824112, 0.8325040679216384, 0.8373297392368316, 0.8308135695934296, 0.8198840294957161, 0.8149787714958191, 0.8037518949151039, 0.8054897474050522, 0.7902022152900696, 0.8027258678674698, 0.7813959865093231, 0.7757828348338603, 0.7976319522380829, 0.8019624327659607, 0.7778959437131882, 0.7837811749458313, 0.7660952563405037, 0.7710922857761383, 0.764170417690277, 0.7754838088035584, 0.7771091002464294, 0.7639746448516845, 0.7536396073579789, 0.7870718006134033, 0.7745101166129112, 0.772151857972145, 0.7770572028040886, 0.8042461374998092, 0.7894715814828873, 0.7682580067396164, 0.7808466106534004, 0.772368951678276, 0.7790550349473954, 0.7748635146439076, 0.7849929935157299, 0.7893183988451957, 0.772231577938795, 0.7980978078961373, 0.7763801577210426, 0.7998105929613113, 0.7886816802382469, 0.8034935464859009, 0.8002948106825352, 0.7806463013768196]\ntest_accuracy_sgd_outer_scaled = [0.2517, 0.3259, 0.3778, 0.4336, 0.4543, 0.49, 0.515, 0.5449, 0.5448, 0.5699, 0.5903, 0.6143, 0.6153, 0.6312, 0.6416, 0.6497, 0.6599, 0.6624, 0.6786, 0.6748, 0.6842, 0.6809, 0.6916, 0.6927, 0.6973, 0.7031, 0.7097, 0.7132, 0.7087, 0.7115, 0.7164, 0.7209, 0.7262, 0.7206, 0.7305, 0.7281, 0.7367, 0.7369, 0.7277, 0.7288, 0.7346, 0.7364, 0.7422, 0.744, 0.744, 0.7402, 0.743, 0.7525, 0.7539, 0.7457, 0.7444, 0.748, 0.7444, 0.7372, 0.7436, 0.7426, 0.7484, 0.7503, 0.7495, 0.7487, 0.7471, 0.7486, 0.7551, 0.7483, 0.7525, 0.7516, 0.7565, 0.7535, 0.7579, 0.7616]\n\ntrain_history_sgd_compressed = [1.262003781697939, 1.223101448677385, 1.1879128589654517, 1.1600233070990618, 1.1296119592378817, 1.104746996289324, 1.0826637997956532, 1.0621871757690253, 1.0411297192658915, 1.0193484957565737, 1.000866011890304, 0.9840863443091702, 0.9695096596732469, 0.9473482031956353, 0.9388513554392568, 0.9190900514802665, 0.9090222118760619, 0.8908636833700683, 0.88465454770476, 0.870956557943388, 0.8583722405726343, 0.8458633360350528, 0.8348961371899871, 0.8234545541236468, 0.8114070848125936, 0.8024053716903452, 0.7906545801540775, 0.7885359961663365, 0.7757551888065874, 0.7670770695294871, 0.7543268478130136, 0.7488519387781772, 0.7429596046962397, 0.7364188171256228, 0.7246501275035732, 0.7163491983852728, 0.7091508850722057, 0.7049947650078923, 0.7033130476999161, 0.6850601770079044, 0.6834907396827512, 0.6740120765955552, 0.6665030678214929, 0.6621027841897267, 0.6527933954735241, 0.6496226404176648, 0.6428894973014627, 0.6365494197591797, 0.6268537208399809, 0.6239985527894686, 0.6182065472730895, 0.6124215079542926, 0.6035195072288708, 0.6001607233758472, 0.5914280528150251, 0.5874868590203698, 0.5789026687364749, 0.5732506464814286, 0.5711625596446455, 0.5659618819766032, 0.5587818294077578, 0.5482278624001671, 0.5451072275333697, 0.5423903735092533, 0.5354117498068554, 0.5302327402564876, 0.5288125928253165, 0.5194215984905467, 0.5123884929415515, 0.509419435659028]\ntest_history_sgd_compressed = [1.1902498121261598, 1.17531905169487, 1.178860270023346, 1.121958181333542, 1.1157148064136506, 1.0599474497795105, 1.0443437398433686, 1.0133783891677857, 1.0329696701526643, 0.9948125669956207, 0.9822675715446472, 0.9994031380653381, 0.9567767726421357, 0.9448026925086975, 0.9422902925491333, 0.9493434011936188, 0.9279708566188812, 0.9059292998790741, 0.9177629492282867, 0.9148192732810975, 0.8809989230155945, 0.8770297095298767, 0.8980734417915345, 0.8659609857559204, 0.8624386568546295, 0.856739469909668, 0.8688057856082916, 0.8387032034873962, 0.8265673158168793, 0.8510627625226974, 0.8436722570419312, 0.8616395692825317, 0.8280350009441376, 0.8234533514499665, 0.8329394807815552, 0.8248732857108116, 0.8146991772651673, 0.8121953356742859, 0.7963076956510544, 0.8229876693725586, 0.7863390752553939, 0.7990563718318939, 0.8112512327194213, 0.7894997272014618, 0.7864150822162628, 0.7968904612302781, 0.8109199880838395, 0.7949734829187394, 0.7823168821454048, 0.7795465737104416, 0.7816813512206078, 0.7704219910740853, 0.7818254251360893, 0.765509595143795, 0.7825155123710632, 0.7709559915304184, 0.7793055036902428, 0.764231663787365, 0.7655105590343475, 0.7531800288438797, 0.7635612778425217, 0.7814367614984512, 0.7747712067365646, 0.7856537578344345, 0.7561029555082321, 0.7683688801646232, 0.7953356580734253, 0.7572839631319046, 0.7572657172203064, 0.7870303845405578]\ntest_accuracy_sgd_compressed = [0.5776, 0.5815, 0.5818, 0.5988, 0.6001, 0.6228, 0.6289, 0.6387, 0.6317, 0.6479, 0.6488, 0.6425, 0.6591, 0.6649, 0.6676, 0.6655, 0.6709, 0.682, 0.6785, 0.6762, 0.6895, 0.6945, 0.6877, 0.6974, 0.6967, 0.6997, 0.6919, 0.7014, 0.7145, 0.7045, 0.7091, 0.704, 0.7117, 0.7116, 0.7114, 0.7133, 0.7196, 0.7161, 0.7259, 0.7143, 0.7283, 0.7251, 0.7223, 0.7255, 0.733, 0.7282, 0.7241, 0.7237, 0.7334, 0.7306, 0.7332, 0.7388, 0.7383, 0.7416, 0.7354, 0.7414, 0.7415, 0.7406, 0.7422, 0.7441, 0.7441, 0.7379, 0.7402, 0.742, 0.7473, 0.7432, 0.7358, 0.7464, 0.7493, 0.7411]\n\ntrain_history_sgd_scaled_compressed_inner = [2.9299774322363423, 2.3182335979188493, 2.042542840208849, 1.8557010071966655, 1.723819945779298, 1.6249066969317854, 1.5505730063104264, 1.4894069091743216, 1.4306088768307814, 1.386749512094366, 1.3381545458303388, 1.2978667549769898, 1.2652394399618554, 1.2183289273315683, 1.1938622862176822, 1.1664252142467157, 1.1358750721682673, 1.1108027449654192, 1.0884181646739735, 1.0691120008678388, 1.0466864862100547, 1.0305020870150203, 1.0092211135513032, 0.9923650640660845, 0.9776458028332352, 0.9610805846846012, 0.9434822995949279, 0.9216390820720312, 0.9133935913710338, 0.8976099713684043, 0.8846075527198479, 0.8649432198775698, 0.8536183029184561, 0.8429512245880674, 0.8351894624702766, 0.8255706496555787, 0.807930937508488, 0.8028958163907766, 0.7918678951995147, 0.7757025976924945, 0.7684524859613775, 0.7607900329563014, 0.7467295637216105, 0.7362739494084702, 0.7236211979785538, 0.7129682922912071, 0.7034459121696784, 0.6966595545296779, 0.6927253548294077, 0.678666245297093, 0.6687679351748103, 0.6624701656496433, 0.6596896309986748, 0.6450569130423124, 0.6409150366588017, 0.6333262559093172, 0.624137683292789, 0.618096661079875, 0.6179886847505789, 0.6007488568115722, 0.6014550159806791, 0.5876053515297678, 0.5788004439505164, 0.5765530931980104, 0.5676990876264889, 0.5646206088688063, 0.5562325837209706, 0.5502551691916288, 0.5444448305212933, 0.5407295774315934]\ntest_history_sgd_scaled_compressed_inner = [2.4531054357528688, 2.0726019412994385, 1.8432706939697265, 1.6877176312446593, 1.5961076974868775, 1.5026969236373902, 1.459201419353485, 1.3991970333099366, 1.3383406701087952, 1.2936905153274536, 1.264103639125824, 1.2404495940208435, 1.1803785563468934, 1.161952561044693, 1.137059510564804, 1.1321731094360352, 1.0956928126811982, 1.0794004732608795, 1.0701948640823364, 1.057050452375412, 1.0314080174922944, 1.0185014241218566, 1.0044352447986602, 0.9926362727165222, 0.9883937211036682, 0.982318452501297, 0.9548725729942322, 0.9498049323558807, 0.9463651374340057, 0.9171054335594178, 0.9155992934465408, 0.9102003805160522, 0.9152330703258514, 0.9028133519172669, 0.8907222744464874, 0.8854277063608169, 0.8735000732898712, 0.8669819157361984, 0.8506418509483338, 0.8521835237979889, 0.8467949129104614, 0.8282693381547928, 0.8307449614048004, 0.8245369570732116, 0.8357137504816056, 0.8261662848234177, 0.8389604692459106, 0.8284739316463471, 0.8347499775290489, 0.8125081326723099, 0.8018894144058227, 0.8226334892868996, 0.8089446242332459, 0.8033887022018432, 0.8120341000676156, 0.813242348074913, 0.7964262612104416, 0.7964708696484566, 0.8005146418571473, 0.7924377792596817, 0.8000310825586319, 0.8049330807209015, 0.7911752352237701, 0.8039636115074158, 0.8008808517336845, 0.7972060086250305, 0.7803333067059517, 0.7923710363149643, 0.7907140140533447, 0.808025567650795]\ntest_accuracy_sgd_scaled_compressed_inner = [0.2064, 0.2864, 0.3514, 0.3901, 0.4182, 0.4568, 0.4717, 0.4926, 0.5211, 0.5295, 0.5447, 0.5527, 0.5781, 0.5832, 0.5921, 0.5989, 0.6115, 0.6163, 0.6233, 0.6253, 0.6391, 0.6403, 0.6414, 0.6516, 0.6499, 0.6556, 0.6659, 0.6655, 0.6674, 0.6802, 0.683, 0.6811, 0.6838, 0.6872, 0.6926, 0.6889, 0.6974, 0.6988, 0.7016, 0.7058, 0.7036, 0.711, 0.7141, 0.7126, 0.715, 0.716, 0.7151, 0.7174, 0.7168, 0.7223, 0.7257, 0.7197, 0.7255, 0.7276, 0.7281, 0.7258, 0.7337, 0.7337, 0.7291, 0.7337, 0.7319, 0.7336, 0.7359, 0.7334, 0.7328, 0.7307, 0.7385, 0.7385, 0.7392, 0.729]\n\ntrain_history_sgd_scaled_compressed_outer = [2.49992680488645, 2.101682760526457, 1.9000719739957843, 1.7442440245767383, 1.6272761977237205, 1.5290931883980245, 1.4466403768495526, 1.3730277191952367, 1.3067695271328588, 1.2537697434730237, 1.1987953707385246, 1.1557007631682374, 1.1121256516107818, 1.0808859648911848, 1.0401781579417646, 1.0114214923375708, 0.9811157215281826, 0.9573851287212518, 0.9329429017308423, 0.9143116233293967, 0.8930251846837876, 0.8640207058328497, 0.8501562511219698, 0.8339528601492763, 0.8152163333600134, 0.8010200865738227, 0.7898385654325071, 0.7717439919481497, 0.7606347351123, 0.7443203234764011, 0.7305673479728991, 0.7194150303635756, 0.7055583790592526, 0.6943786820334852, 0.6795703107896058, 0.6647903301831707, 0.6562352272708093, 0.6469405069375587, 0.6332164556931352, 0.6235401973395092, 0.615927092635723, 0.6011365581199032, 0.598900815562519, 0.5894234614908848, 0.5798887512872896, 0.5624150981378677, 0.5568159599133464, 0.547137860675602, 0.5390439368879704, 0.531501192647173, 0.52461237919605, 0.5125738377766231, 0.5066491615436876, 0.5010809462393642, 0.48966657650440243, 0.48235405421317995, 0.4726671870712124, 0.4755332554545244, 0.464963622791383, 0.44758603335036645, 0.451782730381812, 0.4441072025414928, 0.43610464791050346, 0.4257245273388865, 0.4266557438522958, 0.4221905702367768, 0.410424818582547, 0.4022166066995972, 0.39538034579485576, 0.3899186437239732]\ntest_history_sgd_scaled_compressed_outer = [2.1466391706466674, 1.8978026260375978, 1.7171132694244384, 1.5869399968147277, 1.4970256748199462, 1.406497793674469, 1.3288066453933716, 1.2856683755874634, 1.2124824598312378, 1.1737297410964966, 1.1460332576274872, 1.0948306413173676, 1.0585195725917815, 1.0407940132141114, 1.0141463903903962, 0.9979715213298798, 0.9846687103748322, 0.9701690089702606, 0.9206629390239716, 0.9144797614097595, 0.8992555014133453, 0.8934278646230698, 0.8634964232563972, 0.8647179388046264, 0.8745742149353027, 0.84230951628685, 0.8682110961437225, 0.8160320513010025, 0.8192362357854843, 0.8218272322535515, 0.8300912773609161, 0.7995053346037865, 0.8251189314603805, 0.7900039362430572, 0.8056368995428085, 0.78279034948349, 0.7782809800386429, 0.7685016411423683, 0.7623070328712463, 0.7768510251998901, 0.7722879499197006, 0.7636509130716324, 0.763286954331398, 0.7732146285057068, 0.771544474864006, 0.7693361566424369, 0.7522853110790253, 0.7724723019003868, 0.758720090353489, 0.7544767904639245, 0.776011409175396, 0.7526136765360832, 0.7652492602109909, 0.7617963994562625, 0.7666739051342011, 0.7611970255851745, 0.750373073720932, 0.7659328227519989, 0.7607471852719784, 0.7736482710719108, 0.791342638206482, 0.7587500357389451, 0.7783901552915573, 0.7788047758460045, 0.7826953435301781, 0.7828931076288224, 0.7771333933293819, 0.7911407310247421, 0.809991860628128, 0.7885194109112025]\ntest_accuracy_sgd_scaled_compressed_outer = [0.2699, 0.3302, 0.3829, 0.4246, 0.4554, 0.4902, 0.5152, 0.5393, 0.5646, 0.5811, 0.5845, 0.6077, 0.6293, 0.6356, 0.6439, 0.6445, 0.6514, 0.6617, 0.6793, 0.676, 0.6857, 0.6864, 0.6955, 0.6994, 0.6941, 0.7064, 0.6943, 0.7134, 0.7138, 0.7131, 0.7129, 0.7246, 0.7122, 0.724, 0.7223, 0.7277, 0.7322, 0.7314, 0.7412, 0.7355, 0.7388, 0.737, 0.7432, 0.7367, 0.7426, 0.7428, 0.7455, 0.7402, 0.7494, 0.7492, 0.7449, 0.7553, 0.7468, 0.7548, 0.7501, 0.7507, 0.7553, 0.7514, 0.7526, 0.7529, 0.7483, 0.7569, 0.7543, 0.7482, 0.7521, 0.7603, 0.7598, 0.7571, 0.7518, 0.758]\n\ntrain_history_sgd_scaled_compressed_outer_extra = [2.5406237026614606, 2.106352282302154, 1.8920180660379513, 1.732673030680098, 1.6094242099605862, 1.520004238314031, 1.4407110683753361, 1.3607437881972173, 1.3093612794680973, 1.2491173392061687, 1.1995996854189412, 1.1539327477860024, 1.1143254610278723, 1.077865355917255, 1.0527481084589458, 1.0228082583383526, 0.9906338189568971, 0.9657403401401646, 0.9437456850505546, 0.9205543805876046, 0.9012109074751129, 0.8861921836653024, 0.8706265108664627, 0.8481090081012462, 0.8336787236011242, 0.8115093208030056, 0.7947526150347327, 0.784991906274615, 0.7669946231195689, 0.7500489672736439, 0.7434955658510213, 0.7264112451344805, 0.7155844433533262, 0.7061957442547049, 0.6928111300291613, 0.6833290189428403, 0.673631139499757, 0.661589895444148, 0.6547350529057291, 0.6357486350914402, 0.6274105936212613, 0.6153546519139234, 0.6090226125381791, 0.5941820681247565, 0.5840724661679524, 0.577334982888473, 0.5671892185955096, 0.5630355288305551, 0.5503498981980717, 0.5383966239669439, 0.5338161343046467, 0.5246371835698862, 0.5136868773061601, 0.5067757691263848, 0.49758790650636037, 0.4892131721272188, 0.4831313214948415, 0.4739829385676957, 0.4696992494718498, 0.46152757607457584, 0.4556544613274162, 0.44901513588398007, 0.43934724946765946, 0.43190058318855207, 0.4259889204712475, 0.4217458706530159, 0.412760342059233, 0.40294732565007857, 0.407527161604913, 0.3927165676489511]\ntest_history_sgd_scaled_compressed_outer_extra = [2.1627461359024047, 1.9072267169952393, 1.7241101062774657, 1.5820331926345825, 1.5048310627937318, 1.4130854692459107, 1.3277099289894103, 1.2827889265060424, 1.236320248413086, 1.1617516074180603, 1.1409958519458772, 1.1186600090503693, 1.0761837938308716, 1.044596455335617, 1.0699234938144684, 1.068800223493576, 1.0190847621202468, 0.9801188090324402, 0.9446408075094223, 0.9261144246101379, 0.9194872926712037, 0.9449862409114838, 0.9605080866336823, 0.8920181478500366, 0.8624220116853714, 0.8406127494096756, 0.8401101671457291, 0.8258653702974319, 0.8306030585289002, 0.819345897436142, 0.8107458086490631, 0.8248811614751815, 0.8139328319072724, 0.7936740801095963, 0.8099217563867569, 0.7890357864141464, 0.7791718406438828, 0.7689453128576279, 0.7845496306419373, 0.7662637867808342, 0.7693901278853417, 0.758457387816906, 0.782583720421791, 0.7752035917520523, 0.7791474285840988, 0.768068784070015, 0.7596453118681907, 0.7654211332440376, 0.7649640267729759, 0.7758112319111824, 0.7623658915996552, 0.7729759290933609, 0.7531031841576099, 0.7598042618274689, 0.7686878109931946, 0.7857715218186379, 0.7666722971916199, 0.7770382363080979, 0.7537623678326607, 0.7834407404065132, 0.7661984623074531, 0.7778641127943993, 0.7931705818533897, 0.7786724586606025, 0.7862282859086991, 0.7812559281110764, 0.7782887732923031, 0.7789498527944088, 0.7896790513694286, 0.779297125518322]\ntest_accuracy_sgd_scaled_compressed_outer_extra = [0.2671, 0.3286, 0.3857, 0.4236, 0.456, 0.4863, 0.5214, 0.5393, 0.5585, 0.5829, 0.593, 0.6025, 0.6145, 0.6276, 0.6229, 0.622, 0.6435, 0.6568, 0.672, 0.6755, 0.6805, 0.6765, 0.6697, 0.6856, 0.6981, 0.7088, 0.7041, 0.71, 0.7116, 0.7151, 0.7169, 0.7152, 0.7161, 0.7254, 0.7227, 0.7289, 0.7296, 0.7383, 0.7348, 0.7365, 0.739, 0.7422, 0.7312, 0.7392, 0.7411, 0.7443, 0.7466, 0.7496, 0.7449, 0.7446, 0.7491, 0.7464, 0.7543, 0.756, 0.7513, 0.746, 0.7535, 0.753, 0.7563, 0.75, 0.7599, 0.7548, 0.7488, 0.7552, 0.7537, 0.7525, 0.762, 0.7585, 0.7628, 0.7599]\n\ntrain_history_outer_adam = [2.61121294382588, 2.0552122202675664, 1.8039897402838978, 1.6926750414206853, 1.622790591491153, 1.5690120101889686, 1.498701158691855, 1.440886452984627, 1.3968079550491879, 1.3427969381937286, 1.3063491370976734, 1.275025328559339, 1.2351485335308572, 1.2135902676740875, 1.1821732528679205, 1.1643440134994818, 1.1370730686675556, 1.1120589213907872, 1.0934191751663032, 1.0812930708651043, 1.0642533448651015, 1.043872616022749, 1.0329594799624684, 1.013892228188722, 0.9983281987097562, 0.9921220469352839, 0.9747039128447432, 0.9685549231441429, 0.9450095350785024, 0.9413313169003753, 0.9231784445855319, 0.910128961438718, 0.9023034842422856, 0.8982406534502269, 0.896220871249733, 0.89279964390923, 0.8728800794047773, 0.8688472992623858, 0.8618537405567706, 0.849311768856195, 0.8393275345987676, 0.8306141020087026, 0.8322594949351553, 0.8141357741697365, 0.8130984146271825, 0.8178557360263736, 0.8129791815567504, 0.8101245412588729, 0.7935588012266037, 0.788300954929703, 0.7912907568390107, 0.7756436554062397, 0.7924046685628574, 0.763814682317207, 0.762020858504888, 0.7467218344016453, 0.7534629507442875, 0.7543160752262301, 0.751884259912364, 0.7461772842327957, 0.7533736578796221, 0.7410666450210239, 0.7197149110876996, 0.7341793226006695, 0.7250374714126977, 0.7248866289015621, 0.697563453479801, 0.6974859771216312, 0.6998338611686931, 0.7248233410403552]\ntest_history_outer_adam = [6.093672647666931, 1.9155030939102173, 1.7521262046813966, 1.6626970367431642, 1.7802293769836426, 1.6826351286888122, 1.4456771821022034, 1.4745691892623902, 1.307985607290268, 1.267165793800354, 1.300466541671753, 1.3174608361244202, 1.240172216463089, 1.1586219319343567, 1.1627706894397736, 1.2261880486011505, 1.1233512118816376, 1.1201043322086335, 1.1555092755317689, 1.0745550878047943, 1.0610674878120423, 1.0954926280975341, 1.0352102158546448, 1.0230990593910216, 0.9997872844219208, 1.0535347546339036, 1.0097156684398652, 0.9895632110118866, 0.8794805079221726, 0.9789764636039734, 0.9520318202972412, 0.9368130234718323, 0.8882642211198807, 0.8918604131221771, 0.9449115617990493, 0.9076847891569138, 0.9237361988067627, 0.876261108660698, 0.8897307806491852, 0.8894512924194335, 0.844146372961998, 0.9067493494033814, 0.9217988811731338, 0.8577427137494087, 0.8236721105575562, 0.8322501059055328, 0.9008107842206955, 0.812061683356762, 0.7993037868499756, 0.7909158054828643, 0.7630294699430465, 0.8322348034024238, 0.8044501866936684, 0.832157728266716, 0.7916093698501587, 0.8386656709194183, 0.803033715224266, 0.8571025485515594, 0.8021261031389236, 0.8237546702623367, 0.7644322073698043, 0.8842719767332077, 0.7435595806598664, 0.746710204577446, 0.7665097375512123, 0.7660434710502625, 0.7678096270442009, 0.8288663326263428, 0.730210820555687, 2.5744205490112306]\ntest_accuracy_outer_adam = [0.203, 0.303, 0.3708, 0.3667, 0.4155, 0.4068, 0.4749, 0.4774, 0.5219, 0.5467, 0.542, 0.5565, 0.5789, 0.5886, 0.591, 0.5801, 0.6119, 0.6168, 0.5984, 0.6338, 0.6448, 0.6201, 0.6515, 0.647, 0.6593, 0.6477, 0.6723, 0.6716, 0.6968, 0.6754, 0.6815, 0.6829, 0.6962, 0.7046, 0.6815, 0.7081, 0.7065, 0.7007, 0.701, 0.7148, 0.7243, 0.7002, 0.7081, 0.7166, 0.7362, 0.738, 0.7235, 0.7375, 0.737, 0.7343, 0.7478, 0.74, 0.7361, 0.7338, 0.7454, 0.7286, 0.7499, 0.7315, 0.7504, 0.7402, 0.7582, 0.7508, 0.7617, 0.7648, 0.7563, 0.7521, 0.7578, 0.7498, 0.759, 0.7527]","metadata":{"execution":{"iopub.status.busy":"2023-01-12T22:26:38.639744Z","iopub.execute_input":"2023-01-12T22:26:38.640223Z","iopub.status.idle":"2023-01-12T22:26:38.713759Z","shell.execute_reply.started":"2023-01-12T22:26:38.640178Z","shell.execute_reply":"2023-01-12T22:26:38.712789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.plot(train_history_sgd, '-ro', label='train sgd')\n#plt.plot(test_history_sgd, '-r*', label='test sgd')\nplt.plot(train_history_sgd_inner_scaled, '-g.', label='train sgd inner scaled')\nplt.plot(train_history_sgd_outer_scaled, '-g+', label='train sgd outer scaled')\n#plt.plot(test_history_sgd_scaled, '-b*', label='test sgd scaled')\nplt.plot(train_history_sgd_compressed, '-yo', label='train sgd compressed')\n#plt.plot(test_history_sgd_compressed, '-g*', label='test sgd compressed')\nplt.plot(train_history_sgd_scaled_compressed_inner, '-b.', label='train sgd scaled compressed inner')\n#plt.plot(test_history_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\nplt.plot(train_history_sgd_scaled_compressed_outer, '-b+', label='train sgd scaled compressed outer')\n#plt.plot(test_history_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.plot(train_history_sgd_scaled_compressed_outer_extra, '-co', label='train sgd scaled compressed outer extra')\nplt.plot(train_history_outer_adam, '-ko', label='train outer adam')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-12T22:34:53.596354Z","iopub.execute_input":"2023-01-12T22:34:53.597254Z","iopub.status.idle":"2023-01-12T22:34:53.889485Z","shell.execute_reply.started":"2023-01-12T22:34:53.597205Z","shell.execute_reply":"2023-01-12T22:34:53.886525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.plot(test_history_sgd, '-ro', label='test sgd')\n#plt.plot(test_history_sgd, '-r*', label='test sgd')\nplt.plot(test_history_sgd_inner_scaled, '-g.', label='test sgd inner scaled')\nplt.plot(test_history_sgd_outer_scaled, '-g+', label='test sgd outer scaled')\n#plt.plot(test_history_sgd_scaled, '-b*', label='test sgd scaled')\nplt.plot(test_history_sgd_compressed, '-yo', label='test sgd compressed')\n#plt.plot(test_history_sgd_compressed, '-g*', label='test sgd compressed')\nplt.plot(test_history_sgd_scaled_compressed_inner, '-b.', label='test sgd scaled compressed inner')\n#plt.plot(test_history_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\nplt.plot(test_history_sgd_scaled_compressed_outer, '-b+', label='test sgd scaled compressed outer')\n#plt.plot(test_history_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.plot(test_history_sgd_scaled_compressed_outer_extra, '-co', label='test sgd scaled compressed outer extra')\nplt.plot(test_history_outer_adam[1:-1], '-ko', label='test outer adam')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-12T22:34:48.012512Z","iopub.execute_input":"2023-01-12T22:34:48.013064Z","iopub.status.idle":"2023-01-12T22:34:48.408295Z","shell.execute_reply.started":"2023-01-12T22:34:48.013022Z","shell.execute_reply":"2023-01-12T22:34:48.407188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.plot(test_accuracy_sgd, '-ro', label='test sgd')\n#plt.plot(test_history_sgd, '-r*', label='test sgd')\nplt.plot(test_accuracy_sgd_inner_scaled, '-g.', label='test sgd inner scaled')\nplt.plot(test_accuracy_sgd_outer_scaled, '-g+', label='test sgd outer scaled')\n#plt.plot(test_history_sgd_scaled, '-b*', label='test sgd scaled')\nplt.plot(test_accuracy_sgd_compressed, '-yo', label='test sgd compressed')\n#plt.plot(test_history_sgd_compressed, '-g*', label='test sgd compressed')\nplt.plot(test_accuracy_sgd_scaled_compressed_inner, '-b.', label='test sgd scaled compressed inner')\n#plt.plot(test_history_sgd_scaled_compressed_inner, '-k*', label='test sgd scaled compressed inner')\nplt.plot(test_accuracy_sgd_scaled_compressed_outer, '-b+', label='test sgd scaled compressed outer')\n#plt.plot(test_history_sgd_scaled_compressed_outer, '-c*', label='test sgd scaled compressed outer')\nplt.plot(test_accuracy_sgd_scaled_compressed_outer_extra, '-co', label='test sgd scaled compressed outer extra')\nplt.plot(test_accuracy_outer_adam, '-ko', label='test outer adam')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-12T22:34:39.285448Z","iopub.execute_input":"2023-01-12T22:34:39.285899Z","iopub.status.idle":"2023-01-12T22:34:39.708741Z","shell.execute_reply.started":"2023-01-12T22:34:39.285861Z","shell.execute_reply":"2023-01-12T22:34:39.700348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}