{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Referance\n\nhttps://github.com/ki-ljl/Scaffold-Federated-Learning\n\nhttps://arxiv.org/pdf/1910.06378v4.pdf\n\nhttps://www.kaggle.com/code/vikasbhadoria/cifar10-high-accuracy-model-build-on-pytorch\n\nhttps://www.kaggle.com/code/jainishsavalia/cifar-cnn/notebook\n\nhttps://stackoverflow.com/questions/50544730/how-do-i-split-a-custom-dataset-into-training-and-test-datasets","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport cv2\nimport copy\nimport random\nimport sys\nimport numpy as np\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import datasets\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.optim import Optimizer\nfrom torch.optim.lr_scheduler import StepLR\n\ndevice = 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:52:02.013310Z","iopub.execute_input":"2023-10-14T05:52:02.013756Z","iopub.status.idle":"2023-10-14T05:52:02.021395Z","shell.execute_reply.started":"2023-10-14T05:52:02.013711Z","shell.execute_reply":"2023-10-14T05:52:02.019600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"trainDf = pd.read_csv(\"/kaggle/input/cifar-10/trainLabels.csv\")\nlen(trainDf)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:33:23.049911Z","iopub.execute_input":"2023-10-14T04:33:23.050265Z","iopub.status.idle":"2023-10-14T04:33:23.089227Z","shell.execute_reply.started":"2023-10-14T04:33:23.050232Z","shell.execute_reply":"2023-10-14T04:33:23.087206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Different classes in CIPHAR 10 dataset. \nclasses = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:33:23.090241Z","iopub.execute_input":"2023-10-14T04:33:23.090590Z","iopub.status.idle":"2023-10-14T04:33:23.098156Z","shell.execute_reply.started":"2023-10-14T04:33:23.090558Z","shell.execute_reply":"2023-10-14T04:33:23.096727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Torch DataLoader","metadata":{}},{"cell_type":"code","source":"training_dataset = datasets.CIFAR10(root='./data', train=True, download=True)\nvalidation_dataset = datasets.CIFAR10(root='./data', train=False, download=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:36:09.156470Z","iopub.execute_input":"2023-10-14T04:36:09.156834Z","iopub.status.idle":"2023-10-14T04:36:19.610184Z","shell.execute_reply.started":"2023-10-14T04:36:09.156807Z","shell.execute_reply":"2023-10-14T04:36:19.608823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(training_dataset), len(validation_dataset))","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:39:48.355461Z","iopub.execute_input":"2023-10-14T04:39:48.355839Z","iopub.status.idle":"2023-10-14T04:39:48.363147Z","shell.execute_reply.started":"2023-10-14T04:39:48.355805Z","shell.execute_reply":"2023-10-14T04:39:48.361769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dataset[0]","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:51:52.831210Z","iopub.execute_input":"2023-10-14T04:51:52.831594Z","iopub.status.idle":"2023-10-14T04:51:52.842195Z","shell.execute_reply.started":"2023-10-14T04:51:52.831563Z","shell.execute_reply":"2023-10-14T04:51:52.839757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Assuming for the train set there are 10 clients for training and 2 clients for testing, for each client the data length is 5000","metadata":{}},{"cell_type":"code","source":"2500/50000","metadata":{"execution":{"iopub.status.busy":"2023-10-14T04:55:50.350088Z","iopub.execute_input":"2023-10-14T04:55:50.350470Z","iopub.status.idle":"2023-10-14T04:55:50.357049Z","shell.execute_reply.started":"2023-10-14T04:55:50.350444Z","shell.execute_reply":"2023-10-14T04:55:50.356159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client_train = torch.utils.data.random_split(training_dataset, [0.1]*10)\nprint(len(client_train))\nprint(len(client_train[0]))","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:01:34.642191Z","iopub.execute_input":"2023-10-14T05:01:34.642562Z","iopub.status.idle":"2023-10-14T05:01:34.651156Z","shell.execute_reply.started":"2023-10-14T05:01:34.642540Z","shell.execute_reply":"2023-10-14T05:01:34.650262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client_test = torch.utils.data.random_split(validation_dataset, [0.5]*2)\nprint(len(client_test))\nprint(len(client_test[0]))","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:02:29.501718Z","iopub.execute_input":"2023-10-14T05:02:29.502070Z","iopub.status.idle":"2023-10-14T05:02:29.509395Z","shell.execute_reply.started":"2023-10-14T05:02:29.502046Z","shell.execute_reply":"2023-10-14T05:02:29.507736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"class LeNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 16, 3, 1, padding=1) # input is color image, hence 3 i/p channels. 16 filters, kernal size is tuned to 3 to avoid overfitting, stride is 1 , padding is 1 extract all edge features.\n        self.conv2 = nn.Conv2d(16, 32, 3, 1, padding=1) # We double the feature maps for every conv layer as in pratice it is really good.\n        self.conv3 = nn.Conv2d(32, 64, 3, 1, padding=1)\n        self.fc1 = nn.Linear(4*4*64, 500) # I/p image size is 32*32, after 3 MaxPooling layers it reduces to 4*4 and 64 because our last conv layer has 64 outputs. Output nodes is 500\n        self.dropout1 = nn.Dropout(0.5)\n        self.fc2 = nn.Linear(500, 10) # output nodes are 10 because our dataset have 10 different categories\n    def forward(self, x):\n        x = F.relu(self.conv1(x)) #Apply relu to each output of conv layer.\n        x = F.max_pool2d(x, 2, 2) # Max pooling layer with kernal of 2 and stride of 2\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2, 2)\n        x = F.relu(self.conv3(x))\n        x = F.max_pool2d(x, 2, 2)\n        x = x.view(-1, 4*4*64) # flatten our images to 1D to input it to the fully connected layers\n        x = F.relu(self.fc1(x))\n        x = self.dropout1(x) # Applying dropout b/t layers which exchange highest parameters. This is a good practice\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:23:55.784489Z","iopub.execute_input":"2023-10-14T05:23:55.784872Z","iopub.status.idle":"2023-10-14T05:23:55.793818Z","shell.execute_reply.started":"2023-10-14T05:23:55.784843Z","shell.execute_reply":"2023-10-14T05:23:55.792395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LeNet().to(device) # run our model on cuda GPU for faster results\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:24:01.408011Z","iopub.execute_input":"2023-10-14T05:24:01.408387Z","iopub.status.idle":"2023-10-14T05:24:01.421275Z","shell.execute_reply.started":"2023-10-14T05:24:01.408359Z","shell.execute_reply":"2023-10-14T05:24:01.420615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.train()","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:27:43.311676Z","iopub.execute_input":"2023-10-14T05:27:43.312072Z","iopub.status.idle":"2023-10-14T05:27:43.321261Z","shell.execute_reply.started":"2023-10-14T05:27:43.312046Z","shell.execute_reply":"2023-10-14T05:27:43.319476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Federated training functions","metadata":{}},{"cell_type":"code","source":"# options\n# B = Batch size\n# E = epoch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optimizer","metadata":{}},{"cell_type":"code","source":"class ScaffoldOptimizer(Optimizer):\n    def __init__(self, params, lr, weight_decay):\n        defaults = dict(lr=lr, weight_decay=weight_decay)\n        super(ScaffoldOptimizer, self).__init__(params, defaults)\n\n    def step(self, server_controls, client_controls, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure\n\n        for group in self.param_groups:\n            for p, c, ci in zip(group['params'], server_controls.values(), client_controls.values()):\n                if p.grad is None:\n                    continue\n                dp = p.grad.data + c.data - ci.data\n                p.data = p.data - dp.data * group['lr']\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:19:46.607603Z","iopub.execute_input":"2023-10-14T05:19:46.607930Z","iopub.status.idle":"2023-10-14T05:19:46.615897Z","shell.execute_reply.started":"2023-10-14T05:19:46.607905Z","shell.execute_reply":"2023-10-14T05:19:46.614598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Client train","metadata":{}},{"cell_type":"code","source":"def get_val_loss(model, Val):\n    model.eval()\n    loss_function = nn.CrossEntropyLoss().to(device)\n    val_loss = []\n    for (seq, label) in Val:\n        with torch.no_grad():\n            seq = seq.to(device)\n            label = label.to(device)\n            y_pred = model(seq)\n            loss = loss_function(y_pred, label)\n            val_loss.append(loss.item())\n\n    return np.mean(val_loss)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:28:16.543330Z","iopub.execute_input":"2023-10-14T05:28:16.543777Z","iopub.status.idle":"2023-10-14T05:28:16.551355Z","shell.execute_reply.started":"2023-10-14T05:28:16.543740Z","shell.execute_reply":"2023-10-14T05:28:16.549760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_to_dataloader(data, batch_size):\n    seq = DataLoader(dataset=seq, batch_size=batch_size, shuffle=False, num_workers=0)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:52:15.913416Z","iopub.execute_input":"2023-10-14T05:52:15.913770Z","iopub.status.idle":"2023-10-14T05:52:15.919078Z","shell.execute_reply.started":"2023-10-14T05:52:15.913745Z","shell.execute_reply":"2023-10-14T05:52:15.918086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# options include all the parameters like batch_size, epochs, lr, \ndef train(model, server, options):\n    model.train()\n    # TODO: convert the dataset to batchloader\n    print('training...')\n    \n    loss_function = nn.CrossEntropyLoss().to(device)\n    x = copy.deepcopy(model)\n    \n    optimizer = ScaffoldOptimizer(model.parameters(), lr=options.lr, weight_decay=1e-4)\n    lr_step = StepLR(optimizer, step_size=10, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T05:52:09.230144Z","iopub.execute_input":"2023-10-14T05:52:09.230486Z","iopub.status.idle":"2023-10-14T05:52:09.237602Z","shell.execute_reply.started":"2023-10-14T05:52:09.230455Z","shell.execute_reply":"2023-10-14T05:52:09.235827Z"},"trusted":true},"execution_count":null,"outputs":[]}]}