{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"},{"sourceId":2280,"sourceType":"datasetVersion","datasetId":1272},{"sourceId":283795,"sourceType":"datasetVersion","datasetId":118250}],"dockerImageVersionId":30097,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport cv2\nimport os\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import accuracy_score\n\nimport tensorflow as tf\nfrom tensorflow import expand_dims\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Input, Lambda\nfrom tensorflow.keras.layers import MaxPooling2D\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras.layers import Flatten\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras import backend as K","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-24T18:30:28.578436Z","iopub.execute_input":"2023-10-24T18:30:28.578898Z","iopub.status.idle":"2023-10-24T18:30:34.228786Z","shell.execute_reply.started":"2023-10-24T18:30:28.578812Z","shell.execute_reply":"2023-10-24T18:30:34.228009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imutils\nfrom imutils import paths","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:30:34.230336Z","iopub.execute_input":"2023-10-24T18:30:34.230696Z","iopub.status.idle":"2023-10-24T18:30:44.089242Z","shell.execute_reply.started":"2023-10-24T18:30:34.230665Z","shell.execute_reply":"2023-10-24T18:30:44.088195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Todo\n\ncreate subset of clients\n\n- increase comm rounds 300\n- increase hidden units 400\n- increase no of layers\n- no of clients 20","metadata":{}},{"cell_type":"code","source":"debug = 0","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:30:44.091264Z","iopub.execute_input":"2023-10-24T18:30:44.091582Z","iopub.status.idle":"2023-10-24T18:30:44.096885Z","shell.execute_reply.started":"2023-10-24T18:30:44.091548Z","shell.execute_reply":"2023-10-24T18:30:44.095988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(paths, verbose=-1):\n    '''expects images for each class in seperate dir, \n    e.g all digits in 0 class in the directory named 0 '''\n    data = list()\n    labels = list()\n    # loop over the input images\n    for (i, imgpath) in enumerate(paths):\n        # load the image and extract the class labels        \n        im_gray = cv2.imread(imgpath , cv2.IMREAD_GRAYSCALE)\n        image = np.array(im_gray).flatten() # cv2.imread(imgpath) \n        # print(image.shape)\n        label = imgpath.split(os.path.sep)[-2]\n        # scale the image to [0, 1] and add to list\n        data.append(image/255)\n        labels.append(label)\n        # show an update every `verbose` images\n        if verbose > 0 and i > 0 and (i + 1) % verbose == 0:\n            print(\"[INFO] processed {}/{}\".format(i + 1, len(paths)))\n    # return a tuple of the data and labels\n    \n    return data, labels\n\ndef create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    data = list(zip(image_list, label_list))\n    random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n#     max_y = np.argmax(label_list, axis=-1)\n#     sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n#     data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} \n\n\ndef batch_data(data_shard, bs=32):\n    '''Takes in a clients data shard and create a tfds object off it\n    args:\n        shard: a data, label constituting a client's data shard\n        bs:batch size\n    return:\n        tfds object'''\n    #seperate shard into data and labels lists\n    data, label = zip(*data_shard)\n    dataset = tf.data.Dataset.from_tensor_slices((list(data), list(label)))\n    return dataset.shuffle(len(label)).batch(bs)\n\n\ndef weight_scalling_factor(clients_trn_data, client_name):\n    client_names = list(clients_trn_data.keys())\n    #get the bs\n    bs = list(clients_trn_data[client_name])[0][0].shape[0]\n    #first calculate the total training data points across clinets\n    global_count = sum([tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy() for client_name in client_names])*bs\n    # get the total number of data points held by a client\n    local_count = tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy()*bs\n    \n    \n    if debug:\n        print('global_count', global_count, 'local_count', local_count, 'bs', bs)\n    \n    return local_count/global_count\n\n\ndef scale_model_weights(weight, scalar):\n    '''function for scaling a models weights'''\n    weight_final = []\n    steps = len(weight)\n    for i in range(steps):\n        weight_final.append(scalar * weight[i])\n    return weight_final\n\n\n\ndef sum_scaled_weights(scaled_weight_list):\n    '''Return the sum of the listed scaled weights. The is equivalent to scaled avg of the weights'''\n    avg_grad = list()\n    #get the average grad accross all client gradients\n    for grad_list_tuple in zip(*scaled_weight_list):\n        layer_mean = tf.math.reduce_sum(grad_list_tuple, axis=0)\n        avg_grad.append(layer_mean)\n        \n    return avg_grad\n\n\ndef test_model(X_test, Y_test,  model, comm_round):\n    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n    #logits = model.predict(X_test, batch_size=100)\n    logits = model.predict(X_test)\n    loss = cce(Y_test, logits)\n    acc = accuracy_score(tf.argmax(logits, axis=1), tf.argmax(Y_test, axis=1))\n    print('comm_round: {} | global_acc: {:.3%} | global_loss: {}'.format(comm_round, acc, loss))\n    return acc, loss\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:30:44.098282Z","iopub.execute_input":"2023-10-24T18:30:44.098532Z","iopub.status.idle":"2023-10-24T18:30:44.120918Z","shell.execute_reply.started":"2023-10-24T18:30:44.098508Z","shell.execute_reply":"2023-10-24T18:30:44.119982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleMLP:\n    @staticmethod\n    def build(shape, classes):\n        model = Sequential()\n        model.add(Dense(200, input_shape=(shape,)))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(200))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(classes))\n        model.add(Activation(\"softmax\"))\n        return model\n    \n#     def build(shape, classes):\n#         model = Sequential()\n#         model.add(Input(shape=(shape[0], shape[1], shape[2])))\n#         #model.add(Lambda(lambda x: expand_dims(x, axis=-1)))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Flatten())\n#         model.add(Dense(32))\n#         model.add(Dense(classes))\n#         model.add(Activation(\"softmax\"))\n#         return model","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:30:44.122171Z","iopub.execute_input":"2023-10-24T18:30:44.122530Z","iopub.status.idle":"2023-10-24T18:30:44.136143Z","shell.execute_reply.started":"2023-10-24T18:30:44.122502Z","shell.execute_reply":"2023-10-24T18:30:44.135272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#declear path to your mnist data folder\nimg_path = '../input/mnistasjpg/trainingSet/trainingSet' #'../input/cifar10-pngs-in-folders/cifar10/test'  # <-- test dataset #'../input/mnistasjpg/trainingSample/trainingSample' # <-- smaller dataset\n\n#get the path list using the path object\nimage_paths = list(paths.list_images(img_path))\n\n#apply our function\nimage_list, label_list = load(image_paths, verbose=10000)\n\n#binarize the labels\nlb = LabelBinarizer()\nlabel_list = lb.fit_transform(label_list)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-24T18:30:44.137318Z","iopub.execute_input":"2023-10-24T18:30:44.137656Z","iopub.status.idle":"2023-10-24T18:36:52.811169Z","shell.execute_reply.started":"2023-10-24T18:30:44.137627Z","shell.execute_reply":"2023-10-24T18:36:52.810174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split data into training and test set\nX_train, X_test, y_train, y_test = train_test_split(image_list, \n                                                    label_list, \n                                                    test_size=0.1, \n                                                    random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:36:52.812549Z","iopub.execute_input":"2023-10-24T18:36:52.812936Z","iopub.status.idle":"2023-10-24T18:36:52.840152Z","shell.execute_reply.started":"2023-10-24T18:36:52.812884Z","shell.execute_reply":"2023-10-24T18:36:52.839283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### IID","metadata":{}},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:36:52.841546Z","iopub.execute_input":"2023-10-24T18:36:52.841928Z","iopub.status.idle":"2023-10-24T18:36:52.850716Z","shell.execute_reply.started":"2023-10-24T18:36:52.841877Z","shell.execute_reply":"2023-10-24T18:36:52.849795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:36:52.854446Z","iopub.execute_input":"2023-10-24T18:36:52.854772Z","iopub.status.idle":"2023-10-24T18:36:52.929239Z","shell.execute_reply.started":"2023-10-24T18:36:52.854744Z","shell.execute_reply":"2023-10-24T18:36:52.928285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# client_names = ['{}_{}'.format('client', i+1) for i in range(100)]\n# s = clients['client_1'][0][1]*0\n# for c in client_names:\n#     sum = clients[c][0][1]\n#     for i in range(1,378):\n#         sum = sum + clients[c][i][1]\n        \n#     s = s + sum/378\n# s","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:36:52.930799Z","iopub.execute_input":"2023-10-24T18:36:52.931083Z","iopub.status.idle":"2023-10-24T18:36:52.934765Z","shell.execute_reply.started":"2023-10-24T18:36:52.931055Z","shell.execute_reply":"2023-10-24T18:36:52.933685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:36:52.936097Z","iopub.execute_input":"2023-10-24T18:36:52.936433Z","iopub.status.idle":"2023-10-24T18:37:04.565721Z","shell.execute_reply.started":"2023-10-24T18:36:52.936403Z","shell.execute_reply":"2023-10-24T18:37:04.564798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:37:04.567021Z","iopub.execute_input":"2023-10-24T18:37:04.567396Z","iopub.status.idle":"2023-10-24T18:37:04.572869Z","shell.execute_reply.started":"2023-10-24T18:37:04.567358Z","shell.execute_reply":"2023-10-24T18:37:04.571798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(28, 28, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:37:04.574269Z","iopub.execute_input":"2023-10-24T18:37:04.574660Z","iopub.status.idle":"2023-10-24T18:37:04.999034Z","shell.execute_reply.started":"2023-10-24T18:37:04.574622Z","shell.execute_reply":"2023-10-24T18:37:04.998156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    # print(client_names, len(client_names))\n    random.shuffle(client_names)\n    \n#     if debug: \n#         # print('all_client_names', all_client_names)\n#         print('client_names', client_names, len(client_names))\n                \n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        # print('scaling_factor', scaling_factor)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T18:37:05.000287Z","iopub.execute_input":"2023-10-24T18:37:05.000573Z","iopub.status.idle":"2023-10-24T19:01:52.561029Z","shell.execute_reply.started":"2023-10-24T18:37:05.000544Z","shell.execute_reply":"2023-10-24T19:01:52.560081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('IID | total comm rounds', len(global_acc_list))","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:52.562510Z","iopub.execute_input":"2023-10-24T19:01:52.562839Z","iopub.status.idle":"2023-10-24T19:01:52.875263Z","shell.execute_reply.started":"2023-10-24T19:01:52.562808Z","shell.execute_reply":"2023-10-24T19:01:52.874337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\niid_df.to_csv('MNIST_IID.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:52.876597Z","iopub.execute_input":"2023-10-24T19:01:52.876900Z","iopub.status.idle":"2023-10-24T19:01:53.045429Z","shell.execute_reply.started":"2023-10-24T19:01:52.876868Z","shell.execute_reply":"2023-10-24T19:01:53.044647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Non-IID","metadata":{}},{"cell_type":"code","source":"def create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    # data = list(zip(image_list, label_list))\n    # random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n    max_y = np.argmax(label_list, axis=-1)\n    sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n    data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} ","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:53.046525Z","iopub.execute_input":"2023-10-24T19:01:53.046811Z","iopub.status.idle":"2023-10-24T19:01:53.054914Z","shell.execute_reply.started":"2023-10-24T19:01:53.046782Z","shell.execute_reply":"2023-10-24T19:01:53.054021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:53.056297Z","iopub.execute_input":"2023-10-24T19:01:53.056675Z","iopub.status.idle":"2023-10-24T19:01:53.067778Z","shell.execute_reply.started":"2023-10-24T19:01:53.056638Z","shell.execute_reply":"2023-10-24T19:01:53.066993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:53.068963Z","iopub.execute_input":"2023-10-24T19:01:53.069240Z","iopub.status.idle":"2023-10-24T19:01:53.164370Z","shell.execute_reply.started":"2023-10-24T19:01:53.069213Z","shell.execute_reply":"2023-10-24T19:01:53.163392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:01:53.166140Z","iopub.execute_input":"2023-10-24T19:01:53.166447Z","iopub.status.idle":"2023-10-24T19:02:01.807356Z","shell.execute_reply.started":"2023-10-24T19:01:53.166418Z","shell.execute_reply":"2023-10-24T19:02:01.806402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01 \ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:02:01.808741Z","iopub.execute_input":"2023-10-24T19:02:01.809147Z","iopub.status.idle":"2023-10-24T19:02:01.814538Z","shell.execute_reply.started":"2023-10-24T19:02:01.809109Z","shell.execute_reply":"2023-10-24T19:02:01.813585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(32, 32, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:02:01.816093Z","iopub.execute_input":"2023-10-24T19:02:01.816538Z","iopub.status.idle":"2023-10-24T19:02:01.857078Z","shell.execute_reply.started":"2023-10-24T19:02:01.816497Z","shell.execute_reply":"2023-10-24T19:02:01.856324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    random.shuffle(client_names)\n    if debug: \n        # print('all_client_names', all_client_names)\n        print('client_names', client_names)\n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:02:01.858188Z","iopub.execute_input":"2023-10-24T19:02:01.858450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Non-IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('Non-IID | total comm rounds', len(global_acc_list))           ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noniid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\nnoniid_df.to_csv('CIFAR-10_Non-IID.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}