{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport cv2\nimport os\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import accuracy_score\n\nimport tensorflow as tf\nfrom tensorflow import expand_dims\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Input, Lambda\nfrom tensorflow.keras.layers import MaxPooling2D\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras.layers import Flatten\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras import backend as K","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-22T12:39:53.178560Z","iopub.execute_input":"2023-02-22T12:39:53.178913Z","iopub.status.idle":"2023-02-22T12:39:59.012944Z","shell.execute_reply.started":"2023-02-22T12:39:53.178808Z","shell.execute_reply":"2023-02-22T12:39:59.012098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imutils\nfrom imutils import paths","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:39:59.014445Z","iopub.execute_input":"2023-02-22T12:39:59.014805Z","iopub.status.idle":"2023-02-22T12:40:08.077278Z","shell.execute_reply.started":"2023-02-22T12:39:59.014765Z","shell.execute_reply":"2023-02-22T12:40:08.076295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Todo\n\ncreate subset of clients\n\n- increase comm rounds 300\n- increase hidden units 400\n- increase no of layers\n- no of clients 20","metadata":{}},{"cell_type":"code","source":"debug = 0","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:40:08.079628Z","iopub.execute_input":"2023-02-22T12:40:08.080245Z","iopub.status.idle":"2023-02-22T12:40:08.085207Z","shell.execute_reply.started":"2023-02-22T12:40:08.080199Z","shell.execute_reply":"2023-02-22T12:40:08.083923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(paths, verbose=-1):\n    '''expects images for each class in seperate dir, \n    e.g all digits in 0 class in the directory named 0 '''\n    data = list()\n    labels = list()\n    # loop over the input images\n    for (i, imgpath) in enumerate(paths):\n        # load the image and extract the class labels        \n        im_gray = cv2.imread(imgpath , cv2.IMREAD_GRAYSCALE)\n        image = np.array(im_gray).flatten() # cv2.imread(imgpath) \n        # print(image.shape)\n        label = imgpath.split(os.path.sep)[-2]\n        # scale the image to [0, 1] and add to list\n        data.append(image/255)\n        labels.append(label)\n        # show an update every `verbose` images\n        if verbose > 0 and i > 0 and (i + 1) % verbose == 0:\n            print(\"[INFO] processed {}/{}\".format(i + 1, len(paths)))\n    # return a tuple of the data and labels\n    \n    return data, labels\n\ndef create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    data = list(zip(image_list, label_list))\n    random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n#     max_y = np.argmax(label_list, axis=-1)\n#     sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n#     data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} \n\n\ndef batch_data(data_shard, bs=32):\n    '''Takes in a clients data shard and create a tfds object off it\n    args:\n        shard: a data, label constituting a client's data shard\n        bs:batch size\n    return:\n        tfds object'''\n    #seperate shard into data and labels lists\n    data, label = zip(*data_shard)\n    dataset = tf.data.Dataset.from_tensor_slices((list(data), list(label)))\n    return dataset.shuffle(len(label)).batch(bs)\n\n\ndef weight_scalling_factor(clients_trn_data, client_name):\n    client_names = list(clients_trn_data.keys())\n    #get the bs\n    bs = list(clients_trn_data[client_name])[0][0].shape[0]\n    #first calculate the total training data points across clinets\n    global_count = sum([tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy() for client_name in client_names])*bs\n    # get the total number of data points held by a client\n    local_count = tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy()*bs\n    \n    \n    if debug:\n        print('global_count', global_count, 'local_count', local_count, 'bs', bs)\n    \n    return local_count/global_count\n\n\ndef scale_model_weights(weight, scalar):\n    '''function for scaling a models weights'''\n    weight_final = []\n    steps = len(weight)\n    for i in range(steps):\n        weight_final.append(scalar * weight[i])\n    return weight_final\n\n\n\ndef sum_scaled_weights(scaled_weight_list):\n    '''Return the sum of the listed scaled weights. The is equivalent to scaled avg of the weights'''\n    avg_grad = list()\n    #get the average grad accross all client gradients\n    for grad_list_tuple in zip(*scaled_weight_list):\n        layer_mean = tf.math.reduce_sum(grad_list_tuple, axis=0)\n        avg_grad.append(layer_mean)\n        \n    return avg_grad\n\n\ndef test_model(X_test, Y_test,  model, comm_round):\n    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n    #logits = model.predict(X_test, batch_size=100)\n    logits = model.predict(X_test)\n    loss = cce(Y_test, logits)\n    acc = accuracy_score(tf.argmax(logits, axis=1), tf.argmax(Y_test, axis=1))\n    print('comm_round: {} | global_acc: {:.3%} | global_loss: {}'.format(comm_round, acc, loss))\n    return acc, loss\n","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:40:08.086957Z","iopub.execute_input":"2023-02-22T12:40:08.087381Z","iopub.status.idle":"2023-02-22T12:40:08.105104Z","shell.execute_reply.started":"2023-02-22T12:40:08.087346Z","shell.execute_reply":"2023-02-22T12:40:08.104163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleMLP:\n    @staticmethod\n    def build(shape, classes):\n        model = Sequential()\n        model.add(Dense(200, input_shape=(shape,)))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(200))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(classes))\n        model.add(Activation(\"softmax\"))\n        return model\n    \n#     def build(shape, classes):\n#         model = Sequential()\n#         model.add(Input(shape=(shape[0], shape[1], shape[2])))\n#         #model.add(Lambda(lambda x: expand_dims(x, axis=-1)))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Flatten())\n#         model.add(Dense(32))\n#         model.add(Dense(classes))\n#         model.add(Activation(\"softmax\"))\n#         return model","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:40:08.106303Z","iopub.execute_input":"2023-02-22T12:40:08.106807Z","iopub.status.idle":"2023-02-22T12:40:08.120157Z","shell.execute_reply.started":"2023-02-22T12:40:08.106665Z","shell.execute_reply":"2023-02-22T12:40:08.119346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#declear path to your mnist data folder\nimg_path = '../input/mnistasjpg/trainingSet/trainingSet' #'../input/cifar10-pngs-in-folders/cifar10/test'  # <-- test dataset #'../input/mnistasjpg/trainingSample/trainingSample' # <-- smaller dataset\n\n#get the path list using the path object\nimage_paths = list(paths.list_images(img_path))\n\n#apply our function\nimage_list, label_list = load(image_paths, verbose=10000)\n\n#binarize the labels\nlb = LabelBinarizer()\nlabel_list = lb.fit_transform(label_list)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-22T12:40:08.121223Z","iopub.execute_input":"2023-02-22T12:40:08.121522Z","iopub.status.idle":"2023-02-22T12:46:42.336396Z","shell.execute_reply.started":"2023-02-22T12:40:08.121496Z","shell.execute_reply":"2023-02-22T12:46:42.335218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split data into training and test set\nX_train, X_test, y_train, y_test = train_test_split(image_list, \n                                                    label_list, \n                                                    test_size=0.1, \n                                                    random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:42.337851Z","iopub.execute_input":"2023-02-22T12:46:42.338214Z","iopub.status.idle":"2023-02-22T12:46:42.359740Z","shell.execute_reply.started":"2023-02-22T12:46:42.338168Z","shell.execute_reply":"2023-02-22T12:46:42.358894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### IID","metadata":{}},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:42.361054Z","iopub.execute_input":"2023-02-22T12:46:42.361666Z","iopub.status.idle":"2023-02-22T12:46:42.372345Z","shell.execute_reply.started":"2023-02-22T12:46:42.361629Z","shell.execute_reply":"2023-02-22T12:46:42.371425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:42.375448Z","iopub.execute_input":"2023-02-22T12:46:42.375694Z","iopub.status.idle":"2023-02-22T12:46:42.431074Z","shell.execute_reply.started":"2023-02-22T12:46:42.375669Z","shell.execute_reply":"2023-02-22T12:46:42.430149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# client_names = ['{}_{}'.format('client', i+1) for i in range(100)]\n# s = clients['client_1'][0][1]*0\n# for c in client_names:\n#     sum = clients[c][0][1]\n#     for i in range(1,378):\n#         sum = sum + clients[c][i][1]\n        \n#     s = s + sum/378\n# s","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:42.435002Z","iopub.execute_input":"2023-02-22T12:46:42.435280Z","iopub.status.idle":"2023-02-22T12:46:42.441009Z","shell.execute_reply.started":"2023-02-22T12:46:42.435253Z","shell.execute_reply":"2023-02-22T12:46:42.440169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:42.444122Z","iopub.execute_input":"2023-02-22T12:46:42.444374Z","iopub.status.idle":"2023-02-22T12:46:49.404894Z","shell.execute_reply.started":"2023-02-22T12:46:42.444349Z","shell.execute_reply":"2023-02-22T12:46:49.404045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:49.406101Z","iopub.execute_input":"2023-02-22T12:46:49.406425Z","iopub.status.idle":"2023-02-22T12:46:49.410988Z","shell.execute_reply.started":"2023-02-22T12:46:49.406392Z","shell.execute_reply":"2023-02-22T12:46:49.410155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(28, 28, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:49.412287Z","iopub.execute_input":"2023-02-22T12:46:49.412766Z","iopub.status.idle":"2023-02-22T12:46:49.792411Z","shell.execute_reply.started":"2023-02-22T12:46:49.412729Z","shell.execute_reply":"2023-02-22T12:46:49.791513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    # print(client_names, len(client_names))\n    random.shuffle(client_names)\n    \n#     if debug: \n#         # print('all_client_names', all_client_names)\n#         print('client_names', client_names, len(client_names))\n                \n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        # print('scaling_factor', scaling_factor)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T12:46:49.794758Z","iopub.execute_input":"2023-02-22T12:46:49.795147Z","iopub.status.idle":"2023-02-22T13:07:47.931200Z","shell.execute_reply.started":"2023-02-22T12:46:49.795110Z","shell.execute_reply":"2023-02-22T13:07:47.930284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('IID | total comm rounds', len(global_acc_list))","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:47.932782Z","iopub.execute_input":"2023-02-22T13:07:47.933396Z","iopub.status.idle":"2023-02-22T13:07:48.187314Z","shell.execute_reply.started":"2023-02-22T13:07:47.933354Z","shell.execute_reply":"2023-02-22T13:07:48.186327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\niid_df.to_csv('MNIST_IID.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:48.188787Z","iopub.execute_input":"2023-02-22T13:07:48.189146Z","iopub.status.idle":"2023-02-22T13:07:48.343258Z","shell.execute_reply.started":"2023-02-22T13:07:48.189108Z","shell.execute_reply":"2023-02-22T13:07:48.342494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Non-IID","metadata":{}},{"cell_type":"code","source":"def create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    # data = list(zip(image_list, label_list))\n    # random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n    max_y = np.argmax(label_list, axis=-1)\n    sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n    data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} ","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:48.344492Z","iopub.execute_input":"2023-02-22T13:07:48.344848Z","iopub.status.idle":"2023-02-22T13:07:48.353175Z","shell.execute_reply.started":"2023-02-22T13:07:48.344793Z","shell.execute_reply":"2023-02-22T13:07:48.351081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:48.354467Z","iopub.execute_input":"2023-02-22T13:07:48.354852Z","iopub.status.idle":"2023-02-22T13:07:48.366875Z","shell.execute_reply.started":"2023-02-22T13:07:48.354803Z","shell.execute_reply":"2023-02-22T13:07:48.365867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:48.368147Z","iopub.execute_input":"2023-02-22T13:07:48.368590Z","iopub.status.idle":"2023-02-22T13:07:48.444365Z","shell.execute_reply.started":"2023-02-22T13:07:48.368553Z","shell.execute_reply":"2023-02-22T13:07:48.443460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:48.446192Z","iopub.execute_input":"2023-02-22T13:07:48.446539Z","iopub.status.idle":"2023-02-22T13:07:52.713057Z","shell.execute_reply.started":"2023-02-22T13:07:48.446502Z","shell.execute_reply":"2023-02-22T13:07:52.712199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01 \ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:52.714388Z","iopub.execute_input":"2023-02-22T13:07:52.714726Z","iopub.status.idle":"2023-02-22T13:07:52.721872Z","shell.execute_reply.started":"2023-02-22T13:07:52.714689Z","shell.execute_reply":"2023-02-22T13:07:52.720876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(32, 32, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:52.723343Z","iopub.execute_input":"2023-02-22T13:07:52.723679Z","iopub.status.idle":"2023-02-22T13:07:52.759381Z","shell.execute_reply.started":"2023-02-22T13:07:52.723646Z","shell.execute_reply":"2023-02-22T13:07:52.758646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    random.shuffle(client_names)\n    if debug: \n        # print('all_client_names', all_client_names)\n        print('client_names', client_names)\n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:07:52.762175Z","iopub.execute_input":"2023-02-22T13:07:52.762460Z","iopub.status.idle":"2023-02-22T13:28:43.796248Z","shell.execute_reply.started":"2023-02-22T13:07:52.762435Z","shell.execute_reply":"2023-02-22T13:28:43.795342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Non-IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('Non-IID | total comm rounds', len(global_acc_list))           ","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:28:43.797614Z","iopub.execute_input":"2023-02-22T13:28:43.798142Z","iopub.status.idle":"2023-02-22T13:28:44.030035Z","shell.execute_reply.started":"2023-02-22T13:28:43.798101Z","shell.execute_reply":"2023-02-22T13:28:44.029071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noniid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\nnoniid_df.to_csv('CIFAR-10_Non-IID.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-22T13:28:44.031343Z","iopub.execute_input":"2023-02-22T13:28:44.031710Z","iopub.status.idle":"2023-02-22T13:28:44.069420Z","shell.execute_reply.started":"2023-02-22T13:28:44.031673Z","shell.execute_reply":"2023-02-22T13:28:44.068693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}