{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport cv2\nimport os\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import accuracy_score\n\nimport tensorflow as tf\nfrom tensorflow import expand_dims\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Input, Lambda\nfrom tensorflow.keras.layers import MaxPooling2D\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras.layers import Flatten\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras import backend as K","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-13T12:58:21.633496Z","iopub.execute_input":"2021-07-13T12:58:21.634071Z","iopub.status.idle":"2021-07-13T12:58:29.388008Z","shell.execute_reply.started":"2021-07-13T12:58:21.633933Z","shell.execute_reply":"2021-07-13T12:58:29.386876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imutils\nfrom imutils import paths","metadata":{"execution":{"iopub.status.busy":"2021-07-13T12:59:24.294304Z","iopub.execute_input":"2021-07-13T12:59:24.294718Z","iopub.status.idle":"2021-07-13T12:59:36.795115Z","shell.execute_reply.started":"2021-07-13T12:59:24.294685Z","shell.execute_reply":"2021-07-13T12:59:36.793892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Todo\n\ncreate subset of clients\n\n- increase comm rounds 300\n- increase hidden units 400\n- increase no of layers\n- no of clients 20","metadata":{}},{"cell_type":"code","source":"debug = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-13T12:59:36.797400Z","iopub.execute_input":"2021-07-13T12:59:36.797862Z","iopub.status.idle":"2021-07-13T12:59:36.806834Z","shell.execute_reply.started":"2021-07-13T12:59:36.797815Z","shell.execute_reply":"2021-07-13T12:59:36.805691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(paths, verbose=-1):\n    '''expects images for each class in seperate dir, \n    e.g all digits in 0 class in the directory named 0 '''\n    data = list()\n    labels = list()\n    # loop over the input images\n    for (i, imgpath) in enumerate(paths):\n        # load the image and extract the class labels        \n        im_gray = cv2.imread(imgpath , cv2.IMREAD_GRAYSCALE)\n        image = np.array(im_gray).flatten() # cv2.imread(imgpath) \n        # print(image.shape)\n        label = imgpath.split(os.path.sep)[-2]\n        # scale the image to [0, 1] and add to list\n        data.append(image/255)\n        labels.append(label)\n        # show an update every `verbose` images\n        if verbose > 0 and i > 0 and (i + 1) % verbose == 0:\n            print(\"[INFO] processed {}/{}\".format(i + 1, len(paths)))\n    # return a tuple of the data and labels\n    \n    return data, labels\n\ndef create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    data = list(zip(image_list, label_list))\n    random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n#     max_y = np.argmax(label_list, axis=-1)\n#     sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n#     data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} \n\n\ndef batch_data(data_shard, bs=32):\n    '''Takes in a clients data shard and create a tfds object off it\n    args:\n        shard: a data, label constituting a client's data shard\n        bs:batch size\n    return:\n        tfds object'''\n    #seperate shard into data and labels lists\n    data, label = zip(*data_shard)\n    dataset = tf.data.Dataset.from_tensor_slices((list(data), list(label)))\n    return dataset.shuffle(len(label)).batch(bs)\n\n\ndef weight_scalling_factor(clients_trn_data, client_name):\n    client_names = list(clients_trn_data.keys())\n    #get the bs\n    bs = list(clients_trn_data[client_name])[0][0].shape[0]\n    #first calculate the total training data points across clinets\n    global_count = sum([tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy() for client_name in client_names])*bs\n    # get the total number of data points held by a client\n    local_count = tf.data.experimental.cardinality(clients_trn_data[client_name]).numpy()*bs\n    \n    \n    if debug:\n        print('global_count', global_count, 'local_count', local_count, 'bs', bs)\n    \n    return local_count/global_count\n\n\ndef scale_model_weights(weight, scalar):\n    '''function for scaling a models weights'''\n    weight_final = []\n    steps = len(weight)\n    for i in range(steps):\n        weight_final.append(scalar * weight[i])\n    return weight_final\n\n\n\ndef sum_scaled_weights(scaled_weight_list):\n    '''Return the sum of the listed scaled weights. The is equivalent to scaled avg of the weights'''\n    avg_grad = list()\n    #get the average grad accross all client gradients\n    for grad_list_tuple in zip(*scaled_weight_list):\n        layer_mean = tf.math.reduce_sum(grad_list_tuple, axis=0)\n        avg_grad.append(layer_mean)\n        \n    return avg_grad\n\n\ndef test_model(X_test, Y_test,  model, comm_round):\n    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n    #logits = model.predict(X_test, batch_size=100)\n    logits = model.predict(X_test)\n    loss = cce(Y_test, logits)\n    acc = accuracy_score(tf.argmax(logits, axis=1), tf.argmax(Y_test, axis=1))\n    print('comm_round: {} | global_acc: {:.3%} | global_loss: {}'.format(comm_round, acc, loss))\n    return acc, loss\n","metadata":{"execution":{"iopub.status.busy":"2021-07-13T12:59:36.811327Z","iopub.execute_input":"2021-07-13T12:59:36.811667Z","iopub.status.idle":"2021-07-13T12:59:36.835727Z","shell.execute_reply.started":"2021-07-13T12:59:36.811636Z","shell.execute_reply":"2021-07-13T12:59:36.834192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleMLP:\n    @staticmethod\n    def build(shape, classes):\n        model = Sequential()\n        model.add(Dense(200, input_shape=(shape,)))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(200))\n        model.add(Activation(\"relu\"))\n        model.add(Dense(classes))\n        model.add(Activation(\"softmax\"))\n        return model\n    \n#     def build(shape, classes):\n#         model = Sequential()\n#         model.add(Input(shape=(shape[0], shape[1], shape[2])))\n#         #model.add(Lambda(lambda x: expand_dims(x, axis=-1)))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=64, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\"))\n#         model.add(Activation(\"relu\"))\n#         model.add(MaxPooling2D())\n#         model.add(Flatten())\n#         model.add(Dense(32))\n#         model.add(Dense(classes))\n#         model.add(Activation(\"softmax\"))\n#         return model","metadata":{"execution":{"iopub.status.busy":"2021-07-13T12:59:36.838298Z","iopub.execute_input":"2021-07-13T12:59:36.839213Z","iopub.status.idle":"2021-07-13T12:59:36.850403Z","shell.execute_reply.started":"2021-07-13T12:59:36.839164Z","shell.execute_reply":"2021-07-13T12:59:36.849113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#declear path to your mnist data folder\nimg_path = '../input/mnistasjpg/trainingSet/trainingSet' #'../input/cifar10-pngs-in-folders/cifar10/test'  # <-- test dataset #'../input/mnistasjpg/trainingSample/trainingSample' # <-- smaller dataset\n\n#get the path list using the path object\nimage_paths = list(paths.list_images(img_path))\n\n#apply our function\nimage_list, label_list = load(image_paths, verbose=10000)\n\n#binarize the labels\nlb = LabelBinarizer()\nlabel_list = lb.fit_transform(label_list)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-13T12:59:46.667823Z","iopub.execute_input":"2021-07-13T12:59:46.668278Z","iopub.status.idle":"2021-07-13T13:05:05.052829Z","shell.execute_reply.started":"2021-07-13T12:59:46.668247Z","shell.execute_reply":"2021-07-13T13:05:05.051642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split data into training and test set\nX_train, X_test, y_train, y_test = train_test_split(image_list, \n                                                    label_list, \n                                                    test_size=0.1, \n                                                    random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T13:05:05.054706Z","iopub.execute_input":"2021-07-13T13:05:05.055191Z","iopub.status.idle":"2021-07-13T13:05:05.083818Z","shell.execute_reply.started":"2021-07-13T13:05:05.055145Z","shell.execute_reply":"2021-07-13T13:05:05.082654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### IID","metadata":{}},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T14:30:07.109651Z","iopub.execute_input":"2021-07-13T14:30:07.110052Z","iopub.status.idle":"2021-07-13T14:30:07.119319Z","shell.execute_reply.started":"2021-07-13T14:30:07.110017Z","shell.execute_reply":"2021-07-13T14:30:07.117828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T14:30:07.320448Z","iopub.execute_input":"2021-07-13T14:30:07.320814Z","iopub.status.idle":"2021-07-13T14:30:07.415722Z","shell.execute_reply.started":"2021-07-13T14:30:07.320781Z","shell.execute_reply":"2021-07-13T14:30:07.414583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# client_names = ['{}_{}'.format('client', i+1) for i in range(100)]\n# s = clients['client_1'][0][1]*0\n# for c in client_names:\n#     sum = clients[c][0][1]\n#     for i in range(1,378):\n#         sum = sum + clients[c][i][1]\n        \n#     s = s + sum/378\n# s","metadata":{"execution":{"iopub.status.busy":"2021-07-13T14:30:07.474941Z","iopub.execute_input":"2021-07-13T14:30:07.475360Z","iopub.status.idle":"2021-07-13T14:30:07.480347Z","shell.execute_reply.started":"2021-07-13T14:30:07.475327Z","shell.execute_reply":"2021-07-13T14:30:07.479094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:04:24.748503Z","iopub.execute_input":"2021-07-13T15:04:24.748877Z","iopub.status.idle":"2021-07-13T15:04:33.484238Z","shell.execute_reply.started":"2021-07-13T15:04:24.748844Z","shell.execute_reply":"2021-07-13T15:04:33.483097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:04:33.486245Z","iopub.execute_input":"2021-07-13T15:04:33.486700Z","iopub.status.idle":"2021-07-13T15:04:33.493202Z","shell.execute_reply.started":"2021-07-13T15:04:33.486654Z","shell.execute_reply":"2021-07-13T15:04:33.491691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(28, 28, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:04:33.496054Z","iopub.execute_input":"2021-07-13T15:04:33.496963Z","iopub.status.idle":"2021-07-13T15:04:33.542360Z","shell.execute_reply.started":"2021-07-13T15:04:33.496884Z","shell.execute_reply":"2021-07-13T15:04:33.541297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    # print(client_names, len(client_names))\n    random.shuffle(client_names)\n    \n#     if debug: \n#         # print('all_client_names', all_client_names)\n#         print('client_names', client_names, len(client_names))\n                \n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        # print('scaling_factor', scaling_factor)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:04:33.544251Z","iopub.execute_input":"2021-07-13T15:04:33.544696Z","iopub.status.idle":"2021-07-13T15:34:50.170268Z","shell.execute_reply.started":"2021-07-13T15:04:33.544662Z","shell.execute_reply":"2021-07-13T15:34:50.169127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('IID | total comm rounds', len(global_acc_list))","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:35:47.454477Z","iopub.execute_input":"2021-07-13T15:35:47.454952Z","iopub.status.idle":"2021-07-13T15:35:47.799876Z","shell.execute_reply.started":"2021-07-13T15:35:47.454910Z","shell.execute_reply":"2021-07-13T15:35:47.798723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\niid_df.to_csv('MNIST_IID.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:38:12.629719Z","iopub.execute_input":"2021-07-13T15:38:12.630178Z","iopub.status.idle":"2021-07-13T15:38:12.835156Z","shell.execute_reply.started":"2021-07-13T15:38:12.630136Z","shell.execute_reply":"2021-07-13T15:38:12.834001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Non-IID","metadata":{}},{"cell_type":"code","source":"def create_clients(image_list, label_list, num_clients=100, initial='clients'):\n    ''' return: a dictionary with keys clients' names and value as \n                data shards - tuple of images and label lists.\n        args: \n            image_list: a list of numpy arrays of training images\n            label_list:a list of binarized labels for each image\n            num_client: number of fedrated members (clients)\n            initials: the clients'name prefix, e.g, clients_1 \n            \n    '''\n\n    #create a list of client names\n    client_names = ['{}_{}'.format(initial, i+1) for i in range(num_clients)]\n\n    #randomize the data\n    # data = list(zip(image_list, label_list))\n    # random.shuffle(data)  # <- IID\n    \n    # sort data for non-iid\n    max_y = np.argmax(label_list, axis=-1)\n    sorted_zip = sorted(zip(max_y, label_list, image_list), key=lambda x: x[0])\n    data = [(x,y) for _,y,x in sorted_zip]\n\n    #shard data and place at each client\n    size = len(data)//num_clients\n    shards = [data[i:i + size] for i in range(0, size*num_clients, size)]\n\n    #number of clients must equal number of shards\n    assert(len(shards) == len(client_names))\n\n    return {client_names[i] : shards[i] for i in range(len(client_names))} ","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:53:04.151883Z","iopub.execute_input":"2021-07-13T15:53:04.152306Z","iopub.status.idle":"2021-07-13T15:53:04.162248Z","shell.execute_reply.started":"2021-07-13T15:53:04.152274Z","shell.execute_reply":"2021-07-13T15:53:04.161050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train), len(X_test), len(y_train), len(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:53:05.102106Z","iopub.execute_input":"2021-07-13T15:53:05.102515Z","iopub.status.idle":"2021-07-13T15:53:05.109800Z","shell.execute_reply.started":"2021-07-13T15:53:05.102485Z","shell.execute_reply":"2021-07-13T15:53:05.108520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create clients\nclients = create_clients(X_train, y_train, num_clients=100, initial='client')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:53:07.549218Z","iopub.execute_input":"2021-07-13T15:53:07.549605Z","iopub.status.idle":"2021-07-13T15:53:07.692112Z","shell.execute_reply.started":"2021-07-13T15:53:07.549573Z","shell.execute_reply":"2021-07-13T15:53:07.690119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#process and batch the training data for each client\nclients_batched = dict()\nfor (client_name, data) in clients.items():\n    clients_batched[client_name] = batch_data(data)\n    \n#process and batch the test set  \ntest_batched = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(len(y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-13T16:00:35.667200Z","iopub.execute_input":"2021-07-13T16:00:35.667584Z","iopub.status.idle":"2021-07-13T16:00:44.510649Z","shell.execute_reply.started":"2021-07-13T16:00:35.667552Z","shell.execute_reply":"2021-07-13T16:00:44.509505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01 \ncomms_round = 300\nloss='categorical_crossentropy'\nmetrics = ['accuracy']\noptimizer = SGD(lr=lr, \n                decay=lr / comms_round, \n                momentum=0.9\n               )          ","metadata":{"execution":{"iopub.status.busy":"2021-07-13T16:00:44.512562Z","iopub.execute_input":"2021-07-13T16:00:44.513079Z","iopub.status.idle":"2021-07-13T16:00:44.520911Z","shell.execute_reply.started":"2021-07-13T16:00:44.512981Z","shell.execute_reply":"2021-07-13T16:00:44.519294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#initialize global model\n\nbuild_shape = 784 #(32, 32, 3)  # 1024 <- CIFAR-10    # 784 # for MNIST\n\nsmlp_global = SimpleMLP()\nglobal_model = smlp_global.build(build_shape, 10) \nglobal_acc_list = []\nglobal_loss_list = []","metadata":{"execution":{"iopub.status.busy":"2021-07-13T16:00:44.523534Z","iopub.execute_input":"2021-07-13T16:00:44.524063Z","iopub.status.idle":"2021-07-13T16:00:44.571138Z","shell.execute_reply.started":"2021-07-13T16:00:44.523993Z","shell.execute_reply":"2021-07-13T16:00:44.570049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#commence global training loop\nfor comm_round in range(comms_round):\n            \n    # get the global model's weights - will serve as the initial weights for all local models\n    global_weights = global_model.get_weights()\n    \n    #initial list to collect local model weights after scalling\n    scaled_local_weight_list = list()\n\n    #randomize client data - using keys\n    all_client_names = list(clients_batched.keys())\n           \n    client_names = random.sample(all_client_names, k=10)\n    random.shuffle(client_names)\n    if debug: \n        # print('all_client_names', all_client_names)\n        print('client_names', client_names)\n    \n    #loop through each client and create new local model\n    for client in client_names:\n        smlp_local = SimpleMLP()\n        local_model = smlp_local.build(build_shape, 10)\n        local_model.compile(loss=loss, \n                      optimizer=optimizer, \n                      metrics=metrics)\n        \n        #set local model weight to the weight of the global model\n        local_model.set_weights(global_weights)\n        \n        #fit local model with client's data\n        local_model.fit(clients_batched[client], epochs=1, verbose=0)\n        \n        #scale the model weights and add to list\n        scaling_factor = 0.1 # weight_scalling_factor(clients_batched, client)\n        scaled_weights = scale_model_weights(local_model.get_weights(), scaling_factor)\n        scaled_local_weight_list.append(scaled_weights)\n        \n        #clear session to free memory after each communication round\n        K.clear_session()\n        \n    #to get the average over all the local model, we simply take the sum of the scaled weights\n    average_weights = sum_scaled_weights(scaled_local_weight_list)\n    \n    #update global model \n    global_model.set_weights(average_weights)\n\n    #test global model and print out metrics after each communications round\n    for(X_test, Y_test) in test_batched:\n        global_acc, global_loss = test_model(X_test, Y_test, global_model, comm_round)\n        global_acc_list.append(global_acc)\n        global_loss_list.append(global_loss)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T16:00:44.573131Z","iopub.execute_input":"2021-07-13T16:00:44.573625Z","iopub.status.idle":"2021-07-13T16:30:28.941131Z","shell.execute_reply.started":"2021-07-13T16:00:44.573582Z","shell.execute_reply":"2021-07-13T16:30:28.939902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Non-IID \nimport matplotlib.pyplot as plt\nplt.figure(figsize=(16,4))\nplt.subplot(121)\nplt.plot(list(range(0,len(global_loss_list))), global_loss_list)\nplt.subplot(122)\nplt.plot(list(range(0,len(global_acc_list))), global_acc_list)\nprint('Non-IID | total comm rounds', len(global_acc_list))           ","metadata":{"execution":{"iopub.status.busy":"2021-07-13T16:30:28.943041Z","iopub.execute_input":"2021-07-13T16:30:28.943794Z","iopub.status.idle":"2021-07-13T16:30:29.273959Z","shell.execute_reply.started":"2021-07-13T16:30:28.943748Z","shell.execute_reply":"2021-07-13T16:30:29.272921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noniid_df = pd.DataFrame(list(zip(global_acc_list, global_loss_list)), columns =['global_acc_list', 'global_loss_list'])\nnoniid_df.to_csv('CIFAR-10_Non-IID.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T15:09:46.902904Z","iopub.execute_input":"2021-07-05T15:09:46.903416Z","iopub.status.idle":"2021-07-05T15:09:46.915551Z","shell.execute_reply.started":"2021-07-05T15:09:46.903364Z","shell.execute_reply":"2021-07-05T15:09:46.914382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}