{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras.layers as layers\nfrom keras.models import Sequential\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nfrom typing import List, Tuple","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-27T19:42:24.895689Z","iopub.execute_input":"2022-12-27T19:42:24.896116Z","iopub.status.idle":"2022-12-27T19:42:32.568318Z","shell.execute_reply.started":"2022-12-27T19:42:24.896086Z","shell.execute_reply":"2022-12-27T19:42:32.567252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_unique_labels(label_csv):\n    lbl_df = pd.read_csv(label_csv, index_col='id')\n    unique_labels = {v: i for i, v in enumerate(lbl_df['label'].unique())}\n    return unique_labels\n\n\ndef get_label_dict(label_csv, unique_labels):\n    lbl_df = pd.read_csv(label_csv, index_col='id')\n    lbl_df = lbl_df['label'].to_dict()\n    labels = {k: unique_labels[v] for k, v in lbl_df.items()}\n    return labels\n\n\ndef get_data_dict(train_dir):\n    images = {}\n    for root, _, files in os.walk(train_dir):\n        for file in files:\n            filename, _ = os.path.splitext(file)\n            path = os.path.join(root, file)\n            img = plt.imread(path)\n            images[int(filename)] = img\n\n    return images\n\n\ndef prepare_data(data_dict, label_dict):\n    X, y = [], []\n    for k in data_dict.keys():\n        X.append(data_dict[k])\n        y.append(label_dict[k])\n        \n    return np.array(X), tf.keras.utils.to_categorical(y)\n\n\ndef create_model(n_conv: int, \n                 n_filter: int, \n                 filter_size: int, \n                 dropout_rate: float):\n    \n    model = Sequential()\n    model.add(layers.Input(shape=(32, 32, 3)))\n    for _ in range(n_conv):\n        model.add(layers.Conv2D(n_filter, filter_size, padding='same', activation='relu'))\n        model.add(layers.MaxPooling2D(2, 2))\n        model.add(layers.Dropout(dropout_rate))\n    model.add(layers.Flatten())\n    model.add(layers.Dense(256, activation='relu'))\n    model.add(layers.Dense(10, activation='softmax'))\n\n    # model compilation\n    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:42:32.570781Z","iopub.execute_input":"2022-12-27T19:42:32.571622Z","iopub.status.idle":"2022-12-27T19:42:32.586716Z","shell.execute_reply.started":"2022-12-27T19:42:32.571572Z","shell.execute_reply":"2022-12-27T19:42:32.585346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '/kaggle/input/assignmentcifar/train'\nlabel_file = '/kaggle/input/assignmentcifar/trainLabels.csv'\n\nlabel2idx = get_unique_labels(label_file)\nidx2label = {v: k for k, v in label2idx.items()}\nlabel_dict = get_label_dict(label_file, unique_labels=label2idx)\ndata_dict = get_data_dict(train_dir)\nX, y = prepare_data(data_dict, label_dict)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:42:32.588524Z","iopub.execute_input":"2022-12-27T19:42:32.588880Z","iopub.status.idle":"2022-12-27T19:45:32.231737Z","shell.execute_reply.started":"2022-12-27T19:42:32.588849Z","shell.execute_reply":"2022-12-27T19:45:32.230199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_convs = [3, 5]\nn_filters = [32, 64]\nfilter_sizes = [3, 5]\ndropout_rates = [0.2, 0.5, 0.7]\nbatch_sizes = [32, 64, 128]","metadata":{"execution":{"iopub.status.busy":"2022-12-26T19:36:34.258131Z","iopub.execute_input":"2022-12-26T19:36:34.258513Z","iopub.status.idle":"2022-12-26T19:36:34.269975Z","shell.execute_reply.started":"2022-12-26T19:36:34.258484Z","shell.execute_reply":"2022-12-26T19:36:34.265433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_best_hyperparams(n_convs, n_filters, filter_sizes, dropout_rates):\n    df = pd.DataFrame()\n\n    for n_conv in n_convs:\n        for n_filter in n_filters:\n            for filter_size in filter_sizes:\n                for dropout_rate in dropout_rates:\n                    for trial in range(3):\n\n                        model = create_model(n_conv=n_conv, n_filter=n_filter, filter_size=filter_size, dropout_rate=dropout_rate)\n\n                        if trial == 0:\n                            model.summary()\n\n                        H = model.fit(X_train, y_train, epochs=10, validation_data=(X_test, y_test))\n                        df.at[f'n_conv {n_conv} n_filter {n_filter} filter_size {filter_size} dropout_rate {dropout_rate}', f'Trial {trial+1}'] = max(H.history['val_accuracy'])\n    \n    df['Mean'] = df[['Trial 1', 'Trial 2', 'Trial 3']].mean(axis=1)\n    df['Std'] = df[['Trial 1', 'Trial 2', 'Trial 3']].std(axis=1)\n    \n    return df\n\n\ndf = get_best_hyperparams(n_convs=n_convs, n_filters=n_filters, \n                          filter_sizes=filter_sizes, dropout_rates=dropout_rates)\ndf.to_csv('hyperparameter_results.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-26T19:36:34.272100Z","iopub.execute_input":"2022-12-26T19:36:34.272479Z","iopub.status.idle":"2022-12-26T20:53:30.493819Z","shell.execute_reply.started":"2022-12-26T19:36:34.272442Z","shell.execute_reply":"2022-12-26T20:53:30.492842Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_best_batch(batch_sizes):\n    df = pd.DataFrame()\n    \n    for bs in batch_sizes:\n        for t in range(3):\n            model = create_model(n_conv=3, n_filter=64, filter_size=3, dropout_rate=0.2)\n            H = model.fit(X_train, y_train, batch_size=bs, epochs=10, validation_data=(X_test, y_test))\n\n            df.at[f'batch size {bs}', f'Trial {t+1}'] = max(H.history['val_accuracy'])\n    \n    df['Mean'] = df[['Trial 1', 'Trial 2', 'Trial 3']].mean(axis=1)\n    df['Std'] = df[['Trial 1', 'Trial 2', 'Trial 3']].std(axis=1)\n    \n    return df\n\ndf = get_best_batch(batch_sizes=batch_sizes)\ndf.to_csv('batch_results.csv')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-26T21:10:51.166296Z","iopub.execute_input":"2022-12-26T21:10:51.167271Z","iopub.status.idle":"2022-12-26T21:16:46.407811Z","shell.execute_reply.started":"2022-12-26T21:10:51.167233Z","shell.execute_reply":"2022-12-26T21:16:46.406832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(n_conv=3, n_filter=64, filter_size=3, dropout_rate=0.2)\nH = model.fit(X_train, y_train, batch_size=32, epochs=10, validation_data=(X_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2022-12-26T21:48:45.991594Z","iopub.execute_input":"2022-12-26T21:48:45.991978Z","iopub.status.idle":"2022-12-26T21:49:37.193205Z","shell.execute_reply.started":"2022-12-26T21:48:45.991944Z","shell.execute_reply":"2022-12-26T21:49:37.192275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Training results')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\n\nplt.plot(H.history['accuracy'], label='Training accuracy')\nplt.plot(H.history['val_accuracy'], label='Validation accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T21:50:05.406089Z","iopub.execute_input":"2022-12-26T21:50:05.406446Z","iopub.status.idle":"2022-12-26T21:50:05.593128Z","shell.execute_reply.started":"2022-12-26T21:50:05.406416Z","shell.execute_reply":"2022-12-26T21:50:05.592225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('best_model')","metadata":{"execution":{"iopub.status.busy":"2022-12-26T21:50:12.432331Z","iopub.execute_input":"2022-12-26T21:50:12.432703Z","iopub.status.idle":"2022-12-26T21:50:14.467210Z","shell.execute_reply.started":"2022-12-26T21:50:12.432670Z","shell.execute_reply":"2022-12-26T21:50:14.466269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model('best_model')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:45:32.234158Z","iopub.execute_input":"2022-12-27T19:45:32.234594Z","iopub.status.idle":"2022-12-27T19:45:33.020486Z","shell.execute_reply.started":"2022-12-27T19:45:32.234560Z","shell.execute_reply":"2022-12-27T19:45:33.019520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_class_dict(y, idx2label):    \n    selected_ids = {c: [] for c in idx2label.keys()}\n    \n    for i, c in enumerate(y.argmax(axis=-1)):\n        selected_ids[c].append(i)\n    \n    for k, v in selected_ids.items():\n        selected_ids[k] = random.choices(v, k=3)\n        \n    return selected_ids\n            \nselected_ids = get_class_dict(y_test, idx2label)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:46:55.772118Z","iopub.execute_input":"2022-12-27T19:46:55.772621Z","iopub.status.idle":"2022-12-27T19:46:55.785956Z","shell.execute_reply.started":"2022-12-27T19:46:55.772586Z","shell.execute_reply":"2022-12-27T19:46:55.784168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labelizer = np.vectorize(lambda x: idx2label[x])","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:46:56.747022Z","iopub.execute_input":"2022-12-27T19:46:56.747496Z","iopub.status.idle":"2022-12-27T19:46:56.754052Z","shell.execute_reply.started":"2022-12-27T19:46:56.747445Z","shell.execute_reply":"2022-12-27T19:46:56.752344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction(model, X, y, idx, filename):\n    img = X[idx]\n    actual_label = y[idx].argmax()\n\n    preds = model.predict(img[np.newaxis, ...])\n    pred_confs = sorted(preds[0])\n    pred_labels = labelizer(preds[0].argsort())\n    \n    pred_confs, pred_labels = pred_confs[-5:], pred_labels[-5:]\n    \n    s = f'Actual Label -> {idx2label[actual_label]}\\n\\n'\n    for lbl, conf_level in zip(pred_labels[::-1], pred_confs[::-1]):\n        s += f'{lbl} -> {100*conf_level:.5f}\\n'\n\n    plt.clf()\n    plt.imshow(img)\n    plt.text(32, 32, s, color=\"black\", fontdict={\"fontsize\":16, \"ha\":\"left\", \"va\":\"baseline\"})\n    plt.tight_layout()\n    plt.savefig(filename, dpi=300)\n    \n    return idx2label[actual_label]\n\nos.makedirs('images')\nfor k, v_list in selected_ids.items():\n    class_name = idx2label[k]\n    \n    for v in v_list:\n        get_prediction(model, X_test, y_test, idx=v, filename=f'./images/{class_name}_{v}.png')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:54:40.752952Z","iopub.execute_input":"2022-12-27T19:54:40.753516Z","iopub.status.idle":"2022-12-27T19:54:58.445705Z","shell.execute_reply.started":"2022-12-27T19:54:40.753445Z","shell.execute_reply":"2022-12-27T19:54:58.444498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! rm -r image*","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:54:28.073629Z","iopub.execute_input":"2022-12-27T19:54:28.074065Z","iopub.status.idle":"2022-12-27T19:54:29.196701Z","shell.execute_reply.started":"2022-12-27T19:54:28.074032Z","shell.execute_reply":"2022-12-27T19:54:29.195045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! zip -r images.zip images/","metadata":{"execution":{"iopub.status.busy":"2022-12-27T19:54:58.447588Z","iopub.execute_input":"2022-12-27T19:54:58.447951Z","iopub.status.idle":"2022-12-27T19:54:59.676845Z","shell.execute_reply.started":"2022-12-27T19:54:58.447919Z","shell.execute_reply":"2022-12-27T19:54:59.675375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T17:33:47.592534Z","iopub.execute_input":"2022-12-27T17:33:47.592958Z","iopub.status.idle":"2022-12-27T17:33:51.696753Z","shell.execute_reply.started":"2022-12-27T17:33:47.592921Z","shell.execute_reply":"2022-12-27T17:33:51.695438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_labels = labelizer(preds.argmax(axis=-1))\nactual_labels = labelizer(y_test.argmax(axis=-1))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-27T17:34:49.637607Z","iopub.execute_input":"2022-12-27T17:34:49.638048Z","iopub.status.idle":"2022-12-27T17:34:49.650186Z","shell.execute_reply.started":"2022-12-27T17:34:49.638002Z","shell.execute_reply":"2022-12-27T17:34:49.649096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = list(label2idx.keys())\nconf_mat = confusion_matrix(actual_labels, pred_labels, labels=labels)\nconf_disp = ConfusionMatrixDisplay(conf_mat, display_labels=labels)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T17:48:04.086627Z","iopub.execute_input":"2022-12-27T17:48:04.087043Z","iopub.status.idle":"2022-12-27T17:48:04.108743Z","shell.execute_reply.started":"2022-12-27T17:48:04.087006Z","shell.execute_reply":"2022-12-27T17:48:04.107913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_disp.plot()\nplt.xticks(rotation=90)\nplt.title('Confusion Matrix')\nplt.savefig('confusion_matrix.png')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T17:49:03.479893Z","iopub.execute_input":"2022-12-27T17:49:03.480346Z","iopub.status.idle":"2022-12-27T17:49:04.296127Z","shell.execute_reply.started":"2022-12-27T17:49:03.480308Z","shell.execute_reply":"2022-12-27T17:49:04.294861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.table()","metadata":{},"execution_count":null,"outputs":[]}]}