{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-14T00:20:21.766448Z","iopub.execute_input":"2021-06-14T00:20:21.766788Z","iopub.status.idle":"2021-06-14T00:20:21.778442Z","shell.execute_reply.started":"2021-06-14T00:20:21.766752Z","shell.execute_reply":"2021-06-14T00:20:21.776156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport numpy as np\nimport pandas as pd\n\nimport scikitplot\nimport seaborn as sns\nfrom matplotlib import pyplot\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report\n\nimport tensorflow as tf\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Flatten, Dense, Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, LeakyReLU, Activation\nfrom tensorflow.keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras import regularizers\nfrom keras.utils import np_utils","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:20:21.779865Z","iopub.execute_input":"2021-06-14T00:20:21.780105Z","iopub.status.idle":"2021-06-14T00:20:21.790594Z","shell.execute_reply.started":"2021-06-14T00:20:21.780082Z","shell.execute_reply":"2021-06-14T00:20:21.789548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/expertclass2/train_data.csv')\ntest = pd.read_csv('../input/expertclass2/test_data.csv')\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T01:03:35.011729Z","iopub.execute_input":"2021-06-14T01:03:35.012204Z","iopub.status.idle":"2021-06-14T01:03:38.117362Z","shell.execute_reply.started":"2021-06-14T01:03:35.012156Z","shell.execute_reply":"2021-06-14T01:03:38.116570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 48\nemotion_label_to_text = {0:'anger', 1:'disgust', 2:'fear', 3:'happiness', 4: 'sadness', 5: 'surprise', 6: 'neutral'}","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:20:23.663968Z","iopub.execute_input":"2021-06-14T00:20:23.664337Z","iopub.status.idle":"2021-06-14T00:20:23.669572Z","shell.execute_reply.started":"2021-06-14T00:20:23.664304Z","shell.execute_reply":"2021-06-14T00:20:23.668591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.emotion.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:20:23.670945Z","iopub.execute_input":"2021-06-14T00:20:23.671321Z","iopub.status.idle":"2021-06-14T00:20:23.687462Z","shell.execute_reply.started":"2021-06-14T00:20:23.671286Z","shell.execute_reply":"2021-06-14T00:20:23.686646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process_pixels(df):\n    img_array = df.pixels.apply(lambda x: np.array(x.split(' ')).reshape(48, 48, 1).astype('float32'))\n    img_array = np.stack(img_array, axis=0)\n    return img_array\n\nimg_array = pre_process_pixels(df)\nt_img_array = pre_process_pixels(test)        ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T01:05:46.598064Z","iopub.execute_input":"2021-06-14T01:05:46.598401Z","iopub.status.idle":"2021-06-14T01:06:49.456511Z","shell.execute_reply.started":"2021-06-14T01:05:46.598371Z","shell.execute_reply":"2021-06-14T01:06:49.455657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_labels = df['emotion'].values\nle = LabelEncoder()\nimg_labels = le.fit_transform(df.emotion)\nimg_labels = np_utils.to_categorical(img_labels)\nimg_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.391959Z","iopub.execute_input":"2021-06-14T00:21:14.392295Z","iopub.status.idle":"2021-06-14T00:21:14.403062Z","shell.execute_reply.started":"2021-06-14T00:21:14.392258Z","shell.execute_reply":"2021-06-14T00:21:14.401895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(img_array, img_labels,\n                                                    shuffle=True, stratify=img_labels,\n                                                    test_size=0.1, random_state=42)\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.406047Z","iopub.execute_input":"2021-06-14T00:21:14.406482Z","iopub.status.idle":"2021-06-14T00:21:14.774116Z","shell.execute_reply.started":"2021-06-14T00:21:14.406443Z","shell.execute_reply":"2021-06-14T00:21:14.773119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizing results, as neural networks are very sensitive to unnormalized data.\nX_train = X_train / 255.\nX_valid = X_valid / 255.","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.775810Z","iopub.execute_input":"2021-06-14T00:21:14.776206Z","iopub.status.idle":"2021-06-14T00:21:14.856902Z","shell.execute_reply.started":"2021-06-14T00:21:14.776161Z","shell.execute_reply":"2021-06-14T00:21:14.856030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_net(optim, num_classes):\n    \"\"\"\n    This is a Deep Convolutional Neural Network (DCNN). For generalization purpose I used dropouts in regular intervals.\n    I used `ELU` as the activation because it avoids dying relu problem but also performed well as compared to LeakyRelu\n    atleast in this case. `he_normal` kernel initializer is used as it suits ELU. BatchNormalization is also used for better\n    results.\n    \"\"\"\n    net = Sequential(name='DCNN')\n\n    net.add(\n        Conv2D(\n            filters=64,\n            kernel_size=(5,5),\n            input_shape=(48, 48, 1),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_1'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_1'))\n    net.add(\n        Conv2D(\n            filters=64,\n            kernel_size=(5,5),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_2'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_2'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_1'))\n    net.add(Dropout(0.4, name='dropout_1'))\n\n    net.add(\n        Conv2D(\n            filters=128,\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_3'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_3'))\n    net.add(\n        Conv2D(\n            filters=128,\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_4'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_4'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_2'))\n    net.add(Dropout(0.4, name='dropout_2'))\n\n    net.add(\n        Conv2D(\n            filters=256,\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_5'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_5'))\n    net.add(\n        Conv2D(\n            filters=256,\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_6'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_6'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_3'))\n    net.add(Dropout(0.5, name='dropout_3'))\n\n    net.add(Flatten(name='flatten'))\n        \n    net.add(\n        Dense(\n            128,\n            activation='elu',\n            kernel_initializer='he_normal',\n            name='dense_1'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_7'))\n    \n    net.add(Dropout(0.6, name='dropout_4'))\n    \n    net.add(\n        Dense(\n            num_classes,\n            activation='softmax',\n            name='out_layer'\n        )\n    )\n    \n    net.compile(\n        loss='categorical_crossentropy',\n        optimizer=optim,\n        metrics=['accuracy']\n    )\n    \n    net.summary()\n    \n    return net","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.858264Z","iopub.execute_input":"2021-06-14T00:21:14.858602Z","iopub.status.idle":"2021-06-14T00:21:14.874769Z","shell.execute_reply.started":"2021-06-14T00:21:14.858565Z","shell.execute_reply":"2021-06-14T00:21:14.873963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nI used two callbacks one is `early stopping` for avoiding overfitting training data\nand other `ReduceLROnPlateau` for learning rate.\n\"\"\"\n\nearly_stopping = EarlyStopping(\n    monitor='val_accuracy',\n    min_delta=0.00005,\n    patience=11,\n    verbose=1,\n    restore_best_weights=True,\n)\n\nlr_scheduler = ReduceLROnPlateau(\n    monitor='val_accuracy',\n    factor=0.5,\n    patience=7,\n    min_lr=1e-7,\n    verbose=1,\n)\n\ncallbacks = [\n    early_stopping,\n    lr_scheduler,\n]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.876035Z","iopub.execute_input":"2021-06-14T00:21:14.876405Z","iopub.status.idle":"2021-06-14T00:21:14.889338Z","shell.execute_reply.started":"2021-06-14T00:21:14.876368Z","shell.execute_reply":"2021-06-14T00:21:14.888488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As the data in hand is less as compared to the task so ImageDataGenerator is good to go.\ntrain_datagen = ImageDataGenerator(\n    rotation_range=15,\n    width_shift_range=0.15,\n    height_shift_range=0.15,\n    shear_range=0.15,\n    zoom_range=0.15,\n    horizontal_flip=True,\n)\ntrain_datagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.890573Z","iopub.execute_input":"2021-06-14T00:21:14.890956Z","iopub.status.idle":"2021-06-14T00:21:14.987157Z","shell.execute_reply.started":"2021-06-14T00:21:14.890921Z","shell.execute_reply":"2021-06-14T00:21:14.986259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_net2(optim, num_classes):\n    # Create the model\n    model = Sequential()\n\n    model.add(Conv2D(32, kernel_size=(3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.0001),input_shape=(48,48,1)))\n    # model.add(BatchNormalization())\n\n    model.add(Conv2D(64, kernel_size=(3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(7, kernel_size=(1, 1), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # # model.add(BatchNormalization())\n\n    model.add(Conv2D(7, kernel_size=(4, 4), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n\n    model.add(Flatten())\n\n    model.add(Activation(\"softmax\"))\n    \n    model.compile(loss='categorical_crossentropy',optimizer=optim,metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:21:14.990394Z","iopub.execute_input":"2021-06-14T00:21:14.990661Z","iopub.status.idle":"2021-06-14T00:21:15.000017Z","shell.execute_reply.started":"2021-06-14T00:21:14.990634Z","shell.execute_reply":"2021-06-14T00:21:14.999170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32 #batch size of 32 performs the best.\nepochs = 250\noptims = [\n    optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam'),\n    optimizers.SGD(learning_rate = 0.0001, momentum=0.9, nesterov=False, name=\"SGD\"),\n    optimizers.Adam(0.001),\n    \n]\n\n# I tried both `Nadam` and `Adam`, the difference in results is not different but I finally went with Nadam as it is more popular.\nmodel = build_net(optims[0], 7) \nhistory = model.fit_generator(\n    train_datagen.flow(X_train, y_train, batch_size=batch_size),\n    validation_data=(X_valid, y_valid),\n    steps_per_epoch=len(X_train) / batch_size,\n    epochs=epochs,\n    callbacks=callbacks,\n    use_multiprocessing=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T00:29:39.439468Z","iopub.execute_input":"2021-06-14T00:29:39.440650Z","iopub.status.idle":"2021-06-14T00:58:51.018106Z","shell.execute_reply.started":"2021-06-14T00:29:39.440566Z","shell.execute_reply":"2021-06-14T00:58:51.016994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the test data\npredicted_prices = model.predict(t_img_array)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T01:09:55.927519Z","iopub.execute_input":"2021-06-14T01:09:55.927883Z","iopub.status.idle":"2021-06-14T01:09:57.057301Z","shell.execute_reply.started":"2021-06-14T01:09:55.927851Z","shell.execute_reply":"2021-06-14T01:09:57.056369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = np.argwhere(np.around(predicted_prices)).transpose()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T01:42:18.693223Z","iopub.execute_input":"2021-06-14T01:42:18.693537Z","iopub.status.idle":"2021-06-14T01:42:18.702900Z","shell.execute_reply.started":"2021-06-14T01:42:18.693508Z","shell.execute_reply":"2021-06-14T01:42:18.701876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'ID': label[0], 'Emotion': label[1]})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)\nmy_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-14T01:43:14.917076Z","iopub.execute_input":"2021-06-14T01:43:14.917404Z","iopub.status.idle":"2021-06-14T01:43:15.182988Z","shell.execute_reply.started":"2021-06-14T01:43:14.917375Z","shell.execute_reply":"2021-06-14T01:43:15.182218Z"},"trusted":true},"execution_count":null,"outputs":[]}]}