{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install pycaret[full]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:51:43.355927Z","iopub.execute_input":"2021-06-17T23:51:43.356311Z","iopub.status.idle":"2021-06-17T23:52:37.416174Z","shell.execute_reply.started":"2021-06-17T23:51:43.356277Z","shell.execute_reply":"2021-06-17T23:52:37.415271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-17T23:52:37.419174Z","iopub.execute_input":"2021-06-17T23:52:37.419434Z","iopub.status.idle":"2021-06-17T23:52:37.428474Z","shell.execute_reply.started":"2021-06-17T23:52:37.419404Z","shell.execute_reply":"2021-06-17T23:52:37.427649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport numpy as np\nimport pandas as pd\n\nimport scikitplot\nimport seaborn as sns\nfrom matplotlib import pyplot\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import classification_report\n\nimport tensorflow as tf\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Flatten, Dense, Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dropout, BatchNormalization, LeakyReLU, Activation\nfrom tensorflow.keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras import regularizers\nfrom keras.utils import np_utils","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:52:37.430200Z","iopub.execute_input":"2021-06-17T23:52:37.430520Z","iopub.status.idle":"2021-06-17T23:52:37.521716Z","shell.execute_reply.started":"2021-06-17T23:52:37.430486Z","shell.execute_reply":"2021-06-17T23:52:37.520971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/expertclass2/train_data.csv')\ntest = pd.read_csv('../input/expertclass2/test_data.csv')\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:52:37.523259Z","iopub.execute_input":"2021-06-17T23:52:37.523612Z","iopub.status.idle":"2021-06-17T23:52:42.657082Z","shell.execute_reply.started":"2021-06-17T23:52:37.523577Z","shell.execute_reply":"2021-06-17T23:52:42.656233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 48\nemotion_label_to_text = {0:'anger', 1:'disgust', 2:'fear', 3:'happiness', 4: 'sadness', 5: 'surprise', 6: 'neutral'}","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:52:42.658330Z","iopub.execute_input":"2021-06-17T23:52:42.658665Z","iopub.status.idle":"2021-06-17T23:52:42.663328Z","shell.execute_reply.started":"2021-06-17T23:52:42.658628Z","shell.execute_reply":"2021-06-17T23:52:42.662296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.emotion.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:52:42.664843Z","iopub.execute_input":"2021-06-17T23:52:42.665213Z","iopub.status.idle":"2021-06-17T23:52:42.683222Z","shell.execute_reply.started":"2021-06-17T23:52:42.665176Z","shell.execute_reply":"2021-06-17T23:52:42.682387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process_pixels(df):\n    img_array = df.pixels.apply(lambda x: np.array(x.split(' ')).reshape(48, 48, 1).astype('float32'))\n    img_array = np.stack(img_array, axis=0)\n    return img_array\n\nimg_array = pre_process_pixels(df)\nt_img_array = pre_process_pixels(test)        ","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:52:42.684262Z","iopub.execute_input":"2021-06-17T23:52:42.684496Z","iopub.status.idle":"2021-06-17T23:53:45.543436Z","shell.execute_reply.started":"2021-06-17T23:52:42.684474Z","shell.execute_reply":"2021-06-17T23:53:45.542574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_labels = df['emotion'].values\ntarget_encoder = OneHotEncoder(sparse = False)\nimg_labels = target_encoder.fit_transform(img_labels.reshape(-1,1))\n#img_labels = np_utils.to_categorical(img_labels)\nimg_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:53:45.546454Z","iopub.execute_input":"2021-06-17T23:53:45.546846Z","iopub.status.idle":"2021-06-17T23:53:45.559141Z","shell.execute_reply.started":"2021-06-17T23:53:45.546805Z","shell.execute_reply":"2021-06-17T23:53:45.558102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(img_array, img_labels,\n                                                    shuffle=True, stratify=img_labels,\n                                                    test_size=0.1, random_state=42)\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:53:45.561213Z","iopub.execute_input":"2021-06-17T23:53:45.561656Z","iopub.status.idle":"2021-06-17T23:53:45.949394Z","shell.execute_reply.started":"2021-06-17T23:53:45.561609Z","shell.execute_reply":"2021-06-17T23:53:45.948609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizing results, as neural networks are very sensitive to unnormalized data.\nX_train = X_train / 255.\nX_valid = X_valid / 255.","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:53:45.950533Z","iopub.execute_input":"2021-06-17T23:53:45.950855Z","iopub.status.idle":"2021-06-17T23:53:46.033078Z","shell.execute_reply.started":"2021-06-17T23:53:45.950827Z","shell.execute_reply":"2021-06-17T23:53:46.032110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_net(optim, num_classes, srhink_factor = 1):\n    \"\"\"\n    This is a Deep Convolutional Neural Network (DCNN). For generalization purpose I used dropouts in regular intervals.\n    I used `ELU` as the activation because it avoids dying relu problem but also performed well as compared to LeakyRelu\n    atleast in this case. `he_normal` kernel initializer is used as it suits ELU. BatchNormalization is also used for better\n    results.\n    \"\"\"\n    net = Sequential(name='DCNN')\n\n    net.add(\n        Conv2D(\n            filters= int(srhink_factor*64),\n            kernel_size=(4,4),\n            input_shape=(48, 48, 1),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_1'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_1'))\n    net.add(\n        Conv2D(\n            filters=int(srhink_factor*64),\n            kernel_size=(4,4),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_2'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_2'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_1'))\n    net.add(Dropout(0.1, name='dropout_1'))\n\n    net.add(\n        Conv2D(\n            filters=int(srhink_factor*128),\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_3'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_3'))\n    net.add(\n        Conv2D(\n            filters=int(srhink_factor*128),\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_4'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_4'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_2'))\n    net.add(Dropout(0.1, name='dropout_2'))\n\n    net.add(\n        Conv2D(\n            filters=int(srhink_factor*128),\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_5'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_5'))\n    net.add(\n        Conv2D(\n            filters=int(srhink_factor*256),\n            kernel_size=(3,3),\n            activation='elu',\n            padding='same',\n            kernel_initializer='he_normal',\n            name='conv2d_6'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_6'))\n    \n    net.add(MaxPooling2D(pool_size=(2,2), name='maxpool2d_3'))\n    net.add(Dropout(0.1, name='dropout_3'))\n\n    net.add(Flatten(name='flatten'))\n        \n    net.add(\n        Dense(\n            128,\n            activation='elu',\n            kernel_initializer='he_normal',\n            name='dense_1'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_7'))\n    \n    net.add(Dropout(0.1, name='dropout_4'))\n    \n    net.add(\n        Dense(\n            128,\n            activation='elu',\n            kernel_initializer='he_normal',\n            name='dense_2'\n        )\n    )\n    net.add(BatchNormalization(name='batchnorm_8'))\n    \n    net.add(Dropout(0.1, name='dropout_5'))\n    \n    \n    \n    net.add(\n        Dense(\n            num_classes,\n            activation='softmax',\n            name='out_layer'\n        )\n    )\n    \n    net.compile(\n        loss='categorical_crossentropy',\n        optimizer=optim,\n        metrics=['accuracy']\n    )\n    \n    net.summary()\n    \n    return net","metadata":{"execution":{"iopub.status.busy":"2021-06-18T02:51:55.386332Z","iopub.execute_input":"2021-06-18T02:51:55.386674Z","iopub.status.idle":"2021-06-18T02:51:55.403483Z","shell.execute_reply.started":"2021-06-18T02:51:55.386643Z","shell.execute_reply":"2021-06-18T02:51:55.402463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nI used two callbacks one is `early stopping` for avoiding overfitting training data\nand other `ReduceLROnPlateau` for learning rate.\n\"\"\"\n\nearly_stopping = EarlyStopping(\n    monitor='val_accuracy',\n    min_delta=0.00005,\n    patience=11,\n    verbose=1,\n    restore_best_weights=True,\n)\n\nlr_scheduler = ReduceLROnPlateau(\n    monitor='val_accuracy',\n    factor=0.5,\n    patience=7,\n    min_lr=1e-7,\n    verbose=1,\n)\n\ncallbacks = [\n    early_stopping,\n    lr_scheduler,\n]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T02:51:57.298106Z","iopub.execute_input":"2021-06-18T02:51:57.298429Z","iopub.status.idle":"2021-06-18T02:51:57.303820Z","shell.execute_reply.started":"2021-06-18T02:51:57.298400Z","shell.execute_reply":"2021-06-18T02:51:57.303019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As the data in hand is less as compared to the task so ImageDataGenerator is good to go.\ntrain_datagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n)\ntrain_datagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T02:51:58.595568Z","iopub.execute_input":"2021-06-18T02:51:58.597131Z","iopub.status.idle":"2021-06-18T02:51:58.751588Z","shell.execute_reply.started":"2021-06-18T02:51:58.597085Z","shell.execute_reply":"2021-06-18T02:51:58.750670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_net2(optim, num_classes):\n    # Create the model\n    model = Sequential()\n\n    model.add(Conv2D(32, kernel_size=(3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.0001),input_shape=(48,48,1)))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(64, kernel_size=(3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.0001)))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(7, kernel_size=(1, 1), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # # model.add(BatchNormalization())\n\n    model.add(Conv2D(7, kernel_size=(4, 4), activation='relu', kernel_regularizer=regularizers.l2(0.0001)))\n    # model.add(BatchNormalization())\n\n    model.add(Flatten())\n\n    model.add(Activation(\"softmax\"))\n    \n    model.compile(loss='categorical_crossentropy',optimizer=optim,metrics=['accuracy'])\n    \n    return model\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T02:51:59.302099Z","iopub.execute_input":"2021-06-18T02:51:59.302414Z","iopub.status.idle":"2021-06-18T02:51:59.312100Z","shell.execute_reply.started":"2021-06-18T02:51:59.302386Z","shell.execute_reply":"2021-06-18T02:51:59.311145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32 #batch size of 32 performs the best.\nepochs = 250\noptims = [\n    optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name='Nadam'),\n    optimizers.SGD(learning_rate = 0.0001, momentum=0.9, nesterov=False, name=\"SGD\"),\n    optimizers.Adam(0.001),\n    \n]\n\n# I tried both `Nadam` and `Adam`, the difference in results is not different but I finally went with Nadam as it is more popular.\nmodel = build_net(optims[0], 7, 1) \nhistory = model.fit_generator(\n    train_datagen.flow(X_train, y_train, batch_size=batch_size),\n    validation_data=(X_valid, y_valid),\n    steps_per_epoch=len(X_train) / batch_size ,\n    epochs=epochs,\n    callbacks=callbacks,\n    use_multiprocessing=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T02:52:00.292529Z","iopub.execute_input":"2021-06-18T02:52:00.292846Z","iopub.status.idle":"2021-06-18T03:15:43.421556Z","shell.execute_reply.started":"2021-06-18T02:52:00.292815Z","shell.execute_reply":"2021-06-18T03:15:43.420304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:53:17.802448Z","iopub.execute_input":"2021-06-18T01:53:17.802792Z","iopub.status.idle":"2021-06-18T01:53:18.762296Z","shell.execute_reply.started":"2021-06-18T01:53:17.802760Z","shell.execute_reply":"2021-06-18T01:53:18.761322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_layer_model = Model(inputs=model.input,\n                                 outputs=model.get_layer('flatten').output)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:19:55.891305Z","iopub.execute_input":"2021-06-18T03:19:55.891677Z","iopub.status.idle":"2021-06-18T03:19:55.903174Z","shell.execute_reply.started":"2021-06-18T03:19:55.891633Z","shell.execute_reply":"2021-06-18T03:19:55.902385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flatten_conv_train = conv_layer_model.predict(X_train)\nflatten_conv_valid = conv_layer_model.predict(X_valid)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:19:56.200056Z","iopub.execute_input":"2021-06-18T03:19:56.200365Z","iopub.status.idle":"2021-06-18T03:20:01.604421Z","shell.execute_reply.started":"2021-06-18T03:19:56.200336Z","shell.execute_reply":"2021-06-18T03:20:01.603448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flatten_conv_valid = flatten_conv_valid.reshape(*flatten_conv_valid.shape, 1)\nflatten_conv_train = flatten_conv_train.reshape(*flatten_conv_train.shape, 1)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:20:01.605999Z","iopub.execute_input":"2021-06-18T03:20:01.606315Z","iopub.status.idle":"2021-06-18T03:20:01.610463Z","shell.execute_reply.started":"2021-06-18T03:20:01.606283Z","shell.execute_reply":"2021-06-18T03:20:01.609655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dense_net(num_classes, optim):\n    model = Sequential()\n    \n    model.add(\n        Dense(\n            32,\n            activation='elu',\n            kernel_initializer='he_normal',\n            name='dense_1'\n        )\n    )\n    model.add(BatchNormalization(name='batchnorm_7'))\n    \n    model.add(Dropout(0.1, name='dropout_4'))\n    \n    model.add(\n        Dense(\n            num_classes,\n            activation='softmax',\n            name='out_layer'\n        )\n    )\n    model.compile(loss='categorical_crossentropy',optimizer=optim,metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:20:01.612166Z","iopub.execute_input":"2021-06-18T03:20:01.612735Z","iopub.status.idle":"2021-06-18T03:20:01.624562Z","shell.execute_reply.started":"2021-06-18T03:20:01.612698Z","shell.execute_reply":"2021-06-18T03:20:01.623792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 1024\nepochs = 250\n\nfinal_layer = dense_net(7, optims[0])\nhistory = final_layer.fit(\n    x = flatten_conv_train,\n    y = y_train,\n    validation_data=(flatten_conv_valid, y_valid),\n    steps_per_epoch=len(flatten_conv_train) / batch_size ,\n    epochs=epochs,\n    callbacks=callbacks,\n    use_multiprocessing=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:20:05.204689Z","iopub.execute_input":"2021-06-18T03:20:05.205180Z","iopub.status.idle":"2021-06-18T03:20:06.305842Z","shell.execute_reply.started":"2021-06-18T03:20:05.205134Z","shell.execute_reply":"2021-06-18T03:20:06.303401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the test data\npredicted_prices = model.predict(t_img_array)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:12:22.460972Z","iopub.execute_input":"2021-06-18T01:12:22.461322Z","iopub.status.idle":"2021-06-18T01:12:23.247360Z","shell.execute_reply.started":"2021-06-18T01:12:22.461286Z","shell.execute_reply":"2021-06-18T01:12:23.246394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_ann = np.argwhere(np.around(predicted_prices)).transpose()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:17:45.657582Z","iopub.execute_input":"2021-06-18T00:17:45.657912Z","iopub.status.idle":"2021-06-18T00:17:45.663941Z","shell.execute_reply.started":"2021-06-18T00:17:45.657875Z","shell.execute_reply":"2021-06-18T00:17:45.662799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('keras_model.sav')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:17:45.665434Z","iopub.execute_input":"2021-06-18T00:17:45.665874Z","iopub.status.idle":"2021-06-18T00:17:49.068304Z","shell.execute_reply.started":"2021-06-18T00:17:45.665835Z","shell.execute_reply":"2021-06-18T00:17:49.067310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Boosting with output of neural net","metadata":{}},{"cell_type":"code","source":"import keras\nmodel = keras.models.load_model('keras_model.sav')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:17:49.074835Z","iopub.execute_input":"2021-06-18T00:17:49.075160Z","iopub.status.idle":"2021-06-18T00:17:50.111428Z","shell.execute_reply.started":"2021-06-18T00:17:49.075122Z","shell.execute_reply":"2021-06-18T00:17:50.110540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom keras.models import Model\nfrom sklearn.linear_model import LogisticRegression\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:20:14.993111Z","iopub.execute_input":"2021-06-18T03:20:14.993431Z","iopub.status.idle":"2021-06-18T03:20:14.997694Z","shell.execute_reply.started":"2021-06-18T03:20:14.993401Z","shell.execute_reply":"2021-06-18T03:20:14.996559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_layer_model = Model(inputs=model.input,\n                                 outputs=model.get_layer('dense_2').output)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:29:52.068881Z","iopub.execute_input":"2021-06-18T03:29:52.070137Z","iopub.status.idle":"2021-06-18T03:29:52.095394Z","shell.execute_reply.started":"2021-06-18T03:29:52.070060Z","shell.execute_reply":"2021-06-18T03:29:52.094536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_train = intermediate_layer_model.predict(X_train)\nintermediate_val = intermediate_layer_model.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:29:53.206041Z","iopub.execute_input":"2021-06-18T03:29:53.206383Z","iopub.status.idle":"2021-06-18T03:29:56.783378Z","shell.execute_reply.started":"2021-06-18T03:29:53.206352Z","shell.execute_reply":"2021-06-18T03:29:56.782477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import QuantileTransformer\nfrom sklearn.ensemble import StackingClassifier\n\nfinal_model1 = RandomForestClassifier(verbose = 1)\nfinal_model2 = BaggingClassifier(HistGradientBoostingClassifier(verbose = 0, learning_rate = 0.1), 10)\n\n\nfinal_model = Pipeline([\n    #('qt', QuantileTransformer(output_distribution = 'uniform')),\n    ('model', RandomForestClassifier(min_samples_leaf = 1, verbose = 1))],\n    verbose = True\n)\n#final_model = BaggingClassifier(final_model, 20)\nfinal_model.fit(intermediate_train, target_encoder.inverse_transform(y_train))","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:46:34.320253Z","iopub.execute_input":"2021-06-18T03:46:34.320580Z","iopub.status.idle":"2021-06-18T03:47:06.202856Z","shell.execute_reply.started":"2021-06-18T03:46:34.320549Z","shell.execute_reply":"2021-06-18T03:47:06.201887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = final_model.predict(intermediate_val)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:47:06.204338Z","iopub.execute_input":"2021-06-18T03:47:06.204686Z","iopub.status.idle":"2021-06-18T03:47:06.292014Z","shell.execute_reply.started":"2021-06-18T03:47:06.204648Z","shell.execute_reply":"2021-06-18T03:47:06.291042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(preds == target_encoder.inverse_transform(y_valid).flatten()).mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:47:06.293612Z","iopub.execute_input":"2021-06-18T03:47:06.294117Z","iopub.status.idle":"2021-06-18T03:47:06.300979Z","shell.execute_reply.started":"2021-06-18T03:47:06.294075Z","shell.execute_reply":"2021-06-18T03:47:06.299996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_intermediate = np.vstack([intermediate_train, intermediate_val])\nfull_y = np.vstack([y_train, y_valid])\nfull_y = target_encoder.inverse_transform(full_y)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:39:05.078493Z","iopub.execute_input":"2021-06-18T03:39:05.078954Z","iopub.status.idle":"2021-06-18T03:39:05.103418Z","shell.execute_reply.started":"2021-06-18T03:39:05.078898Z","shell.execute_reply":"2021-06-18T03:39:05.102503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model.fit(full_intermediate, full_y)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:47:19.166585Z","iopub.execute_input":"2021-06-18T03:47:19.166901Z","iopub.status.idle":"2021-06-18T03:47:56.173417Z","shell.execute_reply.started":"2021-06-18T03:47:19.166870Z","shell.execute_reply":"2021-06-18T03:47:56.172617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_sub = t_img_array/255.\nintermediate_sub = intermediate_layer_model.predict(intermediate_sub)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:49:09.233907Z","iopub.execute_input":"2021-06-18T03:49:09.234258Z","iopub.status.idle":"2021-06-18T03:49:10.215330Z","shell.execute_reply.started":"2021-06-18T03:49:09.234226Z","shell.execute_reply":"2021-06-18T03:49:10.214440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_preds = final_model.predict(intermediate_sub)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:49:12.025106Z","iopub.execute_input":"2021-06-18T03:49:12.025427Z","iopub.status.idle":"2021-06-18T03:49:12.198799Z","shell.execute_reply.started":"2021-06-18T03:49:12.025399Z","shell.execute_reply":"2021-06-18T03:49:12.197903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nLABEL_MAPPER = {\n    i:v for i,v in enumerate(['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral'])\n}\n\nidx = np.random.choice(np.arange(len(sub_preds)))\nplt.imshow(t_img_array[idx])\nplt.title(LABEL_MAPPER[sub_preds[idx]])","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:50:03.470888Z","iopub.execute_input":"2021-06-18T03:50:03.471251Z","iopub.status.idle":"2021-06-18T03:50:03.584517Z","shell.execute_reply.started":"2021-06-18T03:50:03.471220Z","shell.execute_reply":"2021-06-18T03:50:03.583567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'ID': np.arange(len(sub_preds)), 'Emotion': sub_preds})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)\nmy_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:49:16.183198Z","iopub.execute_input":"2021-06-18T03:49:16.183516Z","iopub.status.idle":"2021-06-18T03:49:16.255016Z","shell.execute_reply.started":"2021-06-18T03:49:16.183485Z","shell.execute_reply":"2021-06-18T03:49:16.254254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:49:20.718822Z","iopub.execute_input":"2021-06-18T03:49:20.719170Z","iopub.status.idle":"2021-06-18T03:49:20.725392Z","shell.execute_reply.started":"2021-06-18T03:49:20.719139Z","shell.execute_reply":"2021-06-18T03:49:20.724136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T03:43:59.015017Z","iopub.status.idle":"2021-06-18T03:43:59.015709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pycaret.classification import *\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:43:33.202678Z","iopub.execute_input":"2021-06-18T00:43:33.203032Z","iopub.status.idle":"2021-06-18T00:43:37.812597Z","shell.execute_reply.started":"2021-06-18T00:43:33.202999Z","shell.execute_reply":"2021-06-18T00:43:37.811770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:18:30.753938Z","iopub.execute_input":"2021-06-18T01:18:30.754302Z","iopub.status.idle":"2021-06-18T01:18:30.765560Z","shell.execute_reply.started":"2021-06-18T01:18:30.754272Z","shell.execute_reply":"2021-06-18T01:18:30.764708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:19:02.098939Z","iopub.execute_input":"2021-06-18T01:19:02.099392Z","iopub.status.idle":"2021-06-18T01:19:17.173821Z","shell.execute_reply.started":"2021-06-18T01:19:02.099354Z","shell.execute_reply":"2021-06-18T01:19:17.173038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model.predict()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(full_intermediate)\ndf['target'] = full_y","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:43:44.888375Z","iopub.execute_input":"2021-06-18T00:43:44.888707Z","iopub.status.idle":"2021-06-18T00:43:44.894206Z","shell.execute_reply.started":"2021-06-18T00:43:44.888677Z","shell.execute_reply":"2021-06-18T00:43:44.892998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf1 = setup(\n    df,\n    target = 'target',\n    session_id=1,\n    log_experiment=True,\n    experiment_name='emotions1',\n    use_gpu= True,\n    preprocess = False,\n\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:43:58.317314Z","iopub.execute_input":"2021-06-18T00:43:58.317639Z","iopub.status.idle":"2021-06-18T00:44:03.748637Z","shell.execute_reply.started":"2021-06-18T00:43:58.317595Z","shell.execute_reply":"2021-06-18T00:44:03.747683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = compare_models()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:44:03.750231Z","iopub.execute_input":"2021-06-18T00:44:03.750545Z","iopub.status.idle":"2021-06-18T00:47:05.489310Z","shell.execute_reply.started":"2021-06-18T00:44:03.750517Z","shell.execute_reply":"2021-06-18T00:47:05.488272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuned_lr = tune_model(best_model)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:47:28.043558Z","iopub.execute_input":"2021-06-18T00:47:28.044007Z","iopub.status.idle":"2021-06-18T00:47:55.219371Z","shell.execute_reply.started":"2021-06-18T00:47:28.043959Z","shell.execute_reply":"2021-06-18T00:47:55.218529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model = finalize_model(tuned_lr)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:48:00.797016Z","iopub.execute_input":"2021-06-18T00:48:00.797360Z","iopub.status.idle":"2021-06-18T00:48:04.989223Z","shell.execute_reply.started":"2021-06-18T00:48:00.797329Z","shell.execute_reply":"2021-06-18T00:48:04.988313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:48:13.152813Z","iopub.execute_input":"2021-06-18T00:48:13.153165Z","iopub.status.idle":"2021-06-18T00:48:13.936108Z","shell.execute_reply.started":"2021-06-18T00:48:13.153131Z","shell.execute_reply":"2021-06-18T00:48:13.935085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:19:31.468065Z","iopub.execute_input":"2021-06-18T01:19:31.468395Z","iopub.status.idle":"2021-06-18T01:19:31.630941Z","shell.execute_reply.started":"2021-06-18T01:19:31.468365Z","shell.execute_reply":"2021-06-18T01:19:31.630061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:22:22.441467Z","iopub.execute_input":"2021-06-18T01:22:22.441785Z","iopub.status.idle":"2021-06-18T01:22:22.553533Z","shell.execute_reply.started":"2021-06-18T01:22:22.441752Z","shell.execute_reply":"2021-06-18T01:22:22.552549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:21:09.473342Z","iopub.execute_input":"2021-06-18T01:21:09.473652Z","iopub.status.idle":"2021-06-18T01:21:09.501820Z","shell.execute_reply.started":"2021-06-18T01:21:09.473622Z","shell.execute_reply":"2021-06-18T01:21:09.500852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-06-18T01:21:15.881082Z","iopub.execute_input":"2021-06-18T01:21:15.881401Z","iopub.status.idle":"2021-06-18T01:21:15.890071Z","shell.execute_reply.started":"2021-06-18T01:21:15.881372Z","shell.execute_reply":"2021-06-18T01:21:15.889225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T00:51:08.252679Z","iopub.execute_input":"2021-06-18T00:51:08.253039Z","iopub.status.idle":"2021-06-18T00:53:10.362565Z","shell.execute_reply.started":"2021-06-18T00:51:08.253006Z","shell.execute_reply":"2021-06-18T00:53:10.361768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}