{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":7.707653,"end_time":"2023-06-04T05:45:01.178636","exception":false,"start_time":"2023-06-04T05:44:53.470983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:07:38.043283Z","iopub.execute_input":"2023-06-05T07:07:38.043556Z","iopub.status.idle":"2023-06-05T07:07:51.538645Z","shell.execute_reply.started":"2023-06-05T07:07:38.043532Z","shell.execute_reply":"2023-06-05T07:07:51.537709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Required Modules ","metadata":{"papermill":{"duration":0.009772,"end_time":"2023-06-04T05:45:01.198819","exception":false,"start_time":"2023-06-04T05:45:01.189047","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport cv2 as cv\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nimport os","metadata":{"papermill":{"duration":7.717679,"end_time":"2023-06-04T05:45:08.926353","exception":false,"start_time":"2023-06-04T05:45:01.208674","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:07:51.540255Z","iopub.execute_input":"2023-06-05T07:07:51.540617Z","iopub.status.idle":"2023-06-05T07:08:00.329166Z","shell.execute_reply.started":"2023-06-05T07:07:51.540572Z","shell.execute_reply":"2023-06-05T07:08:00.328200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Hyper parameters","metadata":{"papermill":{"duration":0.010148,"end_time":"2023-06-04T05:45:08.946898","exception":false,"start_time":"2023-06-04T05:45:08.936750","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_path = \"/kaggle/input/extracting-attributes-from-fashion-images-2/train\"\nBATCH_SIZE = 64\nEPOCHS = 5\nINPUT_SHAPE = 224\nLR = 0.00000001","metadata":{"papermill":{"duration":0.017997,"end_time":"2023-06-04T05:45:08.974958","exception":false,"start_time":"2023-06-04T05:45:08.956961","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.333641Z","iopub.execute_input":"2023-06-05T07:08:00.336195Z","iopub.status.idle":"2023-06-05T07:08:00.343176Z","shell.execute_reply.started":"2023-06-05T07:08:00.336156Z","shell.execute_reply":"2023-06-05T07:08:00.342372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Importing","metadata":{"papermill":{"duration":0.010225,"end_time":"2023-06-04T05:45:08.997136","exception":false,"start_time":"2023-06-04T05:45:08.986911","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# training data\ntrain_df = pd.read_csv(\"/kaggle/input/extracting-attributes-from-fashion-images-2/train.csv\")\ntrain_df","metadata":{"papermill":{"duration":0.05261,"end_time":"2023-06-04T05:45:09.059736","exception":false,"start_time":"2023-06-04T05:45:09.007126","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.348476Z","iopub.execute_input":"2023-06-05T07:08:00.350956Z","iopub.status.idle":"2023-06-05T07:08:00.410552Z","shell.execute_reply.started":"2023-06-05T07:08:00.350900Z","shell.execute_reply":"2023-06-05T07:08:00.409800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntest_df = pd.read_csv(\"/kaggle/input/extracting-attributes-from-fashion-images-2/sample_submission.csv\")\ntest_df","metadata":{"papermill":{"duration":0.033177,"end_time":"2023-06-04T05:45:09.103806","exception":false,"start_time":"2023-06-04T05:45:09.070629","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.414516Z","iopub.execute_input":"2023-06-05T07:08:00.416850Z","iopub.status.idle":"2023-06-05T07:08:00.445157Z","shell.execute_reply.started":"2023-06-05T07:08:00.416809Z","shell.execute_reply":"2023-06-05T07:08:00.444346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking for Imbalance in the dataset","metadata":{"papermill":{"duration":0.010477,"end_time":"2023-06-04T05:45:09.125859","exception":false,"start_time":"2023-06-04T05:45:09.115382","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df.label.value_counts()","metadata":{"papermill":{"duration":0.02485,"end_time":"2023-06-04T05:45:09.161228","exception":false,"start_time":"2023-06-04T05:45:09.136378","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.449256Z","iopub.execute_input":"2023-06-05T07:08:00.451508Z","iopub.status.idle":"2023-06-05T07:08:00.468043Z","shell.execute_reply.started":"2023-06-05T07:08:00.451471Z","shell.execute_reply":"2023-06-05T07:08:00.466972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.label.value_counts()","metadata":{"papermill":{"duration":0.020059,"end_time":"2023-06-04T05:45:09.192283","exception":false,"start_time":"2023-06-04T05:45:09.172224","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.471886Z","iopub.execute_input":"2023-06-05T07:08:00.474325Z","iopub.status.idle":"2023-06-05T07:08:00.485538Z","shell.execute_reply.started":"2023-06-05T07:08:00.474291Z","shell.execute_reply":"2023-06-05T07:08:00.484595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.label.unique()","metadata":{"papermill":{"duration":0.021388,"end_time":"2023-06-04T05:45:09.224714","exception":false,"start_time":"2023-06-04T05:45:09.203326","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.489937Z","iopub.execute_input":"2023-06-05T07:08:00.492067Z","iopub.status.idle":"2023-06-05T07:08:00.503422Z","shell.execute_reply.started":"2023-06-05T07:08:00.492036Z","shell.execute_reply":"2023-06-05T07:08:00.502574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import class_weight\nclass_weights = list(class_weight.compute_class_weight('balanced',\n                                                 classes=np.unique(train_df.label.to_numpy()),\n                                                 y=train_df.label.to_numpy()))\nclass_weights","metadata":{"papermill":{"duration":0.828887,"end_time":"2023-06-04T05:45:10.064700","exception":false,"start_time":"2023-06-04T05:45:09.235813","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:00.506842Z","iopub.execute_input":"2023-06-05T07:08:00.509648Z","iopub.status.idle":"2023-06-05T07:08:01.040330Z","shell.execute_reply.started":"2023-06-05T07:08:00.509614Z","shell.execute_reply":"2023-06-05T07:08:01.039413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = {x:y for x,y in enumerate(class_weights)}\nweights","metadata":{"papermill":{"duration":0.020244,"end_time":"2023-06-04T05:45:10.096627","exception":false,"start_time":"2023-06-04T05:45:10.076383","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:01.044385Z","iopub.execute_input":"2023-06-05T07:08:01.044677Z","iopub.status.idle":"2023-06-05T07:08:01.051449Z","shell.execute_reply.started":"2023-06-05T07:08:01.044653Z","shell.execute_reply":"2023-06-05T07:08:01.050477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization the dataset","metadata":{"papermill":{"duration":0.011032,"end_time":"2023-06-04T05:45:10.118625","exception":false,"start_time":"2023-06-04T05:45:10.107593","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ntrain_files = os.listdir(train_path)\nimg = cv.imread(train_path+\"/\"+train_files[1])\nplt.imshow(img)\nplt.xticks([])\nplt.yticks([])\nplt.show()","metadata":{"papermill":{"duration":0.212544,"end_time":"2023-06-04T05:45:10.342135","exception":false,"start_time":"2023-06-04T05:45:10.129591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:01.053091Z","iopub.execute_input":"2023-06-05T07:08:01.053831Z","iopub.status.idle":"2023-06-05T07:08:01.280969Z","shell.execute_reply.started":"2023-06-05T07:08:01.053800Z","shell.execute_reply":"2023-06-05T07:08:01.279893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Data Pipelines","metadata":{"papermill":{"duration":0.011704,"end_time":"2023-06-04T05:45:10.378418","exception":false,"start_time":"2023-06-04T05:45:10.366714","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntest_df['label'] = test_df['label'].astype(str)\ntrain_df['label'] = train_df['label'].astype(str)\n\ndatagen = ImageDataGenerator(\n    rescale=1/255.0,\n    rotation_range=20,\n    zoom_range=0.05,\n    width_shift_range=0.05,\n    height_shift_range=0.05,\n    shear_range=0.05,\n    horizontal_flip=True,\n    fill_mode = \"nearest\",\n    validation_split=0.15\n)\n\ntrain_generator=datagen.flow_from_dataframe(\n    dataframe=train_df,\n    directory=\"/kaggle/input/extracting-attributes-from-fashion-images-2/train\",\n    x_col=\"file_name\",\n    y_col=\"label\",\n    subset=\"training\",\n    batch_size=BATCH_SIZE,\n    seed=42,\n    shuffle=True,\n    class_mode=\"sparse\",\n    target_size=(INPUT_SHAPE,INPUT_SHAPE))\n\nvalid_generator=datagen.flow_from_dataframe(\n    dataframe=train_df,\n    directory=\"/kaggle/input/extracting-attributes-from-fashion-images-2/train\",\n    x_col=\"file_name\",\n    y_col=\"label\",\n    subset=\"validation\",\n    batch_size=BATCH_SIZE,\n    seed=42,\n    shuffle=True,\n    class_mode=\"sparse\",\n    target_size=(INPUT_SHAPE,INPUT_SHAPE))\n\n","metadata":{"papermill":{"duration":16.891268,"end_time":"2023-06-04T05:45:27.281556","exception":false,"start_time":"2023-06-04T05:45:10.390288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:01.282119Z","iopub.execute_input":"2023-06-05T07:08:01.282479Z","iopub.status.idle":"2023-06-05T07:08:20.616139Z","shell.execute_reply.started":"2023-06-05T07:08:01.282446Z","shell.execute_reply":"2023-06-05T07:08:20.615250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen=ImageDataGenerator(rescale=1./255.)\n\n\n\ntest_generator=test_datagen.flow_from_dataframe(\ndataframe=test_df,\ndirectory=\"/kaggle/input/extracting-attributes-from-fashion-images-2/test\",\nx_col=\"file_name\",\ny_col=\"label\",\nbatch_size=BATCH_SIZE,\nseed=42,\nshuffle=False,\nclass_mode=\"sparse\",\ntarget_size=(INPUT_SHAPE,INPUT_SHAPE))\n","metadata":{"papermill":{"duration":3.132339,"end_time":"2023-06-04T05:45:30.426334","exception":false,"start_time":"2023-06-04T05:45:27.293995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:20.617350Z","iopub.execute_input":"2023-06-05T07:08:20.617771Z","iopub.status.idle":"2023-06-05T07:08:23.574810Z","shell.execute_reply.started":"2023-06-05T07:08:20.617734Z","shell.execute_reply":"2023-06-05T07:08:23.573919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the Deep learning model","metadata":{"papermill":{"duration":0.011938,"end_time":"2023-06-04T05:45:30.451179","exception":false,"start_time":"2023-06-04T05:45:30.439241","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow import keras\nimport tensorflow as tf\nfrom keras.applications import VGG16\nfrom tensorflow.python.keras import regularizers\n\n# conv_base = VGG16(weights=\"imagenet\",\n#                  include_top=False,\n#                  input_shape=(INPUT_SHAPE,INPUT_SHAPE,3)) # 244\n\n\nconv_base = tf.keras.applications.mobilenet.MobileNet( include_top=False,\n                                                           input_shape=(224, 224,3),\n                                                           weights='imagenet')    \n\n\nmodel = Sequential()\n\nmodel.add(conv_base)\n# conv_base.trainable = False\n\nmodel.add(Flatten())\n\nmodel.add(Dense(512))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\n\n\nmodel.add(Dense(7, activation='softmax'))\n\nmodel.compile(optimizer=keras.optimizers.Adam(lr=LR),loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])","metadata":{"papermill":{"duration":5.06608,"end_time":"2023-06-04T05:45:35.529445","exception":false,"start_time":"2023-06-04T05:45:30.463365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:23.576028Z","iopub.execute_input":"2023-06-05T07:08:23.576820Z","iopub.status.idle":"2023-06-05T07:08:27.651351Z","shell.execute_reply.started":"2023-06-05T07:08:23.576786Z","shell.execute_reply":"2023-06-05T07:08:27.650332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_base.summary()","metadata":{"papermill":{"duration":0.304963,"end_time":"2023-06-04T05:45:35.883625","exception":false,"start_time":"2023-06-04T05:45:35.578662","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:27.652973Z","iopub.execute_input":"2023-06-05T07:08:27.653303Z","iopub.status.idle":"2023-06-05T07:08:27.817248Z","shell.execute_reply.started":"2023-06-05T07:08:27.653272Z","shell.execute_reply":"2023-06-05T07:08:27.816514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"papermill":{"duration":0.070775,"end_time":"2023-06-04T05:45:35.988921","exception":false,"start_time":"2023-06-04T05:45:35.918146","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:27.818603Z","iopub.execute_input":"2023-06-05T07:08:27.818968Z","iopub.status.idle":"2023-06-05T07:08:27.850692Z","shell.execute_reply.started":"2023-06-05T07:08:27.818932Z","shell.execute_reply":"2023-06-05T07:08:27.849986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.029968,"end_time":"2023-06-04T05:45:36.049191","exception":false,"start_time":"2023-06-04T05:45:36.019223","status":"completed"},"tags":[]}},{"cell_type":"code","source":"! mkdir /kaggle/working/model/\n\ncheckpoint=tf.keras.callbacks.ModelCheckpoint(filepath=\"/kaggle/working/model/CNN.h5\", \n                                              monitor='val_accuracy', \n                                              verbose=0, \n                                              save_best_only=True,\n                                              save_weights_only=False, \n                                              mode='auto', \n                                              save_freq='epoch', \n                                              options=None)","metadata":{"papermill":{"duration":1.077189,"end_time":"2023-06-04T05:45:37.156493","exception":false,"start_time":"2023-06-04T05:45:36.079304","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:27.851997Z","iopub.execute_input":"2023-06-05T07:08:27.852343Z","iopub.status.idle":"2023-06-05T07:08:28.934208Z","shell.execute_reply.started":"2023-06-05T07:08:27.852313Z","shell.execute_reply":"2023-06-05T07:08:28.932984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size\nSTEP_SIZE_TEST=test_generator.n//test_generator.batch_size\n\nhistory = model.fit(train_generator,\n                    steps_per_epoch=STEP_SIZE_TRAIN,\n                    validation_data=valid_generator,\n                    validation_steps=STEP_SIZE_VALID,\n                    epochs=EPOCHS,\n                   callbacks=[checkpoint])\n#                    class_weight=weights)\n\n\n\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.grid()\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.grid()\nplt.show()","metadata":{"papermill":{"duration":2006.826886,"end_time":"2023-06-04T06:19:04.014061","exception":false,"start_time":"2023-06-04T05:45:37.187175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:08:28.935982Z","iopub.execute_input":"2023-06-05T07:08:28.936572Z","iopub.status.idle":"2023-06-05T07:14:22.565462Z","shell.execute_reply.started":"2023-06-05T07:08:28.936534Z","shell.execute_reply":"2023-06-05T07:14:22.564393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model(\"/kaggle/working/model/CNN.h5\")","metadata":{"papermill":{"duration":1.649693,"end_time":"2023-06-04T06:19:05.821092","exception":false,"start_time":"2023-06-04T06:19:04.171399","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:14:22.567093Z","iopub.execute_input":"2023-06-05T07:14:22.567453Z","iopub.status.idle":"2023-06-05T07:14:24.534718Z","shell.execute_reply.started":"2023-06-05T07:14:22.567421Z","shell.execute_reply":"2023-06-05T07:14:24.533622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing","metadata":{"papermill":{"duration":0.154651,"end_time":"2023-06-04T06:19:06.131307","exception":false,"start_time":"2023-06-04T06:19:05.976656","status":"completed"},"tags":[]}},{"cell_type":"code","source":"valid_generator.reset()\nmodel.evaluate(valid_generator,\nsteps=STEP_SIZE_TEST)","metadata":{"papermill":{"duration":39.981987,"end_time":"2023-06-04T06:19:46.268509","exception":false,"start_time":"2023-06-04T06:19:06.286522","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:14:24.536297Z","iopub.execute_input":"2023-06-05T07:14:24.536695Z","iopub.status.idle":"2023-06-05T07:15:05.694060Z","shell.execute_reply.started":"2023-06-05T07:14:24.536661Z","shell.execute_reply":"2023-06-05T07:15:05.693128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator.reset()\npred=model.predict(test_generator)","metadata":{"papermill":{"duration":38.970702,"end_time":"2023-06-04T06:20:25.413285","exception":false,"start_time":"2023-06-04T06:19:46.442583","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:15:05.697356Z","iopub.execute_input":"2023-06-05T07:15:05.697678Z","iopub.status.idle":"2023-06-05T07:15:55.503689Z","shell.execute_reply.started":"2023-06-05T07:15:05.697649Z","shell.execute_reply":"2023-06-05T07:15:55.502642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=np.argmax(pred,axis=1)\ny_pred","metadata":{"papermill":{"duration":0.176954,"end_time":"2023-06-04T06:20:25.755106","exception":false,"start_time":"2023-06-04T06:20:25.578152","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:15:55.505459Z","iopub.execute_input":"2023-06-05T07:15:55.505868Z","iopub.status.idle":"2023-06-05T07:15:55.513295Z","shell.execute_reply.started":"2023-06-05T07:15:55.505836Z","shell.execute_reply":"2023-06-05T07:15:55.512140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,accuracy_score\ntest_df['label'] = test_df['label'].astype(int)\n\ncm=confusion_matrix(test_df.label, y_pred)\nprint(cm)\n\naccuracy_score(test_df.label.to_numpy(), y_pred)\n# tn,fp,fn,tp = confusion_matrix(test_df.label, y_pred).ravel()","metadata":{"papermill":{"duration":0.258088,"end_time":"2023-06-04T06:20:26.192663","exception":false,"start_time":"2023-06-04T06:20:25.934575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:15:55.514759Z","iopub.execute_input":"2023-06-05T07:15:55.515381Z","iopub.status.idle":"2023-06-05T07:15:55.612057Z","shell.execute_reply.started":"2023-06-05T07:15:55.515347Z","shell.execute_reply":"2023-06-05T07:15:55.611069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating submission.csv","metadata":{"papermill":{"duration":0.166057,"end_time":"2023-06-04T06:20:26.528949","exception":false,"start_time":"2023-06-04T06:20:26.362892","status":"completed"},"tags":[]}},{"cell_type":"code","source":"filenames=test_generator.filenames\nresults=pd.DataFrame({\"file_name\":filenames,\n                      \"label\":y_pred})\nresults.to_csv(\"submission.csv\",index=False)","metadata":{"papermill":{"duration":0.258451,"end_time":"2023-06-04T06:20:26.958686","exception":false,"start_time":"2023-06-04T06:20:26.700235","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:15:55.613611Z","iopub.execute_input":"2023-06-05T07:15:55.614259Z","iopub.status.idle":"2023-06-05T07:15:55.638912Z","shell.execute_reply.started":"2023-06-05T07:15:55.614226Z","shell.execute_reply":"2023-06-05T07:15:55.638073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df['label'] = test_df['label'].astype(int)","metadata":{"papermill":{"duration":0.224056,"end_time":"2023-06-04T06:20:27.348846","exception":false,"start_time":"2023-06-04T06:20:27.124790","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-05T07:15:55.640337Z","iopub.execute_input":"2023-06-05T07:15:55.640688Z","iopub.status.idle":"2023-06-05T07:15:55.645546Z","shell.execute_reply.started":"2023-06-05T07:15:55.640658Z","shell.execute_reply":"2023-06-05T07:15:55.644559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reasons for low testing accuracy \n**Problem**: Although the training and testing data belong to the same dataset, the split in the dataset is very different, training data has about 1402 belonging to class 0 and the testing data has all the data belonging to class 0. Inorder to make a model that generalizes well on the dataset we need atleast a minimun of 5000 samples per class.\n\n**Solution 1**: Adding more data in the dataset would lead to better learning of the model and would remove any bias in the dataset. But this is not possible in our case.\n\n**Solution 2**: If adding data is not an option then using a pre-trained model could lead to better generalization. This is because pre trained models are trained for good feature extration. This should work for our use case but from what I have learned while training the model, Due to the low amount of class 0 data in the dataset the pretrained model is also not able to learn well even though it generalizes well.\n","metadata":{"papermill":{"duration":0.164704,"end_time":"2023-06-04T06:20:27.680022","exception":false,"start_time":"2023-06-04T06:20:27.515318","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.165709,"end_time":"2023-06-04T06:20:28.677499","exception":false,"start_time":"2023-06-04T06:20:28.511790","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}