{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! unzip -o ../input/dogs-vs-cats-redux-kernels-edition/train.zip -d train > /dev/null\n! unzip -o ../input/dogs-vs-cats-redux-kernels-edition/test.zip -d test > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:33:38.40298Z","iopub.execute_input":"2021-08-14T14:33:38.403267Z","iopub.status.idle":"2021-08-14T14:33:58.125201Z","shell.execute_reply.started":"2021-08-14T14:33:38.403196Z","shell.execute_reply":"2021-08-14T14:33:58.12408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom zipfile import ZipFile\n\nimport math\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nfrom sklearn.model_selection import train_test_split \nfrom sklearn.metrics import (\n    accuracy_score, classification_report, \n    roc_curve,auc, plot_confusion_matrix,\n    plot_precision_recall_curve\n)\n\nimport matplotlib.pyplot as plt\n\ndef plot_hist(hist):\n    plt.plot(hist.history[\"accuracy\"])\n    plt.plot(hist.history[\"val_accuracy\"])\n    plt.title(\"model accuracy\")\n    plt.ylabel(\"accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-14T16:03:22.92152Z","iopub.execute_input":"2021-08-14T16:03:22.921854Z","iopub.status.idle":"2021-08-14T16:03:22.930794Z","shell.execute_reply.started":"2021-08-14T16:03:22.921821Z","shell.execute_reply":"2021-08-14T16:03:22.929913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prep Data\n\nIncludes some small data augmentation\n\nSplits the data into three sets, only augmenting the training data.","metadata":{}},{"cell_type":"code","source":"# creating dataframe\n\n# labels\nfile = os.listdir(\"./train/train\")\nLabels = list(map(lambda x: x.split('.')[0], file))\n\n# for filenames\nf = Path(\"./train/train\")\nFile_Path = list(f.glob(r\"**/*.jpg\"))\n\n# dataframe\nFile_Path = pd.Series(File_Path).astype(str)\nLabels = pd.Series(Labels)\ndf = pd.concat([File_Path,Labels],axis=1)\ndf.columns = ['filename', 'category']\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:03:25.127976Z","iopub.execute_input":"2021-08-14T16:03:25.12831Z","iopub.status.idle":"2021-08-14T16:03:25.369467Z","shell.execute_reply.started":"2021-08-14T16:03:25.128278Z","shell.execute_reply":"2021-08-14T16:03:25.368459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data into train/val/hold\ntrain_set, hold_data = train_test_split(df, test_size= 0.2, random_state = 1)\ntrain_data, val_data = train_test_split(train_set, test_size= 0.4, random_state = 2)\n\nbatch_size = 32\n\ntrain_size = train_data.shape[0]\ntrain_steps_per_epoch = math.ceil(train_size/batch_size)\n\nval_size = val_data.shape[0]\nval_steps_per_epoch = math.ceil(val_size/batch_size)\n\nhold_size = hold_data.shape[0]\nhold_steps_per_epoch = math.ceil(hold_size/batch_size)\n\ntrain_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n        shear_range=0.2,\n        zoom_range=0.2,\n        rotation_range=30,\n        brightness_range=[0.5,1.5],\n        horizontal_flip=True,\n        vertical_flip=True)\n\ntest_datagen = tf.keras.preprocessing.image.ImageDataGenerator()\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    train_data, \n    x_col='filename',\n    y_col='category',\n    target_size=(150, 150),\n    class_mode='binary',\n    batch_size=32,\n    shuffle = True\n)\n\nvalidation_generator = test_datagen.flow_from_dataframe(\n    val_data, \n    x_col='filename',\n    y_col='category',\n    target_size=(150, 150),\n    class_mode='binary',\n    batch_size=32,\n    shuffle = False)\n\nhold_generator = test_datagen.flow_from_dataframe(\n    hold_data, \n    x_col='filename',\n    y_col='category',\n    target_size=(150, 150),\n    class_mode='binary',\n    batch_size=32,\n    shuffle = False)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:03:27.152666Z","iopub.execute_input":"2021-08-14T16:03:27.153022Z","iopub.status.idle":"2021-08-14T16:03:27.404947Z","shell.execute_reply.started":"2021-08-14T16:03:27.152987Z","shell.execute_reply":"2021-08-14T16:03:27.403821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained Model\n\n[Details of model being used for transfer learning](https://keras.io/examples/vision/image_classification_efficientnet_fine_tuning/)","metadata":{}},{"cell_type":"code","source":"pretrained = tf.keras.applications.EfficientNetB5(\n    include_top=False,\n    weights=\"imagenet\"\n)\n\npretrained.trainable = False","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:03:29.518381Z","iopub.execute_input":"2021-08-14T16:03:29.518695Z","iopub.status.idle":"2021-08-14T16:03:31.011811Z","shell.execute_reply.started":"2021-08-14T16:03:29.518665Z","shell.execute_reply":"2021-08-14T16:03:31.010973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transfered_model = tf.keras.Sequential([\n        tf.keras.layers.experimental.preprocessing.Resizing(456, 456),\n        pretrained,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(1, activation='sigmoid')\n    ])\n    \nmetrics = [tf.keras.metrics.BinaryAccuracy(name = 'accuracy', threshold=0.5),\n           tf.keras.metrics.Recall(name = 'recall'),\n           tf.keras.metrics.Precision(name = 'precision'),\n           tf.keras.metrics.AUC(name = 'auc')\n           ]\n\ntransfered_model.compile(\n        loss=\"binary_crossentropy\",\n        metrics=metrics,\n        optimizer=tf.keras.optimizers.Adam(0.001)\n    )","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:03:31.934693Z","iopub.execute_input":"2021-08-14T16:03:31.935037Z","iopub.status.idle":"2021-08-14T16:03:31.981365Z","shell.execute_reply.started":"2021-08-14T16:03:31.935004Z","shell.execute_reply":"2021-08-14T16:03:31.980628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train for a few epochs with only the classifer changing. Keep the pretrained weights frozen.","metadata":{}},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', \n                                            patience=2, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\nearlystop = EarlyStopping(patience=6)\n\n\nresult = transfered_model.fit(\n    train_generator, \n    validation_data=validation_generator,\n    epochs = 10,\n    steps_per_epoch = train_steps_per_epoch,\n    validation_steps = val_steps_per_epoch, \n    callbacks = [earlystop, learning_rate_reduction]\n) \n\nplot_hist(result)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:03:35.434782Z","iopub.execute_input":"2021-08-14T16:03:35.435183Z","iopub.status.idle":"2021-08-14T16:24:06.431876Z","shell.execute_reply.started":"2021-08-14T16:03:35.435148Z","shell.execute_reply":"2021-08-14T16:24:06.431083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train for a few more epochs with the full set up weights training","metadata":{}},{"cell_type":"code","source":"for layer in transfered_model.layers[1].layers:\n    if not isinstance(layer, tf.keras.layers.BatchNormalization):\n        layer.trainable = True\n\ntransfered_model.compile(\n        loss=\"binary_crossentropy\",\n        metrics=metrics,\n        optimizer=tf.keras.optimizers.Adam(0.00001)\n    )\n\nresult = transfered_model.fit(\n    train_generator, \n    validation_data=validation_generator,\n    epochs = 10,\n    steps_per_epoch = train_steps_per_epoch,\n    validation_steps = val_steps_per_epoch, \n    callbacks = [earlystop, learning_rate_reduction]\n) \n\nplot_hist(result)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:24:06.433975Z","iopub.execute_input":"2021-08-14T16:24:06.43453Z","iopub.status.idle":"2021-08-14T16:42:28.490919Z","shell.execute_reply.started":"2021-08-14T16:24:06.434492Z","shell.execute_reply":"2021-08-14T16:42:28.49005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Out Validation","metadata":{}},{"cell_type":"code","source":"# Predict the label of the test_images\np = transfered_model.predict(hold_generator).reshape((-1,))\nlabel = (p > 0.5).astype(int)\ntruth = (hold_data.category == 'dog').astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:28.492918Z","iopub.execute_input":"2021-08-14T16:42:28.49342Z","iopub.status.idle":"2021-08-14T16:42:43.170647Z","shell.execute_reply.started":"2021-08-14T16:42:28.493375Z","shell.execute_reply":"2021-08-14T16:42:43.169813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IC = type('IdentityClassifier', (), \n          {\"predict\": lambda i : i, \n           \"predict_proba\": lambda i : i, \n           \"_estimator_type\": \"classifier\", \n           \"classes_\": ['Cat', 'Dog']})\nplot_confusion_matrix(IC, truth, label);","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:43.172245Z","iopub.execute_input":"2021-08-14T16:42:43.172572Z","iopub.status.idle":"2021-08-14T16:42:43.338809Z","shell.execute_reply.started":"2021-08-14T16:42:43.172537Z","shell.execute_reply":"2021-08-14T16:42:43.337725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfpr, tpr, threshold = roc_curve(truth, p)\nroc_auc = auc(fpr, tpr)\n\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, 'b', \n         label = 'AUC = %0.2f' % roc_auc,\n         linewidth=4.0)\nplt.legend(loc = 'lower right')\nplt.plot([0, 1], [0, 1],'r--')\nplt.xlim([0, 1])\nplt.ylim([0, 1])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:43.3402Z","iopub.execute_input":"2021-08-14T16:42:43.34055Z","iopub.status.idle":"2021-08-14T16:42:43.485606Z","shell.execute_reply.started":"2021-08-14T16:42:43.340512Z","shell.execute_reply":"2021-08-14T16:42:43.484282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(truth, label))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:43.487102Z","iopub.execute_input":"2021-08-14T16:42:43.487467Z","iopub.status.idle":"2021-08-14T16:42:43.506815Z","shell.execute_reply.started":"2021-08-14T16:42:43.487427Z","shell.execute_reply":"2021-08-14T16:42:43.505793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Out of Sample Prediction","metadata":{}},{"cell_type":"code","source":"test_filenames = os.listdir(\"./test/test\")\ntest_df = pd.DataFrame({\n    'filename': test_filenames\n})\n\ntest_gen = test_datagen.flow_from_dataframe(\n    test_df, \n    \"./test/test\",\n    x_col='filename',\n    y_col=None,\n    target_size=(150, 150),\n    class_mode=None,\n    batch_size=32,\n    shuffle = False # not to shuffle the given data\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:43.508195Z","iopub.execute_input":"2021-08-14T16:42:43.508575Z","iopub.status.idle":"2021-08-14T16:42:43.625333Z","shell.execute_reply.started":"2021-08-14T16:42:43.508538Z","shell.execute_reply":"2021-08-14T16:42:43.624286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npred = pd.Series(transfered_model.predict(test_gen).reshape((-1,)))\nindex = pd.Series([int(x.split('.')[0]) for x in test_df.filename])\n\nsubmission = pd.DataFrame({'id':index, 'label':pred})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T16:42:43.627492Z","iopub.execute_input":"2021-08-14T16:42:43.628016Z","iopub.status.idle":"2021-08-14T16:43:18.598918Z","shell.execute_reply.started":"2021-08-14T16:42:43.627974Z","shell.execute_reply":"2021-08-14T16:43:18.598017Z"},"trusted":true},"execution_count":null,"outputs":[]}]}