{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras as krs\nimport os\nimport zipfile\nimport glob\nimport cv2\nimport shutil\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-19T16:31:43.937814Z","iopub.execute_input":"2023-02-19T16:31:43.938357Z","iopub.status.idle":"2023-02-19T16:31:49.799877Z","shell.execute_reply.started":"2023-02-19T16:31:43.938227Z","shell.execute_reply":"2023-02-19T16:31:49.798793Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With this notebook I'm experimenting the TransferLerning technik as well as its fine tuning.\n\nI'm starting with data gathering:","metadata":{}},{"cell_type":"code","source":"train_dir = 'train'\n\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip', 'r') as train_zip:\n    train_zip.extractall('')\n\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\n\nbase_dir = '/kaggle/working'\ntrain_dir = os.path.join(base_dir, 'train')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:31:49.803668Z","iopub.execute_input":"2023-02-19T16:31:49.804260Z","iopub.status.idle":"2023-02-19T16:32:01.759626Z","shell.execute_reply.started":"2023-02-19T16:31:49.804226Z","shell.execute_reply":"2023-02-19T16:32:01.758592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create labels directory\npath = '/kaggle/working'\nos.makedirs(os.path.join(train_dir, 'dogs'))\nos.makedirs(os.path.join(train_dir, 'cats'))\ntrain_dogs = os.path.join(train_dir, 'dogs')\ntrain_cats = os.path.join(train_dir, 'cats')\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:01.760921Z","iopub.execute_input":"2023-02-19T16:32:01.762292Z","iopub.status.idle":"2023-02-19T16:32:01.769902Z","shell.execute_reply.started":"2023-02-19T16:32:01.762251Z","shell.execute_reply":"2023-02-19T16:32:01.768880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for filename in train_list:\n    label=filename.split('/')[1].split('.')[0]\n    id=filename.split('/')[1].split('.')[1]\n    if label=='dog':\n        shutil.move(os.path.join(base_dir, filename),os.path.join(train_dogs, id+'.jpg'))\n    else:\n        shutil.move(os.path.join(base_dir, filename),os.path.join(train_cats, id+'.jpg'))","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:01.775183Z","iopub.execute_input":"2023-02-19T16:32:01.775751Z","iopub.status.idle":"2023-02-19T16:32:02.540242Z","shell.execute_reply.started":"2023-02-19T16:32:01.775715Z","shell.execute_reply":"2023-02-19T16:32:02.539269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I'm preparing two dataset with the collected image, one to be used as training dataset, the second as validation dataset (I will not use data augmentation). Using tensorflow utils will help the training time.","metadata":{}},{"cell_type":"code","source":"img_height,img_width = 224, 224\nbatch_size = 32\n\ntrain_ds = krs.utils.image_dataset_from_directory(\n    '/kaggle/working/train/',\n    labels='inferred',\n    label_mode='int',\n    class_names=None,\n    color_mode='rgb',\n    batch_size=batch_size,\n    image_size=(img_height, img_width),\n    shuffle=True,\n    seed=42,\n    validation_split=0.2,\n    subset=\"training\",\n    interpolation='bilinear',\n)\nval_ds = krs.utils.image_dataset_from_directory(\n    '/kaggle/working/train/',\n    labels='inferred',\n    label_mode='int',\n    class_names=None,\n    color_mode='rgb',\n    batch_size=batch_size,\n    image_size=(img_height, img_width),\n    shuffle=True,\n    seed=42,\n    validation_split=0.2,\n    subset=\"validation\",\n    interpolation='bilinear',\n)\nnormalization_layer = tf.keras.layers.Rescaling(1./255)\nnrmlzd_train_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))\nnrmlzd_val_ds = val_ds.map(lambda x, y: (normalization_layer(x), y))\nnrmlzd_train_ds = nrmlzd_train_ds.prefetch(buffer_size=tf.data.AUTOTUNE)\nnrmlzd_val_ds = nrmlzd_val_ds.prefetch(buffer_size=tf.data.AUTOTUNE)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:02.541739Z","iopub.execute_input":"2023-02-19T16:32:02.542226Z","iopub.status.idle":"2023-02-19T16:32:06.904689Z","shell.execute_reply.started":"2023-02-19T16:32:02.542189Z","shell.execute_reply":"2023-02-19T16:32:06.903695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot some picture to see if all is properly set up.","metadata":{}},{"cell_type":"code","source":"class_names = train_ds.class_names\nplt.figure(figsize=(10, 10))\nfor images, labels in nrmlzd_train_ds.take(1):\n    for i in range(4):\n        ax = plt.subplot(1, 4, i + 1)\n        plt.imshow(images[i])\n        plt.title(class_names[labels[i]])\n        plt.axis(\"off\")\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:06.906307Z","iopub.execute_input":"2023-02-19T16:32:06.906845Z","iopub.status.idle":"2023-02-19T16:32:07.848843Z","shell.execute_reply.started":"2023-02-19T16:32:06.906810Z","shell.execute_reply":"2023-02-19T16:32:07.847948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Definition\nNow I'm downloading the monster: ResNet152V2 as basemodel. Remove the top (the final classifier) and freeze the parameters.","metadata":{}},{"cell_type":"code","source":"base_model=krs.applications.ResNet152V2(include_top=False,\n                                       input_shape=(224,224,3),\n                                       pooling='avg')\nfor layer in base_model.layers:\n        layer.trainable=False\n#base_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:07.849948Z","iopub.execute_input":"2023-02-19T16:32:07.850316Z","iopub.status.idle":"2023-02-19T16:32:13.748518Z","shell.execute_reply.started":"2023-02-19T16:32:07.850281Z","shell.execute_reply":"2023-02-19T16:32:13.747505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I build my model adding a customized final classifier with one single binary exit.","metadata":{}},{"cell_type":"code","source":"model=krs.models.Sequential()\nmodel.add(base_model)\nmodel.add(krs.layers.Flatten())\nmodel.add(krs.layers.Dense(128, activation='relu'))\nmodel.add(krs.layers.BatchNormalization())\nmodel.add(krs.layers.Dense(1, activation='sigmoid'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:13.750058Z","iopub.execute_input":"2023-02-19T16:32:13.750419Z","iopub.status.idle":"2023-02-19T16:32:15.107914Z","shell.execute_reply.started":"2023-02-19T16:32:13.750381Z","shell.execute_reply":"2023-02-19T16:32:15.106801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"compile ...","metadata":{}},{"cell_type":"code","source":"model.compile(loss=tf.keras.losses.binary_crossentropy,\n              optimizer=krs.optimizers.SGD(learning_rate = 0.001, \n                                           momentum = 0.1, decay = 0.0),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:15.109582Z","iopub.execute_input":"2023-02-19T16:32:15.109996Z","iopub.status.idle":"2023-02-19T16:32:15.133897Z","shell.execute_reply.started":"2023-02-19T16:32:15.109956Z","shell.execute_reply":"2023-02-19T16:32:15.132919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"and train quickly the final classifier.","metadata":{}},{"cell_type":"code","source":"num_epoch = 10\nmodel_log = model.fit(nrmlzd_train_ds,\n                      epochs=num_epoch,\n                      verbose=1,\n                      validation_data=nrmlzd_val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:32:15.138756Z","iopub.execute_input":"2023-02-19T16:32:15.139053Z","iopub.status.idle":"2023-02-19T16:34:45.553549Z","shell.execute_reply.started":"2023-02-19T16:32:15.139024Z","shell.execute_reply":"2023-02-19T16:34:45.552511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the results from cross validation","metadata":{}},{"cell_type":"code","source":"epochs=range(1, len(model_log.history['loss'])+1)\nplt.plot(epochs, model_log.history['loss'], 'bo', label = 'Training loss')\nplt.plot(epochs, model_log.history['val_loss'], 'b', label = 'Validation loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:34:45.555444Z","iopub.execute_input":"2023-02-19T16:34:45.556299Z","iopub.status.idle":"2023-02-19T16:34:45.744616Z","shell.execute_reply.started":"2023-02-19T16:34:45.556260Z","shell.execute_reply":"2023-02-19T16:34:45.743667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine Tuning\nAt this point the Fine Tuning, I'm unfreezing the last convolutional block from ResNet and set a smaller learning rate. I need to recompile because the tunable parameters are changed.","metadata":{}},{"cell_type":"code","source":"base_model.trainable = True\nset_trainable = False\nfor layer in base_model.layers:\n    if layer.name=='conv5_block3_1_conv':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False\n        \nmodel.compile(loss=tf.keras.losses.binary_crossentropy,\n              optimizer=krs.optimizers.SGD(learning_rate = 0.0001, \n                                                        momentum = 0.1, decay = 0.0),\n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:34:45.746046Z","iopub.execute_input":"2023-02-19T16:34:45.746390Z","iopub.status.idle":"2023-02-19T16:34:45.816192Z","shell.execute_reply.started":"2023-02-19T16:34:45.746355Z","shell.execute_reply":"2023-02-19T16:34:45.815207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'm defining some callbacks to dynamically change the learning rate or stop the training if make sense. Then all is ready for the long fine tuning.","metadata":{}},{"cell_type":"code","source":"early_stop = krs.callbacks.EarlyStopping(monitor=\"val_loss\",min_delta=0, patience=5,\n                           verbose=0, mode=\"min\", baseline=None, restore_best_weights=True)\ncheck = krs.callbacks.ModelCheckpoint(filepath='bestw.ckpt', save_weights_only = True, save_best_only=True, monitor='val_loss')\nlearning_rate_reduction = krs.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.000003)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:34:45.817537Z","iopub.execute_input":"2023-02-19T16:34:45.817861Z","iopub.status.idle":"2023-02-19T16:34:45.823962Z","shell.execute_reply.started":"2023-02-19T16:34:45.817827Z","shell.execute_reply":"2023-02-19T16:34:45.823043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epoch = 100\nmodel_log = model.fit(nrmlzd_train_ds,\n                      epochs=num_epoch,\n                      verbose=1,\n                      validation_data=nrmlzd_val_ds,\n                      callbacks = [early_stop, check, learning_rate_reduction])","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:34:45.825423Z","iopub.execute_input":"2023-02-19T16:34:45.826055Z","iopub.status.idle":"2023-02-19T16:36:50.466936Z","shell.execute_reply.started":"2023-02-19T16:34:45.826019Z","shell.execute_reply":"2023-02-19T16:36:50.465941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=range(1, len(model_log.history['loss'])+1)\nplt.plot(epochs, model_log.history['loss'], 'bo', label = 'Training loss')\nplt.plot(epochs, model_log.history['val_loss'], 'b', label = 'Validation loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:36:50.468586Z","iopub.execute_input":"2023-02-19T16:36:50.469314Z","iopub.status.idle":"2023-02-19T16:36:50.651285Z","shell.execute_reply.started":"2023-02-19T16:36:50.469275Z","shell.execute_reply":"2023-02-19T16:36:50.650381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Test\nNow it's time to collect the test data.","metadata":{}},{"cell_type":"code","source":"test_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip', 'r') as test_zip:\n    test_zip.extractall('test')\n\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\ntest_dir = os.path.join(base_dir, 'test')","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:36:50.652740Z","iopub.execute_input":"2023-02-19T16:36:50.653108Z","iopub.status.idle":"2023-02-19T16:36:56.729723Z","shell.execute_reply.started":"2023-02-19T16:36:50.653071Z","shell.execute_reply":"2023-02-19T16:36:56.728436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = krs.preprocessing.image.ImageDataGenerator(preprocessing_function=krs.applications.resnet_v2.preprocess_input).flow_from_directory(\n    directory = '/kaggle/working/test/',\n    target_size = (img_height,img_width),\n    batch_size = 100,\n    class_mode = None,\n    shuffle = False,\n#    seed = 42\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:36:56.731282Z","iopub.execute_input":"2023-02-19T16:36:56.731630Z","iopub.status.idle":"2023-02-19T16:36:57.071210Z","shell.execute_reply.started":"2023-02-19T16:36:56.731585Z","shell.execute_reply":"2023-02-19T16:36:57.070106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict over the test data","metadata":{}},{"cell_type":"code","source":"test_data.reset()\n\nmodel.load_weights('bestw.ckpt')\npred = model.predict(test_data, steps = len(test_data), verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:36:57.072658Z","iopub.execute_input":"2023-02-19T16:36:57.073014Z","iopub.status.idle":"2023-02-19T16:37:56.358384Z","shell.execute_reply.started":"2023-02-19T16:36:57.072961Z","shell.execute_reply":"2023-02-19T16:37:56.356353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Potential Error Analysis\nI would like to see the picture where the net reports low confidence","metadata":{}},{"cell_type":"code","source":"confidence=np.abs(pred-0.5)\nworst_conf_idx=np.argsort(np.transpose(confidence))\nprint(worst_conf_idx.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:56.359850Z","iopub.execute_input":"2023-02-19T16:37:56.360238Z","iopub.status.idle":"2023-02-19T16:37:56.368171Z","shell.execute_reply.started":"2023-02-19T16:37:56.360201Z","shell.execute_reply":"2023-02-19T16:37:56.367032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract the proper ID for the rubmission","metadata":{}},{"cell_type":"code","source":"id=[]\nfor i in range(0,len(test_data.filenames)):\n    fname = test_data.filenames[i]\n    fn_id=fname.split('/')[1].split('.')[0]\n    id.append(fn_id)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:56.369846Z","iopub.execute_input":"2023-02-19T16:37:56.370665Z","iopub.status.idle":"2023-02-19T16:37:56.385607Z","shell.execute_reply.started":"2023-02-19T16:37:56.370612Z","shell.execute_reply":"2023-02-19T16:37:56.384944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(5, 5, figsize = (15, 15))\n\nfor i in range(0,25):\n    #idx=np.asscalar(worst_conf_idx[0,i])\n    idx=i\n    fname = test_data.filenames[idx]\n    imgBGR = cv2.imread('/kaggle/working/test/' + fname)\n    imgRGB = cv2.cvtColor(imgBGR, cv2.COLOR_BGR2RGB)\n    \n    # a if condition else b\n    predicted_class = \"Dog \" if pred[idx] > 0.5 else \"Cat\"\n\n    ax[i//5, i%5].imshow(imgRGB)\n    ax[i//5, i%5].axis('off')\n    ax[i//5, i%5].set_title(\"Predicted:{}\\nPrediction:{}\\n{}\".format(predicted_class, str(pred[idx]), id[idx]))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:56.386885Z","iopub.execute_input":"2023-02-19T16:37:56.387605Z","iopub.status.idle":"2023-02-19T16:37:58.476595Z","shell.execute_reply.started":"2023-02-19T16:37:56.387542Z","shell.execute_reply":"2023-02-19T16:37:58.475757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\nFinal submission File with visual ispection of the submission format","metadata":{}},{"cell_type":"code","source":"id_fd=pd.Series(id,name = \"id\")\nid_lb=pd.DataFrame(pred, columns=['label'])\nsubmission = pd.concat([id_fd,id_lb],axis = 1)\nsubmission.to_csv(\"digit_recognizer.csv\",index=False)\nsrt_sub=submission.sort_values(by=['id'])\nsrt_sub.head(10)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:58.477816Z","iopub.execute_input":"2023-02-19T16:37:58.478523Z","iopub.status.idle":"2023-02-19T16:37:58.539387Z","shell.execute_reply.started":"2023-02-19T16:37:58.478487Z","shell.execute_reply":"2023-02-19T16:37:58.538181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"srt_sub.tail(10)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:58.541244Z","iopub.execute_input":"2023-02-19T16:37:58.541673Z","iopub.status.idle":"2023-02-19T16:37:58.552556Z","shell.execute_reply.started":"2023-02-19T16:37:58.541635Z","shell.execute_reply":"2023-02-19T16:37:58.551286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"srt_sub.describe()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:58.554623Z","iopub.execute_input":"2023-02-19T16:37:58.555015Z","iopub.status.idle":"2023-02-19T16:37:58.576051Z","shell.execute_reply.started":"2023-02-19T16:37:58.554959Z","shell.execute_reply":"2023-02-19T16:37:58.575233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"srt_sub.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:37:58.577415Z","iopub.execute_input":"2023-02-19T16:37:58.577753Z","iopub.status.idle":"2023-02-19T16:37:58.604733Z","shell.execute_reply.started":"2023-02-19T16:37:58.577720Z","shell.execute_reply":"2023-02-19T16:37:58.603880Z"},"trusted":true},"execution_count":null,"outputs":[]}]}