{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3362,"databundleVersionId":31148,"sourceType":"competition"}],"dockerImageVersionId":30146,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Transfer Learning via Xception model (Dogs vs. Cats)**\n\n* Part 1 [here](https://www.kaggle.com/imcr00z/intro-to-cnn-dogs-vs-cats)\n* Part 2 [here](https://www.kaggle.com/imcr00z/intro-to-cnn-augmentation-dropout)\n\nIn this part i use transfer learning and pretrained models:\n* Xception (from Franc¸ois Chollet). Original document [here](https://arxiv.org/pdf/1610.02357.pdf)\n* MobileNet via Google","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Dense, Dropout\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport shutil\nimport tqdm","metadata":{"execution":{"iopub.status.busy":"2024-01-26T12:09:48.205954Z","iopub.execute_input":"2024-01-26T12:09:48.206329Z","iopub.status.idle":"2024-01-26T12:09:54.9718Z","shell.execute_reply.started":"2024-01-26T12:09:48.206221Z","shell.execute_reply":"2024-01-26T12:09:54.970985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-26T12:09:58.481763Z","iopub.execute_input":"2024-01-26T12:09:58.482052Z","iopub.status.idle":"2024-01-26T12:09:58.490076Z","shell.execute_reply.started":"2024-01-26T12:09:58.482022Z","shell.execute_reply":"2024-01-26T12:09:58.48883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"execution":{"iopub.status.busy":"2021-11-03T13:47:48.931682Z","iopub.execute_input":"2021-11-03T13:47:48.932348Z","iopub.status.idle":"2021-11-03T13:47:48.938073Z","shell.execute_reply.started":"2021-11-03T13:47:48.932292Z","shell.execute_reply":"2021-11-03T13:47:48.937215Z"}}},{"cell_type":"code","source":"CONTENT_DIR = '/kaggle/content'\nTRAIN_DIR = CONTENT_DIR + '/train'\nVALID_DIR = CONTENT_DIR + '/valid'\n\nif not os.path.exists(CONTENT_DIR):\n    # Extract dataset\n    import zipfile\n    with zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip', 'r') as zipf:\n        zipf.extractall(CONTENT_DIR)\n\n    # Split cats and dogs images to train and valid datasets\n    img_filenames = os.listdir(TRAIN_DIR)\n    dog_filenames = [fn for fn in img_filenames if fn.startswith('dog')]\n    cat_filenames = [fn for fn in img_filenames if fn.startswith('cat')]\n    dataset_filenames = train_test_split(\n        dog_filenames, cat_filenames, test_size=0.1, shuffle=True, random_state=42\n    )\n\n    # Move images\n    make_dirs = [d + a for a in ['/dog', '/cat'] for d in [TRAIN_DIR, VALID_DIR]]\n    for dir, fns in zip(make_dirs, dataset_filenames):\n        os.makedirs(dir, exist_ok=True)\n        for fn in tqdm.tqdm(fns):\n            shutil.move(os.path.join(TRAIN_DIR, fn), dir)\n        print('elements in {}: {}'.format(dir, len(os.listdir(dir))))","metadata":{"execution":{"iopub.status.busy":"2024-01-26T12:10:02.196363Z","iopub.execute_input":"2024-01-26T12:10:02.196659Z","iopub.status.idle":"2024-01-26T12:10:16.188529Z","shell.execute_reply.started":"2024-01-26T12:10:02.196627Z","shell.execute_reply":"2024-01-26T12:10:16.187775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nIMAGE_SHAPE = 128","metadata":{"execution":{"iopub.status.busy":"2021-11-04T12:36:09.125544Z","iopub.execute_input":"2021-11-04T12:36:09.126236Z","iopub.status.idle":"2021-11-04T12:36:09.13647Z","shell.execute_reply.started":"2021-11-04T12:36:09.126188Z","shell.execute_reply":"2021-11-04T12:36:09.135547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"train_generator = ImageDataGenerator(rescale=1./255)\nvalid_generator = ImageDataGenerator(rescale=1./255)\ntrain_data = train_generator.flow_from_directory(\n    directory=TRAIN_DIR,\n    target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n    batch_size=BATCH_SIZE,\n    class_mode='binary',\n    shuffle=False\n)\nvalid_data = valid_generator.flow_from_directory(\n    directory=VALID_DIR,\n    target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n    batch_size=BATCH_SIZE,\n    class_mode='binary',\n    shuffle=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T12:36:09.145461Z","iopub.execute_input":"2021-11-04T12:36:09.14579Z","iopub.status.idle":"2021-11-04T12:36:09.964321Z","shell.execute_reply.started":"2021-11-04T12:36:09.145749Z","shell.execute_reply":"2021-11-04T12:36:09.963466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Xception\nIn this part, I extract 'bottleneck features' from an Xception model without fully connected layers and train a new fully connected model on them.\n1. load and train xception, get bottleneck features\n2. create model with dense layers\n3. train dense layers on bottleneck features","metadata":{}},{"cell_type":"code","source":"# load xception model\nxception_model = tf.keras.applications.xception.Xception(\n    include_top=False,\n    weights='imagenet',\n    input_shape=(IMAGE_SHAPE, IMAGE_SHAPE, 3),\n    pooling='avg'\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:33:16.768303Z","iopub.execute_input":"2021-11-04T10:33:16.768736Z","iopub.status.idle":"2021-11-04T10:33:21.374712Z","shell.execute_reply.started":"2021-11-04T10:33:16.768671Z","shell.execute_reply":"2021-11-04T10:33:21.373933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at this, it's very scary :)\n# change include_top=True -> last layer added\n# tf.keras.utils.plot_model(xception_model, dpi=48, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:33:21.37719Z","iopub.execute_input":"2021-11-04T10:33:21.377639Z","iopub.status.idle":"2021-11-04T10:33:22.588701Z","shell.execute_reply.started":"2021-11-04T10:33:21.3776Z","shell.execute_reply":"2021-11-04T10:33:22.586168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bottleneck features for train dataset\ntrain_bottleneck = xception_model.predict_generator(\n    train_data, train_data.n // BATCH_SIZE, verbose=1\n)\n# np.save(open('train_bottleneck.np', 'wb'), train_bottleneck)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:33:22.590321Z","iopub.execute_input":"2021-11-04T10:33:22.590854Z","iopub.status.idle":"2021-11-04T10:34:45.377759Z","shell.execute_reply.started":"2021-11-04T10:33:22.590816Z","shell.execute_reply":"2021-11-04T10:34:45.376859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bottleneck features for valid dataset\nvalid_bottleneck = xception_model.predict_generator(\n    valid_data, valid_data.n // BATCH_SIZE, verbose=1\n)\n# np.save(open('valid_bottleneck.np', 'wb'), valid_bottleneck)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:34:45.380583Z","iopub.execute_input":"2021-11-04T10:34:45.380886Z","iopub.status.idle":"2021-11-04T10:34:51.544052Z","shell.execute_reply.started":"2021-11-04T10:34:45.380848Z","shell.execute_reply":"2021-11-04T10:34:51.543333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.Sequential([\n    Dense(units=256, activation='relu', input_shape=xception_model.output_shape[1:]),\n    Dropout(0.5),\n    Dense(units=128, activation='relu'),\n    Dropout(0.5),\n    Dense(units=2, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:34:51.545749Z","iopub.execute_input":"2021-11-04T10:34:51.546007Z","iopub.status.idle":"2021-11-04T10:34:51.582732Z","shell.execute_reply.started":"2021-11-04T10:34:51.545973Z","shell.execute_reply":"2021-11-04T10:34:51.582052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:34:51.585596Z","iopub.execute_input":"2021-11-04T10:34:51.586216Z","iopub.status.idle":"2021-11-04T10:34:51.59552Z","shell.execute_reply.started":"2021-11-04T10:34:51.586177Z","shell.execute_reply":"2021-11-04T10:34:51.594755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='adam',\n    loss='sparse_categorical_crossentropy',\n    metrics=['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:34:51.597931Z","iopub.execute_input":"2021-11-04T10:34:51.598196Z","iopub.status.idle":"2021-11-04T10:34:51.611436Z","shell.execute_reply.started":"2021-11-04T10:34:51.598157Z","shell.execute_reply":"2021-11-04T10:34:51.610797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 10\nhistory = model.fit(\n    x=train_bottleneck,\n    y=train_data.labels[:len(train_bottleneck)],\n    batch_size=BATCH_SIZE,\n    epochs=EPOCHS,\n    validation_data=(valid_bottleneck, valid_data.labels[:len(valid_bottleneck)])\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:34:51.612529Z","iopub.execute_input":"2021-11-04T10:34:51.613222Z","iopub.status.idle":"2021-11-04T10:35:12.774994Z","shell.execute_reply.started":"2021-11-04T10:34:51.613176Z","shell.execute_reply":"2021-11-04T10:35:12.774219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look to validation loss and accuracy.","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:20:22.46169Z","iopub.execute_input":"2021-11-03T14:20:22.462169Z","iopub.status.idle":"2021-11-03T14:20:22.494683Z","shell.execute_reply.started":"2021-11-03T14:20:22.462041Z","shell.execute_reply":"2021-11-03T14:20:22.493462Z"}}},{"cell_type":"code","source":"def show_graphs(history):\n    plt.figure(figsize=(12, 8))\n\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['accuracy'], label='train')\n    plt.plot(history.history['val_accuracy'], label='valid')\n    plt.legend(loc='lower right')\n    plt.title('Accuracy')\n\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['loss'], label='train')\n    plt.plot(history.history['val_loss'], label='valid')\n    plt.legend(loc='upper left')\n    plt.title('Loss (sparse_categorical_crossentropy)')\n\n    plt.show()\n    \nshow_graphs(history)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:35:12.776888Z","iopub.execute_input":"2021-11-04T10:35:12.777405Z","iopub.status.idle":"2021-11-04T10:35:13.124591Z","shell.execute_reply.started":"2021-11-04T10:35:12.777367Z","shell.execute_reply":"2021-11-04T10:35:13.123911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MobileNet\nIn this case i use classical path with frozen layers.","metadata":{}},{"cell_type":"code","source":"mobilenet_model = tf.keras.applications.mobilenet.MobileNet(\n    include_top=False,\n    weights='imagenet',\n    input_shape=(IMAGE_SHAPE, IMAGE_SHAPE, 3),\n    pooling='avg'\n)\nmobilenet_model.trainable = False # freeze convolutional layers","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:35:32.0278Z","iopub.execute_input":"2021-11-04T10:35:32.02806Z","iopub.status.idle":"2021-11-04T10:35:32.537731Z","shell.execute_reply.started":"2021-11-04T10:35:32.028032Z","shell.execute_reply":"2021-11-04T10:35:32.536976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new fully connected part of the network\ndense_model = tf.keras.models.Sequential([\n    Dense(units=1000, activation='relu'),\n    Dropout(0.5),\n    Dense(units=128, activation='relu'),\n    Dropout(0.5),\n    Dense(units=2, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:37:39.926903Z","iopub.execute_input":"2021-11-04T10:37:39.927488Z","iopub.status.idle":"2021-11-04T10:37:39.939786Z","shell.execute_reply.started":"2021-11-04T10:37:39.927447Z","shell.execute_reply":"2021-11-04T10:37:39.939019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# build a new model\nmodel2 = tf.keras.models.Sequential([\n    mobilenet_model,\n    dense_model\n])","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:38:15.780729Z","iopub.execute_input":"2021-11-04T10:38:15.780977Z","iopub.status.idle":"2021-11-04T10:38:15.990984Z","shell.execute_reply.started":"2021-11-04T10:38:15.780949Z","shell.execute_reply":"2021-11-04T10:38:15.990301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2.compile(\n    optimizer='adam',\n    loss='sparse_categorical_crossentropy',\n    metrics=['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:38:19.580817Z","iopub.execute_input":"2021-11-04T10:38:19.581613Z","iopub.status.idle":"2021-11-04T10:38:19.593747Z","shell.execute_reply.started":"2021-11-04T10:38:19.581555Z","shell.execute_reply":"2021-11-04T10:38:19.59293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 10\ntrain_data.reset()\nvalid_data.reset()\nhistory = model2.fit_generator(\n    train_data,\n    steps_per_epoch=train_data.n // BATCH_SIZE,\n    epochs=EPOCHS,\n    validation_data=valid_data,\n    validation_steps=valid_data.n // BATCH_SIZE\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:38:25.710556Z","iopub.execute_input":"2021-11-04T10:38:25.711119Z","iopub.status.idle":"2021-11-04T10:48:44.802993Z","shell.execute_reply.started":"2021-11-04T10:38:25.711062Z","shell.execute_reply":"2021-11-04T10:48:44.802268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_graphs(history)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T10:50:35.782166Z","iopub.execute_input":"2021-11-04T10:50:35.782426Z","iopub.status.idle":"2021-11-04T10:50:36.132607Z","shell.execute_reply.started":"2021-11-04T10:50:35.782396Z","shell.execute_reply":"2021-11-04T10:50:36.131939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MobileNet with native output layer","metadata":{}},{"cell_type":"code","source":"IMAGE_SHAPE = 224\nexample_data = train_generator.flow_from_directory(\n    directory=TRAIN_DIR,\n    target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n    batch_size=BATCH_SIZE,\n    class_mode='binary',\n    shuffle=True\n)\nexample_x, example_y = example_data.next()\nexample_classes = list(example_data.class_indices.keys())\nexample_y_classes = [example_classes[int(i)] for i in example_y]","metadata":{"execution":{"iopub.status.busy":"2021-11-04T13:26:15.852087Z","iopub.execute_input":"2021-11-04T13:26:15.852396Z","iopub.status.idle":"2021-11-04T13:26:16.680995Z","shell.execute_reply.started":"2021-11-04T13:26:15.852366Z","shell.execute_reply":"2021-11-04T13:26:16.679985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mobilenet_native = tf.keras.applications.mobilenet.MobileNet(\n    include_top=True,\n    weights='imagenet',\n    input_shape=(IMAGE_SHAPE, IMAGE_SHAPE, 3),\n    pooling='avg'\n)\nmobilenet_native.compile()\nexample_pred = mobilenet_native.predict(\n    example_x\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T13:26:16.683137Z","iopub.execute_input":"2021-11-04T13:26:16.683458Z","iopub.status.idle":"2021-11-04T13:26:18.380062Z","shell.execute_reply.started":"2021-11-04T13:26:16.683413Z","shell.execute_reply":"2021-11-04T13:26:18.379067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_path = tf.keras.utils.get_file(\n    'ImageNetLabels.txt',\n    'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt'\n)\nimagenet_labels = np.array(open(labels_path).read().splitlines())\nresult = imagenet_labels[np.argmax(example_pred, axis=1)]","metadata":{"execution":{"iopub.status.busy":"2021-11-04T13:26:18.810538Z","iopub.execute_input":"2021-11-04T13:26:18.811414Z","iopub.status.idle":"2021-11-04T13:26:18.820563Z","shell.execute_reply.started":"2021-11-04T13:26:18.811376Z","shell.execute_reply":"2021-11-04T13:26:18.819224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_ROWS = 5\nNUM_COLS = 5\nNUM_IMAGES = NUM_COLS * NUM_ROWS\nplt.figure(figsize=(2*NUM_COLS, 2*NUM_ROWS))\nfor i in range(NUM_IMAGES):\n    plt.subplot(NUM_ROWS, NUM_COLS, i + 1)\n    plt.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(example_x[i], cmap=plt.cm.binary)\n    plt.xlabel('{} {:.0%}\\n({})'.format(result[i], np.max(example_pred[i]), example_y_classes[i]))\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T13:27:25.249261Z","iopub.execute_input":"2021-11-04T13:27:25.249646Z","iopub.status.idle":"2021-11-04T13:27:27.897591Z","shell.execute_reply.started":"2021-11-04T13:27:25.249614Z","shell.execute_reply":"2021-11-04T13:27:27.896552Z"},"trusted":true},"execution_count":null,"outputs":[]}]}