{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Transfer learning\n\nThis notebook shows how to do transfer learning using a pre-trained CNN for the MNIST dataset.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom matplotlib import pyplot as plt\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"dftrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndftest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nX_train = dftrain.drop(columns='label').values.reshape((dftrain.shape[0], 28, 28, 1))\ny_train = keras.utils.to_categorical(dftrain['label'].values)\nX_test = dftest.values.reshape((dftest.shape[0], 28, 28, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = tf.image.grayscale_to_rgb(tf.constant(X_train))\nx = tf.pad(x, ((0, 0), (2, 2), (2, 2), (0, 0)))\nx = keras.applications.resnet_v2.preprocess_input(tf.cast(x, tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lambda_ = 1e-3\nmodel = keras.Sequential()\ncore = keras.applications.ResNet152V2(\n    include_top=False,\n    weights='imagenet',\n    input_shape=x.shape[1:]\n)\nfor layer in core.layers[:-2]:\n    layer.trainable = False\nmodel.add(core)\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(10, activation='softmax', \n                             kernel_regularizer=keras.regularizers.l2(lambda_)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n    optimizer=keras.optimizers.Adam(),\n    loss='categorical_crossentropy',\n    metrics=['accuracy']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(x, y_train, validation_split=0.2, batch_size=128, epochs=20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I was expecting the results to be better, to be honest. A plain, small, fully connected NN performs better. This is probably because the dataset is too simple for a too complex NN. I could not unfreeze the last Conv2D layer, as it has 1M+ params - massive overfitting if I did. An option would be to try something simpler like AlexNet or LeNet5, but I haven't seen pre-trained models for these built in keras.","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}