{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Импорты","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nimport keras\nfrom keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\n\nfrom keras.layers import Dense\nfrom keras.models import Model\n\nimport random\nimport zipfile","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.rcParams['figure.figsize'] = (8, 6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Распаковка данных","metadata":{}},{"cell_type":"code","source":"def unzip(file, directory):\n    with zipfile.ZipFile(file, 'r') as zip_ref:\n        zip_ref.extractall(directory)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unzip('../input/dogs-vs-cats/train.zip', './data')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nroot = './data/train/'\nFILES_TO_LOAD = 1000\n\nimg_paths = [x[2] for x in os.walk(root)][0][:FILES_TO_LOAD]\nprint(img_paths[:5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_image(root, path):\n    \"\"\"\n        Возвращает файл, находящийся по адресу root/path, и метку класса (1 - собака, 0 - кот) \n    \"\"\"\n    c = 1 if path[:3] == 'dog' else 0\n    img = image.load_img(root+path, target_size=(224, 224))\n    x = image.img_to_array(img)\n    # Обрабатываем входные данные, приводим числа к диапазону от -1 до 1\n    x = preprocess_input(x).astype('float32') / 255.0\n    return x, c","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\n    'x': [],\n    'y': []\n}\n\nfor img_path in img_paths:\n    img, c = process_image(root, img_path)\n    data['x'].append(img)\n    data['y'].append(c)\n\nX = np.stack(data['x'], axis=0).reshape(-1, 224, 224, 3)\ny = np.array(data['y'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Данные хорошо сбалансированы:","metadata":{}},{"cell_type":"code","source":"print('Баланс классов:\\n0: {}\\n1: {}\\n#0/#1: {:.2f}'.format(len(y_train[y_train == 0]),\n                                                            len(y_train[y_train == 1]),\n                                                            len(y_train[y_train == 0]) / len(y_train[y_train == 1])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Приведем метки классов к категориальному виду (one-hot encoding):","metadata":{}},{"cell_type":"code","source":"y_train = keras.utils.to_categorical(y_train, 2)\ny_test = keras.utils.to_categorical(y_test, 2)\ny_train[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Модель VGG-16","metadata":{}},{"cell_type":"markdown","source":"Подробно о данной модели можно прочитать тут: https://arxiv.org/pdf/1409.1556.pdf","metadata":{}},{"cell_type":"markdown","source":"## Инициализация модели","metadata":{}},{"cell_type":"code","source":"vgg = keras.applications.VGG16(weights='imagenet', include_top=True)\nvgg.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = vgg.input\nnew_classification_layer = Dense(2, activation='softmax')\nout = new_classification_layer(vgg.layers[-2].output)\n\n# Создаем модель, поставив вместо predictions слоя VGG16 собственный слой\nmodel = Model(inp, out)\n\n# Обучаем только последний слой\nfor layer in model.layers[:-1]:\n    layer.trainable = False\nmodel.layers[-1].trainable = True\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Обучение модели","metadata":{}},{"cell_type":"code","source":"def plot_history(history, item, idx=(None, None, None), log=False):\n    sns.set_context(\"paper\")\n    hist_dict = history.history\n    \n    if log:\n        gfg=sns.lineplot(\n            data=np.log10(np.array(hist_dict[item])[idx[0]:idx[1]:idx[2]] + 1e-3)\n        )\n    else:\n        gfg=sns.lineplot(\n            data=np.array(hist_dict[item])[idx[0]:idx[1]:idx[2]]\n        )\n    gfg.set(\n        xlabel='Epochs',\n        ylabel=item[0].upper() + item[1:],\n        title=item[0].upper() + item[1:]+' evolution' + (' [Log]' if log else '')\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer='adam',\n    metrics=['accuracy', keras.metrics.Precision(), keras.metrics.Recall()]\n)\n\nhistory = model.fit(\n    X_train,\n    y_train,\n    epochs = 10,\n    batch_size = 25,\n    verbose = 2,\n    validation_data=(X_test, y_test)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_loss')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_accuracy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_precision')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_recall')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_values = {metric: values[-1] for metric, values in history.history.items()}\nlast_values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Подбор гиперпараметров (fine tuning)","metadata":{}},{"cell_type":"markdown","source":"В качестве гиперпараметров данной модели будем подбирать гиперпараметры тренировки - оптимизатор, начальный коэффициент обучения, размер партии, количество эпох. Всего дано 1000 объектов. Если брать большой размер партии, в партию может попасть много похожих друг на друга объектов, что может привести к снижению качества обучения. Уменьшим размер партии до 10. Более того, вместо Adam будем использовать RMSProp - поскольку партия достаточно маленькая, в партию вряд ли попадется много схожих объектов, что сделает метод моментов неэффективным. Зададим начальный коэффициент обучения 1e-4. Как видно на графиках, 10 эпох достаточно для получения неплохого результата, поэтому оставим количество эпох прежним.","metadata":{}},{"cell_type":"code","source":"%%time\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=keras.optimizers.RMSprop(learning_rate=1e-4),\n    metrics=['accuracy']\n)\n\nhistory = model.fit(\n    X_train,\n    y_train,\n    epochs = 10,\n    batch_size = 10,\n    verbose = 2,\n    validation_data=(X_test, y_test)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_loss')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_accuracy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_values_ft = {metric: values[-1] for metric, values in history.history.items()}\nlast_values_ft","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Как видно, данный метод обучения оказался слишком эффективным - после 6 эпохи модель начала переобучаться. В результате тренировки, модель не смогла превзойти точность 90.4%, и точность стабильно держалась в пределах 90%.. Это говорит о том, что модель достигла своего предела - с текущей архитектурой, модель не может превзойти данного качества, сколько бы она ни обучалась.  \nДля проверки данной гипотезы, построим вторую модель, с добавленным полностью соединенным слоем и попробуем натренировать её на тех же параметрах:","metadata":{}},{"cell_type":"code","source":"inp = vgg.input\nfc = Dense(64, activation='relu')(vgg.layers[-2].output)\npredictions = Dense(2, activation='softmax')(fc)\n\n# Создаем модель, поставив вместо predictions слоя VGG16 собственный слой\nmodel = Model(inp, predictions)\n\n# Обучаем только последние два слоя\nfor layer in model.layers[:-2]:\n    layer.trainable = False\nfor layer in model.layers[-2:]:\n    layer.trainable = True\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=keras.optimizers.RMSprop(learning_rate=1e-4),\n    metrics=['accuracy']\n)\n\nhistory_2 = model.fit(\n    X_train,\n    y_train,\n    epochs = 10,\n    batch_size = 10,\n    verbose = 2,\n    validation_data=(X_test, y_test)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_loss')\nplot_history(history_2, 'val_loss')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 'val_accuracy')\nplot_history(history_2, 'val_accuracy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_values_ft_2 = {metric: values[-1] for metric, values in history_2.history.items()}\nlast_values_ft_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Как видно, новая модель (оранжевая линия) приближалась по точности к исходной модели (синяя линия), однако не достигла её значений. Скорее всего, при большем объеме данных, новая модель обогнала бы старую, однако поскольку новая модель имеет более сложную архитектуру (примерно в 33 раза больше обучаемых параметров), на таком малом объеме обучаемых данных (750 изображений), она не сможет превзойти старую.","metadata":{}},{"cell_type":"markdown","source":"# Модель ResNet\nДанная модель основана на идее об остаточных сетях (Residual Networks). Подробно о данной модели можно прочитать тут: https://arxiv.org/abs/1512.03385","metadata":{}},{"cell_type":"markdown","source":"## Инициализация модели","metadata":{}},{"cell_type":"code","source":"resnet = keras.applications.ResNet50(weights='imagenet', include_top=True)\nresnet.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = resnet.input\nfc = Dense(512, activation='relu')(resnet.layers[-2].output)\npredictions = Dense(2, activation='softmax')(fc)\n\n# Создаем модель, поставив вместо predictions слоя ResNet50 собственный слой\nmodel = Model(inp, predictions)\n\n# Обучаем только последний слой\nfor layer in model.layers[:-2]:\n    layer.trainable = False\nfor layer in model.layers[-2:]:\n    layer.trainable = True\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Обучение модели","metadata":{}},{"cell_type":"code","source":"%%time\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=keras.optimizers.RMSprop(learning_rate=1e-4),\n    metrics=['accuracy']\n)\n\nhistory_resnet = model.fit(\n    X_train,\n    y_train,\n    epochs = 30,\n    batch_size = 3,\n    verbose = 2,\n    validation_data=(X_test, y_test),\n    callbacks=[keras.callbacks.EarlyStopping(patience=4)]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history_resnet, 'val_loss')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history_resnet, 'val_accuracy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_values_resnet = {metric: values[-1] for metric, values in history_resnet.history.items()}\nlast_values_resnet","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Как видно, данная модель не так эффективна, как VGG16.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}