{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# библиотека для определения случайных значений\nimport random\n\n# библиотека для разархивирования  файлов\nimport zipfile\n\n# юиюлиотека для подготовки изображений \nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img\n\n# отображение картинок\nimport matplotlib.pyplot as plt\n\n# разделение выборки на тестовую и валидационную\nfrom sklearn.model_selection import train_test_split\n\n# библиотека работы с НС\nimport tensorflow as tf\n\n# оптимизатор ф потерь \nfrom keras import optimizers\n\nimport re\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:23.161908Z","iopub.execute_input":"2023-04-17T09:12:23.162394Z","iopub.status.idle":"2023-04-17T09:12:23.179682Z","shell.execute_reply.started":"2023-04-17T09:12:23.162347Z","shell.execute_reply":"2023-04-17T09:12:23.178507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# загружвем файлы из архива в папку tmp\nlocal_zip = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip' \nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall('/tmp')\nzip_ref.close()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:28.877485Z","iopub.execute_input":"2023-04-17T09:12:28.878348Z","iopub.status.idle":"2023-04-17T09:12:39.040398Z","shell.execute_reply.started":"2023-04-17T09:12:28.878312Z","shell.execute_reply":"2023-04-17T09:12:39.039355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = '/tmp/train'\nimg_names = os.listdir(os.path.join(base_dir))\nimg_names[:10]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:41.306876Z","iopub.execute_input":"2023-04-17T09:12:41.307586Z","iopub.status.idle":"2023-04-17T09:12:41.329172Z","shell.execute_reply.started":"2023-04-17T09:12:41.307547Z","shell.execute_reply":"2023-04-17T09:12:41.328024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# отобразим случайные картинки из выборки\nsample = random.choice(img_names)\nimage = load_img(\"/tmp/train/\"+sample)\nplt.title(sample, fontsize=12)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:44.071612Z","iopub.execute_input":"2023-04-17T09:12:44.072542Z","iopub.status.idle":"2023-04-17T09:12:44.372957Z","shell.execute_reply.started":"2023-04-17T09:12:44.072489Z","shell.execute_reply":"2023-04-17T09:12:44.372119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# установим метку собака и кошка для изображений (это так для себя)\nlabel = []\n\nfor i in img_names:\n    if(i.split(\".\")[0] == \"dog\"):\n        label.append(\"Dog\")\n    else:\n        label.append(\"Cat\")\n\nlabel[:10]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:47.341651Z","iopub.execute_input":"2023-04-17T09:12:47.342019Z","iopub.status.idle":"2023-04-17T09:12:47.362904Z","shell.execute_reply.started":"2023-04-17T09:12:47.341987Z","shell.execute_reply":"2023-04-17T09:12:47.361980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# сформируем датасет с названиями картинок и мемтками\ndf = pd.DataFrame({\"Image\" : img_names, \"Label\" : label})\nprint(df.shape)\ndf.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:49.200891Z","iopub.execute_input":"2023-04-17T09:12:49.201592Z","iopub.status.idle":"2023-04-17T09:12:49.216920Z","shell.execute_reply.started":"2023-04-17T09:12:49.201555Z","shell.execute_reply":"2023-04-17T09:12:49.215798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# отобразим баланс классов\ndf.Label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:51.096319Z","iopub.execute_input":"2023-04-17T09:12:51.096917Z","iopub.status.idle":"2023-04-17T09:12:51.108754Z","shell.execute_reply.started":"2023-04-17T09:12:51.096881Z","shell.execute_reply":"2023-04-17T09:12:51.107619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# разбиваем тренировочную выборку на трен и валид\ntrain_df, validate_df = train_test_split(df, test_size = 0.2)\ntrain_df = train_df.reset_index(drop = True)\nvalidate_df = validate_df.reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:52.551404Z","iopub.execute_input":"2023-04-17T09:12:52.552069Z","iopub.status.idle":"2023-04-17T09:12:52.562815Z","shell.execute_reply.started":"2023-04-17T09:12:52.552033Z","shell.execute_reply":"2023-04-17T09:12:52.561792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:54.789794Z","iopub.execute_input":"2023-04-17T09:12:54.790785Z","iopub.status.idle":"2023-04-17T09:12:54.800965Z","shell.execute_reply.started":"2023-04-17T09:12:54.790737Z","shell.execute_reply":"2023-04-17T09:12:54.799929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# размер картинки после предобработки (верт и гор)\nimg_size = 150\n\n# Предобработка трен выборки с разбиением на батчи\n\ntrain_datagen = ImageDataGenerator(\n      rescale=1./255,\n      rotation_range=40,\n      width_shift_range=0.2,\n      height_shift_range=0.2,\n      shear_range=0.2,\n      zoom_range=0.2,\n      horizontal_flip=True,\n      fill_mode='nearest')\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    train_df, \n    \"/tmp/train/\", \n    x_col='Image',\n    y_col='Label',\n    target_size = (img_size, img_size),\n    class_mode = 'binary',\n    batch_size = 32\n)\n\n# Предобработка валид выборки с разбиением на батчи\nvalidation_datagen = ImageDataGenerator(rescale=1./255)\n\nvalidation_generator = validation_datagen.flow_from_dataframe(\n    validate_df, \n    \"/tmp/train/\", \n    x_col='Image',\n    y_col='Label',\n    target_size = (img_size, img_size),\n    class_mode = 'binary',\n    batch_size = 10\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:12:56.811996Z","iopub.execute_input":"2023-04-17T09:12:56.812916Z","iopub.status.idle":"2023-04-17T09:12:57.054781Z","shell.execute_reply.started":"2023-04-17T09:12:56.812866Z","shell.execute_reply":"2023-04-17T09:12:57.053844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# получили массив обуч выборки\n# отразил больше для собственного понимания\nprint('батчей',len(train_generator))\nprint('признаки и цели',len(train_generator[0]))\nprint('размер батча', len(train_generator[0][0]))\nprint('размер картинки в пикс верт', len(train_generator[0][0][0]))\nprint('размер картинки в пикс гор', len(train_generator[0][0][0][0]))\nprint('rgb пиксель',            len(train_generator[0][0][0][0][0]))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:13:00.394688Z","iopub.execute_input":"2023-04-17T09:13:00.395208Z","iopub.status.idle":"2023-04-17T09:13:01.467369Z","shell.execute_reply.started":"2023-04-17T09:13:00.395166Z","shell.execute_reply":"2023-04-17T09:13:01.466166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# индикация классов\nprint(train_generator.class_indices)\n\n# целевая функция бля 1го батча\ntrain_generator[0][1]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:13:03.172987Z","iopub.execute_input":"2023-04-17T09:13:03.173570Z","iopub.status.idle":"2023-04-17T09:13:03.396528Z","shell.execute_reply.started":"2023-04-17T09:13:03.173537Z","shell.execute_reply":"2023-04-17T09:13:03.395398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Конструируем НС 2\n\n# За основу берем базовую модель VGG19 с весами для imagenet \nimage_size = 150\n\ninput_shape = (image_size, image_size, 3)\n\nbase_model = tf.keras.applications.VGG19(\n    weights=\"imagenet\",  # загружаем веса ImageNet.\n    input_shape=input_shape,\n    include_top=False,\n)  \n\n\nfor layer in base_model.layers[:15]:\n    layer.trainable = False\n\n# зафиксируем веса 1х 15 слоев без тренировки\nfor layer in base_model.layers[15:]:\n    layer.trainable = True\n    \nlast_layer = base_model.get_layer('block5_pool')\nlast_output = last_layer.output    \n\n# Выровняйте выходной слой до 1 размера\nx = tf.keras.layers.GlobalMaxPooling2D()(last_output)\n\n# Добавляем слой 512 нейронов к скрытым стоям\nx = tf.keras.layers.Dense(256, activation='relu')(x)\n\n# Добавим дропаут 0,5 \nx = tf.keras.layers.Dropout(0.5)(x)\n\n# Финальная ф. сигмоид для классификации на 2 класса\nx = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n\nmodel2 = tf.keras.Model(base_model.input, x)\n\nmodel2.compile(loss='binary_crossentropy',\n              optimizer=optimizers.SGD(learning_rate=0.001, momentum=0.9),\n              metrics=['accuracy'])\n\nmodel2.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:13:05.592938Z","iopub.execute_input":"2023-04-17T09:13:05.593383Z","iopub.status.idle":"2023-04-17T09:13:06.157709Z","shell.execute_reply.started":"2023-04-17T09:13:05.593347Z","shell.execute_reply":"2023-04-17T09:13:06.156906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# запускаем обучение \nepochs = 10\ncallback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n\nmodel2_history = model2.fit(train_generator, \n                            epochs=epochs, \n                            validation_data=validation_generator,\n                            callbacks=[callback])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:13:10.296408Z","iopub.execute_input":"2023-04-17T09:13:10.296979Z","iopub.status.idle":"2023-04-17T09:41:35.099655Z","shell.execute_reply.started":"2023-04-17T09:13:10.296942Z","shell.execute_reply":"2023-04-17T09:41:35.098565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\nmodel2.save('model')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:05:23.638226Z","iopub.execute_input":"2023-04-17T10:05:23.639270Z","iopub.status.idle":"2023-04-17T10:05:26.561874Z","shell.execute_reply.started":"2023-04-17T10:05:23.639219Z","shell.execute_reply":"2023-04-17T10:05:26.560872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# История метрики по эпохам \n\nhist_df=pd.DataFrame(model2_history.history)\nhist_df","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:05:43.890959Z","iopub.execute_input":"2023-04-17T10:05:43.891648Z","iopub.status.idle":"2023-04-17T10:05:43.904517Z","shell.execute_reply.started":"2023-04-17T10:05:43.891612Z","shell.execute_reply":"2023-04-17T10:05:43.903460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(hist_df['accuracy'],label='Accuracy')\nplt.plot(hist_df['loss'],label='Loss')\nplt.title('Accuracy && LOSS')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:05:46.636421Z","iopub.execute_input":"2023-04-17T10:05:46.637520Z","iopub.status.idle":"2023-04-17T10:05:46.860460Z","shell.execute_reply.started":"2023-04-17T10:05:46.637481Z","shell.execute_reply":"2023-04-17T10:05:46.859506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# разархивируем тестовую выборку\nlocal_zip = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall('/tmp')\nzip_ref.close()\n\ntest_dir = '/tmp/test/'\ntest_img = os.listdir(os.path.join(test_dir))\ntest_img[:10]\n\ntest_df = pd.DataFrame({'Image': test_img})\n\n# выделим из названия файла картинки id\ntest_df['id'] = test_df['Image'].apply( lambda x: re.search('\\d+', x)[0]).astype(int)\ntest_df.sort_values(by = 'id')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:17:25.702068Z","iopub.execute_input":"2023-04-17T10:17:25.702681Z","iopub.status.idle":"2023-04-17T10:17:30.445056Z","shell.execute_reply.started":"2023-04-17T10:17:25.702645Z","shell.execute_reply":"2023-04-17T10:17:30.443995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:17:58.997524Z","iopub.execute_input":"2023-04-17T10:17:58.998479Z","iopub.status.idle":"2023-04-17T10:17:59.033330Z","shell.execute_reply.started":"2023-04-17T10:17:58.998429Z","shell.execute_reply":"2023-04-17T10:17:59.032289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# предобрабатываем тест выборку для модели\ntest_gen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_gen.flow_from_dataframe(\n    test_df, \n    \"/tmp/test/\", \n    x_col = 'Image',\n    y_col = None,\n    class_mode = None,\n    target_size = (130, 130),\n    batch_size = 10,\n    shuffle = False\n)\n\ntest_generator.reset()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:18:04.220684Z","iopub.execute_input":"2023-04-17T10:18:04.221049Z","iopub.status.idle":"2023-04-17T10:18:04.329847Z","shell.execute_reply.started":"2023-04-17T10:18:04.221017Z","shell.execute_reply":"2023-04-17T10:18:04.328884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = model2.predict(test_generator, steps = np.ceil(12500/10))\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:18:17.782794Z","iopub.execute_input":"2023-04-17T10:18:17.783250Z","iopub.status.idle":"2023-04-17T10:18:58.928571Z","shell.execute_reply.started":"2023-04-17T10:18:17.783211Z","shell.execute_reply":"2023-04-17T10:18:58.927516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label(predict):\n    if(predict > 0.5):\n        return \"Dog\"\n    else:\n        return \"Cat\"\n    \ntest_df['predict'] = predict\ntest_df['label'] = round(test_df['predict'])\ntest_df['label_name'] = test_df['label'].apply( lambda x: 'Dog' if x== 1 else 'Cat')\ntest_df    ","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:22:39.465639Z","iopub.execute_input":"2023-04-17T10:22:39.466010Z","iopub.status.idle":"2023-04-17T10:22:39.492481Z","shell.execute_reply.started":"2023-04-17T10:22:39.465978Z","shell.execute_reply":"2023-04-17T10:22:39.491404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.label_name.value_counts().plot.bar(color = ['red','blue'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:22:50.733023Z","iopub.execute_input":"2023-04-17T10:22:50.733758Z","iopub.status.idle":"2023-04-17T10:22:50.927166Z","shell.execute_reply.started":"2023-04-17T10:22:50.733721Z","shell.execute_reply":"2023-04-17T10:22:50.926155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# для проверки будем выводить картинки из тестовой выборки и метки к ним (dog/cat) для иллюстрации результата\nv = random.randint(0, 12000)\nsample_test = test_df.iloc[v:(v+4)].reset_index(drop = True)\nsample_test\nplt.figure(figsize=(12, 24))\nfor index, row in sample_test.iterrows():\n    filename = row['Image']\n    category = row['label_name']\n    id_img = row['id']\n    label = row['label']\n    img = load_img(\"/tmp/test/\" + filename, target_size = (150, 150))\n    plt.subplot(4, 4, index + 1)\n    plt.imshow(img)\n    plt.xlabel(str(id_img) + ' ( ' +str(label) + \" {}\".format(category) + ' )' )\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:23:16.470504Z","iopub.execute_input":"2023-04-17T10:23:16.471271Z","iopub.status.idle":"2023-04-17T10:23:17.121362Z","shell.execute_reply.started":"2023-04-17T10:23:16.471234Z","shell.execute_reply":"2023-04-17T10:23:17.120139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = test_df[['id','predict']]\nsubmission.rename(columns={'predict': 'label'}, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:25:37.808054Z","iopub.execute_input":"2023-04-17T10:25:37.808743Z","iopub.status.idle":"2023-04-17T10:25:37.816883Z","shell.execute_reply.started":"2023-04-17T10:25:37.808707Z","shell.execute_reply":"2023-04-17T10:25:37.815763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:25:41.548920Z","iopub.execute_input":"2023-04-17T10:25:41.549836Z","iopub.status.idle":"2023-04-17T10:25:41.565401Z","shell.execute_reply.started":"2023-04-17T10:25:41.549785Z","shell.execute_reply":"2023-04-17T10:25:41.564257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission_dog_v_cat6.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:25:52.093205Z","iopub.execute_input":"2023-04-17T10:25:52.094076Z","iopub.status.idle":"2023-04-17T10:25:52.121819Z","shell.execute_reply.started":"2023-04-17T10:25:52.094039Z","shell.execute_reply":"2023-04-17T10:25:52.120640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ф потерь бинарная кросэнтропия, смотрим accuracy, обучение на 20 эпохах","metadata":{}}]}