{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## モジュールのインポート\nImporting modules","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-29T20:20:50.166681Z","iopub.execute_input":"2023-07-29T20:20:50.167147Z","iopub.status.idle":"2023-07-29T20:20:50.177139Z","shell.execute_reply.started":"2023-07-29T20:20:50.167102Z","shell.execute_reply":"2023-07-29T20:20:50.175696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys\nimport zipfile\nimport tensorflow as tf\nfrom keras.utils import np_utils\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D\nfrom keras.layers.pooling import GlobalAveragePooling2D\nfrom keras.preprocessing.image import ImageDataGenerator\n%matplotlib inline\nimport matplotlib.image as img\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix\nimport plotly.graph_objects as go\nimport itertools\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:21:12.514954Z","iopub.execute_input":"2023-07-29T20:21:12.515391Z","iopub.status.idle":"2023-07-29T20:21:12.527557Z","shell.execute_reply.started":"2023-07-29T20:21:12.515357Z","shell.execute_reply":"2023-07-29T20:21:12.526123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys\nimport zipfile\nimport tensorflow as tf\nfrom keras.utils import np_utils\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D\nfrom keras.layers.pooling import GlobalAveragePooling2D\nfrom keras.preprocessing.image import ImageDataGenerator\n%matplotlib inline\nimport matplotlib.image as img\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix\nimport plotly.graph_objects as go\nimport itertools\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:20:58.773194Z","iopub.execute_input":"2023-07-29T20:20:58.773629Z","iopub.status.idle":"2023-07-29T20:20:58.785334Z","shell.execute_reply.started":"2023-07-29T20:20:58.773599Z","shell.execute_reply":"2023-07-29T20:20:58.783967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## seed値の設定\nモデルの再現性を保つため。","metadata":{}},{"cell_type":"code","source":"seed = 0\nnp.random.seed(seed)\ntf.random.set_seed(3)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:21:16.177706Z","iopub.execute_input":"2023-07-29T20:21:16.178183Z","iopub.status.idle":"2023-07-29T20:21:16.184266Z","shell.execute_reply.started":"2023-07-29T20:21:16.178146Z","shell.execute_reply":"2023-07-29T20:21:16.182936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## zipファイルの解凍\nUnzip zip file","metadata":{}},{"cell_type":"code","source":"zip_files = ['test', 'train']\n\nfor zip_file in zip_files:\n    with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}.zip\".format(zip_file),\"r\") as z:\n         z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:21:18.595672Z","iopub.execute_input":"2023-07-29T20:21:18.596117Z","iopub.status.idle":"2023-07-29T20:21:35.863902Z","shell.execute_reply.started":"2023-07-29T20:21:18.596071Z","shell.execute_reply":"2023-07-29T20:21:35.862891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 指定したフォルダ内のファイル名を取得\nGet file names in the specified folder","metadata":{}},{"cell_type":"code","source":"IMAGE_FOLDER_PATH = \"../working/train\"\nFILE_NAMES = os.listdir(IMAGE_FOLDER_PATH)\n\n\nlabels = []\nfor i in os.listdir(IMAGE_FOLDER_PATH):\n    labels+=[i]\n\ntest_data_dir = \"../working/test/\"\ntest_FILE_NAMES = os.listdir(test_data_dir)\ntest_labels = []\nfor j in os.listdir(test_data_dir):\n    test_labels+=[j]","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:21:41.889139Z","iopub.execute_input":"2023-07-29T20:21:41.889543Z","iopub.status.idle":"2023-07-29T20:21:41.957668Z","shell.execute_reply.started":"2023-07-29T20:21:41.889513Z","shell.execute_reply":"2023-07-29T20:21:41.956480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データフレームの作成\nCreation of data frames","metadata":{}},{"cell_type":"code","source":"targets = list()\nfull_paths = list()\ntrain_cats_dir = list()\ntrain_dogs_dir = list()\n\nfor file_name in FILE_NAMES:\n    target = file_name.split(\".\")[0]\n    full_path = os.path.join(IMAGE_FOLDER_PATH,file_name)\n    \n    if(target == 'dog'):\n        train_dogs_dir.append(full_path)\n    if(target == 'cat'):\n        train_cats_dir.append(full_path)\n        \n    full_paths.append(full_path)\n    targets.append(target)\n    \ndataset = pd.DataFrame()\ndataset['image_path'] = full_paths\ndataset['target'] = targets","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:22:17.162297Z","iopub.execute_input":"2023-07-29T20:22:17.163269Z","iopub.status.idle":"2023-07-29T20:22:17.269790Z","shell.execute_reply.started":"2023-07-29T20:22:17.163210Z","shell.execute_reply":"2023-07-29T20:22:17.268507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_full_paths=list()\n\nfor test_file_name in test_FILE_NAMES:\n    test_full_path = os.path.join(test_data_dir,test_file_name)\n    test_full_paths.append(test_full_path)\ntest_dataset = pd.DataFrame()\ntest_dataset['test_image_path'] = test_full_paths","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:22:21.706408Z","iopub.execute_input":"2023-07-29T20:22:21.706841Z","iopub.status.idle":"2023-07-29T20:22:21.749197Z","shell.execute_reply.started":"2023-07-29T20:22:21.706805Z","shell.execute_reply":"2023-07-29T20:22:21.747966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの分割\n25000個のデータを訓練データと検証用データにそれぞれ同じ数に分割\n\nData Division. Split 25,000 data into the same number of data for training and data for validation, respectively.","metadata":{}},{"cell_type":"code","source":"dataset_train,dataset_val = train_test_split(dataset, test_size=0.5, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:22:32.060753Z","iopub.execute_input":"2023-07-29T20:22:32.061326Z","iopub.status.idle":"2023-07-29T20:22:32.078111Z","shell.execute_reply.started":"2023-07-29T20:22:32.061282Z","shell.execute_reply":"2023-07-29T20:22:32.077125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 訓練データの作成\nCreation of training data","metadata":{}},{"cell_type":"code","source":"WIDTH = 180\nHEIGHT = 180\ntrain_datagen=ImageDataGenerator(\n    rotation_range=15,\n    rescale=1./255,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    width_shift_range=0.1,\n    height_shift_range=0.1)\n\ntrain_datagenerator=train_datagen.flow_from_dataframe(dataframe=dataset_train,\n                                                      x_col=\"image_path\",\n                                                      y_col=\"target\",\n                                                      target_size=(WIDTH,HEIGHT),\n                                                      color_mode=\"rgb\",\n                                                      class_mode='binary',\n                                                      batch_size=100)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:22:39.536598Z","iopub.execute_input":"2023-07-29T20:22:39.536999Z","iopub.status.idle":"2023-07-29T20:22:39.724536Z","shell.execute_reply.started":"2023-07-29T20:22:39.536969Z","shell.execute_reply":"2023-07-29T20:22:39.723425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 検証データの作成\nCreation of validation data","metadata":{}},{"cell_type":"code","source":"val_datagen = ImageDataGenerator(rescale=1./255)\nval_datagenerator = val_datagen.flow_from_dataframe(dataframe=dataset_val,\n                                                      x_col=\"image_path\",\n                                                      y_col=\"target\",\n                                                      target_size=(WIDTH,HEIGHT),\n                                                      color_mode=\"rgb\",\n                                                      class_mode='binary',\n                                                      batch_size=100)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:22:50.481774Z","iopub.execute_input":"2023-07-29T20:22:50.482224Z","iopub.status.idle":"2023-07-29T20:22:50.665219Z","shell.execute_reply.started":"2023-07-29T20:22:50.482187Z","shell.execute_reply":"2023-07-29T20:22:50.664260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## テストデータの作成\nCreation of test data","metadata":{}},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(rescale=1./255)\ntest_datagenerator = test_datagen.flow_from_dataframe( dataframe = test_dataset,\n                                                      x_col=\"test_image_path\",\n                                                      target_size=(WIDTH,HEIGHT),\n                                                      color_mode=\"rgb\",\n                                                      class_mode= None,\n                                                      batch_size=100)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:23:01.360529Z","iopub.execute_input":"2023-07-29T20:23:01.360967Z","iopub.status.idle":"2023-07-29T20:23:01.513876Z","shell.execute_reply.started":"2023-07-29T20:23:01.360932Z","shell.execute_reply":"2023-07-29T20:23:01.512655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNNモデルの設定\nCNN Model Setup","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (3,3), input_shape=(WIDTH, HEIGHT, 3), activation = 'relu'))\nmodel.add(Conv2D(32, (3,3), activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size=2))\nmodel.add(Conv2D(64, (3,3), activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size=2))\nmodel.add(Dropout(0.25))\nmodel.add(Flatten())\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1,activation = 'sigmoid'))\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer='adam', metrics = ['accuracy'])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:23:13.373064Z","iopub.execute_input":"2023-07-29T20:23:13.373500Z","iopub.status.idle":"2023-07-29T20:23:13.699516Z","shell.execute_reply.started":"2023-07-29T20:23:13.373469Z","shell.execute_reply":"2023-07-29T20:23:13.698440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルの実行\nModel Execution","metadata":{}},{"cell_type":"code","source":"History=model.fit(train_datagenerator,\n                  epochs=30,\n                  validation_data=val_datagenerator,\n                  validation_steps=dataset_val.shape[0]/150,\n                  steps_per_epoch=dataset_train.shape[0]/150)","metadata":{"execution":{"iopub.status.busy":"2023-07-29T20:23:19.543837Z","iopub.execute_input":"2023-07-29T20:23:19.544278Z","iopub.status.idle":"2023-07-30T02:33:54.163626Z","shell.execute_reply.started":"2023-07-29T20:23:19.544241Z","shell.execute_reply":"2023-07-30T02:33:54.162209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 学習過程を可視化するためのグラフ作成\nCreating graphs to visualize the learning process","metadata":{}},{"cell_type":"code","source":"acc = History.history['accuracy']\nval_acc = History.history['val_accuracy']\nloss = History.history['loss']\nval_loss = History.history['val_loss']\n\nepochs = range(len(acc))\nplt.figure()\nplt.plot(epochs, acc, 'r', label = 'Training accuracy')\nplt.plot(epochs, val_acc, 'b', label = 'Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.show","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:40:07.822527Z","iopub.execute_input":"2023-07-30T02:40:07.823058Z","iopub.status.idle":"2023-07-30T02:40:08.244062Z","shell.execute_reply.started":"2023-07-30T02:40:07.823020Z","shell.execute_reply":"2023-07-30T02:40:08.242493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\n\nplt.plot(epochs, loss, 'go', label = 'Training Loss')\nplt.plot(epochs, val_loss, 'g', label = 'validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:40:12.280472Z","iopub.execute_input":"2023-07-30T02:40:12.280919Z","iopub.status.idle":"2023-07-30T02:40:12.670250Z","shell.execute_reply.started":"2023-07-30T02:40:12.280885Z","shell.execute_reply":"2023-07-30T02:40:12.669152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 精度の検証\nVerification of accuracy","metadata":{}},{"cell_type":"code","source":"test_loss, test_acc = model.evaluate(val_datagenerator, steps=len(test_datagenerator), verbose=0)\nprint('Loss:%.3f'%(test_loss))\nprint('Accuracy:%.3f'%(test_acc))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:40:16.442747Z","iopub.execute_input":"2023-07-30T02:40:16.443180Z","iopub.status.idle":"2023-07-30T02:43:51.804618Z","shell.execute_reply.started":"2023-07-30T02:40:16.443146Z","shell.execute_reply":"2023-07-30T02:43:51.803548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(x=test_datagenerator, steps = len(test_datagenerator), verbose=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:44:04.228300Z","iopub.execute_input":"2023-07-30T02:44:04.228776Z","iopub.status.idle":"2023-07-30T02:47:41.130940Z","shell.execute_reply.started":"2023-07-30T02:44:04.228741Z","shell.execute_reply":"2023-07-30T02:47:41.129853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:47:45.537906Z","iopub.execute_input":"2023-07-30T02:47:45.538355Z","iopub.status.idle":"2023-07-30T02:47:45.547296Z","shell.execute_reply.started":"2023-07-30T02:47:45.538318Z","shell.execute_reply":"2023-07-30T02:47:45.545932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CSVファイルの作成\nCreation of CSV files","metadata":{}},{"cell_type":"code","source":"def gen_image_label(directory):\n    '''A generator that yields (label, id, jpg_filename) tuple'''\n    for root,dirs, files in os.walk(directory):\n        for f in files:\n            _, ext = os.path.splitext(f)\n            if ext != '.jpg':\n                continue\n            basename = os.path.basename(f)\n            splits = basename.split('.')\n            if len(splits) == 3:\n                label, id_, ext = splits\n            else:\n                label = None\n                id_, ext = splits\n            fullname = os.path.join(root, f)\n            yield label, int(id_), fullname","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:47:53.204559Z","iopub.execute_input":"2023-07-30T02:47:53.205448Z","iopub.status.idle":"2023-07-30T02:47:53.214240Z","shell.execute_reply.started":"2023-07-30T02:47:53.205406Z","shell.execute_reply":"2023-07-30T02:47:53.212880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = list(gen_image_label(test_data_dir))\ntest_df = pd.DataFrame(a, columns=['label', 'id', 'filename'])\ntest_df = test_df.sort_values(by=['label', 'id'])\ntest_df['label_code'] = test_df.label.map({'cat':0, 'dog':1})\ntest_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:47:57.587348Z","iopub.execute_input":"2023-07-30T02:47:57.587772Z","iopub.status.idle":"2023-07-30T02:47:57.747989Z","shell.execute_reply.started":"2023-07-30T02:47:57.587740Z","shell.execute_reply":"2023-07-30T02:47:57.746595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.DataFrame({'id':pd.Series(test_df.id.values[:predictions.shape[0]]),'label':pd.Series(predictions.T[0])})\nresults.to_csv('submission.csv', index=False)\nresults.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T02:48:02.160206Z","iopub.execute_input":"2023-07-30T02:48:02.160712Z","iopub.status.idle":"2023-07-30T02:48:02.223966Z","shell.execute_reply.started":"2023-07-30T02:48:02.160672Z","shell.execute_reply":"2023-07-30T02:48:02.222755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}