{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Starter notebook ”Dogs vs. Cats” with EfficientNet vs My own model\nThis notebook shows a flow of \"Dogs and Cats\" distinguish model development.  \nhttps://www.kaggle.com/c/dogs-vs-cats/  \n\n[Agenda]\n1. Import libraries  \n2.　Data preparation  \n    2-1. Unzip data  \n    2-2. Path and file name setting  \n    2-3. making directory tree for ImageDataGenerator  \n    2-4. Copy picture data to tree for ImageDataGenerator  \n3. Training  \n    3-1. setting for training  \n    3-2. dataset confirmation  \n    3-3. ImageDataGenerator  \n    3-4. Network definition  \n    3-5. train  \n4. Save Model  \n5. Model evaluation  \n    5-1. Confusion matrix  \n6. Prediction with Test data  \n7. Create submission file  ","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-03T16:20:10.981727Z","iopub.execute_input":"2021-07-03T16:20:10.982099Z","iopub.status.idle":"2021-07-03T16:20:10.993798Z","shell.execute_reply.started":"2021-07-03T16:20:10.982067Z","shell.execute_reply":"2021-07-03T16:20:10.992082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check　assined GPU detail\n!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:20:12.037812Z","iopub.execute_input":"2021-07-03T16:20:12.038202Z","iopub.status.idle":"2021-07-03T16:20:12.895666Z","shell.execute_reply.started":"2021-07-03T16:20:12.038171Z","shell.execute_reply":"2021-07-03T16:20:12.894466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Import libraries","metadata":{}},{"cell_type":"code","source":"# File I/O\nimport csv\nimport subprocess\nimport shutil\nimport os\nfrom glob import glob\nfrom datetime import datetime\nfrom PIL import Image\n\n# Data processing\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\n\n# Image processing\nimport cv2\nfrom scipy.ndimage import rotate\nimport scipy.misc\n\n# Graph\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom keras import Model\nfrom keras import optimizers\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers.convolutional import Conv2D\nfrom keras.layers.convolutional import MaxPooling2D\nfrom keras.layers import GlobalAveragePooling2D\nfrom keras.layers.core import Activation\nfrom keras.layers.core import Flatten, Dropout\nfrom keras.layers.core import Dense\nfrom keras.utils.np_utils import to_categorical\nfrom keras.utils import np_utils\nfrom keras.optimizers import Adam\nfrom keras.optimizers import Adagrad\nfrom keras.callbacks import TensorBoard\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.callbacks import EarlyStopping\n# EfficientNet\nfrom tensorflow.keras.applications import EfficientNetB0\n\nfrom tensorflow.keras.optimizers import RMSprop\n\nfrom IPython.display import display\nimport ipywidgets as widgets","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:20:14.194364Z","iopub.execute_input":"2021-07-03T16:20:14.194767Z","iopub.status.idle":"2021-07-03T16:20:14.209179Z","shell.execute_reply.started":"2021-07-03T16:20:14.194734Z","shell.execute_reply":"2021-07-03T16:20:14.207360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.Data preparation  \n## 2-1. Unzip data  \nI will use ImageGenerator to load data for training.  \nTest data poctures shall not be stored in Test data path folder directory.Test data path shall have one folder.  \nIf prediction target pictures are stored in the test data path folder directory, error occurrs.   ","metadata":{}},{"cell_type":"code","source":"!unzip -q '../input/dogs-vs-cats/train.zip' -d '/kaggle/working/' # ->'/kaggle/working/train'\n!unzip -q '../input/dogs-vs-cats/test1.zip' -d '/kaggle/working/test/' # ->'/kaggle/working/test/test1'","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:20:15.138522Z","iopub.execute_input":"2021-07-03T16:20:15.138862Z","iopub.status.idle":"2021-07-03T16:23:15.069624Z","shell.execute_reply.started":"2021-07-03T16:20:15.138832Z","shell.execute_reply":"2021-07-03T16:23:15.068318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.move('/kaggle/working/train/', '/kaggle/working/train_raw/all_pics')","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:15.073577Z","iopub.execute_input":"2021-07-03T16:23:15.073918Z","iopub.status.idle":"2021-07-03T16:23:15.085734Z","shell.execute_reply.started":"2021-07-03T16:23:15.073885Z","shell.execute_reply":"2021-07-03T16:23:15.084632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2-2. Path and file name setting  \nAgain, test path folder shall have one more folder at least.  ","metadata":{}},{"cell_type":"code","source":"root_path = '/kaggle/working'\n\n# Path of Unzipping data stored (root)\ntrain_raw_root_path = '/kaggle/working/train_raw/'\n\n\n# Path of Unzipping data stored \ntrain_raw_path = '/kaggle/working/train_raw/all_pics'\n\n# Path for training\ntrain_root_path = '/kaggle/working/train_root'\ntest_path = '/kaggle/working/test'\nsubmit_file = 'submission.csv'\n\n# Path for Imagegenerator\ntrain_dir = os.path.join(train_root_path, 'train')\nvalid_dir = os.path.join(train_root_path, 'valid')\n\nprint(train_dir)\nprint(valid_dir)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:15.088837Z","iopub.execute_input":"2021-07-03T16:23:15.089881Z","iopub.status.idle":"2021-07-03T16:23:15.099020Z","shell.execute_reply.started":"2021-07-03T16:23:15.089832Z","shell.execute_reply":"2021-07-03T16:23:15.097661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2-3. making directory tree for ImageDataGenerator","metadata":{}},{"cell_type":"markdown","source":"Making each file name list of classificaion target categories","metadata":{}},{"cell_type":"markdown","source":"## 2-4. Copy picture data to tree for ImageDataGenerator","metadata":{}},{"cell_type":"code","source":"# picture file name list\ndata_img_list = [os.path.basename(f) for f in glob(f'{train_raw_path}/*.jpg')]\ntest_img_list = [os.path.basename(f) for f in glob(f'{test_path}/test1/*.jpg')]","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:15.101221Z","iopub.execute_input":"2021-07-03T16:23:15.101773Z","iopub.status.idle":"2021-07-03T16:23:15.295461Z","shell.execute_reply.started":"2021-07-03T16:23:15.101728Z","shell.execute_reply":"2021-07-03T16:23:15.294278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target labels list\ntarget_labels = ['cat', 'dog']\n\n# directories in the train_root path\ndirs = ['train', 'valid']\n\n# make dirctories for data store\nfor d in dirs:\n    for label in target_labels:\n        os.makedirs(os.path.join(train_root_path, d, label), exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:15.297231Z","iopub.execute_input":"2021-07-03T16:23:15.297716Z","iopub.status.idle":"2021-07-03T16:23:15.304769Z","shell.execute_reply.started":"2021-07-03T16:23:15.297638Z","shell.execute_reply":"2021-07-03T16:23:15.303462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initializing instance\nimg_cat_list = []\nimg_dog_list = []\n\n# make dog and cat file name list\nfor f in data_img_list:\n    label = f.split('.')[0]\n    if label == target_labels[0]:\n        img_cat_list.append(f)\n    elif label == target_labels[1]:\n        img_dog_list.append(f)\n    else:\n        print('abnormal file name is found', f)\n\n# Split data to for train and for valid\ntrain_list_cat, val_list_cat, train_list_dog, val_list_dog = train_test_split(img_cat_list, img_dog_list, \n                                                                  test_size=0.3,\n                                                                 random_state=46)\n# directories list to join to train_root path\ncopy_to_dirs = ['train/cat', 'valid/cat', 'train/dog', 'valid/dog']\nimg_lists = [train_list_cat, val_list_cat, train_list_dog, val_list_dog] \n\n# Copy to each directories\nfor to_dir in copy_to_dirs:\n    for f in img_lists[copy_to_dirs.index(to_dir)]:\n        shutil.copyfile(os.path.join(train_raw_path,f), os.path.join(train_root_path, to_dir,f))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:15.306626Z","iopub.execute_input":"2021-07-03T16:23:15.307329Z","iopub.status.idle":"2021-07-03T16:23:22.633388Z","shell.execute_reply.started":"2021-07-03T16:23:15.307283Z","shell.execute_reply":"2021-07-03T16:23:22.632232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check file quantity \nprint('train image quantity:', len(data_img_list))\nprint('cat:', len(img_cat_list))\nprint('dog:', len(img_dog_list))\nprint()\nprint('prediction target test image quantity:', len(test_img_list))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:22.635263Z","iopub.execute_input":"2021-07-03T16:23:22.635708Z","iopub.status.idle":"2021-07-03T16:23:22.646232Z","shell.execute_reply.started":"2021-07-03T16:23:22.635662Z","shell.execute_reply":"2021-07-03T16:23:22.644988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check file number\nfor d in dirs:\n    for label in target_labels:\n        print(os.path.join(train_root_path, d, label))\n        print(len([file for file in os.listdir(os.path.join(train_root_path, d, label))]))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:22.649904Z","iopub.execute_input":"2021-07-03T16:23:22.650656Z","iopub.status.idle":"2021-07-03T16:23:22.681877Z","shell.execute_reply.started":"2021-07-03T16:23:22.650610Z","shell.execute_reply":"2021-07-03T16:23:22.680783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check file quantity in each directory.Above quantity shows correct copy picture process.","metadata":{}},{"cell_type":"code","source":"# Check data directry tree structure\n!tree -d '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:22.683958Z","iopub.execute_input":"2021-07-03T16:23:22.684356Z","iopub.status.idle":"2021-07-03T16:23:23.738475Z","shell.execute_reply.started":"2021-07-03T16:23:22.684329Z","shell.execute_reply":"2021-07-03T16:23:23.737185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test: pictures for test  \ntrain_raw: train data provided from kaggle  \ntrain_root: train pictures copied to place for ImageDataGenerator","metadata":{}},{"cell_type":"markdown","source":"# 3. Training","metadata":{}},{"cell_type":"markdown","source":"## 3-1. setting for training","metadata":{}},{"cell_type":"code","source":"learning_rate = 1e-4\nbatch_size = 20\ninput_height , input_width = 300, 300\nrandom_split = 1\nepochs = 200\n\n# model name setting\nmodel_name_list = ['efficient_net', 'mymodel', ]","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:23.742648Z","iopub.execute_input":"2021-07-03T16:23:23.742975Z","iopub.status.idle":"2021-07-03T16:23:23.753130Z","shell.execute_reply.started":"2021-07-03T16:23:23.742941Z","shell.execute_reply":"2021-07-03T16:23:23.752091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3-2. dataset confirmation","metadata":{}},{"cell_type":"code","source":"# 画像の表示\nim = Image.open(os.path.join(train_raw_path, img_dog_list[0]))\nprint(os.path.join(train_raw_path, img_dog_list[0]))\nplt.imshow(im)\nplt.title(img_dog_list[0])\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:23.754727Z","iopub.execute_input":"2021-07-03T16:23:23.755171Z","iopub.status.idle":"2021-07-03T16:23:24.156677Z","shell.execute_reply.started":"2021-07-03T16:23:23.755122Z","shell.execute_reply":"2021-07-03T16:23:24.155552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['cat', 'dog']\nlabel_description = {\n    '0': 'cat',\n    '1': 'dog',\n}\n\nfor label in labels:\n    f, ax = plt.subplots(figsize=(12,10))\n    if label == 'cat':\n        img_list = img_cat_list\n        \n    elif label == 'dog':\n        img_list = img_dog_list\n        \n    # show pictures 3 x 3 \n    for x in range(9):\n        plt.subplot(3, 3, x+1)\n        im = Image.open(os.path.join(train_raw_path, img_list[x]))\n        plt.axis('off')\n        plt.title(img_list[x])\n        plt.imshow(im)\n        \n    print(f'\\t\\t\\t\\t# {label}')\n    plt.show()\n    print()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:24.158198Z","iopub.execute_input":"2021-07-03T16:23:24.158809Z","iopub.status.idle":"2021-07-03T16:23:26.132709Z","shell.execute_reply.started":"2021-07-03T16:23:24.158763Z","shell.execute_reply":"2021-07-03T16:23:26.131714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3-3. ImageDataGenerator","metadata":{}},{"cell_type":"code","source":"# All images will be rescaled by 1./255\n# train_datagen = ImageDataGenerator(rescale=1./255)\n# Added data augumentation to avoid over fitting (version 8)\ntrain_datagen = ImageDataGenerator(\n    rotation_range = 360,\n    # zca_whitening = True,\n    # zca_epsilon = 1e-06,\n    width_shift_range = 0.2,\n    height_shift_range = 0.2,\n    shear_range = 0.1,\n    zoom_range = [0.5,1.0],\n    channel_shift_range = 0.2,\n    horizontal_flip = True,\n    vertical_flip = True,\n    rescale = 1./255,\n    preprocessing_function = None\n)\nvalid_datagen = ImageDataGenerator(rescale=1./255)\n\n\n# Flow training images in batches of 20 using train_datagen generator\ntrain_generator = train_datagen.flow_from_directory(\n        train_dir,  # This is the source directory for training images\n        target_size=(input_height, input_width),  # All images will be resized to 150x150\n        batch_size=20,\n        # Since we use binary_crossentropy loss, we need binary labels\n        class_mode='binary')\n\n# Flow validation images in batches of 20 using test_datagen generator\nvalid_generator = valid_datagen.flow_from_directory(\n        valid_dir,\n        target_size=(input_height, input_width),\n        batch_size=20,\n        class_mode='binary')","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:26.134238Z","iopub.execute_input":"2021-07-03T16:23:26.134841Z","iopub.status.idle":"2021-07-03T16:23:27.219658Z","shell.execute_reply.started":"2021-07-03T16:23:26.134794Z","shell.execute_reply":"2021-07-03T16:23:27.218505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3-4. Network definition","metadata":{}},{"cell_type":"code","source":"def efficientnet_model(input_height, input_width):\n    model = Sequential()\n    model.add( tf.keras.applications.EfficientNetB3(\n    include_top=False,\n    weights=\"imagenet\", input_shape=(input_height, input_width, 3)))\n    model.add(GlobalAveragePooling2D())\n    model.add(Dense(1, activation=\"sigmoid\"))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:27.221355Z","iopub.execute_input":"2021-07-03T16:23:27.221816Z","iopub.status.idle":"2021-07-03T16:23:27.229829Z","shell.execute_reply.started":"2021-07-03T16:23:27.221769Z","shell.execute_reply":"2021-07-03T16:23:27.227931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mymodel(input_height, input_width):\n    # モデルの作成\n    mymodel = Sequential()\n    mymodel.add(Conv2D(32, kernel_size=3, padding=\"same\", activation='relu', input_shape=(input_height, input_width, 3)))\n    mymodel.add(MaxPooling2D(pool_size=(3, 3)))\n    mymodel.add(Conv2D(64, kernel_size=3, padding=\"same\", activation='relu'))\n    mymodel.add(MaxPooling2D(pool_size=(2, 2)))\n    mymodel.add(Conv2D(128, kernel_size=3, padding=\"same\", activation='relu'))\n    mymodel.add(MaxPooling2D(pool_size=(2, 2)))\n    mymodel.add(Flatten())    #Flatten()により特徴マップをベクトルに変換し、後続の全結合層と繋げられるようにする\n    mymodel.add(Dense(384, activation='relu'))\n    mymodel.add(Dense(128, activation='relu'))\n    mymodel.add(Dense(1, activation='sigmoid'))    #Softmax関数にて、クラス毎の確率として出力\n\n    return mymodel","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:27.231765Z","iopub.execute_input":"2021-07-03T16:23:27.232507Z","iopub.status.idle":"2021-07-03T16:23:27.245384Z","shell.execute_reply.started":"2021-07-03T16:23:27.232445Z","shell.execute_reply":"2021-07-03T16:23:27.243886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Adding some model network definition","metadata":{}},{"cell_type":"code","source":"'''\ndef mymodel2 (input_height, input_width, num_classes):\n  # network layer\n  # write code and define network\n\n  return model\n'''","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:27.247533Z","iopub.execute_input":"2021-07-03T16:23:27.248119Z","iopub.status.idle":"2021-07-03T16:23:27.262706Z","shell.execute_reply.started":"2021-07-03T16:23:27.248073Z","shell.execute_reply":"2021-07-03T16:23:27.261250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_answer(x):\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:27.264877Z","iopub.execute_input":"2021-07-03T16:23:27.265461Z","iopub.status.idle":"2021-07-03T16:23:27.272993Z","shell.execute_reply.started":"2021-07-03T16:23:27.265342Z","shell.execute_reply":"2021-07-03T16:23:27.271890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_selected = get_answer(widgets.RadioButtons(options=model_name_list))\ndisplay(model_selected)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:40.090476Z","iopub.execute_input":"2021-07-03T16:23:40.090844Z","iopub.status.idle":"2021-07-03T16:23:40.106830Z","shell.execute_reply.started":"2021-07-03T16:23:40.090813Z","shell.execute_reply":"2021-07-03T16:23:40.105647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = model_selected.value\nprint(model_name)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:43.548352Z","iopub.execute_input":"2021-07-03T16:23:43.548716Z","iopub.status.idle":"2021-07-03T16:23:43.556114Z","shell.execute_reply.started":"2021-07-03T16:23:43.548684Z","shell.execute_reply":"2021-07-03T16:23:43.554651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def select_model(model_name, input_height, input_width):\n    if model_name == 'mymodel':\n        model = mymodel(input_height, input_width)\n    elif model_name == 'efficient_net':\n        model = efficientnet_model(input_height, input_width)\n\n    return model\n\nmodel = select_model(model_name, input_height, input_width)\n\n# display model summary \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:44.549896Z","iopub.execute_input":"2021-07-03T16:23:44.550300Z","iopub.status.idle":"2021-07-03T16:23:44.647438Z","shell.execute_reply.started":"2021-07-03T16:23:44.550268Z","shell.execute_reply":"2021-07-03T16:23:44.645189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3-5. train","metadata":{}},{"cell_type":"code","source":"# compile model with some setting item\nmodel.compile(loss='binary_crossentropy',\n              optimizer=RMSprop(lr=learning_rate),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:48.655419Z","iopub.execute_input":"2021-07-03T16:23:48.655933Z","iopub.status.idle":"2021-07-03T16:23:48.679901Z","shell.execute_reply.started":"2021-07-03T16:23:48.655889Z","shell.execute_reply":"2021-07-03T16:23:48.678463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_accuracy',\n                                           patiene = 3,\n                                           verbose = 1,\n                                           factor = 0.5,\n                                           min_lr = 0.00001)\n\nearly_stopping = EarlyStopping(monitor='val_loss',\n                               patience=10)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:49.762955Z","iopub.execute_input":"2021-07-03T16:23:49.763381Z","iopub.status.idle":"2021-07-03T16:23:49.769651Z","shell.execute_reply.started":"2021-07-03T16:23:49.763348Z","shell.execute_reply":"2021-07-03T16:23:49.767964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\nhistory = model.fit(\n      train_generator,\n      steps_per_epoch=100,  # 2000 images = batch_size * steps\n      epochs=200,\n      validation_data=valid_generator,\n      validation_steps=50,  # 1000 images = batch_size * steps\n      verbose=2,\n      callbacks = [learning_rate_reduction, early_stopping],\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:23:54.993647Z","iopub.execute_input":"2021-07-03T16:23:54.994075Z","iopub.status.idle":"2021-07-03T16:59:41.078011Z","shell.execute_reply.started":"2021-07-03T16:23:54.993988Z","shell.execute_reply":"2021-07-03T16:59:41.077016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:59:41.083035Z","iopub.execute_input":"2021-07-03T16:59:41.085527Z","iopub.status.idle":"2021-07-03T16:59:41.656567Z","shell.execute_reply.started":"2021-07-03T16:59:41.085480Z","shell.execute_reply":"2021-07-03T16:59:41.655571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"        ","metadata":{}},{"cell_type":"markdown","source":"# 4. Save Model","metadata":{}},{"cell_type":"code","source":"# Save Model\nmodel.save(root_path + '/model.h5' )    # model\nmodel.save_weights(root_path + '/model_weights.h5')    # model parameter","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:59:41.661686Z","iopub.execute_input":"2021-07-03T16:59:41.664229Z","iopub.status.idle":"2021-07-03T16:59:42.471676Z","shell.execute_reply.started":"2021-07-03T16:59:41.664185Z","shell.execute_reply":"2021-07-03T16:59:42.470462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load model\n# from keras.models import load_model\n# model = load_model(root_path + '/model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:59:42.473646Z","iopub.execute_input":"2021-07-03T16:59:42.474125Z","iopub.status.idle":"2021-07-03T16:59:42.479372Z","shell.execute_reply.started":"2021-07-03T16:59:42.474094Z","shell.execute_reply":"2021-07-03T16:59:42.477917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Model evaluation","metadata":{}},{"cell_type":"markdown","source":"## 5-1. Confusion matrix","metadata":{}},{"cell_type":"code","source":"eval_datagen = ImageDataGenerator(rescale=1./255)\neval_generator = eval_datagen.flow_from_directory(\n    train_raw_root_path,\n    target_size=(input_height, input_width),\n    batch_size=1,\n    class_mode=None,\n    shuffle=False)\n\nprint('evaluation(all pictures(raw train dataset) ：', len(data_img_list))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:59:42.481537Z","iopub.execute_input":"2021-07-03T16:59:42.482680Z","iopub.status.idle":"2021-07-03T16:59:43.738465Z","shell.execute_reply.started":"2021-07-03T16:59:42.482632Z","shell.execute_reply":"2021-07-03T16:59:43.737403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_pred_proba = model.predict(\n    eval_generator,\n    steps=len(data_img_list),\n    verbose=0)\n\n# probability value is predicted\nprint(eval_pred_proba.shape)\neval_pred_class = np.where(eval_pred_proba < 0.5, 0, 1)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T16:59:43.740127Z","iopub.execute_input":"2021-07-03T16:59:43.740583Z","iopub.status.idle":"2021-07-03T17:01:43.904881Z","shell.execute_reply.started":"2021-07-03T16:59:43.740538Z","shell.execute_reply":"2021-07-03T17:01:43.903614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_class = []\ntrue_class_name = []\nfor f in os.listdir(train_raw_path):\n    class_name = f.split('.')[0]\n    true_class_name.append(class_name)\n    if class_name == 'cat':\n        true_class.append(0)\n    elif class_name == 'dog':\n        true_class.append(1)\n    else:\n        continue\n        \nprint(true_class[0:10])\nprint(true_class_name[0:10])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:43.906567Z","iopub.execute_input":"2021-07-03T17:01:43.907207Z","iopub.status.idle":"2021-07-03T17:01:43.950484Z","shell.execute_reply.started":"2021-07-03T17:01:43.907145Z","shell.execute_reply":"2021-07-03T17:01:43.949032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\neval_cmx = confusion_matrix(true_class, eval_pred_class)\n\n# DataFrame\neval_cmx_df = pd.DataFrame(eval_cmx, columns=target_labels, index=target_labels)\n\nprint('Confution matrix with train all pics data')\n\n# 結果の表示\neval_cmx_df","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:43.953959Z","iopub.execute_input":"2021-07-03T17:01:43.954401Z","iopub.status.idle":"2021-07-03T17:01:44.029274Z","shell.execute_reply.started":"2021-07-03T17:01:43.954358Z","shell.execute_reply":"2021-07-03T17:01:44.027989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score\n\nprint('accuracy:', accuracy_score(true_class, eval_pred_class))\nprint('precision:', precision_score(true_class, eval_pred_class, average='macro'))\nprint('recall:', recall_score(true_class, eval_pred_class, average='macro'))\nprint('f1:', f1_score(true_class, eval_pred_class, average='macro'))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:44.031346Z","iopub.execute_input":"2021-07-03T17:01:44.031815Z","iopub.status.idle":"2021-07-03T17:01:44.169819Z","shell.execute_reply.started":"2021-07-03T17:01:44.031769Z","shell.execute_reply":"2021-07-03T17:01:44.168473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nsns.heatmap(eval_cmx, annot=True, cmap='Blues')\nplt.savefig(root_path + '/' + 'sklearn_confusion_matrix.png')","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:44.171658Z","iopub.execute_input":"2021-07-03T17:01:44.172188Z","iopub.status.idle":"2021-07-03T17:01:44.562383Z","shell.execute_reply.started":"2021-07-03T17:01:44.172149Z","shell.execute_reply":"2021-07-03T17:01:44.561082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Prediction with Test data","metadata":{}},{"cell_type":"code","source":"# Again, test path folder shall have one more folder at least and test data shall be stored in the folder\ntest_datagen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_datagen.flow_from_directory(\n    test_path,\n    target_size=(input_height, input_width),\n    batch_size=1,\n    class_mode=None,\n    shuffle=False)\n\nprint('test data quantity：', len(test_img_list))","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:44.564003Z","iopub.execute_input":"2021-07-03T17:01:44.564550Z","iopub.status.idle":"2021-07-03T17:01:44.913659Z","shell.execute_reply.started":"2021-07-03T17:01:44.564508Z","shell.execute_reply":"2021-07-03T17:01:44.912565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_proba = model.predict(\n    test_generator,\n    steps=len(test_img_list),\n    verbose=0)\n\nprint(test_pred_proba.shape)\ntest_pred_class = np.where(test_pred_proba < 0.5, 0, 1)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:01:44.916861Z","iopub.execute_input":"2021-07-03T17:01:44.917197Z","iopub.status.idle":"2021-07-03T17:02:45.264566Z","shell.execute_reply.started":"2021-07-03T17:01:44.917165Z","shell.execute_reply":"2021-07-03T17:02:45.263485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Create Submission file","metadata":{}},{"cell_type":"code","source":"test_filename_list=[]\nfor f in test_img_list:\n    test_filename_list.append(f.split('.')[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:02:45.266216Z","iopub.execute_input":"2021-07-03T17:02:45.266674Z","iopub.status.idle":"2021-07-03T17:02:45.279447Z","shell.execute_reply.started":"2021-07-03T17:02:45.266628Z","shell.execute_reply":"2021-07-03T17:02:45.278157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_id = pd.DataFrame(test_filename_list, columns=['id'])\nresult_label = pd.DataFrame(test_pred_class, columns=['label'])\nresult_for_submit = pd.concat([result_id, result_label], axis=1)\nresult_for_submit.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:02:45.280833Z","iopub.execute_input":"2021-07-03T17:02:45.281357Z","iopub.status.idle":"2021-07-03T17:02:45.299579Z","shell.execute_reply.started":"2021-07-03T17:02:45.281310Z","shell.execute_reply":"2021-07-03T17:02:45.298115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_for_submit.to_csv(submit_file, sep=\",\", header=True, index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T17:02:45.301587Z","iopub.execute_input":"2021-07-03T17:02:45.302132Z","iopub.status.idle":"2021-07-03T17:02:45.657236Z","shell.execute_reply.started":"2021-07-03T17:02:45.302087Z","shell.execute_reply":"2021-07-03T17:02:45.656193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"------------------------unimplimented-------------------------------","metadata":{}},{"cell_type":"code","source":"'''\n# ファイル名に付ける日時データの文字列作成\nimport datetime\n\ndt_now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9)))\nd_today = datetime.date.today()\n\nfiletime = str(d_today)+'_'+str(dt_now.hour)+str(dt_now.minute)\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nhist  = model.fit(\n    train_generator,\n    validation_data = valid_generator,\n    steps_per_epoch = len(df_train)*0.9//batch_size,\n    validation_steps = len(df_train)*0.1//batch_size, \n    epochs = epochs,\n    # callbacks = [model_checkpoint, early_stopping])\n    callbacks = [model_checkpoint, learning_rate_reduction, early_stopping])\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# cp_cb = ModelCheckpoint(model_path + '/checkpoint/' + \"{epoch}\"+\"checkpoint.h5\", verbose=1, save_best_only=True)\nmodel_checkpoint = ModelCheckpoint(\n        filepath = f'{root_path}/checkpoint.h5',\n        save_weights_only=True,\n        monitor='val_accuracy',\n        mode='max',\n        save_best_only=True)\n'''","metadata":{},"execution_count":null,"outputs":[]}]}