{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1.Inputデータパスの確認\n◯右記３点を確認①「sample_submission.csv」②「train.zip」③「test.zip」","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:06.003864Z","iopub.execute_input":"2023-07-28T07:29:06.005027Z","iopub.status.idle":"2023-07-28T07:29:06.057307Z","shell.execute_reply.started":"2023-07-28T07:29:06.004982Z","shell.execute_reply":"2023-07-28T07:29:06.056159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.Inputデータの解凍及びパスの確認\n◯「train.zip」「test.zip」を解凍し,解凍先のパスを確認","metadata":{}},{"cell_type":"code","source":"# 解凍&確認に必要なライブラリのインポート\nimport shutil\nimport glob\n\n# Inputデータ内、「train.zip」「test.zip」をOutputデータとして使用ディレクトリで解凍(学習&テスト用)\nshutil.unpack_archive('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip', '/kaggle/working/unpack')\nshutil.unpack_archive('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip', '/kaggle/working/unpack')\n\n# Inputデータ内、「sample_submission.csv」をOutputデータとして使用ディレクトリにコピー(提出用)\nshutil.copyfile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv','/kaggle/working/sample_submission.csv')\n     \n# /kaggle/workingフォルダ以下に入っているファイルを確認\nfiles = glob.glob('/kaggle/working/*')\nfor file in files:\n    print(file)\nfiles = glob.glob('/kaggle/working/unpack/*')\nfor file in files:\n    print(file)\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:06.059866Z","iopub.execute_input":"2023-07-28T07:29:06.060699Z","iopub.status.idle":"2023-07-28T07:29:24.560607Z","shell.execute_reply.started":"2023-07-28T07:29:06.060658Z","shell.execute_reply":"2023-07-28T07:29:24.559182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3.学習データとテストデータの確認\n◯学習データ(訓練データ&検証データ)とテストデータの画像枚数を確認\n* 学習データ：「犬：12500枚」「猫：12500枚」\n* テストデータ：「犬猫混合：12500枚」","metadata":{}},{"cell_type":"code","source":"# Outputデータ内、「train」「test」ディレクトリ内のデータを確認\nTrain_data_list = glob.glob(os.path.join(\"unpack/train\",'*.jpg'))\nprint(f\"Train_data_num: {len(Train_data_list)}\")\n\nTest_data_list = glob.glob(os.path.join(\"unpack/test\",'*.jpg'))\nprint(f\"Test_Data_num: {len(Test_data_list)}\")\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:24.562154Z","iopub.execute_input":"2023-07-28T07:29:24.563156Z","iopub.status.idle":"2023-07-28T07:29:24.805185Z","shell.execute_reply.started":"2023-07-28T07:29:24.563114Z","shell.execute_reply":"2023-07-28T07:29:24.804054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4.Datasetディレクトリの作成&階層構造の確認\n◯データの学習,分類に使用するためのdatasetディレクトリを作成&階層構造の確認","metadata":{}},{"cell_type":"code","source":"# ディレクトリ構造を描画するためのライブラリをインストール\n!pip install Tree\nprint(\"----------------------\")\n\n# ./datasetディレクトリ以下のパス\ntrain_dir = \"./dataset/trainData\"\nvalid_dir = \"./dataset/validData\"\ntest_dir = \"./dataset/testData\"\n\n# 訓練データ\nos.makedirs(\"%s/dogs\" %train_dir, exist_ok=True)\nos.makedirs(\"%s/cats\" %train_dir, exist_ok=True)\n# 検証データ\nos.makedirs(\"%s/dogs\" %valid_dir, exist_ok=True)\nos.makedirs(\"%s/cats\" %valid_dir, exist_ok=True)\n# テストデータ\nos.makedirs(\"%s\" %test_dir, exist_ok=True)\n\n!tree -d ./\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:24.808058Z","iopub.execute_input":"2023-07-28T07:29:24.809201Z","iopub.status.idle":"2023-07-28T07:29:44.369359Z","shell.execute_reply.started":"2023-07-28T07:29:24.809164Z","shell.execute_reply":"2023-07-28T07:29:44.367993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5.学習データとテストデータをDatasetディレクトリに移動\n◯移動元ディレクトリ「unpack」から移動先ディレクトリ「dataset」にデータを移動","metadata":{}},{"cell_type":"code","source":"# データ移動元パスを定義\nsource_dir = \"./unpack\"\n\n# 最初の10000枚をtrainDataフォルダに格納\nfor i in range(10000):\n  os.rename(\"%s/train/dog.%d.jpg\" %(source_dir, i+0), \"%s/dogs/dog%04d.jpg\" %(train_dir, i+0))\n  os.rename(\"%s/train/cat.%d.jpg\" %(source_dir, i+0), \"%s/cats/cat%04d.jpg\" %(train_dir, i+0))\n\n# 次の2500枚をvalidDataフォルダに格納\nfor i in range(2500):\n  os.rename(\"%s/train/dog.%d.jpg\" %(source_dir, 10000+i+0), \"%s/dogs/dog%04d.jpg\" %(valid_dir, i+0))\n  os.rename(\"%s/train/cat.%d.jpg\" %(source_dir, 10000+i+0), \"%s/cats/cat%04d.jpg\" %(valid_dir, i+0))\n\n# /unpack/test/*.jpgをtestDataフォルダに格納\nread_dir = './unpack/test/'\nread_files = os.listdir(read_dir)\nfor read_name in read_files:\n    copy_from_path = read_dir + read_name\n    copy_to_path = test_dir + '/' + read_name \n    shutil.copy(copy_from_path, copy_to_path)\n    \nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:44.371550Z","iopub.execute_input":"2023-07-28T07:29:44.371967Z","iopub.status.idle":"2023-07-28T07:29:46.792878Z","shell.execute_reply.started":"2023-07-28T07:29:44.371926Z","shell.execute_reply":"2023-07-28T07:29:46.791164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6.Datasetの確認\n◯datasetディレクトリの中身を確認","metadata":{}},{"cell_type":"code","source":"# Outputデータ内、「train」「test」ディレクトリ内のデータを確認\nTrain_data_dogs_list = glob.glob(os.path.join(\"dataset/trainData/dogs/\",'*.jpg'))\nTrain_data_cats_list = glob.glob(os.path.join(\"dataset/trainData/cats/\",'*.jpg'))\nprint(f\"trainData_dogs: {len(Train_data_dogs_list)}\")\nprint(f\"trainData_cats: {len(Train_data_cats_list)}\")\n\nValid_data_dogs_list = glob.glob(os.path.join(\"dataset/validData/dogs/\",'*.jpg'))\nValid_data_cats_list = glob.glob(os.path.join(\"dataset/validData/cats/\",'*.jpg'))\nprint(f\"validData_dogs: {len(Valid_data_dogs_list)}\")\nprint(f\"validData_cats: {len(Valid_data_cats_list)}\")\n\nTest_data_list = glob.glob(os.path.join(\"dataset/testData/\",'*.jpg'))\nprint(f\"testData_num: {len(Test_data_list)}\")\n\n!tree -d ./dataset\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:46.796449Z","iopub.execute_input":"2023-07-28T07:29:46.796765Z","iopub.status.idle":"2023-07-28T07:29:48.179323Z","shell.execute_reply.started":"2023-07-28T07:29:46.796737Z","shell.execute_reply":"2023-07-28T07:29:48.177391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7.必要ライブラリのインポート\n◯学習モデルの構築,分類に必要なライブラリをまとめてインポート","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline \n\nimport math\nimport re\nimport csv\nimport urllib.request\nimport tqdm\n\nfrom tensorflow import keras\nfrom tensorflow.keras import models, optimizers, layers\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow.keras.layers as layers\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.layers import Activation, BatchNormalization, Input\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras.callbacks import EarlyStopping\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:48.181351Z","iopub.execute_input":"2023-07-28T07:29:48.181809Z","iopub.status.idle":"2023-07-28T07:29:58.345913Z","shell.execute_reply.started":"2023-07-28T07:29:48.181765Z","shell.execute_reply":"2023-07-28T07:29:58.344554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8.ジェネレータの設定\n◯モデルにデータを読み込ませるためのジェネレータを設定","metadata":{}},{"cell_type":"code","source":"# バッチサイズの指定\nbatch_size = 32\n\n# データ拡張しながら，テンソル画像データのバッチを生成\ntrain_data_generator = ImageDataGenerator(rescale=1.0/255,\n                                          shear_range=0.2,\n                                          zoom_range=0.2,\n                                          horizontal_flip=True,\n                                          vertical_flip = True)\n\nvalid_data_generator = ImageDataGenerator(rescale=1.0/255)\n\n# ディレクトリからデータを生成\ntrain_generator = train_data_generator.flow_from_directory(\n    train_dir,\n    target_size=(224,224),\n    batch_size=batch_size,\n    class_mode=\"binary\",\n    shuffle=True\n)\n\nvalid_generator = valid_data_generator.flow_from_directory(\n    valid_dir,\n    target_size=(224,224),\n    batch_size=batch_size,\n    class_mode=\"binary\",\n    shuffle=True\n)\n\n# ----- {'cats': 0, 'dogs': 1} ------#\nprint(train_generator.class_indices)\nprint(valid_generator.class_indices)\n\nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:58.347620Z","iopub.execute_input":"2023-07-28T07:29:58.348399Z","iopub.status.idle":"2023-07-28T07:29:59.694139Z","shell.execute_reply.started":"2023-07-28T07:29:58.348367Z","shell.execute_reply":"2023-07-28T07:29:59.692967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9.学習モデルの構築\n◯学習済みモデルである「VGG16」を使用,犬・猫の2値分類用に全結合層を作成し従来のVGG16に結合","metadata":{}},{"cell_type":"code","source":"# VGG16をインポート\nmodel_vgg16 = VGG16(include_top=False, weights=\"imagenet\", input_tensor=Input(shape=(224,224,3)))\n\n# 全結合層を作成\nx = model_vgg16.output\nx = Flatten()(x)\nx = Dense(256)(x)\nx = Activation(\"relu\")(x)\nx = Dropout(0.1)(x)\nprediction = Dense(1, activation=\"sigmoid\")(x)\n\n# モデル合体\nmodel = Model(inputs=model_vgg16.input, outputs=prediction)\n\n# モデル構築 \nmodel.compile(loss = 'binary_crossentropy',\n             optimizer = optimizers.Adam(learning_rate = 1e-4),\n             metrics = ['accuracy'])\n\n# 構築したモデルの確認\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:29:59.695891Z","iopub.execute_input":"2023-07-28T07:29:59.696338Z","iopub.status.idle":"2023-07-28T07:30:05.427282Z","shell.execute_reply.started":"2023-07-28T07:29:59.696302Z","shell.execute_reply":"2023-07-28T07:30:05.426464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 10.モデルの学習\n◯学習済みモデルである「VGG16」をファインチューニング","metadata":{}},{"cell_type":"code","source":"# block5_pool(18層目)までの重みの更新をキャンセル\nfor layer in model_vgg16.layers[:15]:\n  layer.trainable=False\n\n# 学習\nhistory = model.fit(train_generator,\n                    epochs=5,\n                    verbose=1,\n                    validation_data=valid_generator,\n                    callbacks=[EarlyStopping(patience=10)])\n\n# モデル保存\nmodel.save('VGG16_cat_vs_dogs.h5')\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T07:30:05.430563Z","iopub.execute_input":"2023-07-28T07:30:05.430949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 11.訓練データ,検証データに対するLoss値,Acc値の確認","metadata":{}},{"cell_type":"code","source":"# lossとaccuracyを可視化\nprint(\"-\"*100)\nplt.figure(1, figsize=(13,4))\nplt.subplots_adjust(wspace=0.5)\n\n# 学習曲線\nplt.subplot(1, 2, 1)\nplt.plot(history.history[\"loss\"], label=\"train\")\nplt.plot(history.history[\"val_loss\"], label=\"valid\")\nplt.title(\"train and valid loss\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"loss\")\nplt.legend()\nplt.grid()\n\n# 精度表示\nplt.subplot(1, 2, 2)\nplt.plot(history.history[\"accuracy\"], label=\"train\")\nplt.plot(history.history[\"val_accuracy\"], label=\"valid\")\nplt.title(\"train and valid accuracy\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"accuracy\")\nplt.legend()\nplt.grid()\n\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 12.テストデータの分類","metadata":{}},{"cell_type":"code","source":"# testData内の写真を番号順にソート\np = re.compile(r'\\d+')\ntest_list = glob.glob('/kaggle/working/dataset/testData/*.jpg')\ntest_list = sorted(test_list, key = lambda s: int(p.search(s).group()))\n# print(test_list)\nimcount = 1\n\n#モデルの読み込み\nmodel = models.load_model('VGG16_cat_vs_dogs.h5')\n\n#csvに書き込み\nwith open('/kaggle/working/sample_submission.csv', 'w') as f:\n    writer = csv.writer(f, lineterminator='\\n')\n    writer.writerow(['id', 'label'])\n    \n    print(\" ~ テスト開始 ~ \")\n    for i in tqdm.tqdm(range(len(test_list))):\n        img = image.load_img(test_list[i], target_size = (224, 224))\n        x = image.img_to_array(img)\n        x = np.expand_dims(x, axis=0)\n        x = x / 255.0\n        \n        result_predict = model.predict(x,verbose=0)\n        writer.writerow([imcount, result_predict[0][0]])\n        imcount += 1\n        \nprint(\"\")\nprint(\"Completed\")\nprint(\"\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}