{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30512,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-19T05:35:28.87149Z","iopub.execute_input":"2024-07-19T05:35:28.872411Z","iopub.status.idle":"2024-07-19T05:35:28.886474Z","shell.execute_reply.started":"2024-07-19T05:35:28.87238Z","shell.execute_reply":"2024-07-19T05:35:28.885499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport glob\nimport shutil\nimport tensorflow as tf  # 追加\n\n\nprint(\"TensorFlow version:\", tf.__version__)\nprint(\"Keras version:\",tf.keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:28.888712Z","iopub.execute_input":"2024-07-19T05:35:28.889422Z","iopub.status.idle":"2024-07-19T05:35:28.895096Z","shell.execute_reply.started":"2024-07-19T05:35:28.889396Z","shell.execute_reply":"2024-07-19T05:35:28.894258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvcc --version\n!cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -A 2\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\nprint(\"TensorFlow version:\", tf.__version__)\nprint(\"CUDA version:\", tf.sysconfig.get_build_info()['cuda_version'])\nprint(\"cuDNN version:\", tf.sysconfig.get_build_info()['cudnn_version'])","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:28.896112Z","iopub.execute_input":"2024-07-19T05:35:28.896403Z","iopub.status.idle":"2024-07-19T05:35:30.955927Z","shell.execute_reply.started":"2024-07-19T05:35:28.89638Z","shell.execute_reply":"2024-07-19T05:35:30.954706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"データの読み込み","metadata":{}},{"cell_type":"code","source":"# Delete existing train and test directories recursively\nif os.path.exists('/kaggle/working/train'):\n    shutil.rmtree('/kaggle/working/train')\nif os.path.exists('/kaggle/working/test'):\n    shutil.rmtree('/kaggle/working/test')\nif os.path.exists('/kaggle/working/val'):\n    shutil.rmtree('/kaggle/working/val') \n\n# zipファイルを自分の作業するディレクトリに解凍\nimport zipfile\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as existing_zip:\n    existing_zip.extractall()\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as existing_zip:\n    existing_zip.extractall()\n\n\n# 自分の作業するディレクトリを確認\nfiles = glob.glob('/kaggle/working/*')\nfor file in files:\n    print(file)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:30.957622Z","iopub.execute_input":"2024-07-19T05:35:30.957945Z","iopub.status.idle":"2024-07-19T05:35:40.763237Z","shell.execute_reply.started":"2024-07-19T05:35:30.957915Z","shell.execute_reply":"2024-07-19T05:35:40.762179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データを正しく読み込めたかを確認","metadata":{}},{"cell_type":"code","source":"# trainファイルの中の確認\nfiles = sorted(glob.glob('/kaggle/working/train/*'))\nprint('trainフォルダの中のtrain用の画像ファイル', len(files), '枚')\n\n# testファイルの中の確認\nfiles = sorted(glob.glob('/kaggle/working/test/*'))\nprint('testフォルダの中のtest用の画像ファイル', len(files), '枚')\n\n# 学習させる犬、猫の画像数の確認\nn_dog = 0# 犬用\nn_cat = 0# 猫用\nfiles = glob.glob('/kaggle/working/train/*')\nfor file in files:\n    if 'cat' in file:\n        n_cat += 1\n    else:\n        n_dog += 1\nprint('学習させる犬の画像数：', n_dog, '枚')\nprint('学習させる猫の画像数：', n_cat, '枚')","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:40.766966Z","iopub.execute_input":"2024-07-19T05:35:40.767312Z","iopub.status.idle":"2024-07-19T05:35:41.002502Z","shell.execute_reply.started":"2024-07-19T05:35:40.767285Z","shell.execute_reply":"2024-07-19T05:35:41.001562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 犬と猫の画像をそれぞれ保存するファイルパスを定義\ndog_dir = '/kaggle/working/train/dog'\ncat_dir = '/kaggle/working/train/cat'\n\n# ファイル作成\nif(os.path.exists(dog_dir) == False):\n    os.mkdir(dog_dir)\nif(os.path.exists(cat_dir) == False):\n    os.mkdir(cat_dir)\n\n# 犬と猫の画像を分ける\nfiles = glob.glob('/kaggle/working/train/*.jpg')\nfor file in files:\n    file_name = os.path.basename(file)\n    if 'cat' in file:\n        shutil.move(file,'/kaggle/working/train/cat/' + file_name)\n    else:\n        shutil.move(file,'/kaggle/working/train/dog/' + file_name)","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:41.003631Z","iopub.execute_input":"2024-07-19T05:35:41.003923Z","iopub.status.idle":"2024-07-19T05:35:41.908688Z","shell.execute_reply.started":"2024-07-19T05:35:41.003898Z","shell.execute_reply":"2024-07-19T05:35:41.907651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データを分ける","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n#バリデーションデータ用のファイルパスを定義\nval = '/kaggle/working/val'\nval_dog = '/kaggle/working/val/dog'\nval_cat = '/kaggle/working/val/cat'\n\n# ファイル作成\nif(os.path.exists(val) == False):\n    os.mkdir(val)\n    os.mkdir(val_dog)\n    os.mkdir(val_cat)\n\n# 犬の訓練データの一部を，バリデーションデータ用に分割\ndog_files = glob.glob('/kaggle/working/train/dog/*.jpg')\ndog_train, dog_val = train_test_split(dog_files, test_size=0.2, random_state=42)\n# 画像の一部をバリデーションフォルダに移動させる\nfor file in dog_val:\n    file_name = os.path.basename(file)\n    shutil.move(file,'/kaggle/working/val/dog/' + file_name)\n\n# 猫の訓練データの一部を，バリデーションデータ用に分割\ncat_files = glob.glob('/kaggle/working/train/cat/*.jpg')\ncat_train, cat_val = train_test_split(cat_files, test_size=0.2, random_state=42)\n# バリデーションフォルダに移動させる\nfor file in cat_val:\n    file_name = os.path.basename(file)\n    shutil.move(file,'/kaggle/working/val/cat/' + file_name)","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:41.909933Z","iopub.execute_input":"2024-07-19T05:35:41.910258Z","iopub.status.idle":"2024-07-19T05:35:42.167985Z","shell.execute_reply.started":"2024-07-19T05:35:41.910232Z","shell.execute_reply":"2024-07-19T05:35:42.167166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習モデルの定義","metadata":{}},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential , Model\nfrom keras.layers import Dense, Dropout, Flatten , Input\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers\n\n#入力画像のサイズ\nimg_width, img_height = 224, 224\n\ntrain_data_dir = '/kaggle/working/train'\ntest_data_dir = '/kaggle/working/val'\n\n# 分類するクラスの設定\nclasses = ['dog','cat']\nnb_classes = len(classes)\n\n\n# VGG16モデルのロード\nvgg_model = VGG16(\n        include_top = False,\n        weights = 'imagenet',\n        input_shape = (img_height, img_width, 3))\n\n\n# VGG16モデルの下に全結合層を追加\ntop_model = Sequential()\ntop_model.add(Flatten(input_shape= vgg_model.output_shape[1:]))\ntop_model.add(Dense(256, activation='relu'))\ntop_model.add(Dropout(0.5))\ntop_model.add(Dense(nb_classes, activation='softmax'))\n\nmodel = Model( vgg_model.input,top_model(vgg_model.output))\n\nfor layer in vgg_model.layers[:15]:\n    layer.trainable = False\n\nmodel.summary()\n\n# 最適化関数のパラメータ設定\nsgd = optimizers.SGD(learning_rate = 0.001, momentum = 0.1, decay = 0.0)\n\n# 損失関数は交差エントロピー、最適化関数は確率的勾配法\nmodel.compile(\n        loss = 'categorical_crossentropy',\n        optimizer = sgd,\n        metrics = ['accuracy'])\n\n# 学習データのデータを正規化\ntrain_datagen = ImageDataGenerator(rescale = 1.0 / 255)\n\n# 検証データのデータを正規化\ntest_datagen = ImageDataGenerator(rescale = 1.0 / 255)\n\n# 訓練データ\ntrain_generator = train_datagen.flow_from_directory(\n        train_data_dir,\n        target_size = (img_height, img_width),\n        classes = classes,\n        batch_size = 32,\n        class_mode = 'categorical')\n\n# 検証データ\ntest_generator = test_datagen.flow_from_directory(\n        test_data_dir,\n        target_size = (img_height, img_width),\n        classes = classes,\n        batch_size = 32,\n        class_mode = 'categorical')\n","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:42.169236Z","iopub.execute_input":"2024-07-19T05:35:42.169537Z","iopub.status.idle":"2024-07-19T05:35:43.318066Z","shell.execute_reply.started":"2024-07-19T05:35:42.169492Z","shell.execute_reply":"2024-07-19T05:35:43.317287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"学習スタート","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint\n# 保存するファイルパス\ncheckpoint_filepath = '/kaggle/working/best_model.keras'\n\n# モデルチェックポイントのコールバックを作成\nmodel_checkpoint_callback = ModelCheckpoint(\n    filepath=checkpoint_filepath,\n    save_weights_only=False,\n    monitor='val_loss',\n    mode='min',\n    save_best_only=True,\n    verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:43.319121Z","iopub.execute_input":"2024-07-19T05:35:43.31944Z","iopub.status.idle":"2024-07-19T05:35:43.324908Z","shell.execute_reply.started":"2024-07-19T05:35:43.319414Z","shell.execute_reply":"2024-07-19T05:35:43.323909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nepoch = 2\n\n# モデルの学習する\nhistory = model.fit_generator(\n    train_generator,\n    epochs=epoch,\n    validation_data = test_generator,\n    callbacks=[model_checkpoint_callback])\n    \n#学習済みモデルの保存\nmodel.save(\"/kaggle/working/dog_cat.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:35:43.326282Z","iopub.execute_input":"2024-07-19T05:35:43.32693Z","iopub.status.idle":"2024-07-19T05:36:44.747249Z","shell.execute_reply.started":"2024-07-19T05:35:43.326905Z","shell.execute_reply":"2024-07-19T05:36:44.745583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習過程のlossとaccuracyの表示","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline \n\nplt.figure(1, figsize=(13,4))\nplt.subplots_adjust(wspace=0.5)\n\n# 学習曲線\nplt.subplot(1, 2, 1)\nplt.plot(history.history[\"loss\"], label=\"train\")\nplt.plot(history.history[\"val_loss\"], label=\"valid\")\nplt.title(\"train and valid loss\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"loss\")\nplt.legend()\nplt.grid()\n\n# 精度表示\nplt.subplot(1, 2, 2)\nplt.plot(history.history[\"accuracy\"], label=\"train\")\nplt.plot(history.history[\"val_accuracy\"], label=\"valid\")\nplt.title(\"train and valid accuracy\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"accuracy\")\nplt.legend()\nplt.grid()\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:36:44.74832Z","iopub.status.idle":"2024-07-19T05:36:44.748837Z","shell.execute_reply.started":"2024-07-19T05:36:44.748573Z","shell.execute_reply":"2024-07-19T05:36:44.748599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras.utils as image\nfrom keras.models import load_model\nimport re\nimport csv\nimport glob","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:36:44.750255Z","iopub.status.idle":"2024-07-19T05:36:44.750599Z","shell.execute_reply.started":"2024-07-19T05:36:44.750422Z","shell.execute_reply":"2024-07-19T05:36:44.750437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 予測スタート","metadata":{}},{"cell_type":"code","source":"import sys\n\np = re.compile(r'\\d+')\nimcount = 1\ntest_list = glob.glob('/kaggle/working/test/*.jpg')\ntest_list = sorted(test_list,key=lambda s: int(p.search(s).group()))\n\n#学習済みモデルの読み込み\nmodel = load_model('../working/best_model.keras')\n\n#csv書き込み\nwith open('/kaggle/working/submission.csv', 'w') as f:\n    writer = csv.writer(f, lineterminator='\\n')\n    writer.writerow(['id', 'label'])\n    print(\"start\")\n    imcount = 1\n    for img_path in test_list:\n        img = image.load_img(img_path, target_size=(224, 224))\n        x = image.img_to_array(img)\n        x = np.expand_dims(x, axis=0)\n        x = x / 255.0  # 正規化\n\n        result_predict = model.predict(x, verbose=0)\n        predicted_class = np.argmax(result_predict, axis=1)\n\n        writer.writerow([imcount, result_predict[0][0]])\n\n        sys.stdout.write(f\"\\rProcessing image {imcount}/{len(test_list)}\")\n        sys.stdout.flush()\n\n#         print(f\"Predicted class: {predicted_class[0]}, Probabilities: {result_predict[0]}\")\n        print(f\"Predicted class: {predicted_class[0]}, Probabilities: {result_predict[0]}\")\n        imcount += 1\n    print(\"end\")\n","metadata":{"execution":{"iopub.status.busy":"2024-07-19T05:36:44.753088Z","iopub.status.idle":"2024-07-19T05:36:44.753468Z","shell.execute_reply.started":"2024-07-19T05:36:44.753301Z","shell.execute_reply":"2024-07-19T05:36:44.753319Z"},"trusted":true},"execution_count":null,"outputs":[]}]}