{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-25T05:16:16.916950Z","iopub.execute_input":"2023-07-25T05:16:16.917549Z","iopub.status.idle":"2023-07-25T05:16:16.932857Z","shell.execute_reply.started":"2023-07-25T05:16:16.917506Z","shell.execute_reply":"2023-07-25T05:16:16.931658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\n\n# train.zipの展開\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip', 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/train')\n\n# test.zipの展開\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip', 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/test')\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:16:16.935183Z","iopub.execute_input":"2023-07-25T05:16:16.935598Z","iopub.status.idle":"2023-07-25T05:16:34.434688Z","shell.execute_reply.started":"2023-07-25T05:16:16.935565Z","shell.execute_reply":"2023-07-25T05:16:34.432127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ディレクトリの作成","metadata":{}},{"cell_type":"code","source":"import os\n\n# 新しいディレクトリのパスを指定\ndirectory_path = [\n    \"/kaggle/working/data/train/dogs\",\n    \"/kaggle/working/data/train/cats\",\n    \"/kaggle/working/data/valid/dogs\",\n    \"/kaggle/working/data/valid/cats\",\n    \"/kaggle/working/data/test\",\n]\n\n# ディレクトリを作成\nfor path in directory_path:\n    os.makedirs(path, exist_ok=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:16:34.451016Z","iopub.execute_input":"2023-07-25T05:16:34.458604Z","iopub.status.idle":"2023-07-25T05:16:34.482353Z","shell.execute_reply.started":"2023-07-25T05:16:34.457751Z","shell.execute_reply":"2023-07-25T05:16:34.479282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの移動","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nimport glob\n\nsource_directory = \"/kaggle/working/train/train\"\ndirectory = [\n    \"/kaggle/working/data/train/dogs\",\n    \"/kaggle/working/data/train/cats\",\n]\ntest_source_directory = \"/kaggle/working/test/test\"\ntest_directory = \"/kaggle/working/data/test/\"\n\nCATEGORIES = [\"dog.\", \"cat.\"]\n\n# 訓練(train)データの画像ファイルを所定の場所へを移動\nfor key,DIR in zip(CATEGORIES,directory):\n    files = [file for file in os.listdir(source_directory) if file.startswith(key)]\n    for file in files:\n        source_path = os.path.join(source_directory, file)\n        destination_path = os.path.join(DIR, file)\n        shutil.move(source_path, destination_path)\n\n# テスト(test)データの画像ファイルを所定の場所へを移動\nfiles = [file for file in os.listdir(test_source_directory) if file.endswith(\".jpg\")]\nfor file in files:\n    source_path = os.path.join(test_source_directory, file)\n    destination_path = os.path.join(test_directory, file)\n    shutil.move(source_path, destination_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:16:34.496747Z","iopub.execute_input":"2023-07-25T05:16:34.503773Z","iopub.status.idle":"2023-07-25T05:16:36.293576Z","shell.execute_reply.started":"2023-07-25T05:16:34.503730Z","shell.execute_reply":"2023-07-25T05:16:36.292539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 確認用","metadata":{}},{"cell_type":"code","source":"dogs_directory = \"/kaggle/working/data/train/dogs\"\ncats_directory = \"/kaggle/working/data/train/cats\"\ntest_directory = \"/kaggle/working/data/test/\"\n\n# dogsディレクトリ内のファイルを表示\nprint(\"Dogs:\")\ndog_files = os.listdir(dogs_directory)\nprint(f'file number : {len(dog_files)}')\nfor file in dog_files[:5]:\n    print(file)\n\n# catsディレクトリ内のファイルを表示\nprint(\"\\nCats:\")\ncat_files = os.listdir(cats_directory)\nprint(f'file number : {len(cat_files)}')\nfor file in cat_files[:5]:\n    print(file)\n\n# dogsディレクトリ内のファイルを表示\nprint(\"\\ntest:\")\ntest_files = os.listdir(test_directory)\nprint(f'file number : {len(test_files)}')\nfor file in test_files[:5]:\n    print(file)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:16:36.295082Z","iopub.execute_input":"2023-07-25T05:16:36.295491Z","iopub.status.idle":"2023-07-25T05:16:36.330312Z","shell.execute_reply.started":"2023-07-25T05:16:36.295452Z","shell.execute_reply":"2023-07-25T05:16:36.329140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習の準備","metadata":{}},{"cell_type":"markdown","source":"## 使用するライブラリのインポート","metadata":{}},{"cell_type":"code","source":"# -------------------------------------------------------------------\n# 各種ライブラリのインポート\n# -------------------------------------------------------------------\n! pip install --upgrade tensorflow-io\nimport matplotlib.pyplot as plt\nimport glob\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.utils import to_categorical","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:16:36.331786Z","iopub.execute_input":"2023-07-25T05:16:36.332255Z","iopub.status.idle":"2023-07-25T05:17:02.011617Z","shell.execute_reply.started":"2023-07-25T05:16:36.332220Z","shell.execute_reply":"2023-07-25T05:17:02.010462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルに入力出来るようにデータの形を変形","metadata":{}},{"cell_type":"code","source":"# -------------------------------------------------------------------\n# 定数の定義\n# -------------------------------------------------------------------\nDATADIR = \"/kaggle/working/data/train\"\n# MODEL_DIR = \"/content/drive/MyDrive\"\nCATEGORIES = [\"dogs\", \"cats\"]\n\n# -------------------------------------------------------------------\n# メイン処理（画像を読込み、犬猫データセットを設定）\n# -------------------------------------------------------------------\n#犬猫データセットを定義\nfeature_data = []\nlabel_data = []\n    \n# 平均画像サイズ\ntarget_size = (128, 128)\nprint(target_size)\n\n#画像を読込み、犬猫データセットを設定\n#（0:Dog、1:Cat)…class_numはCATEGORIESの配列index／categoryはCATEGORIESの中身　\nfor class_num, category in enumerate(CATEGORIES):\n    #ディレクトリを設定（ループ1回目：./PetImages/Dog　2回目：./PetImages/Cat)\n    path = os.path.join(DATADIR, category)\n    for image_name in os.listdir(path):\n        try:\n            img_file = cv2.imread(os.path.join(path, image_name))# 画像読み込み\n            gray_image = cv2.cvtColor(img_file, cv2.COLOR_BGR2GRAY)\n            resize_file = cv2.resize(gray_image, (target_size))# 画像のリサイズ\n            feature_data.append(resize_file)\n            label_data.append(class_num)# 画像データ(犬 or 猫)、ラベル情報(0 or 1)を追加\n        except Exception as e:\n            #画像の読込みエラーはスルー（スキップして次の画像へ）\n            print(\"skiped.\")\n            pass\n    print(f\"{category}finish.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:17:02.015098Z","iopub.execute_input":"2023-07-25T05:17:02.015715Z","iopub.status.idle":"2023-07-25T05:18:05.766147Z","shell.execute_reply.started":"2023-07-25T05:17:02.015686Z","shell.execute_reply":"2023-07-25T05:18:05.764927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 訓練データと検証データに分割","metadata":{}},{"cell_type":"code","source":"train_x, valid_x, train_y, valid_y = train_test_split(feature_data, label_data, test_size=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:18:05.767797Z","iopub.execute_input":"2023-07-25T05:18:05.768220Z","iopub.status.idle":"2023-07-25T05:18:05.788971Z","shell.execute_reply.started":"2023-07-25T05:18:05.768186Z","shell.execute_reply":"2023-07-25T05:18:05.788029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルに入力出来るようにデータの形を変形（最終調整）","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf # TensorFlowライブラリをロード\n# データ変換(正規化とテンソル変換)\ndef convert_data(x,y):\n    batch,a,b = x.shape # データの形（シェイプ）を取得（32×28×28）\n    x = tf.reshape(x,(batch,a,b,1)) # 1チャンネル分増やし，4次元テンソルに変換\n    x = tf.cast(x,tf.float32) # float32に変換\n    x /= 255. # 正規化\n    return x, tf.cast(y,tf.int32)\n\n#\n# tf.Dataによるtf.Tensor変換\n#\n\ntrain_ds = tf.data.Dataset.from_tensor_slices((train_x,train_y)) # 訓練セットの入力と教師ラベルを合体させてTensorに変換\nvalid_ds = tf.data.Dataset.from_tensor_slices((valid_x,valid_y)) # 検証用も同様に処理\nAUTOTUNE = tf.data.experimental.AUTOTUNE # 自動チューニング設定 (おまじない)\n\ntrain_ds = train_ds.shuffle(len(train_x)) # 訓練データをシャッフルする\ntrain_ds = train_ds.repeat(1) # １epochで使われるデータの回数。１の場合１epochで１回しか使われない。引数を空欄にすると無限に使われる\ntrain_ds = train_ds.batch(32) # ミニパッチを作る。１パッチ３２個のデータ。\ntrain_ds = train_ds.map(lambda x, y: tf.py_function(convert_data, [x, y], Tout=[tf.float32, tf.int32]))\ntrain_ds = train_ds.prefetch(buffer_size=AUTOTUNE) # 訓練中に次のパッチを取り出すための処理\n\nvalid_ds = valid_ds.batch(32) # 検証データはシャッフルする必要がないので,パッチ化のみの処理\nvalid_ds = valid_ds.map(lambda x, y: tf.py_function(convert_data, [x, y], Tout=[tf.float32, tf.int32]))","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:18:05.790419Z","iopub.execute_input":"2023-07-25T05:18:05.790779Z","iopub.status.idle":"2023-07-25T05:19:39.810574Z","shell.execute_reply.started":"2023-07-25T05:18:05.790747Z","shell.execute_reply":"2023-07-25T05:19:39.809596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 確認用","metadata":{}},{"cell_type":"code","source":"train_ds","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:19:39.818767Z","iopub.execute_input":"2023-07-25T05:19:39.821175Z","iopub.status.idle":"2023-07-25T05:19:39.834075Z","shell.execute_reply.started":"2023-07-25T05:19:39.821138Z","shell.execute_reply":"2023-07-25T05:19:39.832993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルの作成","metadata":{}},{"cell_type":"code","source":"# 入力層\ninput = tf.keras.layers.Input(shape=(128,128,1), name='input') # ⼊⼒層の定義\n# CNN 第一層\nh = tf.keras.layers.Conv2D(32, (9, 9), activation='relu', name='cnn01')(input)\nh = tf.keras.layers.MaxPooling2D((2, 2), name='pool01')(h)\n# CNN 第二層\nh = tf.keras.layers.Conv2D(32, (5, 5), activation='relu', name='cnn02')(h)\nh = tf.keras.layers.MaxPooling2D((2, 2), name='pool02')(h)\n# CNN 第三層\nh = tf.keras.layers.Conv2D(64, (5, 5), activation='relu', name='cnn03')(h)\nh = tf.keras.layers.MaxPooling2D((2, 2), name='pool03')(h)\n# CNN 第四層\nh = tf.keras.layers.Conv2D(64, (5, 5), activation='relu', name='cnn04')(h)\nh = tf.keras.layers.MaxPooling2D((2, 2), name='pool04')(h)\n# 1次元テンソルに変換する。フラット化。\nh = tf.keras.layers.Flatten(name='flatten')(h)\n# 全結合層 隠れ層のノードは 256\nh = tf.keras.layers.Dense(256, activation='relu', name='dense01')(h)\n# 出力層\noutput = tf.keras.layers.Dense(2, activation='softmax', name='output')(h) # 出⼒層\n# この処理でモデルを実体化する。⼊⼒層と出⼒層を渡すと⾃動的にネットワークを構築してくれる。\nmodel = tf.keras.Model(inputs=input, outputs=output)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:19:39.838883Z","iopub.execute_input":"2023-07-25T05:19:39.839248Z","iopub.status.idle":"2023-07-25T05:19:40.099813Z","shell.execute_reply.started":"2023-07-25T05:19:39.839216Z","shell.execute_reply":"2023-07-25T05:19:40.099058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルの設定","metadata":{}},{"cell_type":"code","source":"# 学習のためのmodelの設定（楽する⽅法）\n# model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics='accuracy')\nmodel.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:19:40.100924Z","iopub.execute_input":"2023-07-25T05:19:40.101531Z","iopub.status.idle":"2023-07-25T05:19:40.153879Z","shell.execute_reply.started":"2023-07-25T05:19:40.101495Z","shell.execute_reply":"2023-07-25T05:19:40.152957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルを学習","metadata":{}},{"cell_type":"code","source":"model.fit(train_ds,epochs=5,validation_data=valid_ds)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:19:40.158378Z","iopub.execute_input":"2023-07-25T05:19:40.161001Z","iopub.status.idle":"2023-07-25T05:20:38.347234Z","shell.execute_reply.started":"2023-07-25T05:19:40.160965Z","shell.execute_reply":"2023-07-25T05:20:38.346268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# テストデータの準備","metadata":{}},{"cell_type":"code","source":"# -------------------------------------------------------------------\n# 定数の定義\n# -------------------------------------------------------------------\nTESTDIR = \"/kaggle/working/data/test\"\n# MODEL_DIR = \"/content/drive/MyDrive\"\nCATEGORIES = [\"dogs\", \"cats\"]\n\n# -------------------------------------------------------------------\n# メイン処理（画像を読込み、犬猫データセットを設定）\n# -------------------------------------------------------------------\n#犬猫データセットを定義\nfeature_test_data = []\nlabel_test_data = []\n    \n# 平均画像サイズ\ntarget_size = (128, 128)\nprint(f'target_size = {target_size}')\n\n#画像を読込み、犬猫データセットを設定\nfor image_name in os.listdir(TESTDIR):\n    try:\n        img_file = cv2.imread(os.path.join(TESTDIR, image_name))# 画像読み込み\n        gray_image = cv2.cvtColor(img_file, cv2.COLOR_BGR2GRAY)\n        resize_file = cv2.resize(gray_image, (target_size))# 画像のリサイズ\n        feature_test_data.append(resize_file)\n        #label_test_data.append(class_num)# 画像データ(犬 or 猫)、ラベル情報(0 or 1)を追加\n    except Exception as e:\n        #画像の読込みエラーはスルー（スキップして次の画像へ）\n        print(\"skiped.\")\n        pass\nprint(f\"finish.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:20:38.348646Z","iopub.execute_input":"2023-07-25T05:20:38.349100Z","iopub.status.idle":"2023-07-25T05:21:10.301839Z","shell.execute_reply.started":"2023-07-25T05:20:38.349064Z","shell.execute_reply":"2023-07-25T05:21:10.300911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルに入力出来るようにデータの形を変形（最終調整）","metadata":{}},{"cell_type":"code","source":"# テストデータの変換\n# X_test = X_test.reshape(X_test.shape[0], 28, 28)\n\ndef convert_data_test(x):\n    batch,a,b = x.shape # データの形（シェイプ）を取得（32×28×28）\n    x = tf.reshape(x,(batch,a,b,1)) # 1チャンネル分増やし，4次元テンソルに変換\n    x = tf.cast(x,tf.float32) # float32に変換\n    x /= 255. # 正規化\n    return x\n\ntest_ds = tf.data.Dataset.from_tensor_slices(feature_test_data) # 検証用も同様に処理\nAUTOTUNE = tf.data.experimental.AUTOTUNE # 自動チューニング設定 (おまじない)\n\ntest_ds = test_ds.batch(32) # 検証データはシャッフルする必要がないので,パッチ化のみの処理\ntest_ds = test_ds.map(lambda x: tf.py_function(convert_data_test, [x], Tout=[tf.float32]))\n\n#X_test, _ = convert_data(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:10.303518Z","iopub.execute_input":"2023-07-25T05:21:10.303985Z","iopub.status.idle":"2023-07-25T05:21:55.572626Z","shell.execute_reply.started":"2023-07-25T05:21:10.303946Z","shell.execute_reply":"2023-07-25T05:21:55.571678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 確認用","metadata":{}},{"cell_type":"code","source":"test_ds","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:55.574210Z","iopub.execute_input":"2023-07-25T05:21:55.574581Z","iopub.status.idle":"2023-07-25T05:21:55.582577Z","shell.execute_reply.started":"2023-07-25T05:21:55.574544Z","shell.execute_reply":"2023-07-25T05:21:55.581462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<_PrefetchDataset element_spec=(TensorSpec(shape=<unknown>, dtype=tf.float32, name=None), TensorSpec(shape=<unknown>, dtype=tf.int32, name=None))>","metadata":{}},{"cell_type":"markdown","source":"# データを予測","metadata":{}},{"cell_type":"code","source":"# テストデータの予測\ny_pred = model.predict(test_ds)\n\n# 予測結果の正解率の計算\npredictions = tf.argmax(y_pred, axis=1)\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:55.584139Z","iopub.execute_input":"2023-07-25T05:21:55.584753Z","iopub.status.idle":"2023-07-25T05:21:58.434166Z","shell.execute_reply.started":"2023-07-25T05:21:55.584720Z","shell.execute_reply":"2023-07-25T05:21:58.433067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 内容確認","metadata":{}},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:58.438732Z","iopub.execute_input":"2023-07-25T05:21:58.439750Z","iopub.status.idle":"2023-07-25T05:21:58.451687Z","shell.execute_reply.started":"2023-07-25T05:21:58.439708Z","shell.execute_reply":"2023-07-25T05:21:58.450715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 結果を提出出来る形に修正","metadata":{}},{"cell_type":"code","source":"pred_pd = pd.DataFrame(y_pred)\n# id列を作成\npred_pd.insert(0, 'id', range(1, len(pred_pd) + 1))\npred_pd.columns\npred_pd = pred_pd.drop([0], axis=1)\npred_pd = pred_pd.rename(columns={1: 'label'})\npred_pd\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:58.455231Z","iopub.execute_input":"2023-07-25T05:21:58.457198Z","iopub.status.idle":"2023-07-25T05:21:58.504744Z","shell.execute_reply.started":"2023-07-25T05:21:58.457154Z","shell.execute_reply":"2023-07-25T05:21:58.503792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 結果を保存","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv') # ひな形ファイルを読み込む\nprint(sub.shape)\n# subのlabel列をtest_predictedのlabel列で上書きする\nsub['label'] = pred_pd['label']\n# 結果を表示\nsub\nsub.to_csv('submission.csv', index=False) # 変換したファイルを保存","metadata":{"execution":{"iopub.status.busy":"2023-07-25T05:21:58.509220Z","iopub.execute_input":"2023-07-25T05:21:58.511584Z","iopub.status.idle":"2023-07-25T05:21:58.597571Z","shell.execute_reply.started":"2023-07-25T05:21:58.511549Z","shell.execute_reply":"2023-07-25T05:21:58.596462Z"},"trusted":true},"execution_count":null,"outputs":[]}]}