{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-26T06:43:34.069167Z","iopub.execute_input":"2021-07-26T06:43:34.069654Z","iopub.status.idle":"2021-07-26T06:43:34.082366Z","shell.execute_reply.started":"2021-07-26T06:43:34.069603Z","shell.execute_reply":"2021-07-26T06:43:34.080887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, cv2, random\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import ticker\nimport seaborn as sns\n%matplotlib inline \n\nfrom keras.models import Sequential\nfrom keras.layers import Input, Dropout, Flatten, Dense, Activation, Conv2D, BatchNormalization, MaxPooling2D\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\nfrom keras.utils import np_utils\n%env KERAS_BACKEND=theano\n\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:04:45.224671Z","iopub.execute_input":"2021-07-26T08:04:45.225316Z","iopub.status.idle":"2021-07-26T08:04:52.286296Z","shell.execute_reply.started":"2021-07-26T08:04:45.225228Z","shell.execute_reply":"2021-07-26T08:04:52.285461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#zipファイルの解凍\n! unzip \"../input/dogs-vs-cats-redux-kernels-edition/train.zip\" -d train\n! unzip \"../input/dogs-vs-cats-redux-kernels-edition/test.zip\" -d test","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-26T08:04:56.467813Z","iopub.execute_input":"2021-07-26T08:04:56.468331Z","iopub.status.idle":"2021-07-26T08:05:15.997724Z","shell.execute_reply.started":"2021-07-26T08:04:56.468281Z","shell.execute_reply":"2021-07-26T08:05:15.996342Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#1.データの準備\nTRAIN_DIR = './train/train/'\nTEST_DIR = './test/test/'\n\n#画像サイズ=ROWS×COLS\nROWS = 64\nCOLS = 64\nCHANNELS = 3\n\n#学習データ数、テストデータ数\nTRAIN_NO = 7500  #(max:7500)\nVALID_NO = 5000  #(max:5000)\n#TEST_NO = 25\n\n\ndef read_image(file_path):\n    img = cv2.imread(file_path, cv2.IMREAD_COLOR) #cv2.IMREAD_GRAYSCALE\n    \n    return cv2.resize(img, (ROWS, COLS), interpolation=cv2.INTER_CUBIC)\n\ndef ex_data(data,ex_type):\n    count = len(data)\n    ex_data = np.ndarray((3,count, CHANNELS, ROWS, COLS), dtype=np.uint8)\n    if ex_type == 0: \n        for j, image in enumerate(data): \n            ex_data[0][j] = cv2.flip(image.T, 0).T\n            ex_data[1][j] = cv2.flip(image.T, 1).T\n            ex_data[2][j] = cv2.flip(image.T, -1).T\n    elif ex_type == 1:\n        for j, image in enumerate(data): \n            ex_data[0][j] = cv2.rotate(image.T, cv2.ROTATE_90_CLOCKWISE).T\n            ex_data[1][j] = cv2.rotate(image.T, cv2.ROTATE_180).T\n            ex_data[2][j] = cv2.rotate(image.T, cv2.ROTATE_90_COUNTERCLOCKWISE).T\n                \n    return ex_data[0].tolist() + ex_data[1].tolist() + ex_data[2].tolist()\n\ndef prep_data(images):\n    count = len(images)\n    data = np.ndarray((count, CHANNELS, ROWS, COLS), dtype=np.uint8)\n    for i, image_file in enumerate(images):\n        image = read_image(image_file)\n        data[i] = image.T\n        #if i%250 == 0: print('Processed {} of {}'.format(i, count))\n\n    return data\n\ndef make_labels(images):\n    labels = []\n    for i in images:\n        if 'dog' in i:\n            labels.append(1)\n        else:\n            labels.append(0)\n            \n    return labels\n\n#train_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] \ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\ntest_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\n#学習用データをから犬と猫をランダムにTRAIN_NOずつ選ぶ\nrandom.seed(123456)\nrandom.shuffle(train_dogs)\nrandom.shuffle(train_cats)\ntrain_images = train_dogs[:TRAIN_NO] + train_cats[:TRAIN_NO]\n\n#検証用データ(学習用データからかぶらないように選ぶ)\nvalid_images = train_dogs[7500:7500+VALID_NO] + train_cats[7500:7500+VALID_NO]\n\n#train_dogsとtrain_catsの開放\ndel train_dogs\ndel train_cats\ngc.collect()\n\n#学習用データの下処理と拡張(いったんlistにしてデータを結合してからndarrayに戻す)\ntrain = prep_data(train_images).tolist()    #original\n#train = train + ex_data(np.array(train),0)    #original + 反転\n#train = train + ex_data(np.array(train),1)    #original + 回転\n#train = train + ex_data(np.array(train),0) + ex_data(np.array(train),1)    #original + 反転 + 回転\ntrain = np.array(train)\n\n#教師データの作成(model.fit()ではnd.array型)\nlabels_train = np.array(make_labels(train_images))    #original\n#labels_train = np.array(make_labels(train_images)*4)    #original + 反転(or回転)\n#labels_train = np.array(make_labels(train_images)*7)    #original + 反転 + 回転\n\n#学習用データとその教師データのシャッフル(ndarray型)\nnp.random.seed(123)\nnp.random.shuffle(train)\nnp.random.seed(123)\nnp.random.shuffle(labels_train)\n\n#検証用データの下処理とその教師データ\nvalid = prep_data(valid_images)\nlabels_valid = np.array(make_labels(valid_images))\n\n#テストデータ(提出用のデータ)\n#test_images =  test_images\n\n#テストデータの下処理\ntest = prep_data(test_images)\n\n#テストデータのid\ntest_id = []\nfor i,path in enumerate(test_images):\n    temp = path.replace('./test/test/','')\n    test_id.append(int(temp.replace('.jpg','')))\n\nprint('(data_no,channels,cols.row)')\nprint(\"Train shape: {}\".format(train.shape))\nprint(\"Valid shape: {}\".format(valid.shape))\nprint(\"labels_train shape: {}\".format(len(labels_train)))\nprint(\"labels_valid shape: {}\".format(len(labels_valid)))\nprint('train:{}'.format(type(train)))\nprint('labels_train:{}'.format(type(labels_train)))\nprint(\"Test shape: {}\".format(test.shape))\nprint(\"Test_id: {}\".format(len(test_id)))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:25:59.123514Z","iopub.execute_input":"2021-07-26T08:25:59.123892Z","iopub.status.idle":"2021-07-26T08:28:38.153374Z","shell.execute_reply.started":"2021-07-26T08:25:59.123858Z","shell.execute_reply":"2021-07-26T08:28:38.152048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2.ニューラルネットワークの構築\noptimizer = RMSprop(lr=1e-4)\nobjective = 'binary_crossentropy'\n\ndef catdog():\n    \n    model = Sequential()\n    \n    model.add(Conv2D(32, (3, 3), activation='relu', strides=(1, 1),padding='same', input_shape=(CHANNELS, ROWS, COLS)))\n    model.add(Conv2D(32, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\n    model.add(Conv2D(64, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(Conv2D(64, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\n    model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\n    model.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1),padding='same'))\n    model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n    \n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n\n    model.add(Dense(1))\n    model.add(Activation('sigmoid'))\n    model.compile(loss=objective, optimizer=optimizer, metrics=['accuracy'])\n    \n    return model\n\n\nmodel = catdog()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:28:46.238025Z","iopub.execute_input":"2021-07-26T08:28:46.238405Z","iopub.status.idle":"2021-07-26T08:28:46.532996Z","shell.execute_reply.started":"2021-07-26T08:28:46.238371Z","shell.execute_reply":"2021-07-26T08:28:46.531987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#3.学習、4.検証\nnb_epoch = 100\nbatch_size = 16\n\ncheckpoint_path = 'XXX.h5'\n\n## Callback for loss logging per epoch\nclass LossHistory(Callback):\n    def on_train_begin(self, logs={}):\n        self.losses = []\n        self.accuracy = []\n        self.val_losses = []\n        self.val_accuracy = []\n        \n    def on_epoch_end(self, batch, logs={}):\n        self.losses.append(logs.get('loss'))\n        self.accuracy.append(logs.get('accuracy'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.val_accuracy.append(logs.get('val_accuracy'))\n        \ndef run_catdog():\n    history = LossHistory()\n    early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto')\n    checkpoint = ModelCheckpoint(filepath=checkpoint_path,monitor='val_loss',verbose=1,save_best_only=True,save_weights_only=False,mode='min',period=1)\n    model.fit(train, labels_train, batch_size=batch_size, epochs=nb_epoch,\n                        validation_data=(valid,labels_valid), verbose=1, shuffle=True,callbacks=[history,early_stopping,checkpoint])\n    #predictions = model.predict(np.array(test), verbose=1)\n    \n    return history\n\nhistory = run_catdog()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:30:14.820848Z","iopub.execute_input":"2021-07-26T08:30:14.821250Z","iopub.status.idle":"2021-07-26T08:46:53.806500Z","shell.execute_reply.started":"2021-07-26T08:30:14.821218Z","shell.execute_reply":"2021-07-26T08:46:53.805481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#結果の保存\n\n#ファイル名\nfname ='T'+str(len(train))+'_V'+str(len(valid))\n\nloss = history.losses\nval_loss = history.val_losses\naccuracy = history.accuracy\nval_accuracy = history.val_accuracy\n\n#データフレームとして保存\ndf = pd.DataFrame({'loss':loss,'val_loss':val_loss,\n                   'accuracy':accuracy,'val_accuracy':val_accuracy})\ndf.to_csv(fname+'_b'+str(batch_size)+'.txt')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:52:00.812393Z","iopub.execute_input":"2021-07-26T08:52:00.812817Z","iopub.status.idle":"2021-07-26T08:52:00.825358Z","shell.execute_reply.started":"2021-07-26T08:52:00.812783Z","shell.execute_reply":"2021-07-26T08:52:00.824226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#val_loss最小となったモデルを読み込み\n\n#検証データで保存できているか確認\nmodel.load_weights(checkpoint_path)\nloss,acc = model.evaluate(valid,  labels_valid, verbose=2)\n\n#テストデータの予測\npredictions = model.predict(np.array(test), verbose=1)\n\n#提出用ファイルに書き込む\nsubmission = pd.read_csv('../input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission['id'] = test_id\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:52:27.035848Z","iopub.execute_input":"2021-07-26T08:52:27.036238Z","iopub.status.idle":"2021-07-26T08:52:39.449003Z","shell.execute_reply.started":"2021-07-26T08:52:27.036206Z","shell.execute_reply":"2021-07-26T08:52:39.448065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ディレクトリの整理\nimport shutil\n\nos.remove('XXX.h5')\nshutil.rmtree('./train/')\nshutil.rmtree('./test/')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T08:52:55.270843Z","iopub.execute_input":"2021-07-26T08:52:55.271199Z","iopub.status.idle":"2021-07-26T08:52:56.619191Z","shell.execute_reply.started":"2021-07-26T08:52:55.271170Z","shell.execute_reply":"2021-07-26T08:52:56.618247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}