{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom keras.preprocessing.image import ImageDataGenerator, load_img # 数据增强\nfrom keras.utils import to_categorical #\nfrom sklearn.model_selection import train_test_split #\nimport matplotlib.pyplot as plt\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 超参数的设定\nFAST_RUN = False\nIMAGE_WIDTH=128\nIMAGE_HEIGHT=128\nIMAGE_SIZE=(IMAGE_WIDTH, IMAGE_HEIGHT)\nIMAGE_CHANNELS=3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 准备训练数据\nfilenames = os.listdir(\"/kaggle/input/dogs-vs-cats/train/train\")\ncategories = [] # 存放类别\nfor filename in filenames:\n    category = filename.split('.')[0] # 取出cat or dog\n    if category == 'dog':\n        categories.append(1)\n    else:\n        categories.append(0)\n# 将文件与分好类别相对应        \ndf = pd.DataFrame({\n    'filename': filenames,\n    'category': categories\n})\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['category'].value_counts().plot.bar() # 查看是否猫狗数量一致","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import *\n\nmodel = Sequential()\n\nmodel.add(Conv2D(32,(3,3),activation = 'relu',input_shape=(IMAGE_WIDTH,IMAGE_HEIGHT,IMAGE_CHANNELS))) #卷积层\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(128, (3, 3), activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\n# 全连接层\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(2, activation='softmax')) # 2 猫狗二分类问题\n\nmodel.compile(loss='categorical_crossentropy',optimizer='rmsprop',metrics=['accuracy'])\n\nmodel.summary() # 输出模型各层的具体参数情况","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nearlystop = EarlyStopping(patience=10) # 如发现loss相比上一个epoch训练没有下降，则经过patience个epoch后停止训练。\n\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', # 被监控的量\n                                            patience=2,        # 当patience个epoch过去而模型性能不提升时，学习率减少的动作会被触发\n                                            verbose=1,         # 日志打印格式： 进度条\n                                            factor=0.5,        # 每次减少学习率的因子，学习率将以lr = lr*factor的形式被减少\n                                            min_lr=0.00001)    # min_lr：学习率的下限\ncallbacks = [earlystop, learning_rate_reduction]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"category\"] = df[\"category\"].replace({0: 'cat', 1: 'dog'})  # 将0，1替换为字符串，后面的image genaretor用得到\n\n# train_test_split函数用于将矩阵随机划分为训练子集和验证子集，并返回划分好的训练集测试集样本和验证集测试集标签。\n# test_size 为测试集样本数目与原始样本数目之比；\n# random_state 随机数种子\ntrain_df, validate_df = train_test_split(df, test_size=0.20, random_state=42)\n# reset_index:重置索引列\ntrain_df = train_df.reset_index(drop=True)\nvalidate_df = validate_df.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['category'].value_counts().plot.bar() # 查看训练集10000cat10000dog","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validate_df['category'].value_counts().plot.bar() # 查看验证集2500cat和dog","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_train = train_df.shape[0] #20000条训练数据\ntotal_validate = validate_df.shape[0]\nbatch_size=15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 对训练集\n# 数据增强操作：ImageDataGenerator()是keras.preprocessing.image模块中的图片生成器，同时也可以在batch中对数据进行增强，扩充数据集大小，增强模型的泛化能力。\ntrain_datagen = ImageDataGenerator(\n    rotation_range=15, # 旋转范围\n    rescale=1./255,    # rescale参数指定将图像张量的数字缩放\n    shear_range=0.1,   # float, 透视变换的范围\n    zoom_range=0.2,    # 缩放范围\n    horizontal_flip=True,   # 水平反转\n    width_shift_range=0.1,  # 水平平移范围\n    height_shift_range=0.1  # 垂直平移范围\n)\n\n# 返回值：一个生成 (x, y) 元组的 DataFrameIterator， 其中：\n#         x 是一个包含一批尺寸为 (batch_size, *target_size, channels) 的图像样本的 numpy 数组，\n#         y 是对应的标签的 numpy 数组。\ntrain_generator = train_datagen.flow_from_dataframe(\n    train_df,                                  # Pandas dataframe\n    \"/kaggle/input/dogs-vs-cats/train/train\",  # 字符串，目标目录的路径，其中包含在 dataframe 中映射的所有图像\n    x_col='filename',                          # 字符串，dataframe 中包含目标图像文件夹的目录的列\n    y_col='category',                          # 字符串或字符串列表，dataframe 中将作为目标数据的列\n    target_size=IMAGE_SIZE,                    # 整数元组 (height, width)。 所有找到的图都会调整到这个维度。\n    class_mode='categorical',                  # \"categorical\" 将是 2D one-hot 编码标签\n    batch_size=batch_size                      # 批量数据的尺寸\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 对验证集\n# 验证集不用数据增强，只需缩放\nvalidation_datagen = ImageDataGenerator(rescale=1./255)\n# 生成对应的图像与标签的（x，y）的数组\nvalidation_generator = validation_datagen.flow_from_dataframe(\n    validate_df, \n    \"/kaggle/input/dogs-vs-cats/train/train\", \n    x_col='filename',\n    y_col='category',\n    target_size=IMAGE_SIZE,\n    class_mode='categorical',\n    batch_size=batch_size\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 将数据fit model\n# Keras中的fit()函数传入的x_train和y_train是被完整的加载进内存的,当然用起来很方便.\n# 但是如果我们数据量很大，那么是不可能将所有数据载入内存的，必将导致内存泄漏，这时候我们可以用fit_generator函数来进行训练\nepochs=3 if FAST_RUN else 20 # 训练轮次\nhistory = model.fit_generator(\n    train_generator,    # 训练数据生成器，20000个（x,y）元组\n    epochs=epochs,      # 轮次\n    validation_data=validation_generator,         # 验证集生成器\n    validation_steps=total_validate//batch_size,  # 仅当 validation_data 是一个生成器时才可用。 在停止前 generator 生成的总步数（样本批数）\n    steps_per_epoch=total_train//batch_size,      # 一个 epoch 完成并开始下一个 epoch 之前从 generator 产生的总步数（批次样本）\n    callbacks=callbacks                           # 在训练时调用的一系列回调函数\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(\"model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = os.listdir(\"/kaggle/input/dogs-vs-cats/test1/test1\")\ntest_df = pd.DataFrame({\n    'filename': test_filenames\n})\nnb_samples = test_df.shape[0]\n\ntest_gen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_gen.flow_from_dataframe(\n    test_df, \n    \"/kaggle/input/dogs-vs-cats/test1/test1\", \n    x_col='filename',\n    y_col=None,\n    class_mode=None,\n    target_size=IMAGE_SIZE,\n    batch_size=batch_size,\n    shuffle=False\n)\n\npredict = model.predict_generator(test_generator, steps=np.ceil(nb_samples/batch_size))\n\n\ntest_df['category'] = np.argmax(predict, axis=-1)\n\nlabel_map = dict((v,k) for k,v in train_generator.class_indices.items())\ntest_df['category'] = test_df['category'].replace(label_map)\n\ntest_df['category'] = test_df['category'].replace({ 'dog': 1, 'cat': 0 })\n\ntest_df['category'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = test_df.copy()\nsubmission_df['id'] = submission_df['filename'].str.split('.').str[0]\nsubmission_df['label'] = submission_df['category']\nsubmission_df.drop(['filename', 'category'], axis=1, inplace=True)\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}