{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"#将train目录下训练集数据和test目录下测试集数据读入列表中（文件路径）\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport math\nimport os\nimport shutil\nimport cv2\nimport sys\n\ntrain_image_path=\"../input/train/\"\ntest_image_path=\"../input/test/\"\ntrain_image_list=[]\ntest_image_list=[]\nrows = 299\ncols = 299\n   \n#从目录中所有文件读入到列表中\ndef get_image_list(path_name, list_name):\n    for file_name in os.listdir(path_name):\n        list_name.append(os.path.join(path_name, file_name))\n\nget_image_list(train_image_path, train_image_list)\nget_image_list(test_image_path, test_image_list)\nprint(\"train image sample:{}\\ntest image sample:{}\".format(len(train_image_list),len(test_image_list)))","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82aa53e411f96eaa9ee4cde4284b122b9abfbdb3"},"cell_type":"code","source":"#实现显示图片函数，并显示训练集中前10个和后10个图片\ndef display_img(img_list, summary = True):\n    fig = plt.figure(figsize=(15, 3 * math.ceil(len(img_list)/5)))\n    for i in range(0, len(img_list)):\n        img = cv2.imread(img_list[i])\n        img = img[:,:,::-1]#BGR->RGB\n        if summary:\n            print(\"---->image: {}  - shape: {}\".format(img_list[i], img.shape))\n        ax = fig.add_subplot(math.ceil(len(img_list)/5),5,i+1)\n        ax.set_title(os.path.basename(img_list[i]))\n        ax.set_xticks([])\n        ax.set_yticks([])\n        img = cv2.resize(img, (128,128))\n        ax.imshow(img)\n    plt.show()\n    \ndis_img_list = train_image_list[:10] + train_image_list[-10:]\ndisplay_img(dis_img_list)","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df66e520a0b9ea0351ddc32440e3c2cd795d3e8c"},"cell_type":"code","source":"#导入Xception模型（载入imagenet预训练权值）\nfrom keras.applications import *\nmodel_pre=xception.Xception(weights='imagenet')","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d7ea7d82e75fd8ca306b17b016dd09f5ec7d78f5"},"cell_type":"code","source":"#ImageNet 1000个类具体内容，参考文献 https://blog.csdn.net/zhangjunbob/article/details/53258524\n#定义猫狗种类\nDogs = [ 'n02085620','n02085782','n02085936','n02086079','n02086240','n02086646','n02086910','n02087046','n02087394','n02088094','n02088238',\n        'n02088364','n02088466','n02088632','n02089078','n02089867','n02089973','n02090379','n02090622','n02090721','n02091032','n02091134',\n        'n02091244','n02091467','n02091635','n02091831','n02092002','n02092339','n02093256','n02093428','n02093647','n02093754','n02093859',\n        'n02093991','n02094114','n02094258','n02094433','n02095314','n02095570','n02095889','n02096051','n02096177','n02096294','n02096437',\n        'n02096585','n02097047','n02097130','n02097209','n02097298','n02097474','n02097658','n02098105','n02098286','n02098413','n02099267',\n        'n02099429','n02099601','n02099712','n02099849','n02100236','n02100583','n02100735','n02100877','n02101006','n02101388','n02101556',\n        'n02102040','n02102177','n02102318','n02102480','n02102973','n02104029','n02104365','n02105056','n02105162','n02105251','n02105412',\n        'n02105505','n02105641','n02105855','n02106030','n02106166','n02106382','n02106550','n02106662','n02107142','n02107312','n02107574',\n        'n02107683','n02107908','n02108000','n02108089','n02108422','n02108551','n02108915','n02109047','n02109525','n02109961','n02110063',\n        'n02110185','n02110341','n02110627','n02110806','n02110958','n02111129','n02111277','n02111500','n02111889','n02112018','n02112137',\n        'n02112350','n02112706','n02113023','n02113186','n02113624','n02113712','n02113799','n02113978']\nCats=['n02123045','n02123159','n02123394','n02123597','n02124075','n02125311','n02127052']","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"72bc845c53ed8cca37741f5603e2c71dc6d551b7"},"cell_type":"code","source":"#实现判断是否为cat/dog判断函数\ndef batch_img(img_path_list, batch_size):\n    '''split img_path_list into batches'''\n    for begin in range(0, len(img_path_list), batch_size):\n        end = min(begin+batch_size, len(img_path_list))\n        yield img_path_list[begin:end]\n        \ndef read_batch_img(batch_imgpath_list):\n    '''read batch img and resize'''\n    images = np.zeros((len(batch_imgpath_list), 299, 299, 3), dtype=np.uint8)\n    for i in range(len(batch_imgpath_list)):\n        img = cv2.imread(batch_imgpath_list[i])\n        img = img[:,:,::-1]\n        img = cv2.resize(img, (299,299))\n        images[i] = img\n    return images\n\ndef pred_pet(model, img_path_list, top_num, preprocess_input, decode_predictions, batch_size = 32):\n    '''predict img\n    #returns\n        the list, will show pet or not\n    '''\n    ret = []\n    for batch_imgpath_list in batch_img(img_path_list, batch_size):\n        X = read_batch_img(batch_imgpath_list)\n        X = preprocess_input(X)\n        preds = model.predict(X)\n        dps = decode_predictions(preds, top = top_num)\n        for index in range(len(dps)):\n            for i, val in enumerate(dps[index]):\n                if (val[0] in Dogs) and ('dog' in batch_imgpath_list[index]):\n                    ret.append(True)\n                    break\n                elif (val[0] in Cats) and ('cat' in batch_imgpath_list[index]):\n                    ret.append(True)\n                    break\n                if i==len(dps[index])-1:\n                    ret.append(False)\n    return ret     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"daf394c1c21675515e515ad39e210e6661b5533a"},"cell_type":"code","source":"#显示并剔除异常值\ndef get_abnormal_v(train_image_list, topN = 50):\n    abnormal_v = []\n    if os.path.exists(\"./abnormal.txt\"):\n        with open(\"./abnormal.txt\", 'r') as f:\n            items = f.readlines()\n            abnormal_v = [item.strip('\\n') for item in items]\n    else:\n        ret =[]\n        ret = pred_pet(model_pre, train_image_list, topN, xception.preprocess_input, xception.decode_predictions)\n        for i,val in enumerate(ret):\n            if not val:\n                abnormal_v.append(train_image_list[i])\n        with open(\"./abnormal.txt\", 'w') as f:\n            for item in abnormal_v:\n                f.write(\"{}\\n\".format(item))\n    return abnormal_v\n\nabnormal_v = get_abnormal_v(train_image_list, topN=50)\n# display_img(abnormal_v, summary = False)\ntrain_image_list = [item for item in train_image_list if item not in abnormal_v]\nfor i in abnormal_v:\n    os.remove(i) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cf564fdc2ab4cc39a03b2ab0406e7c3e5080e0f7"},"cell_type":"code","source":"#shuffle\nimport random\nrandom.shuffle(train_image_list)\n\n#创建train和val目录，将20%图片作为验证集，80%作为训练集，move猫狗图片\nimport shutil\ndef create_dir_and_move_file(image_list):\n    dir_list = [\"./train/dogs\", \"./train/cats\", \"./val/dogs\" ,\"./val/cats\"]\n    # 创建多级目录\n    for dir_path in dir_list:\n        if os.path.exists(dir_path):\n            shutil.rmtree(dir_path)\n        os.makedirs(dir_path)  \n    \n    # 移动文件或目录\n    for file_path in image_list[:math.ceil(len(image_list)/5)]:\n        if \"dog\" in file_path:\n            shutil.move(file_path,\"./val/dogs\")\n        else:\n            shutil.move(file_path, \"./val/cats\") \n    for file_path in image_list[math.ceil(len(image_list)/5):]:\n        if \"dog\" in file_path:\n            shutil.move(file_path,\"./train/dogs\")\n        else:\n            shutil.move(file_path, \"./train/cats\")\ncreate_dir_and_move_file(train_image_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d866153e243c4ed9c3befc109419a95f81949577"},"cell_type":"code","source":"# create the base pre-trained model\nbase_model = xception.Xception(weights='imagenet', input_shape = (299,299,3), include_top=False, pooling='avg')\n\nx = base_model.output\n\nfrom keras.models import Model\nfrom keras.layers import Dense\n# 二分类分类器\npredictions = Dense(1, activation='sigmoid')(x)\n# this is the model we will train\nmodel = Model(inputs=base_model.input, outputs=predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"daea53ae56152b0cfa03e0ebb82995014b350551"},"cell_type":"code","source":"#创建回调函数\nfrom keras import callbacks\nclass LossHistory(callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.losses = []\n        self.val_losses = []\n        self.acc = []\n        self.val_acc = []\n\n    def on_epoch_end(self, batch, logs={}):\n        self.losses.append(logs.get('loss'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.acc.append(logs.get('acc'))\n        self.val_acc.append(logs.get('val_acc'))\n\nhistory = LossHistory()\nearlyStopping = callbacks.EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"dc89d60d27d805964bb0e986d4de35371c6410f3"},"cell_type":"code","source":"#设置超参\nepochs = 20\nbatch_size = 32\nnb_validation_samples = math.ceil(len(train_image_list)/ 5)\nnb_train_samples = 4 * nb_validation_samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b6f96f1cd1c6e5c01ac9fffc48176a55a37b8169"},"cell_type":"code","source":"#冻结Xception所有层，值训练top layers\nfor layer in base_model.layers:\n    layer.trainable = False   \n#编译模型    \nfrom keras.optimizers import SGD\nmodel.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='binary_crossentropy', metrics=['accuracy'])\n#图片数据增强\nfrom keras.preprocessing.image import ImageDataGenerator\n#训练数据增强\ntrain_datagen = ImageDataGenerator( preprocessing_function=xception.preprocess_input, \n                                    shear_range=0.2, \n                                    zoom_range=0.2, \n                                    horizontal_flip=True)\n#验证数据增强\nvalidation_datagen = ImageDataGenerator(preprocessing_function=xception.preprocess_input)\ntrain_generator = train_datagen.flow_from_directory( \"./train/\", \n                                                     target_size = (rows, cols),\n                                                     batch_size = batch_size,\n                                                     class_mode='binary',\n                                                     shuffle=True)\nvalidation_generator = validation_datagen.flow_from_directory( \"./val/\", \n                                                               target_size = (rows, cols),\n                                                               batch_size = batch_size, \n                                                               class_mode='binary',\n                                                               shuffle=True)\n#训练模型\nmodel.fit_generator(train_generator, \n                    steps_per_epoch=math.ceil(nb_train_samples/batch_size),\n                    epochs=epochs, \n                    validation_data=validation_generator, \n                    validation_steps=math.ceil(nb_validation_samples/batch_size),\n                    callbacks=[history, earlyStopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"33d59408fad500ede18544db5c5576620c6af699"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ea27df9fa1b127b9b57201ae5ae6d3c98d243819"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}