{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\n\n# Ignore FutureWarning from numpy\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport keras.backend as K\nimport tensorflow as tf\n\nos.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n \n# The GPU id to use, usually either \"0\" or \"1\";\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\";\n\n# Allow growth of GPU memory, otherwise it will always look like all the memory is being used\nphysical_devices = tf.config.experimental.list_physical_devices('GPU')\ntf.config.experimental.set_memory_growth(physical_devices[0], True) \n\n# 路径\ninput_path = r'/kaggle/input/dogs-vs-cats-redux-kernels-edition/'\nroot_path = r'/kaggle/working/'\ntrain_path = os.path.join(root_path,'train')\ntest_path = os.path.join(root_path,'test')\ncsv_path = os.path.join(root_path,'test.csv')\ntsrbd_path = os.path.join(root_path,'tsrbd')\nmod_path = os.path.join(root_path,'model')\n\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-05-06T09:56:16.976897Z","iopub.execute_input":"2023-05-06T09:56:16.977228Z","iopub.status.idle":"2023-05-06T09:56:23.393481Z","shell.execute_reply.started":"2023-05-06T09:56:16.977203Z","shell.execute_reply":"2023-05-06T09:56:23.392498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取数据集\n设置图片维度为300×300×3，归一化，不使用数据增强。训练集中的20%用作交叉验证集。","metadata":{}},{"cell_type":"code","source":"#读取数据集\nfrom tensorflow.keras.preprocessing.image import load_img,img_to_array\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# 设置一些参数\nimg_width, img_height = 300,300\nbatch_size = 250\nepochs = 100\n\n# 使用ImageDataGenerator类来进行数据增强(training set中的20%用于验证集)\ntrain_datagen = ImageDataGenerator(rescale=1./255,validation_split=0.2)\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\n# 使用flow_from_directory方法来读取训练和验证数据集，epoch使用50，所以batchsize为25000/100=250\ntrain_generator = train_datagen.flow_from_directory(train_path, target_size=(img_width, img_height), batch_size=batch_size, class_mode='binary',subset='training')\nval_generator = train_datagen.flow_from_directory(train_path, target_size=(img_width, img_height), batch_size=batch_size, class_mode='binary',subset='validation')\n#不打乱test data数据集,设置shuffle为False\ntest_generator = test_datagen.flow_from_directory(root_path,target_size=(img_width, img_height), batch_size=batch_size, shuffle=False, classes=['test'])","metadata":{"execution":{"iopub.status.busy":"2023-05-06T00:46:38.833971Z","iopub.execute_input":"2023-05-06T00:46:38.834690Z","iopub.status.idle":"2023-05-06T00:46:39.870986Z","shell.execute_reply.started":"2023-05-06T00:46:38.834656Z","shell.execute_reply":"2023-05-06T00:46:39.870180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 构建CNN模型","metadata":{}},{"cell_type":"code","source":"#2D CNN\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Input, Conv2D, BatchNormalization, MaxPooling2D, Flatten, Dense, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import *\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\n\n# Set seed from random number generator, for better comparisons\nfrom numpy.random import seed\nseed(123)\n\ndef build_CNN(input_shape, n_conv_layers=2, n_filters=16, n_dense_layers=0, n_nodes=50, use_dropout=False, learning_rate=0.01):\n\n    # Setup a sequential model\n    model = Sequential()\n\n    # Add first convolutional layer to the model, requires input shape\n    model.add(Conv2D(n_filters, (3, 3), activation='relu', padding='same', input_shape=input_shape))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D((2, 2)))\n    \n    # Add remaining convolutional layers to the model, the number of filters should increase a factor 2 for each layer\n    for i in range(n_conv_layers-1):\n        model.add(Conv2D(n_filters*(2**(i+1)), (3, 3), activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(MaxPooling2D((2, 2)))\n    \n    # Add flatten layer\n    model.add(Flatten())\n    \n    # Add intermediate dense layers\n    for i in range(n_dense_layers):\n        model.add(Dense(n_nodes, activation='relu'))\n        model.add(BatchNormalization())\n        if(use_dropout):\n            model.add(Dropout(0.5))\n        \n        \n    # Add final dense layer\n    model.add(Dense(1, activation='sigmoid'))\n   \n    # Compile model\n    model.compile(optimizer=Adam(learning_rate=learning_rate), loss='binary_crossentropy', metrics=['accuracy']，\n                 callbacks=[reduce_lr])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-05T20:24:58.678237Z","iopub.execute_input":"2023-05-05T20:24:58.679205Z","iopub.status.idle":"2023-05-05T20:24:58.689041Z","shell.execute_reply.started":"2023-05-05T20:24:58.679171Z","shell.execute_reply":"2023-05-05T20:24:58.688040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets define a help function for plotting the training results\nimport matplotlib.pyplot as plt\ndef plot_results(history):\n    \n    loss = history.history['loss']\n    acc = history.history['accuracy']\n    val_loss = history.history['val_loss']\n    val_acc = history.history['val_accuracy']\n    \n    plt.figure(figsize=(10,4))\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.plot(loss)\n    plt.plot(val_loss)\n    plt.legend(['Training','Validation'])\n\n    plt.figure(figsize=(10,4))\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.plot(acc)\n    plt.plot(val_acc)\n    plt.legend(['Training','Validation'])\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-06T09:56:47.703850Z","iopub.execute_input":"2023-05-06T09:56:47.704529Z","iopub.status.idle":"2023-05-06T09:56:47.711461Z","shell.execute_reply.started":"2023-05-06T09:56:47.704498Z","shell.execute_reply":"2023-05-06T09:56:47.710635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **训练简单的8层CNN模型**\n（batch size=250,5层卷积层，过滤器3*3，第一层36个，随后每一层增加一倍，2层FC层，50个node，batchNorm，不使用droupout，学习率0.02，Adam优化器）","metadata":{}},{"cell_type":"code","source":"#训练神经网络\n# Setup some training parameters\nvalidation_split=0.2\nbatch_size = 250\nepochs = 100\ninput_shape = (300, 300, 3)\n\n# Build model (your best config)\nmodel = build_CNN(input_shape, n_conv_layers=5, n_filters=36, n_dense_layers=2, n_nodes=50, use_dropout=False, learning_rate=0.02)\n\nvalidation_split=0.2\n    \n# Train the model using on the fly augmentation\nhistory = model.fit(train_generator,batch_size=batch_size,epochs=epochs,\n                      steps_per_epoch=25000*(1 - validation_split)/batch_size,\n                      validation_steps=25000*validation_split/batch_size,\n                      validation_data=val_generator)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T20:29:01.857154Z","iopub.execute_input":"2023-05-05T20:29:01.857529Z","iopub.status.idle":"2023-05-05T23:23:35.525420Z","shell.execute_reply.started":"2023-05-05T20:29:01.857490Z","shell.execute_reply":"2023-05-05T23:23:35.524517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Summary\nmodel.summary()\n\n# Plot the history from the training run\nplot_results(history)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T23:24:10.854900Z","iopub.execute_input":"2023-05-05T23:24:10.855232Z","iopub.status.idle":"2023-05-05T23:24:11.370455Z","shell.execute_reply.started":"2023-05-05T23:24:10.855203Z","shell.execute_reply":"2023-05-05T23:24:11.369621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **使用CNN模型预测**\n**（注：因为模型训练过程中使用了归一化，所以需要预测的图像也应该归一化，且维度应为300×300×3）**","metadata":{}},{"cell_type":"code","source":"# 对测试数据进行预测\npreds = model.predict(test_generator)\n\n# 打印预测结果\n# print(preds)\n\n#写入文件提交\nresult=[0 if i <0.5 else 1 for i in preds]\nresult=np.array(result)\nid=np.array(range(1,12501))\n\nimport pandas as pd\ndf_result = pd.DataFrame({'id': id, 'label': result})\n\nprint(df_result)\ndf_result.to_csv('\\kaggle\\working\\my_data.csv', index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T23:52:19.304316Z","iopub.execute_input":"2023-05-05T23:52:19.304682Z","iopub.status.idle":"2023-05-05T23:53:07.926523Z","shell.execute_reply.started":"2023-05-05T23:52:19.304650Z","shell.execute_reply":"2023-05-05T23:53:07.925248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_result.head(50))","metadata":{"execution":{"iopub.status.busy":"2023-05-05T23:47:14.907706Z","iopub.execute_input":"2023-05-05T23:47:14.908052Z","iopub.status.idle":"2023-05-05T23:47:14.915263Z","shell.execute_reply.started":"2023-05-05T23:47:14.908022Z","shell.execute_reply":"2023-05-05T23:47:14.914353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#导出模型\nmodel.save('\\kaggle\\working\\my_cnn_dogcat_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-05-05T23:51:54.921811Z","iopub.execute_input":"2023-05-05T23:51:54.922202Z","iopub.status.idle":"2023-05-05T23:51:55.091071Z","shell.execute_reply.started":"2023-05-05T23:51:54.922167Z","shell.execute_reply":"2023-05-05T23:51:55.090207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **使用ResNet50迁移学习**","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Input, Conv2D, BatchNormalization, MaxPooling2D, Flatten, Dense, Dropout,GlobalAveragePooling2D\nfrom tensorflow.keras.applications.resnet50 import ResNet50,preprocess_input,decode_predictions\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import *\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\n\n# Set seed from random number generator, for better comparisons\nfrom numpy.random import seed\nseed(123)\n\n\ndef build_ResNet(input_shape,n_dense_layers=0, n_nodes=50, use_dropout=False, learning_rate=0.01):\n\n    #ResNet,不包括全连接层，使用imagenet预训练权重\n    resnet=ResNet50(weights='imagenet', include_top=False, input_shape=input_shape)\n    # 冻结所有卷积层\n    for layer in resnet.layers:\n        layer.trainable = False\n    \n    # Setup a sequential model\n    model = Sequential()\n    \n    #Add resnet\n    model.add(resnet)\n    \n    # Add global average pooling layer\n    model.add(GlobalAveragePooling2D())\n    \n    # Add intermediate dense layers\n    for i in range(n_dense_layers):\n        model.add(Dense(n_nodes, activation='relu'))\n        model.add(BatchNormalization())\n        if(use_dropout):\n            model.add(Dropout(0.25))\n        \n        \n    # Add final dense layer\n    model.add(Dense(1, activation='sigmoid'))\n   \n    # Compile model\n    model.compile(optimizer=Adam(learning_rate=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:04:30.062619Z","iopub.execute_input":"2023-05-06T10:04:30.063558Z","iopub.status.idle":"2023-05-06T10:04:30.072870Z","shell.execute_reply.started":"2023-05-06T10:04:30.063517Z","shell.execute_reply":"2023-05-06T10:04:30.071859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**对ResNet做迁移学习时不应当对训练数据集归一化，因为ResNet训练时并没有这样做。**","metadata":{}},{"cell_type":"code","source":"#重新读取未归一化的数据\nfrom tensorflow.keras.preprocessing.image import load_img,img_to_array\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# 设置一些参数\nimg_width, img_height = 300,300\nbatch_size = 250\nepochs = 100\n\n# 使用ImageDataGenerator类来进行数据增强(training set中的20%用于验证集)\ntrain_datagen = ImageDataGenerator(validation_split=0.2)\ntest_datagen = ImageDataGenerator()\n\n# 使用flow_from_directory方法来读取训练和验证数据集，epoch使用50，所以batchsize为25000/100=250\ntrain_generator = train_datagen.flow_from_directory(train_path, target_size=(img_width, img_height), batch_size=batch_size, class_mode='binary',subset='training')\nval_generator = train_datagen.flow_from_directory(train_path, target_size=(img_width, img_height), batch_size=batch_size, class_mode='binary',subset='validation')\ntest_generator = test_datagen.flow_from_directory(root_path,target_size=(img_width, img_height), batch_size=batch_size, shuffle=False,classes=['test'])","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:04:36.568686Z","iopub.execute_input":"2023-05-06T10:04:36.569029Z","iopub.status.idle":"2023-05-06T10:04:37.257794Z","shell.execute_reply.started":"2023-05-06T10:04:36.569003Z","shell.execute_reply":"2023-05-06T10:04:37.256915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **训练模型**\n当模型的val_loss训练多轮不再下降时，提前结束训练。","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping,ReduceLROnPlateau\n#训练神经网络\n# Setup some training parameters\nvalidation_split=0.2\nbatch_size = 250\nepochs = 100\ninput_shape = (300, 300, 3)\n\n# Build model (your best config)\nRes_model = build_ResNet(input_shape, n_dense_layers=1, n_nodes=1024, use_dropout=True, learning_rate=0.02)\n\nvalidation_split=0.2\n\n# Early stopping\nearly_stop = EarlyStopping(monitor='val_loss', patience=13)\n# 使用动态学习率\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=7, mode='auto', factor=0.2)\n    \n# Train the model using on the fly augmentation\nRes_history = Res_model.fit(train_generator,batch_size=batch_size,epochs=epochs,\n                      steps_per_epoch=25000*(1 - validation_split)/batch_size,\n                      validation_steps=25000*validation_split/batch_size,\n                      validation_data=val_generator,\n                      callbacks=[early_stop,reduce_lr])","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:04:59.939783Z","iopub.execute_input":"2023-05-06T10:04:59.940121Z","iopub.status.idle":"2023-05-06T10:41:12.714377Z","shell.execute_reply.started":"2023-05-06T10:04:59.940096Z","shell.execute_reply":"2023-05-06T10:41:12.713197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Summary\nRes_model.summary()\n\n# Plot the history from the training run\nplot_results(Res_history)","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:43:29.058977Z","iopub.execute_input":"2023-05-06T10:43:29.059350Z","iopub.status.idle":"2023-05-06T10:43:52.134308Z","shell.execute_reply.started":"2023-05-06T10:43:29.059324Z","shell.execute_reply":"2023-05-06T10:43:52.133378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **使用ResNet模型预测**\n\n预测数据也不应当归一化","metadata":{}},{"cell_type":"code","source":"# 对测试数据进行预测\nRes_preds = Res_model.predict(test_generator)\n\n# 打印预测结果\nprint(Res_preds)\n\n#写入文件提交\nRes_result=[0 if i <0.5 else 1 for i in Res_preds]\nRes_result=np.array(Res_result)\nid=np.array(range(1,12501))\n\nimport pandas as pd\nRes_df_result = pd.DataFrame({'id': id, 'label': Res_result})\n\nprint(Res_df_result)\nRes_df_result.to_csv('\\kaggle\\working\\my_Res_data.csv', index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:47:23.077162Z","iopub.execute_input":"2023-05-06T10:47:23.077531Z","iopub.status.idle":"2023-05-06T10:48:15.801320Z","shell.execute_reply.started":"2023-05-06T10:47:23.077501Z","shell.execute_reply":"2023-05-06T10:48:15.800374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Res_df_result.head(50))","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:48:26.748500Z","iopub.execute_input":"2023-05-06T10:48:26.749365Z","iopub.status.idle":"2023-05-06T10:48:26.758340Z","shell.execute_reply.started":"2023-05-06T10:48:26.749308Z","shell.execute_reply":"2023-05-06T10:48:26.755565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#导出模型\nRes_model.save('\\kaggle\\working\\my_res_dogcat_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-05-06T10:49:28.095612Z","iopub.execute_input":"2023-05-06T10:49:28.095960Z","iopub.status.idle":"2023-05-06T10:49:28.651358Z","shell.execute_reply.started":"2023-05-06T10:49:28.095932Z","shell.execute_reply":"2023-05-06T10:49:28.650495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **你的PyTorch代码**","metadata":{}},{"cell_type":"code","source":"class CatDogDataSet(Dataset):\n    \"\"\"\n    生成标准预处理图像数据集\n    \n    :inherits DataSet:\n    :param data_path: 数据路径\n    :param train: 是否为训练集\n    :param transform: 图像预处理方式\n    \"\"\"\n    \n    def __init__(self, data_path:str, train=True, transform=None):\n        self.data_path = data_path\n        self.train_flag = train\n        self.path_list = os.listdir(self.data_path)\n        \n        if transform is None:\n            # 设置图像默认预处理方式：调整尺寸为 300x300、转为张量并进行归一化、标准化处理\n            # 标准化：(x - 均值)/标准差。此处是为了使取值范围转为(-1,1)             \n            self.transform = transforms.Compose(\n            [\n                transforms.Resize((300,300)),\n                transforms.ToTensor(),\n                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n            ])\n        else:\n            self.transform = transform\n            \n    def __getitem__(self, idx: int):\n        \"\"\"\n        根据索引返回预处理后的图像与标记，如果是测试集，则将图片序号作为标记\n\n        :param idx: 索引\n        :return img: 图像\n        :return tag: 标记\n        \"\"\"\n        img_path = self.path_list[idx]\n        \n        label = img_path.split('.')[0]\n        label = (\n            int(label) if self.train_flag is False \n            else 1 if label == 'dog' else 0\n        )\n        \n        label = torch.as_tensor(label, dtype=torch.int64) \n        img = self.transform(Image.open(os.path.join(self.data_path, img_path)))\n        \n        return img, label\n    \n    def __len__(self) -> int:\n        return len(self.path_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:17:26.787810Z","iopub.execute_input":"2023-05-05T17:17:26.788162Z","iopub.status.idle":"2023-05-05T17:17:26.798472Z","shell.execute_reply.started":"2023-05-05T17:17:26.788133Z","shell.execute_reply":"2023-05-05T17:17:26.797547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#获得训练集和无标签测试集\ntrain_data = CatDogDataSet(train_path)\ntest_data = CatDogDataSet(test_path,train=False)\n\n#维数\nprint(len(train_data))        #训练集25000张图片\nprint(len(test_data))         #测试集12500张图片\nprint(train_data[0][0])\nprint(train_data[0][0].shape) #3*300*300\nprint(train_data[13600][1])   #标签，猫为1，狗为0\n\n#看一眼\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,4))\nfor i in range(18):\n    idx = np.random.randint(7500)\n    label = \"dog\" if train_data[idx][1] else \"cat\"\n    \n    plt.subplot(3,6,i+1)\n    plt.tight_layout()\n    plt.imshow(train_data[idx][0].T)\n    plt.title(\"Class: {} \".format(label))\n    plt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T17:34:42.433001Z","iopub.execute_input":"2023-05-05T17:34:42.433355Z","iopub.status.idle":"2023-05-05T17:34:45.198124Z","shell.execute_reply.started":"2023-05-05T17:34:42.433326Z","shell.execute_reply":"2023-05-05T17:34:45.196303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取训练集与无标签测试集\ntrain_ds = CatDogDataSet(train_path)\ntest_ds = CatDogDataSet(test_path,train=False)\n\n# 将数据集按比例随机划分为训练集与有标签测试集，以检验准确率\nsub_train_size = int(0.8 * len(train_ds))\nsub_test_size = len(train_ds) - sub_train_size\n\nsub_train_ds, sub_test_ds = random_split(\n    train_ds,[sub_train_size, sub_test_size]\n)\n\n# 定义数据加载器，用于分批加载数据集\n# 使用 enumerate 迭代获取批次\ntrain_loader = DataLoader(\n    train_ds, batch_size=32, pin_memory=True, shuffle=True\n)\ntest_loader = DataLoader(\n    test_ds, batch_size=32, pin_memory=True, shuffle=True\n)\nsub_train_loader  = DataLoader(\n    sub_train_ds, batch_size=32, pin_memory=True, shuffle=True\n)\nsub_test_loader = DataLoader(\n    sub_train_ds, batch_size=32, pin_memory=True, shuffle=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:32.352735Z","iopub.execute_input":"2023-05-04T05:06:32.353092Z","iopub.status.idle":"2023-05-04T05:06:32.403467Z","shell.execute_reply.started":"2023-05-04T05:06:32.353064Z","shell.execute_reply":"2023-05-04T05:06:32.402502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 测试输出图片\nrandom_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure()\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = transforms.ToPILImage()(train_ds[idx][0]).convert('RGB')\n    plt.imshow(img)\n    i+=1\n\nplt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 使用 ResNet50 深度残差网络 作为预训练模型，默认权重\nmodel = models.resnet50(weights='DEFAULT')\n\n# 将 ResNet50 连接层的输出维度（分类）由 2048 改为 2\nmodel.fc = nn.Linear(2048,2)\n\n# 选择损失函数（交叉熵）和优化函数（随机梯度下降）\nloss = nn.CrossEntropyLoss()\nopt = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:37.840382Z","iopub.execute_input":"2023-05-04T05:06:37.840715Z","iopub.status.idle":"2023-05-04T05:06:38.992346Z","shell.execute_reply.started":"2023-05-04T05:06:37.840688Z","shell.execute_reply":"2023-05-04T05:06:38.991402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, device, loader, lossfunc, optizer, epoch):\n    \"\"\"\n    训练模型\n    \n    param: model: 模型\n    param: device: 设备标识\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    param: optizer: 优化函数\n    param: epoch: 迭代次数\n    \"\"\"\n    # 生成并训练模型   \n    model = model.to(device)\n    # 转换成 tqdm 类型，便于输出\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    for e in range(epoch):\n        losses = 0.0\n        model.train()\n        loader.set_description(f'迭代次数 [{e+1}/{epoch}]')\n        for i, (images, labels) in loader:\n            # 重置下降梯度\n            opt.zero_grad()\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 误差\n            loss = lossfunc(outputs,labels)\n            losses += loss.item()\n            # 误差逆传播\n            loss.backward()\n            # 迭代优化\n            opt.step()\n            \n    print('训练结束')","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:08:46.557345Z","iopub.execute_input":"2023-05-04T06:08:46.558029Z","iopub.status.idle":"2023-05-04T06:08:46.565221Z","shell.execute_reply.started":"2023-05-04T06:08:46.557997Z","shell.execute_reply":"2023-05-04T06:08:46.564302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(model, device, loader, lossfunc):\n    \"\"\"\n    测试模型\n    \n    param: model: 模型\n    param: device: 设备\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    \"\"\"\n    losses = 0.0\n    correct = 0\n    total = 0\n    \n    # 生成并测试模型\n    model = model.to(device)\n    model.eval()\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    \n    # 关闭梯度计算\n    with torch.no_grad():\n        for i, (images, labels) in loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 误差\n            loss = lossfunc(outputs,labels)\n            losses += loss.item()\n            # 预测结果\n            _,pred = torch.max(outputs.data,1)\n            # 准确率\n            total += labels.size(0)\n            correct += (pred == labels).sum().item()\n            \n            loader.set_postfix(loss=losses/total, acc=correct/total)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:06:54.623245Z","iopub.execute_input":"2023-05-04T05:06:54.623835Z","iopub.status.idle":"2023-05-04T05:06:54.630292Z","shell.execute_reply.started":"2023-05-04T05:06:54.623803Z","shell.execute_reply":"2023-05-04T05:06:54.629422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(model, device, loader, lossfunc, csv_path):\n    \"\"\"\n    测试模型并输出预测结果\n    \n    param: model: 模型\n    param: device: 设备\n    param: loader: 数据集\n    param: lossfunc: 损失函数\n    param: csv_path: 输出路径\n    \"\"\"\n    rs = []\n    \n    # 生成并测试模型\n    model = model.to(device)\n    model.eval()\n    loader = tqdm(\n        enumerate(loader), \n        total=len(loader), \n        leave=False\n    )\n    \n    # 关闭梯度计算\n    with torch.no_grad():\n        for i, (images, labels) in loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            # 预测结果\n            _,pred = nn.Softmax(1)(outputs).max(dim=1)\n            # 写入列表\n            for p in range(len(pred)):\n                rs.append({\n                    \"id\": labels[p].item(),\n                    \"label\": pred[p].item()\n                })\n    # 转成 dataframe 并保存为 csv\n    pd.DataFrame(rs,columns=rs[0].keys()).sort_values(\"id\").to_csv(csv_path, index=None)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T05:41:21.159473Z","iopub.execute_input":"2023-05-04T05:41:21.162542Z","iopub.status.idle":"2023-05-04T05:41:21.174882Z","shell.execute_reply.started":"2023-05-04T05:41:21.162499Z","shell.execute_reply":"2023-05-04T05:41:21.172259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(model,device,sub_train_loader,loss,opt,60)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:08:50.750536Z","iopub.execute_input":"2023-05-04T06:08:50.750879Z","iopub.status.idle":"2023-05-04T06:12:04.618273Z","shell.execute_reply.started":"2023-05-04T06:08:50.750852Z","shell.execute_reply":"2023-05-04T06:12:04.617236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(model,device,sub_test_loader,loss)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(csv_path):\n    os.remove(csv_path)\nsubmit(model, device, test_loader, loss, csv_path)","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:02:17.059041Z","iopub.execute_input":"2023-05-04T06:02:17.059471Z","iopub.status.idle":"2023-05-04T06:04:00.647254Z","shell.execute_reply.started":"2023-05-04T06:02:17.059437Z","shell.execute_reply":"2023-05-04T06:04:00.646267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(root_path,'test.csv')).sample(n=10)\n\nfig, axs = plt.subplots(nrows=2, ncols=5, figsize=(12, 6),\n                        subplot_kw={'xticks': [], 'yticks': []})\n\nfor i, ax in enumerate(axs.flat):\n    img = df.iloc[i, 0]\n    label = df.iloc[i, 1]\n\n    # 加载图像并将其转换为numpy数组\n    img = Image.open(os.path.join(test_path,f'{img}.jpg'))\n\n    # 显示图像并添加标签\n    ax.imshow(np.array(img))\n    ax.set_title('dog' if label == 0 else 'cat')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-04T06:04:05.256343Z","iopub.execute_input":"2023-05-04T06:04:05.256702Z","iopub.status.idle":"2023-05-04T06:04:06.257834Z","shell.execute_reply.started":"2023-05-04T06:04:05.256674Z","shell.execute_reply":"2023-05-04T06:04:06.255967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}