{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-17T01:44:39.118443Z","iopub.execute_input":"2023-08-17T01:44:39.119651Z","iopub.status.idle":"2023-08-17T01:44:39.124525Z","shell.execute_reply.started":"2023-08-17T01:44:39.119600Z","shell.execute_reply":"2023-08-17T01:44:39.123422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vgg16():\n    model=models.Sequential()    \n    \n    model.add(layers.Conv2D(64,3, padding='same',activation='relu', input_shape=(224, 224, 1)))\n    model.add(layers.Conv2D(64,3, padding='same',activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n    \n    model.add(layers.Conv2D(128,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(128,3, padding='same',activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n    \n    model.add(layers.Conv2D(256,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(256,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(256,3, padding='same',activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n    \n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n    \n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.Conv2D(512,3, padding='same',activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n\n    model.add(layers.Flatten())\n    \n    model.add(layers.Dense(4096, activation='relu'))\n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(4096, activation='relu'))\n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(10, activation='softmax'))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-16T06:42:23.255985Z","iopub.execute_input":"2023-08-16T06:42:23.256758Z","iopub.status.idle":"2023-08-16T06:42:23.275438Z","shell.execute_reply.started":"2023-08-16T06:42:23.256725Z","shell.execute_reply":"2023-08-16T06:42:23.273971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import optimizers\n\nopt=optimizers.Adam(learning_rate=0.001)\n\nmodel=vgg16()\n\nmodel.compile(opt, loss='categorical_crossentropy', metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T06:42:23.277273Z","iopub.execute_input":"2023-08-16T06:42:23.277740Z","iopub.status.idle":"2023-08-16T06:42:25.165556Z","shell.execute_reply.started":"2023-08-16T06:42:23.277698Z","shell.execute_reply":"2023-08-16T06:42:25.164624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#files=['data_batch_1','data_batch_2','data_batch_3','data_batch_4','data_batch_5','test_batch']\nfiles=['data_batch_3','data_batch_4','data_batch_5']","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:51:58.885481Z","iopub.execute_input":"2023-08-17T03:51:58.886369Z","iopub.status.idle":"2023-08-17T03:51:58.891312Z","shell.execute_reply.started":"2023-08-17T03:51:58.886335Z","shell.execute_reply":"2023-08-17T03:51:58.890441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read(file):\n    import pickle\n    with open('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/'+file, 'rb') as fo:\n        dict = pickle.load(fo, encoding='bytes')\n    return dict","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:01.546295Z","iopub.execute_input":"2023-08-17T03:52:01.547107Z","iopub.status.idle":"2023-08-17T03:52:01.553605Z","shell.execute_reply.started":"2023-08-17T03:52:01.547075Z","shell.execute_reply":"2023-08-17T03:52:01.552317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split(batch):\n    return batch[b'labels'],batch[b'data'].reshape((batch[b'data'].shape[0], 32, 32, 3))","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:03.443295Z","iopub.execute_input":"2023-08-17T03:52:03.443673Z","iopub.status.idle":"2023-08-17T03:52:03.449404Z","shell.execute_reply.started":"2023-08-17T03:52:03.443646Z","shell.execute_reply":"2023-08-17T03:52:03.448208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def processImages(data):\n    import tensorflow as tf\n    return tf.image.resize(data, (224, 224))/255.0","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:05.763232Z","iopub.execute_input":"2023-08-17T03:52:05.763621Z","iopub.status.idle":"2023-08-17T03:52:05.768731Z","shell.execute_reply.started":"2023-08-17T03:52:05.763593Z","shell.execute_reply":"2023-08-17T03:52:05.767821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def processLabels(labels):\n    import tensorflow as tf\n    return tf.keras.utils.to_categorical(labels, 10)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:07.301212Z","iopub.execute_input":"2023-08-17T03:52:07.302484Z","iopub.status.idle":"2023-08-17T03:52:07.308663Z","shell.execute_reply.started":"2023-08-17T03:52:07.302435Z","shell.execute_reply":"2023-08-17T03:52:07.307110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def store(data,path):\n    import pickle\n    \n    with open(path, 'wb') as file:\n        pickle.dump(data, file)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:09.363243Z","iopub.execute_input":"2023-08-17T03:52:09.364437Z","iopub.status.idle":"2023-08-17T03:52:09.370430Z","shell.execute_reply.started":"2023-08-17T03:52:09.364394Z","shell.execute_reply":"2023-08-17T03:52:09.368785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nfor file in files:\n    batch=read(file)\n    labels,data=split(batch)\n    data=processImages(data)\n    labels=processLabels(labels)\n    store(labels,file+'_labels')\n    store(data,file+'_data')\n    print(file)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:52:11.824460Z","iopub.execute_input":"2023-08-17T03:52:11.824818Z","iopub.status.idle":"2023-08-17T03:53:56.421888Z","shell.execute_reply.started":"2023-08-17T03:52:11.824792Z","shell.execute_reply":"2023-08-17T03:53:56.420634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\n\n# 生成一个指向输出文件的链接\nFileLink('/kaggle/working/test_batch_labels')\n","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:22:33.884441Z","iopub.execute_input":"2023-08-17T03:22:33.884889Z","iopub.status.idle":"2023-08-17T03:22:33.892943Z","shell.execute_reply.started":"2023-08-17T03:22:33.884861Z","shell.execute_reply":"2023-08-17T03:22:33.891880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/data_batch_1')\ntrain_2=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/data_batch_2')\ntrain_3=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/data_batch_3')\ntrain_4=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/data_batch_4')\ntrain_5=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/data_batch_5')\ntest=unpickle('/kaggle/input/d/yangyanpei/cifar-10/cifar-10-batches-py/test_batch')","metadata":{"execution":{"iopub.status.busy":"2023-08-16T09:08:08.171912Z","iopub.execute_input":"2023-08-16T09:08:08.172379Z","iopub.status.idle":"2023-08-16T09:08:11.624646Z","shell.execute_reply.started":"2023-08-16T09:08:08.172345Z","shell.execute_reply":"2023-08-16T09:08:11.623362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1_labels,train_1_data=split(train_1)","metadata":{"execution":{"iopub.status.busy":"2023-08-16T09:19:31.029732Z","iopub.execute_input":"2023-08-16T09:19:31.031413Z","iopub.status.idle":"2023-08-16T09:19:31.037324Z","shell.execute_reply.started":"2023-08-16T09:19:31.031356Z","shell.execute_reply":"2023-08-16T09:19:31.035889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1_data=upsize(train_1_data)\nprint(train_1_data.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store(train_1_data,'/kaggle/working/train_1_data.vb')","metadata":{"execution":{"iopub.status.busy":"2023-08-16T09:30:17.427648Z","iopub.execute_input":"2023-08-16T09:30:17.428137Z","iopub.status.idle":"2023-08-16T09:30:56.427634Z","shell.execute_reply.started":"2023-08-16T09:30:17.428103Z","shell.execute_reply":"2023-08-16T09:30:56.424906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1_labels,train_1_data=train_1[b'labels'],train_1[b'data']\ntrain_2_labels,train_2_data=train_2[b'labels'],train_2[b'data']\ntrain_3_labels,train_3_data=train_3[b'labels'],train_3[b'data']\ntrain_4_labels,train_4_data=train_4[b'labels'],train_4[b'data']\ntrain_5_labels,train_5_data=train_5[b'labels'],train_5[b'data']\ntest_labels,test_data=test[b'labels'],test[b'data']","metadata":{"execution":{"iopub.status.busy":"2023-08-16T09:11:14.224309Z","iopub.execute_input":"2023-08-16T09:11:14.224814Z","iopub.status.idle":"2023-08-16T09:11:14.231771Z","shell.execute_reply.started":"2023-08-16T09:11:14.224777Z","shell.execute_reply":"2023-08-16T09:11:14.230728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-08-16T09:06:39.366788Z","iopub.execute_input":"2023-08-16T09:06:39.368652Z","iopub.status.idle":"2023-08-16T09:06:39.376584Z","shell.execute_reply.started":"2023-08-16T09:06:39.368585Z","shell.execute_reply":"2023-08-16T09:06:39.374933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import requests\n\nurl = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'\nr = requests.get(url)\n\nwith open('cifar-10-python.tar.gz', 'wb') as f:\n    f.write(r.content)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T08:46:18.512656Z","iopub.execute_input":"2023-08-16T08:46:18.513315Z","iopub.status.idle":"2023-08-16T08:46:23.728349Z","shell.execute_reply.started":"2023-08-16T08:46:18.513270Z","shell.execute_reply":"2023-08-16T08:46:23.727030Z"}}},{"cell_type":"markdown","source":"from keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(rescale=1./255)\ntrain_generator = train_datagen.flow_from_directory(\n        'test',\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='binary')\n\nprint(type(train_generator),train_generator)","metadata":{"execution":{"iopub.status.busy":"2023-08-16T08:30:21.945741Z","iopub.execute_input":"2023-08-16T08:30:21.946239Z","iopub.status.idle":"2023-08-16T08:30:24.768625Z","shell.execute_reply.started":"2023-08-16T08:30:21.946198Z","shell.execute_reply":"2023-08-16T08:30:24.766876Z"}}},{"cell_type":"markdown","source":"import tarfile\n\nwith tarfile.open('cifar-10-python.tar.gz', 'r:gz') as tar:\n    tar.extractall()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T08:50:37.895794Z","iopub.execute_input":"2023-08-16T08:50:37.896215Z","iopub.status.idle":"2023-08-16T08:50:39.796760Z","shell.execute_reply.started":"2023-08-16T08:50:37.896183Z","shell.execute_reply":"2023-08-16T08:50:39.795403Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.datasets import cifar10\nimport numpy as np\nimport sys\nimport gc\n(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()\n\n# 调整图像大小\nresized_train_images=None\nj=1\nfor i in range(0,train_images.shape[0]-1,99):    \n    temp=tf.image.resize(train_images[i:i+99], (224, 224))\n    if i == 0:\n        resized_train_images=temp\n    else:\n        resized_train_images=np.concatenate((resized_train_images, temp), axis=0)    \n    if j%10==0:\n        gc.collect()\n        print(j)\n    j=j+1\n    \nprint('训练尺寸调整完成')\n\nj=1\nresized_test_images=None\nfor i in range(0,test_images.shape[0]-1,99):\n    temp=tf.image.resize(test_images[i:i+99], (224, 224))\n    if i == 0:\n        resized_test_images=temp\n    else:\n        resized_test_images=np.concatenate((resized_test_images, temp), axis=0)\n    if j%10==0:\n        gc.collect()\n        print(j)\n    j=j+1\n    \nprint('测试尺寸调整完成')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-17T01:44:52.951930Z","iopub.execute_input":"2023-08-17T01:44:52.952404Z","iopub.status.idle":"2023-08-17T02:31:20.393308Z","shell.execute_reply.started":"2023-08-17T01:44:52.952345Z","shell.execute_reply":"2023-08-17T02:31:20.391896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gzip\nimport pickle\n\n\n# 打开一个文件用于写入\nwith gzip.open('resized_train_images.pkl.gz', 'wb') as file:\n    # 使用 pickle.dump 函数将变量存储到文件中\n    pickle.dump(resized_train_images, file)","metadata":{"execution":{"iopub.status.busy":"2023-08-17T02:32:55.137311Z","iopub.execute_input":"2023-08-17T02:32:55.138764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import psutil\n\n# 获取 CPU 信息\nprint('物理CPU个数:', psutil.cpu_count(logical=False))\nprint('逻辑CPU个数:', psutil.cpu_count())\nprint('CPU使用情况:', psutil.cpu_percent(percpu=True))\n\n# 获取内存信息\nmemory = psutil.virtual_memory()\nprint('总内存:', memory.total)\nprint('可用内存:', memory.available)\nprint('内存使用率:', memory.percent)\nprint('已使用内存:', memory.used)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-17T03:07:00.373951Z","iopub.execute_input":"2023-08-17T03:07:00.374351Z","iopub.status.idle":"2023-08-17T03:07:00.383449Z","shell.execute_reply.started":"2023-08-17T03:07:00.374322Z","shell.execute_reply":"2023-08-17T03:07:00.382431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nimport py7zr\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-16T07:53:23.834815Z","iopub.execute_input":"2023-08-16T07:53:23.835236Z","iopub.status.idle":"2023-08-16T08:16:43.774836Z","shell.execute_reply.started":"2023-08-16T07:53:23.835204Z","shell.execute_reply":"2023-08-16T08:16:43.773187Z"}}},{"cell_type":"markdown","source":"print(resized_test_images.shape)","metadata":{}},{"cell_type":"code","source":"# 归一化像素值到 0 到 1 之间\ntrain_images, test_images = train_images / 255.0, test_images / 255.0\nprint('归一化完成')\n\n\n# 对标签进行 One-hot 编码\ntrain_labels = tf.keras.utils.to_categorical(train_labels, 10)\ntest_labels = tf.keras.utils.to_categorical(test_labels, 10)\nprint('标签编码完成')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_images, train_labels, epochs=10, validation_data=(test_images, test_labels))","metadata":{"execution":{"iopub.status.busy":"2023-08-16T06:46:58.212866Z","iopub.status.idle":"2023-08-16T06:46:58.213405Z","shell.execute_reply.started":"2023-08-16T06:46:58.213179Z","shell.execute_reply":"2023-08-16T06:46:58.213202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}