{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 解压\n# import zipfile\n# FROM_PATH = r'/kaggle/input/dogs-vs-cats-redux-kernels-edition'\n# flist = [os.path.join(FROM_PATH,'test.zip'),os.path.join(FROM_PATH,'train.zip')]\n\n# for fpath in flist:\n#     with zipfile.ZipFile(fpath, 'r') as zf:\n#         zf.extractall(ROOT_PATH)\n\n# 关闭警告\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:38:26.275459Z","iopub.execute_input":"2023-05-12T11:38:26.275823Z","iopub.status.idle":"2023-05-12T11:38:26.286733Z","shell.execute_reply.started":"2023-05-12T11:38:26.275792Z","shell.execute_reply":"2023-05-12T11:38:26.285603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\nfrom tensorflow.data import Dataset \nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import ResNet50\n\n# 设备\n# DEVICE = tf.config.list_physical_devices('GPU')\n# if len(DEVICE) > 0:\n#     os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n#     os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n#     tf.config.experimental.set_memory_growth(DEVICE[0], True)\n# else:\n#     os.environ['CUDA_VISIBLE_DEVICES'] = ''\n\nos.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n \n# The GPU id to use, usually either \"0\" or \"1\";\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\";\n\n# Allow growth of GPU memory, otherwise it will always look like all the memory is being used\nphysical_devices = tf.config.experimental.list_physical_devices('GPU')\ntf.config.experimental.set_memory_growth(physical_devices[0], True) \n\n# 路径\n# ROOT_PATH = r'D:\\kyu\\data\\ML\\DogNCat'\nROOT_PATH = r'/kaggle/working/'\n# DS_PATH = os.path.join(ROOT_PATH,'dataset')\nMODEL_PATH = os.path.join(ROOT_PATH,'model')\nCSV_PATH = os.path.join(ROOT_PATH,'test.csv')\nTSRBD_PATH = os.path.join(ROOT_PATH,'tsrbd')\nTRAIN_PATH = os.path.join(ROOT_PATH,'train')\nTEST_PATH = os.path.join(ROOT_PATH,'test')\n\n# 随机数种子\nSEED = int(time.time())","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:38:29.672962Z","iopub.execute_input":"2023-05-12T11:38:29.673328Z","iopub.status.idle":"2023-05-12T11:38:36.403791Z","shell.execute_reply.started":"2023-05-12T11:38:29.673297Z","shell.execute_reply":"2023-05-12T11:38:36.402804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\ndevice_lib.list_local_devices()\n# os.environ['CUDA_VISIBLE_DEVICES']\n# len(DEVICE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 定义数据生成器与输出类型\ndef gen(path: str,train: bool):\n    for fname in os.listdir(path):\n        label = fname.split('.')[0]\n        label = int(label) if not train else 0 if label == 'cat' else 1\n        img = tf.io.read_file(os.path.join(path,fname))\n        img = tf.image.decode_jpeg(img,3)\n        img = tf.image.resize(img,(224,224)) / 255.0\n        yield img,label\n\nout_types = (\n    tf.TensorSpec((224,224,3),tf.float32),\n    tf.TensorSpec((),tf.int32)\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:38:41.39559Z","iopub.execute_input":"2023-05-12T11:38:41.39628Z","iopub.status.idle":"2023-05-12T11:38:41.40596Z","shell.execute_reply.started":"2023-05-12T11:38:41.396248Z","shell.execute_reply":"2023-05-12T11:38:41.40494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 创建数据集\ntrain_ds = tf.data.Dataset.from_generator(lambda: gen(TRAIN_PATH,True),output_signature=out_types)\ntest_ds = tf.data.Dataset.from_generator(lambda: gen(TEST_PATH,False),output_signature=out_types)\n\n# 划分验证集\ntrain_size = int(0.8*len(os.listdir(TRAIN_PATH)))\ntrain_ds = train_ds.take(train_size)\nvalid_ds = train_ds.skip(train_size)\n\n# 随机分批（32份/批）缓存与异步预读数据，以提高数据加载速度\ntrain_ds = train_ds.batch(32).prefetch(tf.data.AUTOTUNE)\nvalid_ds = valid_ds.batch(32).prefetch(tf.data.AUTOTUNE)\ntest_ds = test_ds.batch(32).prefetch(tf.data.AUTOTUNE)\n\n# .shuffle(1000).cache()\n\n# 测试图片读取\n# for img,label in train_ds.take(1):\n#     plt.axis('off')\n#     plt.title('cat' if label.numpy() == 0 else 'dog')\n#     plt.imshow((img*255).numpy().astype(\"uint8\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:39:37.125669Z","iopub.execute_input":"2023-05-12T11:39:37.126084Z","iopub.status.idle":"2023-05-12T11:39:37.453309Z","shell.execute_reply.started":"2023-05-12T11:39:37.126051Z","shell.execute_reply":"2023-05-12T11:39:37.452352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 从 ResNet50 创建模型\n# 使用 ImageNet 权重，移除默认分类层\nbase_model = ResNet50(\n    weights='imagenet', \n    input_shape=(224,224,3),\n    include_top=False, \n)\n# 冻结预训练参数\nbase_model.trainable = False\n\n# 自定义分类层\nx = base_model.output\n# 池化层\nx = layers.GlobalAveragePooling2D()(x)\n# 全连接层（256 维，ReLU 激活）\nx = layers.Dense(256, activation='relu')(x)\n# 全连接层（2 维，softmax 激活）\npred = layers.Dense(2, activation='softmax')(x)\n# 构建迁移学习模型\nmodel = Model(\n    inputs=base_model.input, \n    outputs=pred\n)\n# 预览模型结构\n# tf.keras.utils.plot_model(model,'CatDogModel.jpg', show_shapes=True)\n# 编译模型（adam 优化器，学习率 0.001、整数编码交叉熵损失函数、准确率评估）\nmodel.compile(\n    optimizer=Adam(1e-3), \n    loss='sparse_categorical_crossentropy', \n    metrics=['accuracy']\n)\n# 训练模型（10 轮）\nmodel.fit(epochs=10,x=train_ds,validation_data=valid_ds)\n# 保存模型\n# model.save('cat_dog_resnet50_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-05-12T11:39:39.783726Z","iopub.execute_input":"2023-05-12T11:39:39.784815Z","iopub.status.idle":"2023-05-12T12:01:39.137188Z","shell.execute_reply.started":"2023-05-12T11:39:39.784773Z","shell.execute_reply":"2023-05-12T12:01:39.136153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}