{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:55.531325Z","iopub.execute_input":"2025-06-23T14:01:55.531604Z","iopub.status.idle":"2025-06-23T14:01:55.552668Z","shell.execute_reply.started":"2025-06-23T14:01:55.531578Z","shell.execute_reply":"2025-06-23T14:01:55.552149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:55.553654Z","iopub.execute_input":"2025-06-23T14:01:55.553858Z","iopub.status.idle":"2025-06-23T14:01:58.572110Z","shell.execute_reply.started":"2025-06-23T14:01:55.553844Z","shell.execute_reply":"2025-06-23T14:01:58.571438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport pandas as pd\nimport numpy as np\nimport os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport time\nimport py7zr  # 用于解压7z文件\n\nprint(\"所有必要的库已导入\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.573128Z","iopub.execute_input":"2025-06-23T14:01:58.573380Z","iopub.status.idle":"2025-06-23T14:01:58.578941Z","shell.execute_reply.started":"2025-06-23T14:01:58.573355Z","shell.execute_reply":"2025-06-23T14:01:58.578197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 设置随机种子保证可复现性\ntorch.manual_seed(42)\nnp.random.seed(42)\n\n# Kaggle数据集路径\nKAGGLE_DATA_DIR = \"/kaggle/input/cifar-10\"\nWORKING_DIR = \"/kaggle/working\"\n\nprint(f\"数据集目录: {KAGGLE_DATA_DIR}\")\nprint(f\"工作目录: {WORKING_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.579837Z","iopub.execute_input":"2025-06-23T14:01:58.580101Z","iopub.status.idle":"2025-06-23T14:01:58.595091Z","shell.execute_reply.started":"2025-06-23T14:01:58.580080Z","shell.execute_reply":"2025-06-23T14:01:58.594509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_data():\n    # 确保工作目录存在\n    os.makedirs(os.path.join(WORKING_DIR, \"data\"), exist_ok=True)\n    \n    # 检查数据是否已解压\n    train_dir = os.path.join(WORKING_DIR, \"data/train\")\n    test_dir = os.path.join(WORKING_DIR, \"data/test\")\n    \n    if not os.path.exists(train_dir) or not os.path.exists(test_dir):\n        print(\"开始解压数据集...\")\n        \n        # 解压训练集\n        train_7z_path = os.path.join(KAGGLE_DATA_DIR, \"train.7z\")\n        print(f\"解压训练集: {train_7z_path}\")\n        with py7zr.SevenZipFile(train_7z_path, mode='r') as z:\n            z.extractall(path=os.path.join(WORKING_DIR, \"data\"))\n        print(\"训练集解压完成\")\n        \n        # 解压测试集\n        test_7z_path = os.path.join(KAGGLE_DATA_DIR, \"test.7z\")\n        print(f\"解压测试集: {test_7z_path}\")\n        with py7zr.SevenZipFile(test_7z_path, mode='r') as z:\n            z.extractall(path=os.path.join(WORKING_DIR, \"data\"))\n        print(\"测试集解压完成\")\n        \n        # 重命名解压后的文件夹\n        os.rename(os.path.join(WORKING_DIR, \"data/train\"), train_dir)\n        os.rename(os.path.join(WORKING_DIR, \"data/test\"), test_dir)\n        print(f\"数据已保存到: {WORKING_DIR}/data\")\n    else:\n        print(\"数据集已存在，跳过解压\")\n    \n    return train_dir, test_dir\n\n# 执行解压\ntrain_dir, test_dir = extract_data()\nprint(f\"训练集目录: {train_dir}\")\nprint(f\"测试集目录: {test_dir}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.597148Z","iopub.execute_input":"2025-06-23T14:01:58.597353Z","iopub.status.idle":"2025-06-23T14:01:58.618170Z","shell.execute_reply.started":"2025-06-23T14:01:58.597337Z","shell.execute_reply":"2025-06-23T14:01:58.617456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_data():\n    # 加载训练标签\n    labels_path = os.path.join(KAGGLE_DATA_DIR, \"trainLabels.csv\")\n    print(f\"加载标签文件: {labels_path}\")\n    labels_df = pd.read_csv(labels_path)\n    \n    # 创建标签映射\n    label_names = sorted(labels_df['label'].unique())\n    label_to_idx = {name: idx for idx, name in enumerate(label_names)}\n    \n    print(f\"发现 {len(label_names)} 个类别: {label_names}\")\n    return label_to_idx\n\n# 执行数据准备\nlabel_to_idx = prepare_data()\nclass_names = list(label_to_idx.keys())\nprint(\"标签映射创建完成\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.618888Z","iopub.execute_input":"2025-06-23T14:01:58.619140Z","iopub.status.idle":"2025-06-23T14:01:58.650240Z","shell.execute_reply.started":"2025-06-23T14:01:58.619125Z","shell.execute_reply":"2025-06-23T14:01:58.649674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CIFAR10Dataset(Dataset):\n    def __init__(self, root_dir, transform=None, mode='train'):\n        self.root_dir = root_dir\n        self.transform = transform\n        self.mode = mode\n        self.image_files = []\n        self.labels = []\n        \n        if mode == 'train':\n            # 加载训练标签\n            labels_path = os.path.join(KAGGLE_DATA_DIR, \"trainLabels.csv\")\n            labels_df = pd.read_csv(labels_path)\n            \n            # 创建ID到标签的映射\n            id_to_label = dict(zip(labels_df['id'], labels_df['label']))\n            \n            for img_file in os.listdir(root_dir):\n                if img_file.endswith('.png'):\n                    img_id = int(img_file.split('.')[0])\n                    label = id_to_label.get(img_id)\n                    if label is not None:\n                        img_path = os.path.join(root_dir, img_file)\n                        self.image_files.append(img_path)\n                        self.labels.append(label_to_idx[label])\n            print(f\"训练集加载完成: {len(self.image_files)} 张图片\")\n        else:  # test模式\n            for img_file in os.listdir(root_dir):\n                if img_file.endswith('.png'):\n                    img_path = os.path.join(root_dir, img_file)\n                    self.image_files.append(img_path)\n                    self.labels.append(-1)  # 测试集无标签\n            print(f\"测试集加载完成: {len(self.image_files)} 张图片\")\n    \n    def __len__(self):\n        return len(self.image_files)\n    \n    def __getitem__(self, idx):\n        img_path = self.image_files[idx]\n        image = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        label = self.labels[idx]\n        # 如果是测试集，返回图像和文件名\n        if self.mode == 'test':\n            img_id = os.path.splitext(os.path.basename(img_path))[0]\n            return image, img_id\n        else:\n            return image, label\n\nprint(\"自定义数据集类定义完成\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.650821Z","iopub.execute_input":"2025-06-23T14:01:58.651018Z","iopub.status.idle":"2025-06-23T14:01:58.659305Z","shell.execute_reply.started":"2025-06-23T14:01:58.650981Z","shell.execute_reply":"2025-06-23T14:01:58.658702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_augmentation():\n    # 训练集增强\n    train_transform = transforms.Compose([\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomRotation(10),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n        transforms.RandomResizedCrop(32, scale=(0.8, 1.0)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))\n    ])\n    \n    # 验证集和测试集转换\n    test_transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))\n    ])\n    \n    return train_transform, test_transform\n\ndef create_data_loaders(batch_size=128):\n    train_transform, test_transform = get_augmentation()\n    \n    # 创建数据集\n    print(\"创建训练/验证数据集...\")\n    train_val_dataset = CIFAR10Dataset(\n        root_dir=train_dir,\n        transform=train_transform,\n        mode='train'\n    )\n    \n    # 分割训练集和验证集 (80%训练, 20%验证)\n    train_size = int(0.8 * len(train_val_dataset))\n    val_size = len(train_val_dataset) - train_size\n    train_set, val_set = random_split(\n        train_val_dataset, [train_size, val_size]\n    )\n    print(f\"训练集大小: {len(train_set)}, 验证集大小: {len(val_set)}\")\n    \n    # 测试集\n    print(\"创建测试数据集...\")\n    test_set = CIFAR10Dataset(\n        root_dir=test_dir,\n        transform=test_transform,\n        mode='test'\n    )\n    print(f\"测试集大小: {len(test_set)}\")\n    \n    # 创建数据加载器\n    train_loader = DataLoader(\n        train_set, batch_size=batch_size, shuffle=True, num_workers=2\n    )\n    val_loader = DataLoader(\n        val_set, batch_size=batch_size, shuffle=False, num_workers=2\n    )\n    test_loader = DataLoader(\n        test_set, batch_size=batch_size, shuffle=False, num_workers=2\n    )\n    \n    return train_loader, val_loader, test_loader\n\n# 创建数据加载器\nbatch_size = 128\ntrain_loader, val_loader, test_loader = create_data_loaders(batch_size)\nprint(\"数据加载器创建完成\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:58.660089Z","iopub.execute_input":"2025-06-23T14:01:58.660317Z","iopub.status.idle":"2025-06-23T14:01:59.185215Z","shell.execute_reply.started":"2025-06-23T14:01:58.660297Z","shell.execute_reply":"2025-06-23T14:01:59.184609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CIFAR10Model(nn.Module):\n    def __init__(self, num_classes=10):\n        super(CIFAR10Model, self).__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(3, 32, kernel_size=3, padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            \n            nn.Conv2d(32, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            \n            nn.Conv2d(64, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n        \n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 4 * 4, 512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, num_classes)\n        )\n    \n    def forward(self, x):\n        x = self.features(x)\n        x = self.classifier(x)\n        return x\n\n# 初始化模型\nmodel = CIFAR10Model(num_classes=len(class_names))\nprint(\"模型初始化完成\")\nprint(f\"模型参数数量: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:59.185906Z","iopub.execute_input":"2025-06-23T14:01:59.186134Z","iopub.status.idle":"2025-06-23T14:01:59.213253Z","shell.execute_reply.started":"2025-06-23T14:01:59.186110Z","shell.execute_reply":"2025-06-23T14:01:59.212705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, criterion, optimizer, scheduler, num_epochs=20):\n    \"\"\"训练模型并记录性能指标\"\"\"\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"使用设备: {device}\")\n    model = model.to(device)\n    \n    # 记录训练过程\n    train_losses = []\n    train_accuracies = []  # 训练准确率\n    val_losses = []\n    val_accuracies = []  # 验证准确率\n    \n    best_val_acc = 0.0\n    best_model = None\n    \n    for epoch in range(num_epochs):\n        start_time = time.time()\n        model.train()\n        running_loss = 0.0\n        running_correct = 0  # 训练正确样本数\n        total_samples = 0    # 总样本数\n        \n        # 训练阶段\n        for inputs, labels in train_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            # 更新统计信息\n            running_loss += loss.item() * inputs.size(0)\n            _, predicted = torch.max(outputs.data, 1)\n            running_correct += (predicted == labels).sum().item()\n            total_samples += labels.size(0)\n        \n        # 计算训练指标\n        epoch_loss = running_loss / total_samples\n        epoch_acc = 100.0 * running_correct / total_samples\n        train_losses.append(epoch_loss)\n        train_accuracies.append(epoch_acc)\n        \n        # 验证阶段\n        model.eval()\n        val_loss = 0.0\n        val_correct = 0\n        val_total = 0\n        \n        with torch.no_grad():\n            for inputs, labels in val_loader:\n                inputs, labels = inputs.to(device), labels.to(device)\n                outputs = model(inputs)\n                loss = criterion(outputs, labels)\n                \n                val_loss += loss.item() * inputs.size(0)\n                _, predicted = torch.max(outputs.data, 1)\n                val_total += labels.size(0)\n                val_correct += (predicted == labels).sum().item()\n        \n        # 计算验证指标\n        val_loss = val_loss / val_total\n        val_accuracy = 100 * val_correct / val_total\n        val_losses.append(val_loss)\n        val_accuracies.append(val_accuracy)\n        \n        # 更新学习率\n        scheduler.step(val_loss)\n        \n        # 计算时间\n        end_time = time.time()\n        epoch_time = end_time - start_time\n        \n        # 打印进度\n        print(f'Epoch {epoch+1}/{num_epochs} - {epoch_time:.2f}s')\n        print(f'Train Loss: {epoch_loss:.4f} | Train Acc: {epoch_acc:.2f}%')\n        print(f'Val Loss: {val_loss:.4f} | Val Acc: {val_accuracy:.2f}%')\n        \n        # 保存最佳模型\n        if val_accuracy > best_val_acc:\n            best_val_acc = val_accuracy\n            best_model = model.state_dict()\n            best_model_path = os.path.join(WORKING_DIR, 'best_cifar10_model.pth')\n            torch.save(best_model, best_model_path)\n            print(f\"保存最佳模型到 {best_model_path}, 验证准确率: {val_accuracy:.2f}%\")\n    \n    # 最终保存模型\n    final_model_path = os.path.join(WORKING_DIR, 'final_cifar10_model.pth')\n    torch.save(model.state_dict(), final_model_path)\n    print(f\"最终模型保存到 {final_model_path}\")\n    \n    # 加载最佳模型\n    if best_model is not None:\n        model.load_state_dict(best_model)\n        print(\"加载最佳模型用于测试\")\n    \n    return model, train_losses, train_accuracies, val_losses, val_accuracies","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:59.213850Z","iopub.execute_input":"2025-06-23T14:01:59.214031Z","iopub.status.idle":"2025-06-23T14:01:59.224259Z","shell.execute_reply.started":"2025-06-23T14:01:59.214017Z","shell.execute_reply":"2025-06-23T14:01:59.223542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_training_curves(train_losses, train_accuracies, val_accuracies, num_epochs):\n    \"\"\"绘制训练曲线图\"\"\"\n    plt.figure(figsize=(12, 6))\n    \n    # 1. 训练损失图表\n    plt.subplot(1, 2, 1)\n    plt.plot(range(1, num_epochs+1), train_losses, 'b-o', linewidth=2, markersize=8)\n    plt.xlabel('Epoch')\n    plt.ylabel('Train Loss')\n    plt.title('Training Loss')\n    plt.grid(True, linestyle='--', alpha=0.7)\n    plt.xticks([5, 10, 15, 20] if num_epochs >= 20 else \n               [5, 10, 15, 20][:num_epochs//5])  # 设置X轴刻度\n    \n    # 设置Y轴范围和刻度\n    max_loss = max(train_losses)\n    plt.ylim(0, max_loss * 1.1)\n    plt.yticks(np.linspace(0, max_loss, 5))\n    \n    # 2. 准确率图表\n    plt.subplot(1, 2, 2)\n    plt.plot(range(1, num_epochs+1), train_accuracies, 'g-s', linewidth=2, markersize=8, label='Train Acc')\n    plt.plot(range(1, num_epochs+1), val_accuracies, 'r-^', linewidth=2, markersize=8, label='Valid Acc')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy (%)')\n    plt.title('Training and Validation Accuracy')\n    plt.grid(True, linestyle='--', alpha=0.7)\n    plt.legend(loc='lower right')\n    plt.xticks([5, 10, 15, 20] if num_epochs >= 20 else \n               [5, 10, 15, 20][:num_epochs//5])  # 设置X轴刻度\n    \n    # 设置Y轴范围和刻度\n    min_acc = min(min(train_accuracies), min(val_accuracies))\n    plt.ylim(max(0, min_acc - 5), 100)\n    plt.yticks(range(0, 101, 20))\n    \n    # 保存图表\n    plt.tight_layout()\n    plt.savefig(os.path.join(WORKING_DIR, 'training_curves.png'))\n    plt.show()\n\nprint(\"训练函数定义完成\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:59.224945Z","iopub.execute_input":"2025-06-23T14:01:59.225198Z","iopub.status.idle":"2025-06-23T14:01:59.239256Z","shell.execute_reply.started":"2025-06-23T14:01:59.225173Z","shell.execute_reply":"2025-06-23T14:01:59.238503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 超参数设置\nnum_epochs = 20\nlearning_rate = 0.001\n\n# 损失函数和优化器\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode='min', factor=0.5, patience=3, verbose=True\n)\n\nprint(\"开始训练模型...\")\nprint(f\"训练周期: {num_epochs}\")\nprint(f\"学习率: {learning_rate}\")\nprint(f\"批量大小: {batch_size}\")\n\n# 训练模型\nmodel, train_losses, train_accuracies, val_losses, val_accuracies = train_model(\n    model, train_loader, val_loader, \n    criterion, optimizer, scheduler, num_epochs\n)\n\nprint(\"训练完成\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:01:59.239960Z","iopub.execute_input":"2025-06-23T14:01:59.240218Z","iopub.status.idle":"2025-06-23T14:10:31.634341Z","shell.execute_reply.started":"2025-06-23T14:01:59.240203Z","shell.execute_reply":"2025-06-23T14:10:31.633477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 绘制训练曲线 \nprint(\"绘制训练曲线...\")\nplot_training_curves(train_losses, train_accuracies, val_accuracies, num_epochs)\nprint(\"训练曲线已保存\")\n\n# 生成测试集预测 \nprint(\"生成测试集预测...\")\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel.eval()\n\npredictions = []\nimg_ids = []\n\nwith torch.no_grad():\n    for i, (images, ids) in enumerate(test_loader):\n        images = images.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs, 1)\n        \n        # 转换预测标签为类别名称\n        for idx, pred in enumerate(predicted.cpu().numpy()):\n            img_ids.append(ids[idx])\n            predictions.append(class_names[pred])\n        \n        if (i + 1) % 10 == 0:\n            print(f\"已处理 {len(img_ids)}/{len(test_loader.dataset)} 张测试图片\")\n\n# 创建提交文件\nsubmission_df = pd.DataFrame({\n    'id': img_ids,\n    'label': predictions\n})\n\n# 按id排序\nsubmission_df['id'] = submission_df['id'].astype(int)\nsubmission_df = submission_df.sort_values('id')\n\n# 保存为CSV文件\nsubmission_path = os.path.join(WORKING_DIR, 'submission.csv')\nsubmission_df.to_csv(submission_path, index=False)\nprint(f\"提交文件已保存为 '{submission_path}'\")\nprint(f\"前5行提交内容:\\n{submission_df.head()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T14:10:31.635308Z","iopub.execute_input":"2025-06-23T14:10:31.635620Z","iopub.status.idle":"2025-06-23T14:11:34.674037Z","shell.execute_reply.started":"2025-06-23T14:10:31.635592Z","shell.execute_reply":"2025-06-23T14:11:34.673269Z"}},"outputs":[],"execution_count":null}]}