{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:52:00.098012Z","iopub.execute_input":"2025-06-24T07:52:00.098289Z","iopub.status.idle":"2025-06-24T07:52:00.763327Z","shell.execute_reply.started":"2025-06-24T07:52:00.098265Z","shell.execute_reply":"2025-06-24T07:52:00.762490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr  # 安装解压 7z 依赖\nimport os\nimport shutil\nimport pandas as pd\nimport torch\nimport torchvision\nfrom torch import nn\nfrom torchvision import transforms, datasets\nfrom torch.utils.data import DataLoader\nimport collections\nimport math\nimport matplotlib.pyplot as plt  # 可视化依赖\nfrom tqdm import tqdm  # 进度条依赖","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:52:00.764846Z","iopub.execute_input":"2025-06-24T07:52:00.765160Z","iopub.status.idle":"2025-06-24T07:52:13.369552Z","shell.execute_reply.started":"2025-06-24T07:52:00.765142Z","shell.execute_reply":"2025-06-24T07:52:13.368972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Accumulator:\n    \"\"\"累加器：统计损失、准确率等指标\"\"\"\n    def __init__(self, n):\n        self.data = [0.0] * n\n\n    def add(self, *args):\n        self.data = [a + float(b) for a, b in zip(self.data, args)]\n\n    def reset(self):\n        self.data = [0.0] * len(self.data)\n\n    def __getitem__(self, idx):\n        return self.data[idx]\n\ndef accuracy(y_hat, y):\n    \"\"\"计算预测准确率（分类任务）\"\"\"\n    if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:\n        y_hat = y_hat.argmax(dim=1)  # 取概率最大的类别\n    cmp = y_hat.type(y.dtype) == y\n    return float(cmp.type(y.dtype).sum())\n\ndef evaluate_accuracy_gpu(net, data_iter, device=None):\n    \"\"\"GPU 版准确率评估（支持多 GPU）\"\"\"\n    if device is None:\n        device = next(iter(net.parameters())).device\n    net.eval()\n    metric = Accumulator(2)\n    with torch.no_grad():\n        for X, y in data_iter:\n            X, y = X.to(device), y.to(device)\n            metric.add(accuracy(net(X), y), y.numel())\n    return metric[0] / metric[1]\n\nclass Animator:\n    \"\"\"动态绘图类（训练过程可视化）\"\"\"\n    def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,\n                 ylim=None, xscale='linear', yscale='linear',\n                 fmts=('-', 'm--', 'g-.', 'r:'), figsize=(8, 5)):\n        plt.figure(figsize=figsize)\n        self.axes = plt.gca()\n        self.xlabel = xlabel\n        self.ylabel = ylabel\n        self.legend = legend\n        self.xlim = xlim\n        self.ylim = ylim\n        self.xscale = xscale\n        self.yscale = yscale\n        self.fmts = fmts\n        self.lines = []\n        self.annotations = []\n        self.curr_x = 0\n\n    def add(self, x, y):\n        \"\"\"动态添加数据并更新图表\"\"\"\n        if not hasattr(y, \"__len__\"):\n            y = [y]\n        if not self.lines:\n            self.axes.set_xlabel(self.xlabel)\n            self.axes.set_ylabel(self.ylabel)\n            self.axes.set_xscale(self.xscale)\n            self.axes.set_yscale(self.yscale)\n            if self.xlim:\n                self.axes.set_xlim(self.xlim)\n            if self.ylim:\n                self.axes.set_ylim(self.ylim)\n            self.axes.legend(self.legend)\n        \n        for i, (yi, fmt) in enumerate(zip(y, self.fmts)):\n            if i >= len(self.lines):\n                line, = self.axes.plot([], [], fmt)\n                self.lines.append(line)\n            self.lines[i].set_data(list(self.lines[i].get_xdata()) + [x],\n                                   list(self.lines[i].get_ydata()) + [yi])\n        self.axes.relim()\n        self.axes.autoscale_view()\n        plt.draw()\n        plt.pause(0.001)  # 短暂暂停以更新图表","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:52:13.370365Z","iopub.execute_input":"2025-06-24T07:52:13.370780Z","iopub.status.idle":"2025-06-24T07:52:13.383258Z","shell.execute_reply.started":"2025-06-24T07:52:13.370746Z","shell.execute_reply":"2025-06-24T07:52:13.382447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\n# 数据集路径配置（Kaggle 输入路径 + 解压路径）\ndata_root = '/kaggle/input/cifar-10'  # Kaggle 竞赛数据集输入路径\nunzip_dir = '/kaggle/working/cifar-10-unzipped'  # 解压到可写目录（Kaggle 工作区）\n\n# 解压 train.7z\nwith py7zr.SevenZipFile(os.path.join(data_root, 'train.7z'), 'r') as z:\n    z.extractall(path=unzip_dir)\n\n# 解压 test.7z\nwith py7zr.SevenZipFile(os.path.join(data_root, 'test.7z'), 'r') as z:\n    z.extractall(path=unzip_dir)\n\n# 定义关键路径（解压后的数据目录）\ntrain_dir = os.path.join(unzip_dir, 'train')  # 训练集原始图像目录\ntest_dir = os.path.join(unzip_dir, 'test')    # 测试集原始图像目录\ntrain_labels_path = os.path.join(data_root, 'trainLabels.csv')  # 标签文件路径","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:52:13.384071Z","iopub.execute_input":"2025-06-24T07:52:13.384789Z","iopub.status.idle":"2025-06-24T07:54:32.156282Z","shell.execute_reply.started":"2025-06-24T07:52:13.384766Z","shell.execute_reply":"2025-06-24T07:54:32.155496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"读取标签文件，返回 {文件名: 标签} 字典\"\"\"\n    with open(fname, 'r') as f:\n        lines = f.readlines()[1:]  # 跳过表头\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict((name, label) for name, label in tokens)\n\ndef copyfile(filename, target_dir):\n    \"\"\"复制文件到目标目录（自动创建目录）\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"拆分训练集为：训练集 + 验证集\"\"\"\n    label_count = collections.Counter(labels.values())\n    min_label_count = min(label_count.values())  # 样本最少的类别数量\n    n_valid_per_label = max(1, math.floor(min_label_count * valid_ratio))  # 每个类别保留的验证集样本数\n\n    label_count = {}\n    train_valid_test_root = os.path.join(unzip_dir, 'train_valid_test')  # 新数据目录\n    for train_file in tqdm(os.listdir(data_dir), desc=\"整理训练集\"):\n        label = labels[train_file.split('.')[0]]  # 假设文件名格式：xxxx.png\n        fname = os.path.join(data_dir, train_file)\n        \n        # 复制到 train_valid（合并训练+验证集，用于最终训练）\n        copyfile(fname, os.path.join(train_valid_test_root, 'train_valid', label))\n        \n        # 分配验证集样本\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(train_valid_test_root, 'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(train_valid_test_root, 'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"整理测试集目录结构\"\"\"\n    train_valid_test_root = os.path.join(unzip_dir, 'train_valid_test')\n    for test_file in tqdm(os.listdir(data_dir), desc=\"整理测试集\"):\n        copyfile(\n            os.path.join(data_dir, test_file),\n            os.path.join(train_valid_test_root, 'test', 'unknown')\n        )\n\ndef reorg_cifar10_data(valid_ratio=0.1):\n    \"\"\"主函数：整理 CIFAR-10 数据集\"\"\"\n    labels = read_csv_labels(train_labels_path)\n    reorg_train_valid(train_dir, labels, valid_ratio)\n    reorg_test(test_dir)\n\n# 执行数据整理（仅需运行一次）\nreorg_cifar10_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:54:32.157191Z","iopub.execute_input":"2025-06-24T07:54:32.157415Z","iopub.status.idle":"2025-06-24T07:55:29.001480Z","shell.execute_reply.started":"2025-06-24T07:54:32.157397Z","shell.execute_reply":"2025-06-24T07:55:29.000707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据增强与归一化（训练集）\ntransform_train = transforms.Compose([\n    transforms.Resize(40),  # 放大到 40x40\n    transforms.RandomResizedCrop(32, scale=(0.64, 1.0), ratio=(1.0, 1.0)),  # 随机裁剪 + 缩放\n    transforms.RandomHorizontalFlip(),  # 随机水平翻转\n    transforms.ToTensor(),  # 转换为 Tensor\n    transforms.Normalize([0.4914, 0.4822, 0.4465],  # 归一化（CIFAR-10 均值/方差）\n                         [0.2023, 0.1994, 0.2010])\n])\n\n# 测试集仅归一化（无增强）\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.4914, 0.4822, 0.4465],\n                         [0.2023, 0.1994, 0.2010])\n])\n\n# 数据集加载（基于整理后的目录）\ntrain_valid_test_root = os.path.join(unzip_dir, 'train_valid_test')\ntrain_ds = datasets.ImageFolder(\n    os.path.join(train_valid_test_root, 'train'),\n    transform=transform_train\n)\ntrain_valid_ds = datasets.ImageFolder(\n    os.path.join(train_valid_test_root, 'train_valid'),\n    transform=transform_train\n)\nvalid_ds = datasets.ImageFolder(\n    os.path.join(train_valid_test_root, 'valid'),\n    transform=transform_test\n)\ntest_ds = datasets.ImageFolder(\n    os.path.join(train_valid_test_root, 'test'),\n    transform=transform_test\n)\n\n# 数据加载器（Batch + Shuffle）\nbatch_size = 32  # 根据 GPU 显存调整（Kaggle 免费 GPU 建议 32/64）\ntrain_iter = DataLoader(train_ds, batch_size, shuffle=True, drop_last=True)\ntrain_valid_iter = DataLoader(train_valid_ds, batch_size, shuffle=True, drop_last=True)\nvalid_iter = DataLoader(valid_ds, batch_size, shuffle=False, drop_last=True)\ntest_iter = DataLoader(test_ds, batch_size, shuffle=False, drop_last=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:55:29.003153Z","iopub.execute_input":"2025-06-24T07:55:29.003370Z","iopub.status.idle":"2025-06-24T07:55:30.097307Z","shell.execute_reply.started":"2025-06-24T07:55:29.003352Z","shell.execute_reply":"2025-06-24T07:55:30.096381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_net():\n    \"\"\"构建模型：MobileNetV2（预训练 + 微调分类头）\"\"\"\n    # 加载预训练 MobileNetV2\n    net = torchvision.models.mobilenet_v2(pretrained=True)\n    # 替换最后一层分类器（适配 CIFAR-10 的 10 分类）\n    net.classifier[1] = nn.Linear(net.classifier[1].in_features, 10)\n    return net","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:55:30.098228Z","iopub.execute_input":"2025-06-24T07:55:30.098530Z","iopub.status.idle":"2025-06-24T07:55:30.105303Z","shell.execute_reply.started":"2025-06-24T07:55:30.098504Z","shell.execute_reply":"2025-06-24T07:55:30.104540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period, lr_decay):\n    \"\"\"训练函数：支持动态可视化 + 最终结果绘图\"\"\"\n    # 优化器 + 学习率调度\n    trainer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=wd)\n    scheduler = torch.optim.lr_scheduler.StepLR(trainer, lr_period, lr_decay)\n    \n    num_batches = len(train_iter)\n    # 动态绘图初始化\n    animator = Animator(xlabel='epoch', xlim=[1, num_epochs],\n                        legend=['train loss', 'train acc', 'valid acc'],\n                        figsize=(8, 5))\n    \n    # 多 GPU 支持\n    net = nn.DataParallel(net, device_ids=devices).to(devices[0])\n    loss_fn = nn.CrossEntropyLoss(reduction=\"none\")\n    \n    # 记录指标（用于最终绘图）\n    train_loss_list = []\n    train_acc_list = []\n    valid_acc_list = []\n\n    for epoch in range(num_epochs):\n        net.train()  # 训练模式（启用 Dropout、BatchNorm 等）\n        metric = Accumulator(3)  # 统计：损失总和、正确数、样本数\n\n        # 进度条显示训练过程\n        with tqdm(train_iter, desc=f\"Epoch {epoch + 1}/{num_epochs}\") as pbar:\n            for i, (features, labels) in enumerate(pbar):\n                features, labels = features.to(devices[0]), labels.to(devices[0])\n                \n                # 前向传播 + 反向传播\n                trainer.zero_grad()\n                y_hat = net(features)\n                l = loss_fn(y_hat, labels).mean()\n                l.backward()\n                trainer.step()\n                \n                # 更新指标\n                metric.add(l.item(), accuracy(y_hat, labels), labels.numel())\n                \n                # 进度条显示当前批次指标\n                pbar.set_postfix({\n                    'loss': f'{metric[0] / metric[2]:.3f}',\n                    'acc': f'{metric[1] / metric[2]:.3f}'\n                })\n\n        # 计算当前 Epoch 指标\n        train_loss = metric[0] / metric[2]\n        train_acc = metric[1] / metric[2]\n        train_loss_list.append(train_loss)\n        train_acc_list.append(train_acc)\n\n        # 验证集评估\n        valid_acc = None\n        if valid_iter is not None:\n            valid_acc = evaluate_accuracy_gpu(net, valid_iter)\n            valid_acc_list.append(valid_acc)\n            animator.add(epoch + 1, (train_loss, train_acc, valid_acc))\n        else:\n            animator.add(epoch + 1, (train_loss, train_acc, None))\n\n        # 更新学习率\n        scheduler.step()\n\n    # 训练结束后，绘制最终结果图（与示例图一致）\n    plt.figure(figsize=(8, 5))\n    epochs = range(1, num_epochs + 1)\n    plt.plot(epochs, train_loss_list, label='train loss', color='blue', linestyle='-')\n    plt.plot(epochs, train_acc_list, label='train acc', color='pink', linestyle='--')\n    if valid_iter is not None:\n        plt.plot(epochs, valid_acc_list, label='valid acc', color='green', linestyle='-.')\n    plt.xlabel('epoch')\n    plt.ylabel('value')\n    plt.legend(loc='upper right')\n    plt.grid(alpha=0.3)\n    plt.show()\n\n    # 打印最终结果\n    print(f'\\n训练结束 | train loss: {train_loss:.3f}, train acc: {train_acc:.3f}')\n    if valid_iter is not None:\n        print(f'验证集 acc: {valid_acc:.3f}')\n\n    return net","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:55:30.106008Z","iopub.execute_input":"2025-06-24T07:55:30.106297Z","iopub.status.idle":"2025-06-24T07:55:30.124347Z","shell.execute_reply.started":"2025-06-24T07:55:30.106269Z","shell.execute_reply":"2025-06-24T07:55:30.123622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 确保导入NumPy库\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\n\n# 设备配置（自动检测 GPU/CPU）\ndevices = [torch.device('cuda')] if torch.cuda.is_available() else [torch.device('cpu')]\n\n# ========== 新增：预测结果可视化函数 ==========\ndef visualize_predictions(model, dataset, class_names, num_samples=8):\n    \"\"\"可视化模型预测结果\"\"\"\n    model.eval()\n    indices = np.random.choice(len(dataset), num_samples, replace=False)\n    \n    fig, axes = plt.subplots(1, num_samples, figsize=(16, 3))\n    for i, idx in enumerate(indices):\n        img, _ = dataset[idx]\n        img_tensor = img.unsqueeze(0).to(devices[0])\n        \n        with torch.no_grad():\n            pred = model(img_tensor)\n            pred_class = pred.argmax(dim=1).item()\n        \n        # 反归一化图像用于显示\n        img_np = img.permute(1, 2, 0).cpu().numpy()\n        img_np = img_np * np.array([0.2023, 0.1994, 0.2010]) + np.array([0.4914, 0.4822, 0.4465])\n        img_np = np.clip(img_np, 0, 1)\n        \n        axes[i].imshow(img_np)\n        axes[i].set_title(f\"预测: {class_names[pred_class]}\", fontsize=10)\n        axes[i].axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# ========== 以下是你提供的代码 ==========\n# 训练参数（可根据需求调整）\nnum_epochs, lr, wd = 20, 2e-4, 5e-4  # 轮次、学习率、权重衰减\nlr_period, lr_decay = 4, 0.9         # 学习率衰减周期、衰减率\n\n# ========== 第一阶段训练：训练集 + 验证集评估 ==========\nprint(\"开始第一阶段训练（训练集 + 验证集评估）...\")\nnet = get_net()\nnet = train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period, lr_decay)\n\n# 保存第一阶段模型\ntorch.save(net.state_dict(), 'cifar10_model_stage1.pth')\n\n# ========== 第二阶段训练：合并训练+验证集（为测试集预测做准备） ==========\nprint(\"\\n开始第二阶段训练（合并训练+验证集）...\")\nnet_final = get_net()\nnet_final = train(net_final, train_valid_iter, None, num_epochs, lr, wd, devices, lr_period, lr_decay)\n\n# 保存最终模型\ntorch.save(net_final.state_dict(), 'cifar10_model_final.pth')\n\n# ========== 测试集预测 + 生成提交文件 ==========\nprint(\"\\n开始测试集预测...\")\nnet_final.eval()  # 切换到评估模式\npreds = []\nwith torch.no_grad(), tqdm(test_iter, desc=\"测试集预测\") as pbar:\n    for X, _ in pbar:\n        y_hat = net_final(X.to(devices[0]))\n        preds.extend(y_hat.argmax(dim=1).type(torch.int32).cpu().numpy())\n\n# 生成 Kaggle 提交文件\nsample_submission = pd.read_csv(os.path.join(data_root, 'sampleSubmission.csv'))\nsorted_ids = sample_submission['id'].tolist()  # 严格按示例提交文件的 ID 顺序\n\n# 标签映射（修正后的完整代码）\nlabel_to_idx = train_valid_ds.class_to_idx\nidx_to_label = {v: k for k, v in label_to_idx.items()}  # 修复字典推导式\n\n# 写入结果\nwith open('submission.csv', 'w') as f:\n    f.write('id,label\\n')\n    for img_id, pred_idx in zip(sorted_ids, preds):\n        f.write(f'{img_id},{idx_to_label[pred_idx]}\\n')\n\nprint(\"提交文件已生成：submission.csv\")\n\n# ========== 可视化测试集预测结果 ==========\nprint(\"\\n可视化测试集预测结果:\")\nvisualize_predictions(net_final, test_ds, train_valid_ds.classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-24T07:55:30.125258Z","iopub.execute_input":"2025-06-24T07:55:30.125502Z","iopub.status.idle":"2025-06-24T08:41:08.190980Z","shell.execute_reply.started":"2025-06-24T07:55:30.125482Z","shell.execute_reply":"2025-06-24T08:41:08.190193Z"}},"outputs":[],"execution_count":null}]}