{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":113984,"databundleVersionId":13595424,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1.配置环境并准备数据集\n1. 请确保在右侧**Input**中已经添加本次作业所需要的数据集**深度学习与地理应用-2025秋-作业3-PM2.5浓度估计**（默认会自动添加，如未，则搜索添加即可）\n2. 若以上方法无法添加数据，请手动上传所需数据（可在作业首页下载数据）\n3. 若要使用GPU或TPU，需要手动在右侧**Session options**中指定**ACCELERATOR**，Kaggle为免费用户每周提供30个小时的GPU和20个小时的TPU免费时长，已经足够\n4. 若要托管代码，需要在右侧**Schedule a notebook to run**中保存或者直接点击右上角**Save Version**按钮，Kaggle会后台托管代码自动运行\n5. 详细介绍和其余操作参见课程资源网站","metadata":{}},{"cell_type":"code","source":"import pandas                                       # 提供dataframe，用来处理csv数据\nimport torch                                        # pytorch，目前主流的深度学习框架\nfrom torch.utils import data                        # 从torch.utils导入data模块，用来构建pytorch中的数据结构--->dataloader\nfrom torch import nn                                # 从torch导入nn模块，用来提供基本的神经网络接口，如全连接层nn.Linear()\nimport numpy as np                                  # 导入numpy，并将其重命名为np，numpy是一个高效的用来处理矩阵的库\nfrom tqdm import tqdm                               # tqdm是一个可视化代码进程的模块\nimport os                                           # os库用来处理磁盘读写过程\nfrom sklearn.preprocessing import StandardScaler    # 这是一个用来数据标准化的模块\nimport csv                                          # 读写csv文件的库\n\nnormalize = StandardScaler()                        # 实例化StandardScaler\n\n# 为了处理大批量数据，在目前主流的深度学习框架下，数据都是以矩阵的形式表示的\n# 而对于CPU来讲，其对于矩阵数据处理效率欠佳，而GPU(显卡）则擅长处理矩阵，因此深度学习模型都会部署到GPU上\n# 这段代码的意思是，判断目前的设备中是否有安装CUDA（GPU加速模块）。\n# 如果有的话，我们就让模型部署到GPU上，如果没有，我们继续用CPU（速度会慢一点）\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_path = '/kaggle/input/dl-geo-2025fall-homework3/train.csv'\nval_path = '/kaggle/input/dl-geo-2025fall-homework3/val.csv'\ntest_path = '/kaggle/input/dl-geo-2025fall-homework3/test.csv'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2.构建数据流","metadata":{}},{"cell_type":"code","source":"# 这段代码用来创建pytorch所需的基本数据结构，data.Dataset\nclass PM2_5Dataset(data.Dataset):\n    \n    def __init__(self, csv_path, mode=\"train\"):\n        super(PM2_5Dataset, self).__init__()\n        self.csv_content = pandas.read_csv(csv_path)\n        self.used_column = [\"AOD\", \"ET\", \"BLH\", \"TEM\", \"NDVI\", \"SP\", \"RH\", \"DEM\", \"NTL\", \"PRE\", \"WS\", \"WD\"] # TODO：这些特征都是有用的吗？修改这行代码，可以修改模型输入的特征\n        self.target_column = \"PM2.5\"\n        self.mode = mode\n        self.dim = len(self.used_column)\n        if mode in [\"train\", \"val\"]:\n            self.input_data, self.target_data = self.process_csv_with_gt()\n\n        else:\n            self.input_data = self.process_csv_without_gt()\n\n    def process_csv_with_gt(self):\n        input_data = [list(self.csv_content[i]) for i in self.used_column]\n        input_data = np.array(input_data)\n        input_data = input_data.T\n        input_data = normalize.fit_transform(input_data)\n        target_data = np.array(self.csv_content[self.target_column])\n        return input_data, target_data\n\n    def process_csv_without_gt(self):\n        input_data = [list(self.csv_content[i]) for i in self.used_column]\n        input_data = np.array(input_data)\n        input_data = input_data.T\n        input_data = normalize.fit_transform(input_data)\n        return input_data\n\n    def __getitem__(self, index):\n        if self.mode in [\"train\", \"val\"]:\n            data, gt = self.input_data[index, :], self.target_data[index]\n            data = torch.from_numpy(data)\n            gt = torch.tensor(gt)\n            return data, gt\n        else:\n            data = self.input_data[index, :]\n            return torch.from_numpy(data)\n\n    def __len__(self):\n        return self.input_data.shape[0]\n\ndef prepare_dataset(csv_path, batch_size, mode):\n    dataset = PM2_5Dataset(csv_path, mode)\n    # 在pytorch中，dataset需要用dataloader封装\n    if mode in [\"train\", \"val\"]:\n        dataloader = data.DataLoader(dataset, batch_size, num_workers=0, pin_memory=True)\n    else:\n        dataloader = data.DataLoader(dataset, 1, num_workers=0, pin_memory=True, shuffle=False)\n    return dataloader","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3.构建模型","metadata":{}},{"cell_type":"code","source":"# 这是一个基本的神经网络模型，包含三个全连接层，激活函数为Relu\nclass MyNeuralNet(nn.Module):\n    # TODO：增加约束，如添加dropput，batchnorm等。增删全连接层数。修改神经元数量。修改连接方式等。\n    def __init__(self, input_dim):\n        super(MyNeuralNet, self).__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, 64),\n            nn.ReLU(),\n            nn.Linear(64, 32),\n            nn.ReLU(),\n            nn.Linear(32, 1),\n        )\n\n    def forward(self, input):\n        predict = self.net(input)\n        return predict","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4.编写训练、验证和测试等Pipeline代码","metadata":{}},{"cell_type":"code","source":"def train(train_dataset, val_dataset, config, model):\n    \"\"\"训练代码\"\"\"\n    n_epochs = config['n_epochs'] # 训练次数\n    optimizer = getattr(torch.optim, config['optimizer'])(model.parameters(), **config['optim_hparas']) # 优化器\n    model.train() # 设置模型为训练模式（梯度可以回传）\n    pbar = tqdm(total=n_epochs, desc=\"Train Mode\", unit=\"epoch\")\n    loss = nn.MSELoss() # 损失函数为MSE(均方误差)\n    loss_log = {\"train_loss\": [], \"val_loss\": []}\n    min_mse = 1e16\n    for epoch in range(n_epochs):\n        pbar.update()\n        epoch_mse = 0.0\n        for x, y in train_dataset:\n            optimizer.zero_grad() # 将优化器里储存的梯度清零\n            x, y = x.to(device).float(), y.to(device).float()\n            y = y.unsqueeze(1)\n            pred = model(x)\n            mse_loss = loss(pred, y) # 计算损失\n            mse_loss.backward() # 损失回传\n            optimizer.step() # 梯度更新\n            loss_log[\"train_loss\"].append(mse_loss.detach().cpu().item())\n            epoch_mse += mse_loss.detach().cpu().item()\n        pbar.set_postfix(epoch=epoch + 1, mse=epoch_mse / len(train_dataset))\n        val_mse = val(val_dataset, model)\n        if val_mse < min_mse:\n            min_mse = val_mse\n            torch.save(model.state_dict(), config['save_path'])\n            print(f\"best model saved! val mse:{val_mse},epoch:{epoch + 1}\")\n        loss_log[\"val_loss\"].append(val_mse)\n    return min_mse, loss_log\n\ndef val(val_dataset, model):\n    \"\"\"验证代码\"\"\"\n    model.eval() # 设置为验证模式（梯度不回传）\n    total_loss = 0\n    loss = nn.MSELoss()\n    for x, y in val_dataset:\n        x, y = x.to(device).float(), y.to(device).float()\n        y = y.unsqueeze(1)\n        with torch.no_grad():\n            pred = model(x) # 预测结果\n            mse_loss = loss(pred, y) # 计算损失\n        total_loss += mse_loss.detach().cpu().item()\n    total_loss = total_loss / len(val_dataset)\n    return total_loss\n\n\ndef test(test_set, model):\n    \"\"\"测试代码\"\"\"\n    model.eval() #\n    preds = []\n    for x in test_set:\n        x = x.to(device).float()\n        with torch.no_grad():\n            pred = model(x)\n            preds.append(pred.detach().cpu())\n    return preds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 这部分不需要修改\ndef process_preds_to_csv(preds):\n    with open(\"MyPred.csv\", \"w\", newline='') as f:\n        writer = csv.writer(f)\n        writer.writerow(['id', 'PM2.5'])\n        for i, p in enumerate(preds):\n            p = p.cpu().item()\n            writer.writerow([i, p])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5.模型训练和评估","metadata":{}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/models', exist_ok=True)\n# 模型超参数设置\nconfig = {\n    'n_epochs': 200,  # 训练次数\n    'batch_size': 128,  # mini-batch 的大小\n    'optimizer': 'SGD',  # 优化算法 (optimizer in torch.optim)\n    'optim_hparas': {  # 优化器的超参数(取决于你用了什么优化算法)\n        'lr': 0.001,\n        'momentum':0.9                # SGD的学习率  # SGD的动量\n    },\n    'save_path': 'models/model.pth'  # 存储路径\n}\n# 构建dataset\ntrain_dataset = prepare_dataset(train_path, config[\"batch_size\"], mode=\"train\")\nval_dataset = prepare_dataset(val_path, config[\"batch_size\"], mode=\"val\")\ntest_dataset = prepare_dataset(test_path, config[\"batch_size\"], mode=\"test\")\n# 实例化模型\nmodel = MyNeuralNet(train_dataset.dataset.dim).to(device)\n# 开始训练与验证\nmin_mse, loss_log = train(train_dataset, val_dataset, config, model)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt # 用来绘制曲线\nfrom matplotlib.pyplot import figure\n\ndef plot_learning_curve(loss_record, title=''):\n    ''' 绘制损失函数曲线 '''\n    total_steps = len(loss_record['train_loss'])\n    x_1 = range(total_steps)\n    x_2 = x_1[::len(loss_record['train_loss']) // len(loss_record['val_loss'])]\n    figure(figsize=(6, 4))\n    plt.plot(x_1, loss_record['train_loss'], c='tab:red', label='train')\n    plt.plot(x_2, loss_record['val_loss'], c='tab:cyan', label='val')\n    plt.ylim(0.0, 1000.)\n    plt.xlabel('Training steps')\n    plt.ylabel('MSE loss')\n    plt.title('Learning curve of {}'.format(title))\n    plt.legend()\n    plt.show()\n\ndef plot_pred(dv_set, model, device, lim=35., title=\"\", preds=None, targets=None):\n    ''' 绘制你的预测结果和真实结果之间的分布情况 '''\n    if preds is None or targets is None:\n        model.eval()\n        preds, targets = [], []\n        for x, y in dv_set:\n            x, y = x.to(device), y.to(device)\n            with torch.no_grad():\n                pred = model(x.float())\n                preds.append(pred.detach().cpu())\n                targets.append(y.detach().cpu())\n        preds = torch.cat(preds, dim=0).numpy()\n        targets = torch.cat(targets, dim=0).numpy()\n\n    figure(figsize=(5, 5))\n    plt.scatter(targets, preds, c='r', alpha=0.5)\n    plt.plot([-0.2, lim], [-0.2, lim], c='b')\n    plt.xlim(-0.2, lim)\n    plt.ylim(-0.2, lim)\n    plt.xlabel('ground truth value')\n    plt.ylabel('predicted value')\n    plt.title(title)\n    plt.show()\n\n\nplot_learning_curve(loss_log,\"DNN\")\n\nplot_pred(train_dataset,model,device,35., 'Ground Truth of Train v.s. Prediction')\n\nplot_pred(val_dataset,model,device,35.,'Ground Truth of Val v.s. Prediction')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5.使用训练好的模型进行测试","metadata":{}},{"cell_type":"code","source":"# 测试模型\npreds = test(test_dataset, model)\n# 将测试的结果保存到CSV文件里\nprocess_preds_to_csv(preds)\n# 执行完成后，可以从左侧下载预测结果(MyPred.csv)，并将其提交到Kaggle上，Kaggle会反馈给你测试集的评分","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}