{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd#导入csv文件的库\nimport numpy as np#进行矩阵运算的库\nimport matplotlib.pyplot as plt#作图的库\nimport torch#一个深度学习的库Pytorch\nimport torch.nn as nn#neural network,神经网络\nimport torch.optim as optim#一个实现了各种优化算法的库\nimport torch.nn.functional as F#神经网络函数库\nimport warnings#避免一些可以忽略的报错\nwarnings.filterwarnings('ignore')#filterwarnings()方法是用于设置警告过滤器的方法，它可以控制警告信息的输出方式和级别。","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-18T07:47:07.892412Z","iopub.execute_input":"2023-07-18T07:47:07.892941Z","iopub.status.idle":"2023-07-18T07:47:11.252585Z","shell.execute_reply.started":"2023-07-18T07:47:07.892871Z","shell.execute_reply":"2023-07-18T07:47:11.251529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#设置随机种子\nimport random\ntorch.backends.cudnn.deterministic = True#将cudnn框架中的随机数生成器设为确定性模式\ntorch.backends.cudnn.benchmark = False#关闭CuDNN框架的自动寻找最优卷积算法的功能，以避免不同的算法对结果产生影响\ntorch.manual_seed(42)\nnp.random.seed(42)\nrandom.seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:11.255719Z","iopub.execute_input":"2023-07-18T07:47:11.256731Z","iopub.status.idle":"2023-07-18T07:47:11.267307Z","shell.execute_reply.started":"2023-07-18T07:47:11.256692Z","shell.execute_reply":"2023-07-18T07:47:11.266331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv(\"/kaggle/input/ece597697-sp2023/train.csv/train.csv\")\nprint(f\"len(train_df):{len(train_df)}\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:11.268988Z","iopub.execute_input":"2023-07-18T07:47:11.269623Z","iopub.status.idle":"2023-07-18T07:47:17.914042Z","shell.execute_reply.started":"2023-07-18T07:47:11.269588Z","shell.execute_reply":"2023-07-18T07:47:17.912944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datay=train_df['label'].values\ntrain=train_df.drop(['label','id'],axis=1,inplace=True)\ndataX=train_df.values.reshape(-1,1,28,28)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:17.915530Z","iopub.execute_input":"2023-07-18T07:47:17.916544Z","iopub.status.idle":"2023-07-18T07:47:18.042012Z","shell.execute_reply.started":"2023-07-18T07:47:17.916505Z","shell.execute_reply":"2023-07-18T07:47:18.040776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#划分训练集和测试集的函数\ndef train_test_split(dataX,datay,shuffle=True,percentage=0.8):\n    \"\"\"\n    将训练数据X和标签y以numpy.array数组的形式传入\n    划分的比例定为训练集:测试集=8:2\n    \"\"\"\n    if shuffle:\n        random_num=[index for index in range(len(dataX))]\n        np.random.shuffle(random_num)\n        dataX=dataX[random_num]\n        datay=datay[random_num]\n    split_num=int(len(dataX)*percentage)\n    train_X=dataX[:split_num]\n    train_y=datay[:split_num]\n    test_X=dataX[split_num:]\n    test_y=datay[split_num:]\n    return train_X,train_y,test_X,test_y","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.045987Z","iopub.execute_input":"2023-07-18T07:47:18.046447Z","iopub.status.idle":"2023-07-18T07:47:18.057004Z","shell.execute_reply.started":"2023-07-18T07:47:18.046412Z","shell.execute_reply":"2023-07-18T07:47:18.055928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X,train_y,valid_X,valid_y=train_test_split(dataX,datay,percentage=0.9)\nlabel_count=np.max(train_y)+1\nfor index in range(label_count):\n    print(f\"np.sum(train_y==index)={np.sum(train_y==index)}\")\nprint(f\"train_X.shape:{train_X.shape},valid_X.shape:{valid_X.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.058639Z","iopub.execute_input":"2023-07-18T07:47:18.059024Z","iopub.status.idle":"2023-07-18T07:47:18.663107Z","shell.execute_reply.started":"2023-07-18T07:47:18.058992Z","shell.execute_reply":"2023-07-18T07:47:18.661565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset:\n    #储存数据集的信息\n    def __init__(self,raw_data):\n        self.raw_data=raw_data\n    #获取数据集的长度\n    def __len__(self):\n        return len(self.raw_data)\n    #获取数据集某特定条目的内容\n    def __getitem__(self,index):\n        \n        image,label=self.raw_data[index]\n        \n        return image,label\n\nclass DataLoader:\n    #从数据集中随机加载数据,并拼接成一个batch.\n    def __init__(self,dataset,batch_size=1,shuffle=False):\n        self.dataset=dataset\n        self.batch_size=batch_size\n        self.shuffle=False\n    #实现迭代器,使用时,迭代获取数据内容\n    def __iter__(self):\n        #每次准备迭代的时候,打乱数据,并清空指针\n        self.cursor=0\n        #正常序列\n        self.indexs=np.arange(len(self.dataset))\n\n        #打乱序列\n        if self.shuffle:\n            np.random.shuffle(self.indexs)\n            \n        return self\n\n    def __next__(self):\n        #预期在这里返回一个batch数据,batch是从dataset中随机抓取的\n        begin=self.cursor\n        end=self.cursor+self.batch_size\n\n        if end >len(self.dataset):\n            #如果end超出长度了,那迭代到底了\n            raise StopIteration()#抛出异常\n        #更新指针\n        self.cursor=end\n        #选择begin到end的部分\n        batched_data=[]\n        for index in self.indexs[begin:end]:\n            item=self.dataset[index]\n            batched_data.append(item)\n        return batched_data","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.666100Z","iopub.execute_input":"2023-07-18T07:47:18.666728Z","iopub.status.idle":"2023-07-18T07:47:18.679103Z","shell.execute_reply.started":"2023-07-18T07:47:18.666688Z","shell.execute_reply":"2023-07-18T07:47:18.677683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=50\n\ntrain_data=np.array([[train_X[index],train_y[index]] for index in range(len(train_X))])\nvalid_data=np.array([[valid_X[index],valid_y[index]] for index in range(len(valid_X))])\n\ntrain_dataset=Dataset(train_data)\n\nvalid_dataset=Dataset(valid_data)\n\ntrain_loader=DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\nvalid_loader=DataLoader(valid_dataset,batch_size=batch_size,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.681200Z","iopub.execute_input":"2023-07-18T07:47:18.681996Z","iopub.status.idle":"2023-07-18T07:47:18.785222Z","shell.execute_reply.started":"2023-07-18T07:47:18.681957Z","shell.execute_reply":"2023-07-18T07:47:18.784114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#多头注意力机制\nclass MultiHeadSelfAttention(nn.Module):\n    #定义初始化函数\n    def __init__(self,dim_in,d_model,num_heads=4):\n\n        super(MultiHeadSelfAttention,self).__init__()\n        \n        self.dim_in=dim_in\n        \n        self.d_model=d_model\n        \n        self.num_heads=num_heads\n        \n        #向量的维度必须被头的个数整除,否则会抛出异常.\n        assert d_model %num_heads==0,\"d_model must be multiple of num_heads\"\n        \n        #定义线性变换矩阵\n        self.linear_q=nn.Linear(dim_in,d_model)\n        self.linear_k=nn.Linear(dim_in,d_model)\n        self.linear_v=nn.Linear(dim_in,d_model)\n        self.scale=1/np.sqrt(d_model//num_heads)\n        \n        #最后的线性层\n        self.fc=nn.Linear(d_model,d_model)\n        \n    def forward(self,x):\n        batch,n,dim_in=x.shape\n        \n        assert dim_in==self.dim_in\n        \n        nh=self.num_heads\n        \n        dk=self.d_model//nh\n        \n        q=self.linear_q(x).reshape(batch,n,nh,dk).transpose(1,2)\n        k=self.linear_k(x).reshape(batch,n,nh,dk).transpose(1,2)\n        v=self.linear_v(x).reshape(batch,n,nh,dk).transpose(1,2)\n        \n        \n        dist=torch.matmul(q,k.transpose(2,3))*self.scale\n        dist=torch.softmax(dist,dim=-1)\n        \n        att=torch.matmul(dist,v)\n        \n        att=att.transpose(1,2).reshape(batch,n,self.d_model)\n        \n        output=self.fc(att)\n        \n        return output ","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.787537Z","iopub.execute_input":"2023-07-18T07:47:18.788275Z","iopub.status.idle":"2023-07-18T07:47:18.800446Z","shell.execute_reply.started":"2023-07-18T07:47:18.788240Z","shell.execute_reply":"2023-07-18T07:47:18.799398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SELayer(nn.Module):\n    def __init__(self,channel=256,reduction=16):\n        super(SELayer,self).__init__()\n        self.avg_pool=nn.AdaptiveAvgPool2d(1)#输出为1*1\n        self.fc=nn.Sequential(\n            nn.Linear(channel,channel//reduction,bias=True),\n            nn.BatchNorm1d(channel//reduction),\n            nn.GELU(),\n            nn.Linear(channel//reduction,channel,bias=True),\n            nn.Sigmoid(),\n        )\n    def forward(self,x):\n        #batch_size,channels,\n        b,c,_,_=x.shape\n        \n        y=self.avg_pool(x).view(b,c)\n        y=self.fc(y).view(b,c,1,1)\n        return x*y.expand_as(x)#将y扩展到x同样的shape","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.804126Z","iopub.execute_input":"2023-07-18T07:47:18.804546Z","iopub.status.idle":"2023-07-18T07:47:18.815228Z","shell.execute_reply.started":"2023-07-18T07:47:18.804518Z","shell.execute_reply":"2023-07-18T07:47:18.814075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#对特征图的每个位置也提取一次权重\nclass SEConv(nn.Module):\n    def __init__(self,area=36,reduction=4):\n        super(SEConv,self).__init__()\n        self.fc=nn.Sequential(\n            nn.Linear(area,area//reduction,bias=True),\n            nn.BatchNorm1d(area//reduction),\n            nn.GELU(),\n            nn.Linear(area//reduction,area,bias=True),\n            nn.Sigmoid(),\n        )\n    def forward(self,x):\n        #batch_size,channels,(height*width)\n        b,c,area=x.shape\n        y=torch.mean(x,dim=1)\n        y=self.fc(y)\n        y=y.view(-1,1,area)\n        return x*y.expand_as(x)#将y扩展到x同样的shape","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.816978Z","iopub.execute_input":"2023-07-18T07:47:18.817587Z","iopub.status.idle":"2023-07-18T07:47:18.832188Z","shell.execute_reply.started":"2023-07-18T07:47:18.817551Z","shell.execute_reply":"2023-07-18T07:47:18.831139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Active_Rotating_Filters(nn.Module):\n    #定义初始化函数\n    def __init__(self,in_channels=3,out_channels=64,kernel_size=3,stride=1,padding=0):\n        super(Active_Rotating_Filters,self).__init__()\n        self.in_channels=in_channels\n        self.out_channels=out_channels\n        self.kernel_size=kernel_size\n        self.stride=stride\n        self.padding=padding\n        \n        self.conv=nn.Conv2d(in_channels,out_channels,kernel_size,stride,padding)\n        self.rot_conv=nn.Conv2d(in_channels,out_channels,kernel_size,stride,padding)#用于存放旋转后卷积的参数\n    \n    #对卷积核的权重进行旋转\n    def rotate_weight(self,index,weight):#out_channel*in_channel*kernel_size*kernel_size\n        #逆时针旋转90度旋转一次\n        rot_weight=torch.rot90(weight,index,(2,3))\n        return rot_weight\n        \n    def forward(self,x):\n        rot_x=self.conv(x)\n        for index in range(1,4):\n            #旋转后的参数\n            self.rot_conv.weight=nn.Parameter(self.rotate_weight(index,self.conv.weight))\n            rot_x=torch.cat((rot_x,self.rot_conv(x)),dim=1)\n        b,c,h,w=rot_x.shape\n        rot_x=rot_x.reshape(b,c//4,4,h,w)\n        rot_x,_=torch.max(rot_x,dim=2)#,_忽略索引\n        rot_x=rot_x.view(b,-1,h,w)\n        return rot_x","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.833973Z","iopub.execute_input":"2023-07-18T07:47:18.834769Z","iopub.status.idle":"2023-07-18T07:47:18.847368Z","shell.execute_reply.started":"2023-07-18T07:47:18.834728Z","shell.execute_reply":"2023-07-18T07:47:18.846271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#score:0.92976\nclass CNN(nn.Module):\n\n    def __init__(self):\n        \n        super(CNN,self).__init__()\n        \n        self.flag=0\n        self.center_array=[]\n        \n        #创建自注意机制的类\n        self.Self_Attention=MultiHeadSelfAttention(dim_in=36,d_model=36,num_heads=3)\n        \n        self.selayer=SELayer()\n        \n        self.seconv=SEConv()\n        \n        self.conv=nn.Sequential(\n        Active_Rotating_Filters(1,16,3,1,0),#30*30->28*28\n        nn.BatchNorm2d(16),\n        nn.SiLU(),\n        Active_Rotating_Filters(16,32,3,1,0),#28*28->26*26\n        nn.BatchNorm2d(32),\n        nn.ReLU(),\n        Active_Rotating_Filters(32,72,5,1,0),#26*26->22*22\n        nn.BatchNorm2d(72),\n        nn.Tanh(),\n        nn.Conv2d(72,128,5,1,1),#20*20\n        nn.GELU(),\n        nn.MaxPool2d(2,2,0),#10*10\n        nn.BatchNorm2d(128),\n        nn.Conv2d(128,256,5,1,0),#6*6\n        nn.BatchNorm2d(256),\n        nn.SiLU(),\n        nn.Conv2d(256,256,1,1,0),#6*6\n        nn.BatchNorm2d(256),\n        nn.SiLU(),\n        self.selayer,\n        )\n        \n        self.fc=nn.Sequential( \n        nn.Linear(256*6*6,128),\n        nn.BatchNorm1d(128),\n        nn.Dropout(p=0.2),\n        nn.GELU(),\n        nn.Linear(128,10),\n        )\n    def circle_padding(sellf,image, pad=1):\n        \"\"\"\n        input: torch.Tensor image (batch_size, channels, height, width)\n        output: pad_image (batch_size, channels, height+2*pad, width+2*pad)\n        \"\"\"\n        b, c, h, w = image.shape\n        pad_image = torch.zeros((b, c, h+2*pad, w+2*pad), device=image.device)\n\n        pad_image[:, :, pad:h+pad, pad:w+pad] = image\n        pad_image[:, :, 0:pad, pad:w+pad] = image[:, :, -pad:, :]\n        pad_image[:, :, -pad:, pad:w+pad] = image[:, :, 0:pad, :]\n        pad_image[:, :, pad:h+pad, 0:pad] = image[:, :, :, -pad:]\n        pad_image[:, :, pad:h+pad, -pad:] = image[:, :, :, 0:pad]\n        pad_image[:, :, 0:pad, 0:pad] = image[:, :, -pad:, -pad:]\n        pad_image[:, :, -pad:, -pad:] = image[:, :, 0:pad, 0:pad]\n        pad_image[:, :, 0:pad, -pad:] = image[:, :, -pad:, 0:pad]\n        pad_image[:, :, -pad:, 0:pad] = image[:, :, 0:pad, -pad:]\n\n        return pad_image\n    \n    def get_centering_array(self,image):\n        if self.flag==0:\n            #传入的是np.array的数组\n            batch_size,channels,row,col=image.shape\n            center_x=row/2\n            center_y=col/2\n            array=torch.randn(row,col).to(device)\n            for i in range(row):\n                for j in range(col):\n                    array[i][j]=np.sqrt(1- ( (i-center_x)**2+(j-center_y)**2)/( center_x**2+center_y**2 )/10)\n            self.flag=1\n            self.center_array=array\n        return image*self.center_array\n    #噪音注入\n    def noise_injection(self,features,noise_factor=0.002):\n        noise = torch.randn_like(features) * noise_factor\n        return features + noise\n    \n    def forward(self,input): \n        output=self.circle_padding(input)\n        output=self.get_centering_array(output)\n        output=self.noise_injection(output)\n        output=self.conv(output)\n        output=output.view(-1,256,6*6)#reshape\n        output=self.seconv(output)\n        output=output+self.Self_Attention(output)#跳连接\n        output=output.view(-1,256*6*6)#reshape\n        output=self.fc(output)\n        return F.log_softmax(output/5,dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.849005Z","iopub.execute_input":"2023-07-18T07:47:18.849458Z","iopub.status.idle":"2023-07-18T07:47:18.875294Z","shell.execute_reply.started":"2023-07-18T07:47:18.849425Z","shell.execute_reply":"2023-07-18T07:47:18.874193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms#Pytorch下面的图像处理库,用于对图像进行数据增强\nfrom PIL import Image#图像处理库\n#定义数据增强\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),  # 随机水平翻转\n    #transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1),  # 随机调整图像亮度、对比度、饱和度和色调\n    #transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),  # 对图像进行标准化操作\n    #transforms.ToTensor(),  # 转换为张量\n])\n\n\ndef get_batch_transform(batch_data,is_train=True):\n    \"\"\"\n    input:batch_data batch*(image,label)\n          numpy.array\n    output,tensor:X,y\n    \"\"\"\n    X=[]\n    y=[]\n    batch_size=len(batch_data)\n    for i in range(batch_size):\n        dataX,datay=batch_data[i]\n        image=torch.Tensor(dataX)\n        if is_train:\n            image=train_transform(image)\n        X.append(image.cpu().numpy())\n        y.append(datay)\n    X=torch.Tensor(X)\n    y=torch.Tensor(y)\n    return X,y#返回经过处理后的数据X和y","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:18.878810Z","iopub.execute_input":"2023-07-18T07:47:18.879149Z","iopub.status.idle":"2023-07-18T07:47:19.152447Z","shell.execute_reply.started":"2023-07-18T07:47:18.879122Z","shell.execute_reply":"2023-07-18T07:47:19.151319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndevice='cuda' if torch.cuda.is_available() else\"cpu\"\nprint(f\"device:{device}\")\n\nnetC=CNN()\n#训练次数\nnum_epochs=20\n#优化器\noptimizer=optim.Adam(netC.parameters(),lr=0.00025,betas=(0.5,0.999))\n#损失函数\ncriterion=nn.NLLLoss()\n\nnetC.to(device)\n\nweight_decay=0.001\n\nfor epoch in range(num_epochs):\n    #模型的训练\n    netC.train()\n    for index,batched_data in enumerate(train_loader):\n        X,y=get_batch_transform(batched_data,is_train=True)\n        X,y=X.to(device),y.long().to(device)\n        #将梯度清空\n        optimizer.zero_grad()\n        #将数据放进去训练\n        output=netC(X).to(device)\n        #计算每次的损失函数\n        loss=criterion(output,y).to(device)\n        \n         #L2正则化\n        l2_regularization = torch.tensor(0.).to(device)\n        for param in netC.parameters():\n            l2_regularization += torch.norm(param, 2)\n        loss += weight_decay * l2_regularization\n        #反向传播\n        loss.backward()\n        \n        #优化器进行优化(梯度下降,降低误差)\n        optimizer.step()\n    weight_decay*=1.05\n    print(f\"epoch:{epoch},weight_decay:{weight_decay}\")\n    netC.eval()\n    with torch.no_grad():\n        train_acc=0\n        for index,batched_data in enumerate(train_loader):\n            X,y=get_batch_transform(batched_data,is_train=False)\n            X,y=X.to(device),y.long().to(device)\n            #将数据放进去训练\n            output=netC(X).to(device)\n            output=torch.argmax(output,dim=1)\n            train_acc+=torch.sum(output==y)/len(train_X)\n        valid_acc=0\n        for index,batched_data in enumerate(valid_loader):\n            X,y=get_batch_transform(batched_data,is_train=False)\n            X,y=X.to(device),y.long().to(device)\n            #将数据放进去训练\n            output=netC(X).to(device)\n            output=torch.argmax(output,dim=1)\n            valid_acc+=torch.sum(output==y)/len(valid_X)\n        print(f\"train_acc:{train_acc},valid_acc:{valid_acc}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-18T07:47:19.155905Z","iopub.execute_input":"2023-07-18T07:47:19.156607Z"},"trusted":true},"execution_count":null,"outputs":[]}]}