{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30474,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-26T12:36:02.928716Z","iopub.execute_input":"2023-07-26T12:36:02.929175Z","iopub.status.idle":"2023-07-26T12:36:02.941094Z","shell.execute_reply.started":"2023-07-26T12:36:02.929131Z","shell.execute_reply":"2023-07-26T12:36:02.939629Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import  transforms, datasets\nfrom torch.utils.data import DataLoader, random_split, Dataset\nimport random\nimport torch\nimport numpy as np\nfrom IPython.display import clear_output\nfrom collections import defaultdict\nimport matplotlib.pyplot as plt\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:02.944026Z","iopub.execute_input":"2023-07-26T12:36:02.945391Z","iopub.status.idle":"2023-07-26T12:36:02.955821Z","shell.execute_reply.started":"2023-07-26T12:36:02.945350Z","shell.execute_reply":"2023-07-26T12:36:02.954489Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_random_seed(seed):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n\nset_random_seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:02.957576Z","iopub.execute_input":"2023-07-26T12:36:02.957970Z","iopub.status.idle":"2023-07-26T12:36:02.976406Z","shell.execute_reply.started":"2023-07-26T12:36:02.957939Z","shell.execute_reply":"2023-07-26T12:36:02.975241Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\", dtype=np.float32)\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\", dtype=np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:02.979943Z","iopub.execute_input":"2023-07-26T12:36:02.980750Z","iopub.status.idle":"2023-07-26T12:36:09.404829Z","shell.execute_reply.started":"2023-07-26T12:36:02.980691Z","shell.execute_reply":"2023-07-26T12:36:09.403613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GetData(Dataset):\n    def __init__(self, X_Train, Y_Train, Transform):\n        self.X = X_Train\n        self.transform = Transform\n        self.Y = Y_Train\n        \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        return self.transform(self.X[index]).permute((1, 2, 0)).contiguous(), self.Y[index]","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.406170Z","iopub.execute_input":"2023-07-26T12:36:09.406521Z","iopub.status.idle":"2023-07-26T12:36:09.414631Z","shell.execute_reply.started":"2023-07-26T12:36:09.406492Z","shell.execute_reply":"2023-07-26T12:36:09.413423Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_Train = train_df.label.values\nX0 = train_df.loc[:, train_df.columns != 'label'].values/255\n\nX0.shape, Y_Train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.415952Z","iopub.execute_input":"2023-07-26T12:36:09.416270Z","iopub.status.idle":"2023-07-26T12:36:09.513485Z","shell.execute_reply.started":"2023-07-26T12:36:09.416243Z","shell.execute_reply":"2023-07-26T12:36:09.512676Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_Train = torch.from_numpy(Y_Train).type(torch.LongTensor)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.514898Z","iopub.execute_input":"2023-07-26T12:36:09.515237Z","iopub.status.idle":"2023-07-26T12:36:09.525417Z","shell.execute_reply.started":"2023-07-26T12:36:09.515210Z","shell.execute_reply":"2023-07-26T12:36:09.524276Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_Train = X0.reshape((-1, 1, 28, 28))\nX_Train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.526980Z","iopub.execute_input":"2023-07-26T12:36:09.527758Z","iopub.status.idle":"2023-07-26T12:36:09.543819Z","shell.execute_reply.started":"2023-07-26T12:36:09.527717Z","shell.execute_reply":"2023-07-26T12:36:09.542562Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import  transforms, datasets\nfrom torch.utils.data import DataLoader, random_split\n\ntransform = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize( (0.5), (0.5))\n     ])\n\ntrainset = GetData(X_Train, Y_Train, transform)\n\ntrain_set, val_set  = random_split(trainset, [41500, 500])\n\ntrain_loader = DataLoader(train_set, batch_size=64, shuffle=True, num_workers=4)\nval_loader = DataLoader(val_set, batch_size=64, shuffle=False, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.548547Z","iopub.execute_input":"2023-07-26T12:36:09.549089Z","iopub.status.idle":"2023-07-26T12:36:09.565461Z","shell.execute_reply.started":"2023-07-26T12:36:09.549027Z","shell.execute_reply":"2023-07-26T12:36:09.564279Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"next(iter(train_loader))[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.566919Z","iopub.execute_input":"2023-07-26T12:36:09.567372Z","iopub.status.idle":"2023-07-26T12:36:09.770707Z","shell.execute_reply.started":"2023-07-26T12:36:09.567329Z","shell.execute_reply":"2023-07-26T12:36:09.769559Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Code for training","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\n# compute on cpu or gpu\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef train_epoch(model,\n                optimizer,\n                criterion,\n                train_loader):\n    loss_history = []\n    for batch in train_loader: \n        optimizer.zero_grad()\n        x_train, y_train = batch # parse data\n        x_train, y_train = x_train.to(device), y_train.to(device) # compute on gpu\n        y_pred = model(x_train) # get predictions\n        loss = criterion(y_pred, y_train) # compute loss\n        loss_history.append(loss.cpu().detach().numpy()) # write loss to log\n        loss.backward()\n        optimizer.step()\n    return loss_history\n\n\ndef validate(model,\n             criterion,\n             val_loader):\n    cumloss = 0\n    loss_history = []\n    with torch.no_grad():\n        for batch in val_loader:\n            x_train, y_train = batch # parse data\n            x_train, y_train = x_train.to(device), y_train.to(device) # compute on gpu\n            y_pred = model(x_train) # get predictions\n            loss = criterion(y_pred, y_train) # compute loss\n            loss_history.append(loss.cpu().detach().numpy()) # write loss to log\n            cumloss += loss.cpu().detach().numpy()\n    return cumloss / len(val_loader)# mean loss and history\n\n\n\ndef train_model(model, optimizer, model_name=None, n_epochs=5):\n  \n    criterion = nn.CrossEntropyLoss().to(device)\n\n    train_history = {}\n    train_history['model_name'] = model_name\n    train_history['loss_on_train'] = []\n    train_history['loss_on_test'] = []\n    acc_history = []\n\n    for epoch in range(n_epochs):\n        loss_on_train = train_epoch(model,\n                                    optimizer,\n                                    criterion,\n                                    train_loader)\n        loss_on_test = validate(model,\n                                   criterion,\n                                   val_loader)\n        train_history['loss_on_train'].extend(loss_on_train)\n        train_history['loss_on_test'].append(loss_on_test)\n        acc_history.append(accuracy(model, val_loader, device))\n\n    return train_history, acc_history\n\n\ndef accuracy(model, test_loader, device = \"cpu\"):\n    correct, total  = 0, 0\n    for imgs, labels in val_loader:\n        pred = model(imgs.to(device))        \n        total += labels.size(0)\n        correct += get_correct_count(pred,labels)\n    return correct / total\n\ndef get_correct_count(pred,labels):\n  _, predicted = torch.max(pred.data, 1)\n  return (predicted.cpu() == labels.cpu()).sum().item()","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:36:09.772811Z","iopub.execute_input":"2023-07-26T12:36:09.773224Z","iopub.status.idle":"2023-07-26T12:36:09.793246Z","shell.execute_reply.started":"2023-07-26T12:36:09.773186Z","shell.execute_reply":"2023-07-26T12:36:09.792004Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.optim as optim\n\nclass LeNetDrop(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.activation = nn.ReLU()\n        self.conv1 = nn.Conv2d(1, 6, 5)\n        self.pool = nn.MaxPool2d(2)\n        self.conv2 = nn.Conv2d(6, 12, 5)\n        self.pool = nn.MaxPool2d(2)\n        self.drop1 = nn.Dropout(0.5) # 50 % probability\n        self.fc1 = nn.Linear(12 * 4 * 4, 120)  # 5*5 from image dimension\n        self.drop2 = nn.Dropout(0.2)  # 20% probability\n        self.fc2 = nn.Linear(120, 50)\n        self.fc3 = nn.Linear(50, 10)        \n\n    def forward(self, x):\n        x = self.activation(self.pool(self.conv1(x)))\n        x = self.activation(self.pool(self.conv2(self.drop1(x))))\n        x = self.activation(self.fc1(x.flatten(start_dim=1)))\n        x = self.activation(self.fc2(self.drop2(x)))\n        x = self.fc3(x)\n        return x\n\nset_random_seed(42)\n\nmodel = LeNetDrop().to(device)\noptimizer = optim.SGD(model.parameters(), lr=0.03)\nhistory, acc = train_model(model, optimizer,  n_epochs=30)\nprint(\"Accuracy of model=\", acc[-1])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:41:28.149233Z","iopub.execute_input":"2023-07-26T12:41:28.149733Z","iopub.status.idle":"2023-07-26T12:46:04.329096Z","shell.execute_reply.started":"2023-07-26T12:41:28.149699Z","shell.execute_reply":"2023-07-26T12:46:04.327479Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nfig, axs = plt.subplots(2)\nl = len(acc)\naxs[0].plot(range(0,l), history['loss_on_test'])\naxs[0].set(xlabel='epoch', ylabel='loss')\naxs[0].label_outer()\naxs[1].plot(range(0,l), acc)\naxs[1].set(xlabel='epoch', ylabel='acc')","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:46:36.521000Z","iopub.execute_input":"2023-07-26T12:46:36.521452Z","iopub.status.idle":"2023-07-26T12:46:36.935333Z","shell.execute_reply.started":"2023-07-26T12:46:36.521414Z","shell.execute_reply":"2023-07-26T12:46:36.933718Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Xt = (test_df.values/255).reshape((-1, 1, 28, 28))\nXt.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:46:43.304132Z","iopub.execute_input":"2023-07-26T12:46:43.304548Z","iopub.status.idle":"2023-07-26T12:46:43.342763Z","shell.execute_reply.started":"2023-07-26T12:46:43.304516Z","shell.execute_reply":"2023-07-26T12:46:43.341495Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_sub = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\nwith torch.no_grad():\n    model.eval()\n    sample_sub['Label'] = model(torch.from_numpy(Xt).to(device)).cpu().argmax(dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:46:45.115004Z","iopub.execute_input":"2023-07-26T12:46:45.115425Z","iopub.status.idle":"2023-07-26T12:46:46.724024Z","shell.execute_reply.started":"2023-07-26T12:46:45.115394Z","shell.execute_reply":"2023-07-26T12:46:46.722971Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:46:46.848682Z","iopub.execute_input":"2023-07-26T12:46:46.849121Z","iopub.status.idle":"2023-07-26T12:46:46.859438Z","shell.execute_reply.started":"2023-07-26T12:46:46.849088Z","shell.execute_reply":"2023-07-26T12:46:46.858637Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T12:46:48.828130Z","iopub.execute_input":"2023-07-26T12:46:48.829392Z","iopub.status.idle":"2023-07-26T12:46:48.902598Z","shell.execute_reply.started":"2023-07-26T12:46:48.829347Z","shell.execute_reply":"2023-07-26T12:46:48.901300Z"},"trusted":true},"outputs":[],"execution_count":null}]}