{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.datasets as dsets\nimport matplotlib.pylab as plt\nimport numpy as np\nfrom random import randrange\nfrom torch import sigmoid\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:32:44.635019Z","iopub.execute_input":"2022-04-25T22:32:44.635378Z","iopub.status.idle":"2022-04-25T22:32:44.641797Z","shell.execute_reply.started":"2022-04-25T22:32:44.635342Z","shell.execute_reply":"2022-04-25T22:32:44.641189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:37:14.364235Z","iopub.execute_input":"2022-04-25T22:37:14.364674Z","iopub.status.idle":"2022-04-25T22:37:14.379870Z","shell.execute_reply.started":"2022-04-25T22:37:14.364630Z","shell.execute_reply":"2022-04-25T22:37:14.378868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_URL = '/kaggle/input/digit-recognizer/'","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:38:05.101183Z","iopub.execute_input":"2022-04-25T22:38:05.102126Z","iopub.status.idle":"2022-04-25T22:38:05.106988Z","shell.execute_reply.started":"2022-04-25T22:38:05.102044Z","shell.execute_reply":"2022-04-25T22:38:05.106170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_work = pd.read_csv(DATA_URL + 'train.csv')\n\nsplit_index = round(df_work.shape[0] * 0.9)\ndf_train = df_work.iloc[0:split_index,:]\ndf_test  = df_work.iloc[split_index:,:]\nprint(df_train.shape)\nprint(df_test.shape)\nassert df_work.shape[0] == df_train.shape[0] + df_test.shape[0]\n\ndf_challenge  = pd.read_csv(DATA_URL +  'test.csv')\nprint(df_challenge.shape )","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:38:49.657346Z","iopub.execute_input":"2022-04-25T22:38:49.658295Z","iopub.status.idle":"2022-04-25T22:38:54.445653Z","shell.execute_reply.started":"2022-04-25T22:38:49.658249Z","shell.execute_reply":"2022-04-25T22:38:54.444768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:38:56.134261Z","iopub.execute_input":"2022-04-25T22:38:56.134863Z","iopub.status.idle":"2022-04-25T22:38:56.167750Z","shell.execute_reply.started":"2022-04-25T22:38:56.134813Z","shell.execute_reply":"2022-04-25T22:38:56.167122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:38:59.325662Z","iopub.execute_input":"2022-04-25T22:38:59.326156Z","iopub.status.idle":"2022-04-25T22:38:59.342115Z","shell.execute_reply.started":"2022-04-25T22:38:59.326100Z","shell.execute_reply":"2022-04-25T22:38:59.341190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = torch.utils.data.TensorDataset(torch.tensor(df_train.drop('label', axis=1).values).type(torch.float32), torch.tensor(df_train['label'].values).type(torch.LongTensor))\nds_test = torch.utils.data.TensorDataset(torch.tensor(df_test.drop('label', axis=1).values).type(torch.float32), torch.tensor(df_test['label'].values).type(torch.LongTensor))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:39:01.405203Z","iopub.execute_input":"2022-04-25T22:39:01.405520Z","iopub.status.idle":"2022-04-25T22:39:01.802145Z","shell.execute_reply.started":"2022-04-25T22:39:01.405483Z","shell.execute_reply":"2022-04-25T22:39:01.801180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl_train = torch.utils.data.DataLoader(dataset=ds_train, batch_size=50, shuffle=True)\ndl_test  = torch.utils.data.DataLoader(dataset=ds_test)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:39:04.444488Z","iopub.execute_input":"2022-04-25T22:39:04.444804Z","iopub.status.idle":"2022-04-25T22:39:04.449983Z","shell.execute_reply.started":"2022-04-25T22:39:04.444769Z","shell.execute_reply":"2022-04-25T22:39:04.449136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_img(pixels_list):\n    fig = px.imshow(torch.tensor(pixels_list).reshape(28, 28), binary_format=\"jpeg\", binary_compression_level=0)\n    fig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_img(df_train.iloc[0, 1:].values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_batch(nn.Module):\n    \n    # Contructor\n    def __init__(self, out_1=16, out_2=32, out_3=64, p=0.2, number_of_classes=10):\n        super(CNN_batch, self).__init__()\n        self.layer1 = torch.nn.Sequential(\n            nn.Conv2d(in_channels=1, out_channels=out_1, kernel_size=3, padding=3),\n            nn.BatchNorm2d(out_1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2),\n            nn.Dropout(p=p)\n        )\n        \n        self.layer2 = torch.nn.Sequential(\n            nn.Conv2d(in_channels=out_1, out_channels=out_2, kernel_size=3, stride=1, padding=2),\n            nn.BatchNorm2d(out_2),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2),\n            nn.Dropout(p=p)\n        )\n        \n        self.layer3 = torch.nn.Sequential(\n            nn.Conv2d(in_channels=out_2, out_channels=out_3, kernel_size=3, stride=1, padding=4),\n            nn.BatchNorm2d(out_3),\n            nn.ReLU(),            \n            nn.MaxPool2d(kernel_size=2),\n            nn.Dropout(p=p)\n        )        \n        \n        self.fc1 = nn.Linear(out_3 * 49, number_of_classes)\n        self.bn_fc1 = nn.BatchNorm1d(10)\n    \n    # Prediction\n    def forward(self, x):\n        x = self.layer1(x)\n        # print(x.shape)\n        \n        x = self.layer2(x)\n        # print(x.shape)\n        \n        x = self.layer3(x)\n        # print(x.shape)             \n        \n        x = x.view(x.size(0), -1)\n        x = self.fc1(x)\n        x = self.bn_fc1(x)\n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_batch=CNN_batch(out_1=32, out_2=64, out_3=128)\ncriterion = nn.CrossEntropyLoss()\nlearning_rate = 0.1\noptimizer = torch.optim.SGD(model_batch.parameters(), lr = learning_rate)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model,dl_train,dl_test,optimizer,n_epochs=4):\n    \n    loss_list = []\n    accuracy_list = []\n    \n    for epoch in range(n_epochs):\n        print(\"Epoch: %s \" % str(epoch))\n        \n        ll_train = []\n        for x, y in dl_train:\n            model.train()\n            optimizer.zero_grad()\n#             print(x.reshape(x.shape[0], 1, 28, 28).shape)\n            \n            yhat = model.forward(x.reshape(x.shape[0], 1, 28, 28))\n            loss = criterion(yhat, y)\n            loss.backward()\n            optimizer.step()\n            ll_train.append(loss.data)\n            \n        l = torch.mean(torch.stack(ll_train)).item()\n        loss_list.append(l)\n        print('loss data: ', l)\n            \n        correct = 0\n        # perform a prediction on the dl_test  \n        for x, y in dl_test:\n            model.eval()\n            z = model(x.reshape(x.shape[0], 1, 28, 28))\n            _, yhat = torch.max(z.data, 1)\n            correct += (yhat == y).sum().item()\n        accuracy = correct / len(dl_test)\n        accuracy_list.append(accuracy)\n        \n    return loss_list, accuracy_list, yhat\n        \nloss_list, accuracy_list, yhat = train_model(model_batch, dl_train, dl_test, optimizer, n_epochs=20) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create figure with secondary y-axis\nfig = make_subplots(specs=[[{\"secondary_y\": True}]])\n\n# Add traces\nfig.add_trace(\n    go.Scatter(x=list(range(0, len(loss_list))), y=loss_list, name=\"loss\"),\n    secondary_y=False,\n)\n\nfig.add_trace(\n    go.Scatter(x=list(range(0, len(loss_list))), y=accuracy_list, name=\"accuracy\"),\n    secondary_y=True,\n)\n\n# Add figure title\nfig.update_layout(\n    title_text=\"Loss and Accuracy\"\n)\n\n# Set x-axis title\nfig.update_xaxes(title_text=\"Epoch\")\n\n# Set y-axes titles\nfig.update_yaxes(title_text=\"<b>Loss</b>\", secondary_y=False)\nfig.update_yaxes(title_text=\"<b>Accuracy</b>\", secondary_y=True)\n\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_challenge = torch.utils.data.TensorDataset(torch.tensor(df_challenge.values).type(torch.float32))\ndl_challenge = torch.utils.data.DataLoader(dataset=ds_challenge)\n\ndf_submission  = pd.read_csv(DATA_URL + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:39:34.958404Z","iopub.execute_input":"2022-04-25T22:39:34.959065Z","iopub.status.idle":"2022-04-25T22:39:35.082324Z","shell.execute_reply.started":"2022-04-25T22:39:34.958999Z","shell.execute_reply":"2022-04-25T22:39:35.081384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_submission.shape, len(dl_challenge))\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T22:39:37.237174Z","iopub.execute_input":"2022-04-25T22:39:37.237671Z","iopub.status.idle":"2022-04-25T22:39:37.248028Z","shell.execute_reply.started":"2022-04-25T22:39:37.237606Z","shell.execute_reply":"2022-04-25T22:39:37.247237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, x in enumerate(dl_challenge):\n    x = x[0]\n    z = model_batch(x.reshape(x.shape[0], 1, 28, 28))\n    _, yhat = torch.max(z.data, 1)\n    df_submission.at[i, 'Label'] = yhat.item()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_submission.to_csv('./kaggle/data/submission.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# further updates\n1. Data augmenation\n2. Exploring failing cases and improve\n3. improve number of eprochs and import/export","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}