{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#DATA FROM KAGGLE\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n#IMPORTS\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport pandas\nimport numpy\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom IPython.display import clear_output","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-24T06:24:56.489888Z","iopub.execute_input":"2021-09-24T06:24:56.490269Z","iopub.status.idle":"2021-09-24T06:24:56.831126Z","shell.execute_reply.started":"2021-09-24T06:24:56.49019Z","shell.execute_reply":"2021-09-24T06:24:56.830125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        self.drop1 = nn.Dropout(p=0.0) \n        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 70, kernel_size = 3, stride = 2, padding=True)\n        self.bn1 = nn.BatchNorm2d(70)\n        \n        self.drop2 = nn.Dropout(p=0.1)\n        self.conv2 = nn.Conv2d(in_channels = 70, out_channels = 150, kernel_size = 3, stride = 2, padding=True)\n        self.bn2   = nn.BatchNorm2d(150)\n        \n        self.drop3 = nn.Dropout(p=0.1)\n        self.conv3 = nn.Conv2d(in_channels = 150, out_channels = 240, kernel_size = 3, stride = 2, padding=True)\n        self.bn3   = nn.BatchNorm2d(240)\n        \n        #self.drop4 = nn.Dropout(p=0.1)\n        #self.conv4 = nn.Conv2d(in_channels = 240, out_channels = 340, kernel_size = 3, stride = 2, padding=True)\n        #self.bn4   = nn.BatchNorm2d(340)\n        \n        self.drop5 = nn.Dropout(p=0.1)\n        self.conv5 = nn.Conv2d(in_channels = 240, out_channels = 120, kernel_size = 1, stride = 1, padding=0)\n        self.bn5   = nn.BatchNorm2d(120)\n        \n        self.drop6 = nn.Dropout(p=0.1)\n        self.conv6 = nn.Conv2d(in_channels = 120, out_channels = 70, kernel_size = 1, stride = 1, padding=0)\n        self.bn6 = nn.BatchNorm2d(70)\n        \n        self.drop7 = nn.Dropout(p=0.2)\n        self.conv7 = nn.Conv2d(in_channels = 70, out_channels = 30, kernel_size = 1, stride = 1, padding=0)\n        self.bn7 = nn.BatchNorm2d(30)\n        \n        self.out = nn.Linear(30*4*4, 10)\n        \n    def forward(self, tensor):\n        \n        #Layer\n        tensor = self.drop1(tensor)\n        tensor = self.conv1(tensor)\n        tensor = self.bn1(tensor)\n        tensor = F.leaky_relu(tensor)\n        #tensor  = self.pool1(tensor)\n        \n        #Layer\n        tensor = self.drop2(tensor)\n        tensor = self.conv2(tensor)\n        tensor = self.bn2(tensor)\n        tensor = F.leaky_relu(tensor)\n        #tensor = self.pool2(tensor)\n        \n        #Layer\n        tensor = self.drop3(tensor)\n        tensor = self.conv3(tensor)\n        tensor = self.bn3(tensor)\n        tensor = F.leaky_relu(tensor)\n        \n        #Layer\n        #tensor = self.drop4(tensor)\n        #tensor = self.conv4(tensor)\n        #tensor = self.bn4(tensor)\n        #tensor = F.leaky_relu(tensor)\n        \n        #Layer\n        tensor = self.drop5(tensor)\n        tensor = self.conv5(tensor)\n        tensor = self.bn5(tensor)\n        tensor = F.leaky_relu(tensor)\n        \n        #Layer\n        tensor = self.drop6(tensor)\n        tensor = self.conv6(tensor)\n        tensor = self.bn6(tensor)\n        tensor = F.leaky_relu(tensor)\n        \n        #Layer\n        tensor = self.drop7(tensor)\n        tensor = self.conv7(tensor)\n        tensor = self.bn7(tensor)\n        tensor = F.leaky_relu(tensor)\n        \n        #output\n        tensor = torch.flatten(tensor, 1)\n        tensor = self.out(tensor)\n        return tensor","metadata":{"execution":{"iopub.status.busy":"2021-09-24T06:24:56.833128Z","iopub.execute_input":"2021-09-24T06:24:56.833448Z","iopub.status.idle":"2021-09-24T06:24:56.855358Z","shell.execute_reply.started":"2021-09-24T06:24:56.833417Z","shell.execute_reply":"2021-09-24T06:24:56.854229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#DEVICE\ndevice1 = 'cpu'; device2 = 'cpu'# device2 = xm.xla_device()\n#Network - Class Instance\n\n#print('DONE - network implmentation')\n# DATA PROCESSING\npath = '/kaggle/input/digit-recognizer/train.csv'; ref = pandas.read_csv(path)\nprint('Done - csv')\n\ndef shuffle_FUNCTION(a, b):\n    assert len(a) == len(b)\n    shuffled_a = numpy.empty(a.shape, dtype=a.dtype); shuffled_b = numpy.empty(b.shape, dtype=b.dtype)\n    permutation = numpy.random.permutation(len(a))\n    for old_index, new_index in enumerate(permutation):shuffled_a[new_index] = a[old_index]; shuffled_b[new_index] = b[old_index]\n    return shuffled_a, shuffled_b\n\nim_size=28; image_list = list(); label_list = list(); i = 0\nreal_data = numpy.reshape((ref.iloc[:42000, 1:].to_numpy()), (42000, im_size, im_size)); real_label = numpy.reshape((ref.iloc[:42000, 0].to_numpy()), 42000)\n\nfor img in real_data:\n    img = Image.fromarray(img.astype(numpy.uint8)); img1 = img.rotate(13); img2 = img.rotate(-13); img1 = numpy.array(img1); img2 = numpy.array(img2)\n    img1 = numpy.expand_dims(img1, axis=0); img2 = numpy.expand_dims(img2, axis=0)\n    image_list.append(img1); image_list.append(img2); label_list.append(real_label[i]); label_list.append(real_label[i]); i = i+1   \n\nprint('DONE - totensor')\ndata1 = numpy.concatenate((numpy.expand_dims(real_data, axis=1), numpy.array(image_list, dtype=numpy.int16)), axis=0)\nlabel1 = numpy.concatenate((real_label, numpy.array(label_list, dtype=numpy.int16)))\ndata1, label1 = shuffle_FUNCTION(data1, label1)\ndata = torch.tensor(data1[:110000], dtype = torch.float32, device = device1); label = torch.tensor(label1[:110000], device = device1)\nval_data = torch.tensor(data1[110000:], dtype = torch.float32, device = device1); val_label = torch.tensor(label1[110000:], device = device1)\ndata = data - data.min(); data = data / data.max()\nval_data = val_data - val_data.min(); val_data = val_data / val_data.max()\nprint('ALL DONE')\nprint(f'Training :\\nImage Shape : {data.shape}\\nDevice : {data.device}')\nprint(f'Validation :\\nImage Shape : {val_data.shape}\\nDevice : {val_data.device}')\n#SHOW BATCH\nnum = 310\nplt.imshow(data[num][0].to('cpu').detach().numpy(), cmap='gray')\nprint(label[num].item())\ndel data1, label1, real_data, real_label","metadata":{"execution":{"iopub.status.busy":"2021-09-24T06:24:56.857249Z","iopub.execute_input":"2021-09-24T06:24:56.857592Z","iopub.status.idle":"2021-09-24T06:25:07.166432Z","shell.execute_reply.started":"2021-09-24T06:24:56.857559Z","shell.execute_reply":"2021-09-24T06:25:07.165475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"network = Network().to(device2)\n#HYPERPARAMETERS\ndata_shape = label.shape[0]\nval_shape = val_label.shape[0]\nlearning_rate = 0.01\nbatch_size = 1000#500#10#2#1\nepoch = 6\noptimizer = torch.optim.Adam(network.parameters(), lr=learning_rate, weight_decay=0)","metadata":{"execution":{"iopub.status.busy":"2021-09-24T06:25:07.167833Z","iopub.execute_input":"2021-09-24T06:25:07.168292Z","iopub.status.idle":"2021-09-24T06:25:07.185012Z","shell.execute_reply.started":"2021-09-24T06:25:07.168247Z","shell.execute_reply":"2021-09-24T06:25:07.184189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoche in range (epoch):\n    accuracy = 0\n    train_loss = 0\n    val_loss = 0\n    val_acc = 0; train_acc12 = 0\n    learning_rate = round((1 / ((epoche+1)*70)), 4)\n    optimizer = torch.optim.Adam(network.parameters(), lr=learning_rate, weight_decay=0)\n    for batch in range((data_shape//batch_size)):\n        network.train()\n        pred = network((data[batch_size*batch : (batch_size*batch)+batch_size]).to(device2))#forward propodation\n        loss = F.cross_entropy(pred, label[batch_size*batch : (batch_size*batch)+batch_size].to(device2))#clculate loss\n        #print('>', label[batch_size*batch : (batch_size*batch)+batch_size].shape, '\\n>',pred.shape)\n        #break\n        #network.zero_grad()\n        optimizer.zero_grad()\n        loss.backward()#calculate slope\n        optimizer.step()#update weights and bias\n        \n        train_loss = train_loss + loss.item()\n    t = (batch*batch_size)+batch_size\n    with torch.no_grad():\n        print(f'Epoche : {epoche+1}    Learning rate : {learning_rate}');network.eval()\n        for val_batch in range((val_shape//batch_size)):\n            val_pred = network((val_data[batch_size*val_batch : (batch_size*val_batch)+batch_size].to(device2)))\n            val_acc1 = val_pred.argmax(dim=1).eq((val_label[batch_size*val_batch:(batch_size*val_batch)+batch_size]).to(device2)).sum().item()\n            valid_loss = F.cross_entropy(val_pred, (val_label[batch_size*val_batch : (batch_size*val_batch)+batch_size].to(device2)))\n            val_loss = val_loss + valid_loss.item()\n            val_acc = val_acc + val_acc1\n            \n        for train_batch in range((data_shape//batch_size)):\n            train_pred1 = network((data[batch_size*train_batch : (batch_size*train_batch)+batch_size].to(device2)))\n            train_acc1 = train_pred1.argmax(dim=1).eq((label[batch_size*train_batch:(batch_size*train_batch)+batch_size]).to(device2)).sum().item()\n            train_acc12 = train_acc12 + train_acc1\n    \n    print(f'Training Loss : {round(train_loss, 2)}\\t\\tTraining Accuracy : {round(train_acc12/data_shape, 3)}\\tValidation Accuracy : {round(val_acc/val_shape, 3)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-24T06:25:11.25709Z","iopub.execute_input":"2021-09-24T06:25:11.257665Z","iopub.status.idle":"2021-09-24T06:25:26.675524Z","shell.execute_reply.started":"2021-09-24T06:25:11.257618Z","shell.execute_reply":"2021-09-24T06:25:26.673996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.cuda.memory_snapshot()\n#torch.cuda.empty_cache()\n#pred[0]\nnetwork.eval()\nnum = 3\nplt.imshow(test_data[num][0].to('cpu').detach().numpy(), cmap='gray')\nprint(network(test_data[num:num+1].to('cuda')).argmax(dim=1))\nprint(network(test_data[num:num+1].to('cuda')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    network.eval\n    val_pred = torch.tensor([], device=device2)\n    for batch in range((val_shape//batch_size)):\n        val_pred_tmp = network(val_data[batch_size*batch : (batch_size*batch)+batch_size].to(device2))\n        val_pred = torch.cat((val_pred, val_pred_tmp), dim=0)\n    print('\\t\\t\\t>>>CONFUTION MATRIX<<<')\n    val_pred = val_pred.argmax(dim=1)\n    stacked = torch.stack((val_label.to('cpu'), val_pred.to('cpu')), dim=1)\n    cmt = torch.zeros(10, 10, dtype = torch.int32, device ='cpu')\n    for p in stacked:\n        tl, pl = p.tolist()\n        cmt[tl, pl] = cmt[tl, pl]+1\n    print(cmt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_path = '/kaggle/input/digit-recognizer/test.csv'; test_data = pandas.read_csv(test_data_path)\ntest_data = numpy.array(test_data); test_data = numpy.reshape(test_data, (28000, 1, 28, 28))\ntest_data = torch.tensor(test_data)\ntest_data = test_data - test_data.min(); test_data = test_data / test_data.max()\nprint(test_data.mean(), test_data.std()); print(f'FILE SHAPE : {test_data.shape}')\n\n#Prediction\nwith torch.no_grad():\n    network.eval()\n    test_data_shape = test_data.shape[0]; test_pred = torch.tensor([], device='cuda')\n    for batch in range((test_data_shape//batch_size)):\n        test_pred_tmp = network(test_data[batch_size*batch : (batch_size*batch)+batch_size].to('cuda'))\n        test_pred = torch.cat((test_pred, test_pred_tmp), dim=0)\ntest_pred = test_pred.argmax(dim=1)\nprint(f'PREDICTION DATA SHAPE : {test_pred.shape}')\nprint(f'SOME PREDICTION : {test_pred[0:10]}')\n#SAVING PREDICTION TO CSV\nnumber = list()\nfor i in range(1, 28001): number.append(i)\nnumber = numpy.array(number, dtype=numpy.int32)\ntest_pred = test_pred.to('cpu')\nsubmission = pandas.DataFrame({'ImageId' : number, 'Label': test_pred}); submission.to_csv('submission13.csv',index=False)\npandas.read_csv('./submission13.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_path = '/kaggle/input/digit-recognizer/test.csv'; test_data = pandas.read_csv(test_data_path)\ntest_data = numpy.array(test_data); test_data = numpy.reshape(test_data, (28000, 1, 28, 28))\ntest_data = torch.tensor(test_data)\ntest_data = test_data - test_data.min(); test_data = test_data / test_data.max()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle competitions submit -c digit-recognizer -f submission.csv -m \"Message\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}