{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Loading MNIST dataset.\nLoads the MNIST dataset from CSV files. Defines the functions used to sample batches from dataset, and in order to do so the dataset is reorganized in a more convenient way with a dictionary, where the keys access all the samples from each class","metadata":{"_cell_guid":"26e816f3-784f-4b45-838b-7f27757d1e5f","_uuid":"7ea88036b082f18e7456446bfa4d207a5acc30de"}},{"cell_type":"code","source":"# For some reason tensorflow needs to be imported before other modules, otherwise the\n# kernel crashes...\nimport tensorflow as tf \nimport pandas as pd\nimport random\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n# Loading dataset\ntrain_images = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n#del train_images[\"label\"]\n#print(train_images.shape)\n#train_images = train_images.values.reshape(-1,28,28)\n#print(train_images.shape)\n#train_labels = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")[\"label\"]\n#test_images = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\nX_train = dict()\ny_train = dict()\nX_test = dict()\ny_test =dict()\nfor i in train_images[\"label\"].unique():\n    local = train_images[train_images[\"label\"] == i].copy()\n    local_images = local.copy()\n    del local_images[\"label\"]\n    local_images = local_images.values.reshape(-1,28,28)\n    local_labels = local[\"label\"]\n    X_train[i], y_train[i], X_test[i], y_test[i] = train_test_split(local_images, local_labels) \nfor i in X_train.keys():\n    print(i, X_train[i].shape)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-04-04T11:31:02.464008Z","iopub.execute_input":"2023-04-04T11:31:02.464329Z","iopub.status.idle":"2023-04-04T11:31:09.658530Z","shell.execute_reply.started":"2023-04-04T11:31:02.464269Z","shell.execute_reply":"2023-04-04T11:31:09.657659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n\nfor i in range(1):\n    plt.matshow(X_train[i][1,:,:])","metadata":{"execution":{"iopub.status.busy":"2023-04-04T11:31:09.659626Z","iopub.execute_input":"2023-04-04T11:31:09.659884Z","iopub.status.idle":"2023-04-04T11:31:09.796211Z","shell.execute_reply.started":"2023-04-04T11:31:09.659840Z","shell.execute_reply":"2023-04-04T11:31:09.795381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.nn import Module\nfrom torch.nn import Conv2d\nfrom torch.nn import Linear\nfrom torch.nn import MaxPool2d\nfrom torch.nn import ReLU\nfrom torch.nn import LogSoftmax\n#from torch import flatten\n\nclass LeNet(Module):\n    def __init__(self, numChannels=1, classes=10):\n        # call the parent constructor\n        super(LeNet, self).__init__()\n        # initialize first set of CONV => RELU => POOL layers\n        self.conv1 = Conv2d(in_channels=numChannels, out_channels=20, kernel_size=(5, 5))\n        self.relu1 = ReLU()\n        self.maxpool1 = MaxPool2d(kernel_size=(2, 2), stride=(2, 2))\n        # initialize second set of CONV => RELU => POOL layers\n        self.conv2 = Conv2d(in_channels=20, out_channels=1, kernel_size=(5, 5))\n        self.relu2 = ReLU()\n        self.maxpool2 = MaxPool2d(kernel_size=(2, 2), stride=(2, 2))\n        # initialize first (and only) set of FC => RELU layers\n        \n    def forward(self, x):\n        # pass the input through our first set of CONV => RELU =>\n        # POOL layers\n        x = self.conv1(x)\n        x = self.relu1(x)\n        x = self.maxpool1(x)\n        # pass the output from the previous layer through the second\n        # set of CONV => RELU => POOL layers\n        x = self.conv2(x)\n        x = self.relu2(x)\n        output = self.maxpool2(x)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-04-04T11:31:09.797502Z","iopub.execute_input":"2023-04-04T11:31:09.798035Z","iopub.status.idle":"2023-04-04T11:31:09.846303Z","shell.execute_reply.started":"2023-04-04T11:31:09.797950Z","shell.execute_reply":"2023-04-04T11:31:09.845512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import optim\nfrom torch.autograd import Variable\nfrom random import randint, sample\n\n\n\ndef trace(y1):\n    s = 0\n    for i in range(min(y1.shape[-1],y1.shape[-2])):\n        s += y1[-1,i,i]\n    return s\n\ndef diff(y1,y2):\n    prod = torch.bmm((y1-y2).resize(batch_size,4,4), (y1-y2).transpose(-1,-2).resize(batch_size,4,4))\n    return trace(prod).sum()\n\ndef loss(y1,y2,y3):\n    s = diff(y1,y2)-diff(y1,y3)\n    return s\n\nmodel = LeNet()\n\noptimizer = optim.Adam(model.parameters(), lr=0.0001)\nbatch_size = 1000\n\nfor epoch in range(100):\n    cumsum = 0\n    for i in range(train_images.shape[0]//batch_size):\n        optimizer.zero_grad()\n        label1 = randint(0,9)\n        label2 = randint(0,9)\n        while label2 == label1:\n            label2 = randint(0,9)\n        img1 = X_train[label1][np.random.randint(X_train[label1].shape[0],size=batch_size),:,:]\n        img1 = Variable(torch.Tensor(img1.reshape(-1,1,28,28)))\n        y_pred1 = model(img1)\n\n        img2 = X_train[label1][np.random.randint(X_train[label1].shape[0],size=batch_size),:,:]\n        img2 = Variable(torch.Tensor(img2.reshape(-1,1,28,28)))\n        y_pred2 = model(img2)\n        \n        img3 = X_train[label2][np.random.randint(X_train[label2].shape[0],size=batch_size),:,:]\n        img3 = Variable(torch.Tensor(img3.reshape(-1,1,28,28)))\n        y_pred3 = model(img3)\n    \n        loss_var = loss(y_pred1,y_pred2,y_pred3)\n        loss_var.backward()\n        optimizer.step()\n        cumsum += loss_var.detach()\n        #if i*100/train_images.shape[0]//batch_size != (i+1)*100/train_images.shape[0]//batch_size:\n        print(i)\n    print(epoch, cumsum)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-04-04T11:33:59.495064Z","iopub.execute_input":"2023-04-04T11:33:59.495352Z"},"trusted":true},"execution_count":null,"outputs":[]}]}