{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n\nfrom fastai.vision.all import *\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-07T03:43:52.818662Z","iopub.execute_input":"2023-04-07T03:43:52.819120Z","iopub.status.idle":"2023-04-07T03:43:52.830874Z","shell.execute_reply.started":"2023-04-07T03:43:52.819081Z","shell.execute_reply":"2023-04-07T03:43:52.829440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## helpers","metadata":{}},{"cell_type":"code","source":"def row_pixels(row): \n#     return tensor(row[1:]).reshape((28, 28))\n    return tensor(row[1:].values.astype('int64').reshape((28, 28)))\n\ndef mnist_distance(x1, x2): \n    return (x1 - x2).abs().mean((-1, -2))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:43:52.833856Z","iopub.execute_input":"2023-04-07T03:43:52.835737Z","iopub.status.idle":"2023-04-07T03:43:52.844841Z","shell.execute_reply.started":"2023-04-07T03:43:52.835686Z","shell.execute_reply":"2023-04-07T03:43:52.843871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a validation set","metadata":{}},{"cell_type":"code","source":"MNIST = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nsplit = int(len(MNIST) * 0.8)\nMNIST_TRAINING = MNIST[0:split]\nMNIST_VALIDATION = MNIST[split:]\n\nprint('total number of labeled samples:', len(MNIST))\nprint(\"training set size:\", len(MNIST_TRAINING))\nprint(\"validation set size:\", len(MNIST_VALIDATION))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:43:52.846062Z","iopub.execute_input":"2023-04-07T03:43:52.846392Z","iopub.status.idle":"2023-04-07T03:43:55.231217Z","shell.execute_reply.started":"2023-04-07T03:43:52.846361Z","shell.execute_reply":"2023-04-07T03:43:55.229805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sort training data by label. \n\nSo that we can derive statistics from it","metadata":{}},{"cell_type":"code","source":"# sort training data into torch stacks\ntraining_stacks = [ [] for x in range(10) ]\ntraining_means = [ [] for x in range(10) ]\n\nfor row_index, row in MNIST_TRAINING.iterrows(): # .sample(n=1000, random_state=0)\n    label = row[0]\n    training_stacks[label].append(row_pixels(row))\n\nfor i in range(10):\n    training_stacks[i] = torch.stack(training_stacks[i]).float() / 255\n    training_means[i] = training_stacks[i].mean(0)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:43:55.236127Z","iopub.execute_input":"2023-04-07T03:43:55.238280Z","iopub.status.idle":"2023-04-07T03:43:59.254542Z","shell.execute_reply.started":"2023-04-07T03:43:55.238224Z","shell.execute_reply":"2023-04-07T03:43:59.253510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_stacks = [ [] for x in range(10) ]\nvalidation_means = [ [] for x in range(10) ]\n\nfor row_index, row in MNIST_VALIDATION.iterrows(): # .sample(n=1000, random_state=0)\n\n    label = row[0]\n    validation_stacks[label].append(row_pixels(row))\n    \nfor i in range(10):\n    validation_stacks[i] = torch.stack(validation_stacks[i]).float() / 255\n    validation_means[i] = validation_stacks[i].mean(0)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:43:59.256247Z","iopub.execute_input":"2023-04-07T03:43:59.257004Z","iopub.status.idle":"2023-04-07T03:44:00.255541Z","shell.execute_reply.started":"2023-04-07T03:43:59.256962Z","shell.execute_reply":"2023-04-07T03:44:00.254530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing MNIST data","metadata":{}},{"cell_type":"markdown","source":"## Random selection from all data","metadata":{}},{"cell_type":"code","source":"numrows = 5\nnumcols = 20\nfig, ax = plt.subplots(nrows=numrows, ncols=numcols)\n\nsample_size = numrows * numcols\nsamples = MNIST.iterrows() # .sample(n=sample_size)\n\nfor i in range(numrows):\n    for j in range(numcols):\n        row_index, row = next(samples)\n        label = row[0]\n        pixels = row_pixels(row)\n        \n        ax[i, j].set_title(label)\n        ax[i, j].imshow(pixels)\n        ax[i, j].axis('off')\n        \nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-07T03:44:00.257347Z","iopub.execute_input":"2023-04-07T03:44:00.258186Z","iopub.status.idle":"2023-04-07T03:44:03.235008Z","shell.execute_reply.started":"2023-04-07T03:44:00.258143Z","shell.execute_reply":"2023-04-07T03:44:03.233502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of data - training and validation","metadata":{}},{"cell_type":"code","source":"distribution_training = [len(stack) for stack in training_stacks]\nplt.bar(range(10), distribution_training)\nplt.xticks(range(10))\nplt.title('Distribution of handwritten digits training set')\nplt.show()\n\ndistribution_validation = [len(stack) for stack in validation_stacks]\nplt.bar(range(10), distribution_validation)\nplt.xticks(range(10))\nplt.title('Distribution of handwritten digits validation set')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:03.236673Z","iopub.execute_input":"2023-04-07T03:44:03.237164Z","iopub.status.idle":"2023-04-07T03:44:03.686055Z","shell.execute_reply.started":"2023-04-07T03:44:03.237115Z","shell.execute_reply":"2023-04-07T03:44:03.684863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Idealized handwritten digits","metadata":{}},{"cell_type":"markdown","source":"### Idealized digits - training data","metadata":{}},{"cell_type":"code","source":"show_images(training_means)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:03.687485Z","iopub.execute_input":"2023-04-07T03:44:03.687813Z","iopub.status.idle":"2023-04-07T03:44:04.123095Z","shell.execute_reply.started":"2023-04-07T03:44:03.687779Z","shell.execute_reply":"2023-04-07T03:44:04.121927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Idealized digits - validation data","metadata":{}},{"cell_type":"code","source":"show_images(validation_means)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:04.124555Z","iopub.execute_input":"2023-04-07T03:44:04.125194Z","iopub.status.idle":"2023-04-07T03:44:04.569152Z","shell.execute_reply.started":"2023-04-07T03:44:04.125150Z","shell.execute_reply":"2023-04-07T03:44:04.566434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Identity of digits","metadata":{}},{"cell_type":"code","source":"sample_digits = [x[0] for x in training_stacks]\nshow_images(sample_digits)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:04.574884Z","iopub.execute_input":"2023-04-07T03:44:04.575254Z","iopub.status.idle":"2023-04-07T03:44:05.069930Z","shell.execute_reply.started":"2023-04-07T03:44:04.575212Z","shell.execute_reply":"2023-04-07T03:44:05.068807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error = []\nfor i in range(10):\n    digit_error = []\n    for j in range(10):\n        digit_error.append(F.l1_loss(sample_digits[i], sample_digits[j]))\n    error.append(digit_error)\n\nplt.imshow(error)\nplt.xticks(range(10))\nplt.yticks(range(10))\nplt.title('L1 loss: single digit x single digit')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:05.071449Z","iopub.execute_input":"2023-04-07T03:44:05.072653Z","iopub.status.idle":"2023-04-07T03:44:05.304374Z","shell.execute_reply.started":"2023-04-07T03:44:05.072606Z","shell.execute_reply":"2023-04-07T03:44:05.303048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error = []\nfor i in range(10):\n    stack = training_stacks[i]\n    digit_error = []\n    digits = []\n    for j in range(10):\n        sample_digit = training_stacks[j][0]\n        digits.append(sample_digit)\n        digit_error.append(F.l1_loss(training_means[i], sample_digit))\n    error.append(digit_error)\n\nplt.imshow(error)\nplt.xticks(range(10))\nplt.yticks(range(10))\nplt.title('L1 loss: single digit x ideal digit')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:05.305861Z","iopub.execute_input":"2023-04-07T03:44:05.306553Z","iopub.status.idle":"2023-04-07T03:44:05.549560Z","shell.execute_reply.started":"2023-04-07T03:44:05.306511Z","shell.execute_reply":"2023-04-07T03:44:05.547500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error = []\nfor i in range(10):\n    digit_error = []\n    for j in range(10):\n        digit_error.append(F.l1_loss(training_means[i], training_means[j]))\n    error.append(digit_error)\n\nplt.imshow(error)\nplt.xticks(range(10))\nplt.yticks(range(10))\nplt.title('L1 loss: ideal digit x ideal digit')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:05.552522Z","iopub.execute_input":"2023-04-07T03:44:05.554286Z","iopub.status.idle":"2023-04-07T03:44:05.762687Z","shell.execute_reply.started":"2023-04-07T03:44:05.554222Z","shell.execute_reply":"2023-04-07T03:44:05.761164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compare every idealized digit against every other. How similar or different are idealized digits? This is indeed an identity matrix.","metadata":{}},{"cell_type":"markdown","source":"# Validation - measuring **accuracy**\n\nSample some rows from our validation set against ideal digits of training set. How close are we with just a L1 or mean squared error comparison? What is the baseline?","metadata":{}},{"cell_type":"markdown","source":"### L1 loss","metadata":{}},{"cell_type":"code","source":"# validation_samples = MNIST_VALIDATION.sample(n=3, random_state=5).iterrows()\nvalidation_samples = MNIST_VALIDATION.iterrows()\n\nerror = []\ncorrect, wrong = 0, 0\n\nfor row_index, row in validation_samples:\n    label = row[0]\n    pixels = row_pixels(row)\n    sample_error = []\n    \n    for i in range(10):\n        sample_error.append(F.l1_loss(training_means[i], pixels))\n\n    prediction = sample_error.index(min(sample_error))    \n    if prediction == label:\n        correct += 1\n    else:\n        wrong += 1\n\nprint(\"VALIDATION SET PERFORMANCE (L1 loss):\", correct/(correct+wrong))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:05.764892Z","iopub.execute_input":"2023-04-07T03:44:05.765764Z","iopub.status.idle":"2023-04-07T03:44:10.356102Z","shell.execute_reply.started":"2023-04-07T03:44:05.765717Z","shell.execute_reply":"2023-04-07T03:44:10.353621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### mean squared error","metadata":{}},{"cell_type":"code","source":"# validation_samples = MNIST_VALIDATION.sample(n=10, random_state=1).iterrows()\nvalidation_samples = MNIST_VALIDATION.iterrows()\nerror = []\ncorrect, wrong = 0, 0\n\nfor row_index, row in validation_samples:\n    label = row[0]\n    pixels = row_pixels(row).float()\n    sample_error = []\n    \n    for i in range(10):\n        sample_error.append(F.mse_loss(training_means[i], pixels))\n\n    prediction = sample_error.index(min(sample_error))    \n    if prediction == label:\n        correct += 1\n    else:\n        wrong += 1\n        \nprint(\"VALIDATION SET PERFORMANCE (mean squared error):\", correct/(correct+wrong))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:10.357621Z","iopub.execute_input":"2023-04-07T03:44:10.358092Z","iopub.status.idle":"2023-04-07T03:44:14.409729Z","shell.execute_reply.started":"2023-04-07T03:44:10.358044Z","shell.execute_reply":"2023-04-07T03:44:14.408490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### cross entropy","metadata":{}},{"cell_type":"code","source":"# validation_samples = MNIST_VALIDATION.sample(n=10, random_state=1).iterrows()\nvalidation_samples = MNIST_VALIDATION.iterrows()\nerror = []\ncorrect, wrong = 0, 0\nloss_func = nn.CrossEntropyLoss()\n\nfor row_index, row in validation_samples:\n    label = row[0]\n    pixels = row_pixels(row).float()\n    sample_error = []\n    \n    for i in range(10):\n        sample_error.append(loss_func(training_means[i], pixels))\n\n    prediction = sample_error.index(min(sample_error))    \n    if prediction == label:\n        correct += 1\n    else:\n        wrong += 1\n        \nprint(\"VALIDATION SET PERFORMANCE (cross entropy):\", correct/(correct+wrong))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T03:44:14.411281Z","iopub.execute_input":"2023-04-07T03:44:14.411728Z","iopub.status.idle":"2023-04-07T03:44:19.134712Z","shell.execute_reply.started":"2023-04-07T03:44:14.411684Z","shell.execute_reply":"2023-04-07T03:44:19.133872Z"},"trusted":true},"execution_count":null,"outputs":[]}]}