{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://towardsdatascience.com/how-to-make-an-autoencoder-2f2d99cd5103","metadata":{}},{"cell_type":"markdown","source":"# Notes on AutoEncoders\n\nInitially I did it all wrong:\n* No division by 255\n* No dropout\n* No sigmoid\n* Adam instead of NAdam\n\n\n\n1. Sigmoid made a huge difference in reducing blur, predictably \n2. The tensor flow one trained in fewer epochs (larger dataset maybe)\n3. The TF one seems blurrier but more accurate, while PT one is sharper but less accurate; possibly due to TF one's larger dropout rate?\n4. Nope adding any amt of dropout causing degeneracy\n5. Dividing by 255 helps duh.\n\nConditions:\n* Initially NAdam\n* Zero dropout\n\nLoss plateauing around 400k everytime\n\nNeed to search for a good learning rate tho using some hp tuning.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom IPython import display\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfrom tqdm.auto import tqdm \nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-21T21:04:49.753593Z","iopub.execute_input":"2023-06-21T21:04:49.753963Z","iopub.status.idle":"2023-06-21T21:04:49.770611Z","shell.execute_reply.started":"2023-06-21T21:04:49.753931Z","shell.execute_reply":"2023-06-21T21:04:49.769487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-06-21T16:58:31.287839Z","iopub.execute_input":"2023-06-21T16:58:31.288382Z","iopub.status.idle":"2023-06-21T16:58:31.293143Z","shell.execute_reply.started":"2023-06-21T16:58:31.288348Z","shell.execute_reply":"2023-06-21T16:58:31.292234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-06-21T16:58:31.305502Z","iopub.execute_input":"2023-06-21T16:58:31.306283Z","iopub.status.idle":"2023-06-21T16:58:31.336820Z","shell.execute_reply.started":"2023-06-21T16:58:31.306250Z","shell.execute_reply":"2023-06-21T16:58:31.335907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing data","metadata":{}},{"cell_type":"code","source":"MNIST_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nMNIST_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T16:58:31.380872Z","iopub.execute_input":"2023-06-21T16:58:31.381476Z","iopub.status.idle":"2023-06-21T16:58:36.042697Z","shell.execute_reply.started":"2023-06-21T16:58:31.381442Z","shell.execute_reply":"2023-06-21T16:58:36.041656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datatensor = torch.tensor(MNIST_df.iloc[:,1:].values/255, dtype = torch.float32).to(device)\nMNIST_train = torch.utils.data.TensorDataset(datatensor)\ntrainloader = torch.utils.data.DataLoader(MNIST_train,\n                                          batch_size=64,\n                                          shuffle=True,\n                                          drop_last = True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:11:11.192640Z","iopub.execute_input":"2023-06-21T18:11:11.192995Z","iopub.status.idle":"2023-06-21T18:11:11.435441Z","shell.execute_reply.started":"2023-06-21T18:11:11.192966Z","shell.execute_reply":"2023-06-21T18:11:11.434458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"datatensor.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:39:06.631504Z","iopub.execute_input":"2023-06-21T17:39:06.631868Z","iopub.status.idle":"2023-06-21T17:39:06.638442Z","shell.execute_reply.started":"2023-06-21T17:39:06.631838Z","shell.execute_reply":"2023-06-21T17:39:06.637452Z"}}},{"cell_type":"markdown","source":"# Vanilla AutoEncoder","metadata":{}},{"cell_type":"code","source":"class DeepNN(nn.Module):\n    def __init__(self, num_layers, og_size, hidden_size, final_size, dropout_rate=0.2):\n        super().__init__()\n        self.num_layers = num_layers\n        self.fc = nn.ModuleList()\n        self.dropout_rate = dropout_rate\n        \n        if type(hidden_size) == int:\n            self.fc.append(nn.Linear( og_size, hidden_size))\n            for i in range(num_layers-1):\n                self.fc.append(nn.Linear(hidden_size,hidden_size))\n            self.fc.append(nn.Linear(hidden_size, final_size))\n            \n        elif type(hidden_size) == list:\n            self.fc.append(nn.Linear(og_size, hidden_size[0]))\n            for i in range(num_layers-1):\n                self.fc.append(nn.Linear(hidden_size[i] ,hidden_size[i+1]))\n            self.fc.append(nn.Linear(hidden_size[-1], final_size))\n        \n        self.dropout = nn.Dropout(p= self.dropout_rate)\n        \n    def forward(self, X):\n        x = X\n        for i in range(self.num_layers):\n            x = torch.relu(self.fc[i](x))\n            x = self.dropout(x)\n        \n        x = self.fc[-1](x)\n        return x\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:32:27.529947Z","iopub.execute_input":"2023-06-21T18:32:27.531043Z","iopub.status.idle":"2023-06-21T18:32:27.543065Z","shell.execute_reply.started":"2023-06-21T18:32:27.530999Z","shell.execute_reply":"2023-06-21T18:32:27.542024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, enc, dec):\n        super().__init__()\n        self.enc = enc\n        self.dec = dec\n    \n    def forward(self, X):\n        x = self.enc(X)\n        x = self.dec(x)\n        x = torch.sigmoid(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-21T21:29:00.088893Z","iopub.execute_input":"2023-06-21T21:29:00.089267Z","iopub.status.idle":"2023-06-21T21:29:00.096002Z","shell.execute_reply.started":"2023-06-21T21:29:00.089236Z","shell.execute_reply":"2023-06-21T21:29:00.095093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, optimizer, loss_fn, n_epochs, dataloader):\n    model.train()\n    #writer = SummaryWriter()\n    \n    progress = torch.zeros(n_epochs)\n    for epoch in tqdm(range(n_epochs)):\n        \n        epoch_loss = 0\n        \n        for data in dataloader:\n            data = data[0]\n            optimizer.zero_grad()\n            preds = model(data)\n            loss = loss_fn(preds, data)\n            loss.backward()\n            optimizer.step()\n            \n            epoch_loss += loss.item()\n            \n        progress[epoch] = epoch_loss\n        fig, axs = plt.subplots(4, 4)\n        rand = datatensor[np.random.randint(0, 10000, 16)]\n        display.clear_output() # If you imported display from IPython\n    \n        for i in range(4):\n            for j in range(4):\n                axs[i, j].imshow(model(rand[4*i+j]).reshape(28,28).cpu().detach().numpy(), cmap = \"gray\")\n                axs[i, j].axis(\"off\")\n    \n        plt.subplots_adjust(wspace = 0, hspace = 0)\n        plt.show()\n        print(\"-----------\", \"EPOCH\", epoch, \"-----------\")\n        \n   #     writer.add_scalar(\"Training loss\", epoch_loss, epoch)    \n    \n   #writer.close()\n    plt.plot(progress)\n    plt.xlabel(\"Time\")\n    plt.ylabel(\"Loss\")\n    plt.title(\"Training loss over time\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:54:35.304634Z","iopub.execute_input":"2023-06-21T17:54:35.305010Z","iopub.status.idle":"2023-06-21T17:54:35.315560Z","shell.execute_reply.started":"2023-06-21T17:54:35.304979Z","shell.execute_reply":"2023-06-21T17:54:35.314594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc1 = DeepNN(4, 784, [ 512, 256, 128, 64], 32)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:56:05.009918Z","iopub.execute_input":"2023-06-21T17:56:05.010617Z","iopub.status.idle":"2023-06-21T17:56:05.022067Z","shell.execute_reply.started":"2023-06-21T17:56:05.010557Z","shell.execute_reply":"2023-06-21T17:56:05.020869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dec1 = DeepNN(4, 32, [ 64, 128, 256, 512], 784)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:56:06.615959Z","iopub.execute_input":"2023-06-21T17:56:06.616650Z","iopub.status.idle":"2023-06-21T17:56:06.628015Z","shell.execute_reply.started":"2023-06-21T17:56:06.616607Z","shell.execute_reply":"2023-06-21T17:56:06.627050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fullAE = AutoEncoder(enc1, dec1).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:13:01.522450Z","iopub.execute_input":"2023-06-21T18:13:01.522807Z","iopub.status.idle":"2023-06-21T18:13:01.529378Z","shell.execute_reply.started":"2023-06-21T18:13:01.522776Z","shell.execute_reply":"2023-06-21T18:13:01.528288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fullAE","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:56:10.840450Z","iopub.execute_input":"2023-06-21T17:56:10.842492Z","iopub.status.idle":"2023-06-21T17:56:10.849562Z","shell.execute_reply.started":"2023-06-21T17:56:10.842447Z","shell.execute_reply":"2023-06-21T17:56:10.848538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.NAdam(fullAE.parameters())","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:56:52.968934Z","iopub.execute_input":"2023-06-21T17:56:52.969325Z","iopub.status.idle":"2023-06-21T17:56:52.975659Z","shell.execute_reply.started":"2023-06-21T17:56:52.969293Z","shell.execute_reply":"2023-06-21T17:56:52.974700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"hardly any visible improvement after the first few epochs, in nfact seems to have worsened","metadata":{}},{"cell_type":"code","source":"train(fullAE, optimizer, loss_fn, 50, trainloader)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:13:10.884774Z","iopub.execute_input":"2023-06-21T18:13:10.885133Z","iopub.status.idle":"2023-06-21T18:17:50.400349Z","shell.execute_reply.started":"2023-06-21T18:13:10.885103Z","shell.execute_reply":"2023-06-21T18:17:50.399379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# enc2 = DeepNN(4, 784, [ 512, 256, 128, 64], 32, 0.2)\n# dec2 = DeepNN(4, 32, [ 64, 128, 256, 512], 784, 0.2)\n# new_model = fullAutoEncDec(enc2, dec2).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:48:05.498697Z","iopub.execute_input":"2023-06-21T18:48:05.499070Z","iopub.status.idle":"2023-06-21T18:48:05.518871Z","shell.execute_reply.started":"2023-06-21T18:48:05.499039Z","shell.execute_reply":"2023-06-21T18:48:05.517998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optimizer = torch.optim.NAdam(new_model.parameters())\n# train(new_model, optimizer, loss_fn, 30, trainloader)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T18:48:07.010445Z","iopub.execute_input":"2023-06-21T18:48:07.010814Z","iopub.status.idle":"2023-06-21T18:48:30.897309Z","shell.execute_reply.started":"2023-06-21T18:48:07.010784Z","shell.execute_reply":"2023-06-21T18:48:30.896006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Behold the code of the blogpost now.\nno wonder retards like tensorflow","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.layers import Dense, Input, Flatten,\\\n                                    Reshape, LeakyReLU as LR,\\\n                                    Activation, Dropout\nfrom tensorflow.keras.models import Model, Sequential\nfrom matplotlib import pyplot as plt\nfrom IPython import display # If using IPython, Colab or Jupyter\nimport numpy as np\n(x_train, y_train), (x_test, y_test) = mnist.load_data()\nx_train = x_train/255.0\nx_test = x_test/255.0\n# Plot image data from x_train\nplt.imshow(x_train[0], cmap = \"gray\")\nplt.show()\nLATENT_SIZE = 32\nencoder = Sequential([\n    Flatten(input_shape = (28, 28)),\n    Dense(512),\n    LR(),\n    Dropout(0.5),\n    Dense(256),\n    LR(),\n    Dropout(0.5),\n    Dense(128),\n    LR(),\n    Dropout(0.5),\n    Dense(64),\n    LR(),\n    Dropout(0.5),\n    Dense(LATENT_SIZE),\n    LR()\n])\ndecoder = Sequential([\n    Dense(64, input_shape = (LATENT_SIZE,)),\n    LR(),\n    Dropout(0.5),\n    Dense(128),\n    LR(),\n    Dropout(0.5),\n    Dense(256),\n    LR(),\n    Dropout(0.5),\n    Dense(512),\n    LR(),\n    Dropout(0.5),\n    Dense(784),\n    Activation(\"sigmoid\"),\n    Reshape((28, 28))\n])\nimg = Input(shape = (28, 28))\nlatent_vector = encoder(img)\noutput = decoder(latent_vector)\nmodel = Model(inputs = img, outputs = output)\nmodel.compile(\"nadam\", loss = \"binary_crossentropy\")\nEPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:30:11.999472Z","iopub.execute_input":"2023-06-21T17:30:11.999850Z","iopub.status.idle":"2023-06-21T17:30:23.831292Z","shell.execute_reply.started":"2023-06-21T17:30:11.999821Z","shell.execute_reply":"2023-06-21T17:30:23.830268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(EPOCHS):\n    fig, axs = plt.subplots(4, 4)\n    rand = x_test[np.random.randint(0, 10000, 16)].reshape((4, 4, 1, 28, 28))\n    \n    display.clear_output() # If you imported display from IPython\n    \n    for i in range(4):\n        for j in range(4):\n            axs[i, j].imshow(model.predict(rand[i, j])[0], cmap = \"gray\")\n            axs[i, j].axis(\"off\")\n    \n    plt.subplots_adjust(wspace = 0, hspace = 0)\n    plt.show()\n    print(\"-----------\", \"EPOCH\", epoch, \"-----------\")\n    model.fit(x_train, x_train)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T17:30:33.547040Z","iopub.execute_input":"2023-06-21T17:30:33.547403Z","iopub.status.idle":"2023-06-21T17:38:41.547013Z","shell.execute_reply.started":"2023-06-21T17:30:33.547375Z","shell.execute_reply":"2023-06-21T17:38:41.543781Z"},"trusted":true},"execution_count":null,"outputs":[]}]}