{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Libraries\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split\nimport time","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare Dataset\n# load data\ntrain = pd.read_csv(r\"../input/digit-recognizer/train.csv\",dtype = np.float32)\n\n# split data into features(pixels) and labels(numbers from 0 to 9)\ntargets_numpy = train.label.values\nfeatures_numpy = train.loc[:,train.columns != \"label\"].values/255 # normalization\n\n# train test split. Size of train data is 80% and size of test data is 20%. \nfeatures_train, features_test, targets_train, targets_test = train_test_split(features_numpy,\n                                                                             targets_numpy,\n                                                                             test_size = 0.2,\n                                                                             random_state = 42) \n\n# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable\nfeaturesTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(targets_train).type(torch.LongTensor) # data type is long\n\n# create feature and targets tensor for test set.\nfeaturesTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(targets_test).type(torch.LongTensor) # data type is long\n\n# batch_size, epoch and iteration\nbatch_size = 100\nn_iters = 10000\nnum_epochs = n_iters / (len(features_train) / batch_size) + 1\nnum_epochs = int(num_epochs)\n\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)\nvalid = torch.utils.data.TensorDataset(featuresTest,targetsTest)\n\n\n# data loader\ntrain_loader = DataLoader(train, batch_size = batch_size, shuffle = True)\nvalid_loader = DataLoader(valid, batch_size = batch_size, shuffle = False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CNN Model\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n        '''\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n        self.relu1 = nn.ReLU()\n        \n        # Max pool 1\n        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n     \n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n        self.relu2 = nn.ReLU()\n        \n        # Max pool 2\n        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n        \n        # Fully connected 1\n        self.fc1 = nn.Linear(32 * 4 * 4 , 10) \n        '''\n        self.features_extract = nn.Sequential(\n            nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2),\n            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2)\n        )\n        \n        self.classify = nn.Sequential(\n            nn.Linear(512,50),\n            nn.ReLU(),\n            nn.Linear(50,10)\n        )\n        \n    def forward(self, x):\n        \n        x = self.features_extract(x)\n        x = torch.flatten(x,1)\n        logits = self.classify(x)\n        probs = F.softmax(logits, dim = 1)\n\n        '''\n        out = self.cnn1(x)\n        out = self.relu1(out)\n        out = self.maxpool1(out)\n        \n        out = self.cnn2(out)\n        out = self.relu2(out)\n        out =self.maxpool2(out)\n        \n        out = self.fc1(out)\n        \n        logits = out\n        probs = F.softmax(logits, dim =1)\n        return logits,probs\n        '''\n    \n# Create CNN\nmodel = CNNModel()\n\n# Cross Entropy Loss \nloss_function = nn.CrossEntropyLoss()\n\n# SGD Optimizer\n\nlearning_rate = 0.05\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_accuracy(model, data_loader):\n    '''\n    Function for computing the accuracy of the predictions over the entire data_loader\n    '''\n    \n    correct_pred = 0 \n    n = 0\n    \n    with torch.no_grad():\n        model.eval()\n        for X, y_true in data_loader:\n            X = X.reshape(100,1,28,28)\n            _, y_prob = model(X)\n            _, predicted_labels = torch.max(y_prob, 1)\n\n            n += y_true.size(0)\n            correct_pred += (predicted_labels == y_true).sum()\n\n    return correct_pred.float() / n\n\ndef plot_losses(train_losses, valid_losses):\n    '''\n    Function for plotting training and validation losses\n    '''\n    \n    # temporarily change the style of the plots to seaborn \n    plt.style.use('seaborn')\n\n    train_losses = np.array(train_losses) \n    valid_losses = np.array(valid_losses)\n\n    fig, ax = plt.subplots(figsize = (8, 4.5))\n\n    ax.plot(train_losses, color='blue', label='Training loss') \n    ax.plot(valid_losses, color='red', label='Validation loss')\n    ax.set(title=\"Loss over epochs\", \n            xlabel='Epoch',\n            ylabel='Loss') \n    ax.legend()\n    fig.show()\n    \n    # change the plot style to default\n    plt.style.use('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(train_loader, model, criterion, optimizer):\n\n    model.train()\n    running_loss = 0\n    \n    for X, y_true in train_loader:\n\n        optimizer.zero_grad()\n        # Forward pass\n        X = X.reshape(100,1,28,28)\n        y_hat, _ = model(X) \n        loss = criterion(y_hat, y_true) \n        running_loss += loss.item() * X.size(0)\n\n        # Backward pass\n        loss.backward()\n        optimizer.step()\n\n    epoch_loss = running_loss / len(train_loader.dataset)\n    return model, optimizer, epoch_loss\n\ndef validate(valid_loader, model, criterion):\n   \n    model.eval()\n    running_loss = 0\n    \n    for X, y_true in valid_loader:\n\n        # Forward pass and record loss\n        X = X.reshape(100,1,28,28)\n        y_hat, _ = model(X) \n        loss = criterion(y_hat, y_true) \n        running_loss += loss.item() * X.size(0)\n\n    epoch_loss = running_loss / len(valid_loader.dataset)\n        \n    return model, epoch_loss\n\ndef training_loop(model, criterion, optimizer, train_loader, valid_loader, epochs):\n    \n    # set objects for storing metrics\n    best_loss = 1e10\n    train_losses = []\n    valid_losses = []\n \n    # Train model\n    for epoch in range(0, epochs):\n        \n        # training\n        model, optimizer, train_loss = train(train_loader, model, criterion, optimizer)\n        train_losses.append(train_loss)\n\n        # validation\n        with torch.no_grad():\n            model, valid_loss = validate(valid_loader, model, criterion)\n            valid_losses.append(valid_loss)\n        #scheduler.step(valid_loss)    \n            \n        train_acc = get_accuracy(model, train_loader)\n        valid_acc = get_accuracy(model, valid_loader)\n\n        print(f'Epoch: {epoch}\\t'\n              f'Train loss: {train_loss:.4f}\\t'\n              f'Valid loss: {valid_loss:.4f}\\t'\n              f'Train accuracy: {100 * train_acc:.2f}\\t'\n              f'Valid accuracy: {100 * valid_acc:.2f}')\n\n    plot_losses(train_losses, valid_losses)\n    \n    return model, optimizer, (train_losses, valid_losses)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, optimizer, _ = training_loop(model,loss_function, optimizer,train_loader,valid_loader,num_epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(r\"../input/digit-recognizer/test.csv\",dtype = np.float32)\n\n# split data into features(pixels) and labels(numbers from 0 to 9)\ntests_feat_numpy = test_df.loc[:].values/255 # normalization\n#print(len(tests_feat_numpy))\ntests_features = torch.from_numpy(tests_feat_numpy)\n#test_output = model(tests_features)\noutputs = torch.zeros(len(tests_features),1)\n\nfor i, images in enumerate(tests_features):\n        tests = Variable(images.view(1,1,28,28))\n        outputs[i] = model(tests).argmax(dim = 1)\noutputs[0:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_bc = outputs.tolist()\nstr_output = [str(int(output[0])) for output in output_bc]\n#print(str_output)\nstr_str = \" \".join(str_output)\n#print(str_str)\noutput_bc = [int(str_str_int) for str_str_int in str_str.split(\" \")]\n#output_bc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_result = pd.DataFrame(\n    {\n        \"Label\": output_bc\n    }\n)\ndf_result.index += 1\n\ndf_result\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result.to_csv(\"submission.csv\", index_label=\"ImageId\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}