{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing header files\nimport os\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting path\ntrain_data_path = \"../input/digit-recognizer/train.csv\"\ntest_data_path = \"../input/digit-recognizer/test.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load data frames\ntrain_df = pd.read_csv(train_data_path)\ntest_df = pd.read_csv(test_data_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info(verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.info(verbose = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### The given data is complete and there are no missing values.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Max value in train_df :\", max(train_df.max()))\nprint(\"Min value in train_df :\", min(train_df.min()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Max value in test_df :\", max(test_df.max()))\nprint(\"Min value in test_df :\", min(test_df.min()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = train_df.loc[:, 'pixel0':].to_numpy()\ntrain_y = train_df.loc[:, 'label'].to_numpy().reshape(-1,1).ravel()\ntest_X = test_df.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = np.divide(train_X, 255)\ntest_X = np.divide(test_X, 255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_X.shape)\nprint(train_y.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Making Predictive Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to get submission_file\ndef get_submission_file(model, filename):\n    predictions = model.predict(test_X)\n    df_submission = pd.DataFrame({'ImageId' : test_df.index + 1, 'Label' : predictions})\n    # submission file\n    submission_data_path = os.path.join(os.path.pardir, 'Data', 'Predictions')\n    submission_file_path = os.path.join(submission_data_path, filename)\n    # write to the file\n    df_submission.to_csv(submission_file_path, index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to test model\nfrom sklearn.model_selection import train_test_split\n\ndef test_model(model, trials):\n    total_score = 0\n    for trial in range(trials):\n        X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size = 0.2)\n        model.fit(X_train,  y_train)\n        total_score += model.score(X_test, y_test)\n    print(f\"Average Accuracy of the model : {round(total_score / trials, 3)}\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dummy Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.dummy import DummyClassifier\n\ndummy_model = DummyClassifier(strategy = \"most_frequent\")\n\ntest_model(dummy_model, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logistic Regression Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlr_model_1 = LogisticRegression(max_iter = 1000)\n\ntest_model(lr_model_1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Kth Nearest Neighbour","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\n\nKCN_1 = KNeighborsClassifier()\n\ntest_model(KCN_1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Forest Classifier ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nRFC_1 = RandomForestClassifier()\n\ntest_model(RFC_1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Naive Bayes Classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\n\nGNB = GaussianNB();\n\ntest_model(GNB, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Neural Network Classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\n\nclf_1 = MLPClassifier()\n\ntest_model(clf_1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\n\nclf_2 = MLPClassifier(hidden_layer_sizes = (100, 100))\n\ntest_model(clf_2, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\n\nclf_3 = MLPClassifier(hidden_layer_sizes = (62, 62))\n\ntest_model(clf_3, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decision Tree Classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\n\nDTC = DecisionTreeClassifier()\n\ntest_model(DTC, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Forest Classifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nRFC = RandomForestClassifier()\n\ntest_model(RFC, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TensforFlow","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size = 0.2)\nX_train = X_train.reshape(-1, 28, 28)\nX_test = X_test.reshape(-1, 28, 28)\nX = test_X.reshape(-1, 28, 28)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to get submission_file\ndef get_submission_file_TF(model, filename):\n    y = model.predict(X)\n    predictions = []\n    for prediction in y:\n        predictions.append(np.argmax(prediction))\n    df_submission = pd.DataFrame({'ImageId' : test_df.index + 1, 'Label' : predictions})\n    # submission file\n    submission_data_path = os.path.join(os.path.pardir, 'Data', 'Predictions')\n    submission_file_path = os.path.join(submission_data_path, filename)\n    # write to the file\n    df_submission.to_csv(submission_file_path, index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_submission_file_TF_cnn(model, filename):\n    y = model.predict(X.reshape(-1, 28, 28, 1))\n    predictions = []\n    for prediction in y:\n        predictions.append(np.argmax(prediction))\n    df_submission = pd.DataFrame({'ImageId' : test_df.index + 1, 'Label' : predictions})\n    # submission file\n    submission_data_path = os.path.join(os.path.pardir, 'Data', 'Predictions')\n    submission_file_path = os.path.join(submission_data_path, filename)\n    # write to the file\n    df_submission.to_csv(submission_file_path, index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(model, X):\n    y = model.predict(X)\n    predictions = []\n    for prediction in y:\n        predictions.append(np.argmax(prediction))\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_model(model, X, y):\n    val_loss, val_acc = model.evaluate(X, y)\n    print(val_loss, val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_image(image):\n    plt.imshow(image, cmap = 'gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_image(X_train[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dense Layer Neural Network","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Flatten, Dense\n\nmodel = Sequential()\nmodel.add(Flatten())\nmodel.add(Dense(128, activation = tf.nn.sigmoid))\nmodel.add(Dense(128, activation = tf.nn.sigmoid))\nmodel.add(Dense(10, activation = tf.nn.softmax))\n\nmodel.compile(optimizer = 'adam' , loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])\n\nmodel.fit(X_train, y_train, epochs = 20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(model, X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We expect a similar validation score.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = get_predictions(model, X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_wrong_predictions(predictons):\n    wrong_predictions = []\n    for i in range(len(predictions)):\n        if(y_test[i] != predictions[i]):\n             wrong_predictions.append(i)\n    return wrong_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wrong_predictions = get_wrong_predictions(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndef get_a_wrong_prediction(wrong_predictions):\n    n = random.randint(0, len(wrong_predictions))\n    n = wrong_predictions[n]\n    print_image(X_test[n])\n    print(\"Prediction :\", predictions[n])\n    print(\"Expected :\", y_test[n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_a_wrong_prediction(wrong_predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convolutional Neural Network","execution_count":null},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D\n\nmodel = Sequential()\n\n# Layer 1\nmodel.add(Conv2D(128, (3, 3), input_shape = (28, 28, 1)))\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPooling2D(pool_size = (2, 2)))\n\n# Layer 2\nmodel.add(Conv2D(128, (3, 3)))\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPooling2D(pool_size = (2, 2)))\n       \n# Layer 3\nmodel.add(Flatten())\nmodel.add(Dense(128))\n          \n# Output Layer\nmodel.add(Dense(10))\nmodel.add(Activation('sigmoid'))\n          \nmodel.compile(optimizer = 'adam' , loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])\n          \nmodel.fit(X_train.reshape(-1, 28, 28, 1), y_train, batch_size = 32, epochs = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(model, X_test.reshape(-1, 28, 28, 1), y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We expect similar accuracy scores.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2)\n\ndatagen.fit(X_train.reshape(-1, 28, 28, 1))\n\nmodel.fit(datagen.flow(X_train.reshape(-1, 28, 28, 1), y_train, batch_size = 32), batch_size = 32, steps_per_epoch = (len(X_train) / 32), epochs = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(model, X_test.reshape(-1, 28, 28, 1), y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### This is not improving my accuracy.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### My best score so far is 98.87% (rank 1427).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# This is the work of Anirudh Achal.","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}