{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score, classification_report\nfrom sklearn.model_selection import KFold, cross_val_score, RandomizedSearchCV\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train['label'])\n# target distribution is not skewed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop('label', axis=1)\ny = train['label']\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values\ny_train = y_train.values\nX_test = X_test.values\ny_test = y_test.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dt_model = DecisionTreeClassifier()\ndt_model.fit(X_train, y_train)\ny_pred_dt = dt_model.predict(X_test)\naccuracy = accuracy_score(y_test, y_pred_dt)\nprint(accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rf_model = RandomForestClassifier()\nrf_model.fit(X_train, y_train)\ny_pred_rf = rf_model.predict(X_test)\naccuracy = accuracy_score(y_test, y_pred_rf)\nprint(accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_model = LogisticRegression(max_iter=100)\nlr_model.fit(X_train, y_train)\ny_pred_lr = lr_model.predict(X_test)\naccuracy = accuracy_score(y_test, y_pred_lr)\nprint(accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_models(models):\n    for model in models:\n        print(\"Evaluation for {}\".format(type(model).__name__))\n        print(\"----\"*20)\n        y_pred = model.predict(X_test)\n        cm = confusion_matrix(y_test,y_pred)\n        print(\"\\nConfusion Matrix:\\n\",cm)\n        ac = accuracy_score(y_test,y_pred)\n        print(\"\\nAccuracy:\\n\",ac)\n        print(\"\\nClassification Report:\\n\")\n        print(classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [lr_model, dt_model, rf_model]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_models(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cross_validate_models(models, splits):\n    kf = KFold(n_splits=splits,shuffle=True)\n    for model in models:\n        scores = cross_val_score(model,\n                                 X_train,\n                                 y_train,\n                                 cv=kf,\n                                 n_jobs=12,\n                                 scoring=\"accuracy\")\n        print(\"Cross-Validation for {}:\\n\".format(type(model).__name__))\n        print(\"Mean score: \", np.mean(scores))\n        print(\"Variance of score: \", np.std(scores)**2)\n        fig = plt.figure(figsize = (10,5))\n        ax = fig.add_subplot(111)\n        ax = sns.distplot(scores)\n        ax.set_xlabel(\"Cross-Validated Accuracy scores\")\n        ax.set_ylabel(\"Frequency\")\n        ax.set_title('Frequency Distribution of Cross-Validated Accuracy scores for {}'.format(type(model).__name__), fontsize = 15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_rf = [rf_model]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cross_validate_models(model_rf,10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rf_params = {'bootstrap': [True, False],\n 'max_depth': [10, 15, 20, 25, 30, None],\n 'max_features': ['auto', 'sqrt'],\n 'min_samples_leaf': [1, 2, 4],\n 'min_samples_split': [2, 5, 10],\n 'n_estimators': [100, 200, 400, 600, 800, 1000]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = [rf_params]\n# params_rf = [rf_params]\n\ntuned_models = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def hyper_param_tuning(models,params,splits,scorer):\n    for i in range(len(models)):\n        gsearch = RandomizedSearchCV(estimator=models[i],\n                               param_distributions=params[i],\n                               scoring=scorer,\n                               verbose=2,\n                               n_jobs=-1,\n                               cv=5)\n        start = time.time()\n        gsearch.fit(X_train,y_train)\n        end = time.time()\n        \n        print(\"Grid Search Results for {}:\\n\".format(type(models[i]).__name__))\n        print(\"Time taken for tuning (in secs): \\n\", end-start)\n        print(\"Best parameters: \\n\",gsearch.best_params_)\n        print(\"Best score: \\n\",gsearch.best_score_)\n        tuned_models.append(gsearch.best_estimator_)\n        print(\"\\n\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyper_param_tuning(model_rf,params,10,\"accuracy\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Parameter tuning takes a lot of time. Output is not shown here and the parameters tuned are as follows : \n\nparams_tuned = {'n_estimators': 1000, 'min_samples_split': 5, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'max_depth': 30, 'bootstrap': False}"},{"metadata":{"trusted":true},"cell_type":"code","source":"params_tuned = {'n_estimators': 1000, 'min_samples_split': 5, 'min_samples_leaf': 2, 'max_features': 'sqrt', 'max_depth': 30, 'bootstrap': False}\nrf_tuned_model = RandomForestClassifier(**params_tuned)\nrf_tuned_model.fit(X_train, y_train)\ny_pred_rf_tuned = rf_tuned_model.predict(X_test)\naccuracy = accuracy_score(y_test, y_pred_rf_tuned)\nprint(accuracy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Random Forest gives accuracy of 0.967 on validation data."},{"metadata":{},"cell_type":"markdown","source":"## Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nimport tensorflow as tf\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, Lambda\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X.values.reshape(X.shape[0], 28, 28,1)\nX.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\n\ny = to_categorical(y, num_classes = 10)\n\nfrom sklearn.model_selection import train_test_split\nXtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.1, random_state=101)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean_px = Xtrain.mean().astype(np.float32)\nstd_px = Xtrain.std().astype(np.float32)\n\ndef standardize(x): \n    return (x-mean_px)/std_px","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = RMSprop(lr=0.001)\nmodel = Sequential([\n    Lambda(standardize, input_shape=(28,28,1)),\n    Flatten(),\n    Dense(512, activation='relu'),\n    Dense(10, activation='softmax')\n    ])\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\n\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nepochs = 25\n\nmodel.fit(Xtrain, ytrain, batch_size = batch_size, epochs = epochs, \n         validation_data = (Xtest, ytest), verbose = 2, callbacks=[learning_rate_reduction, es])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(Xtest, ytest)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = Sequential()\n\nmodel1.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape=(28,28,1)))\nmodel1.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel1.add(MaxPool2D(pool_size=(2,2)))\nmodel1.add(Dropout(0.25))\n\n\nmodel1.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel1.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel1.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel1.add(Dropout(0.25))\n\n\nmodel1.add(Flatten())\nmodel1.add(Dense(256, activation = \"relu\"))\nmodel1.add(Dropout(0.5))\nmodel1.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtrain = Xtrain/255\nXtest = Xtest/255\nepochs1 = 15\nbatch_size1 = 32\nmodel1.fit(Xtrain, ytrain, batch_size = batch_size1, epochs = epochs1, \n         validation_data = (Xtest, ytest), verbose = 2, callbacks=[learning_rate_reduction, es])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_vals = test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = test.index.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = image_id + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_vals = test_vals.values.reshape(test_vals.shape[0], 28, 28,1)\ntest_vals.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_vals = test_vals/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model1.predict(test_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.argmax(predictions,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1 = pd.DataFrame({\n        \"ImageId\": image_id,\n        \"Label\": predictions\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1.to_csv('mysubmission1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}