{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-19T05:04:00.432875Z","iopub.execute_input":"2023-03-19T05:04:00.433294Z","iopub.status.idle":"2023-03-19T05:04:00.466273Z","shell.execute_reply.started":"2023-03-19T05:04:00.433256Z","shell.execute_reply":"2023-03-19T05:04:00.465280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the Dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:00.468551Z","iopub.execute_input":"2023-03-19T05:04:00.469402Z","iopub.status.idle":"2023-03-19T05:04:06.957395Z","shell.execute_reply.started":"2023-03-19T05:04:00.469349Z","shell.execute_reply":"2023-03-19T05:04:06.956286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:29.966213Z","iopub.execute_input":"2023-03-19T05:04:29.966626Z","iopub.status.idle":"2023-03-19T05:04:29.992287Z","shell.execute_reply.started":"2023-03-19T05:04:29.966593Z","shell.execute_reply":"2023-03-19T05:04:29.991045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:30.113246Z","iopub.execute_input":"2023-03-19T05:04:30.113903Z","iopub.status.idle":"2023-03-19T05:04:30.127792Z","shell.execute_reply.started":"2023-03-19T05:04:30.113867Z","shell.execute_reply":"2023-03-19T05:04:30.126454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_df.drop(['label'],axis=1)\ny = train_df['label']","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:30.287482Z","iopub.execute_input":"2023-03-19T05:04:30.288198Z","iopub.status.idle":"2023-03-19T05:04:30.374216Z","shell.execute_reply.started":"2023-03-19T05:04:30.288128Z","shell.execute_reply":"2023-03-19T05:04:30.372845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X/255","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:30.554233Z","iopub.execute_input":"2023-03-19T05:04:30.555386Z","iopub.status.idle":"2023-03-19T05:04:30.680455Z","shell.execute_reply.started":"2023-03-19T05:04:30.555340Z","shell.execute_reply":"2023-03-19T05:04:30.679221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X['pixel50'].max()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:31.969588Z","iopub.execute_input":"2023-03-19T05:04:31.970397Z","iopub.status.idle":"2023-03-19T05:04:31.975195Z","shell.execute_reply.started":"2023-03-19T05:04:31.970358Z","shell.execute_reply":"2023-03-19T05:04:31.974082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting the Dataset","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.2,random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:56.289004Z","iopub.execute_input":"2023-03-19T05:04:56.289383Z","iopub.status.idle":"2023-03-19T05:04:56.529489Z","shell.execute_reply.started":"2023-03-19T05:04:56.289350Z","shell.execute_reply":"2023-03-19T05:04:56.528435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape,X_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:57.214019Z","iopub.execute_input":"2023-03-19T05:04:57.214687Z","iopub.status.idle":"2023-03-19T05:04:57.221787Z","shell.execute_reply.started":"2023-03-19T05:04:57.214650Z","shell.execute_reply":"2023-03-19T05:04:57.220608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training ML Models","metadata":{}},{"cell_type":"markdown","source":"### 1. K-Nearest Neighbors","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:04:59.496235Z","iopub.execute_input":"2023-03-19T05:04:59.496677Z","iopub.status.idle":"2023-03-19T05:04:59.502546Z","shell.execute_reply.started":"2023-03-19T05:04:59.496637Z","shell.execute_reply":"2023-03-19T05:04:59.501275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nknn.fit(X_train,y_train)\nprint(knn.score(X_train,y_train),knn.score(X_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:05:01.235624Z","iopub.execute_input":"2023-03-19T05:05:01.236372Z","iopub.status.idle":"2023-03-19T05:06:10.588577Z","shell.execute_reply.started":"2023-03-19T05:05:01.236332Z","shell.execute_reply":"2023-03-19T05:06:10.586434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Support Vector Machine","metadata":{}},{"cell_type":"code","source":"# from sklearn.svm import SVC\n# sv = SVC()\n# sv.fit(X_train,y_train)\n# print(sv.score(X_train,y_train),sv.score(X_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T06:41:27.352755Z","iopub.execute_input":"2023-03-19T06:41:27.353829Z","iopub.status.idle":"2023-03-19T06:41:27.358932Z","shell.execute_reply.started":"2023-03-19T06:41:27.353789Z","shell.execute_reply":"2023-03-19T06:41:27.357351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. RandomForest Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf =RandomForestClassifier()\nrf.fit(X_train,y_train)\nprint(rf.score(X_train,y_train),rf.score(X_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:12:28.089249Z","iopub.execute_input":"2023-03-19T05:12:28.089645Z","iopub.status.idle":"2023-03-19T05:12:50.849198Z","shell.execute_reply.started":"2023-03-19T05:12:28.089609Z","shell.execute_reply":"2023-03-19T05:12:50.847970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XgBoost Classifier","metadata":{}},{"cell_type":"code","source":"# import xgboost as xgb\n# model = xgb.XGBClassifier()\n# model.fit(X_train,y_train)\n# print(model.score(X_train,y_train),model.score(X_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:19:56.941068Z","iopub.execute_input":"2023-03-19T05:19:56.941683Z","iopub.status.idle":"2023-03-19T05:19:56.947312Z","shell.execute_reply.started":"2023-03-19T05:19:56.941642Z","shell.execute_reply":"2023-03-19T05:19:56.946010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training the CNN Model","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:00.847814Z","iopub.execute_input":"2023-03-19T05:20:00.848271Z","iopub.status.idle":"2023-03-19T05:20:07.252392Z","shell.execute_reply.started":"2023-03-19T05:20:00.848234Z","shell.execute_reply":"2023-03-19T05:20:07.251255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:07.254384Z","iopub.execute_input":"2023-03-19T05:20:07.255126Z","iopub.status.idle":"2023-03-19T05:20:10.731209Z","shell.execute_reply.started":"2023-03-19T05:20:07.255093Z","shell.execute_reply":"2023-03-19T05:20:10.730146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_df.drop(['label'],axis = 1).astype('int32')\ny = train_df['label'].astype('float32')\nx_test = test_df.astype('float32')\nX.shape, y.shape, x_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:13.037262Z","iopub.execute_input":"2023-03-19T05:20:13.038018Z","iopub.status.idle":"2023-03-19T05:20:13.204341Z","shell.execute_reply.started":"2023-03-19T05:20:13.037975Z","shell.execute_reply":"2023-03-19T05:20:13.203120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.values.reshape(-1, 28, 28, 1) / 255.0\nx_test = x_test.values.reshape(-1, 28, 28, 1) / 255.0\nX.shape, x_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:13.443978Z","iopub.execute_input":"2023-03-19T05:20:13.445001Z","iopub.status.idle":"2023-03-19T05:20:13.573277Z","shell.execute_reply.started":"2023-03-19T05:20:13.444954Z","shell.execute_reply":"2023-03-19T05:20:13.571963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = tf.keras.utils.to_categorical(y, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:13.860530Z","iopub.execute_input":"2023-03-19T05:20:13.862466Z","iopub.status.idle":"2023-03-19T05:20:13.869238Z","shell.execute_reply.started":"2023-03-19T05:20:13.862415Z","shell.execute_reply":"2023-03-19T05:20:13.867903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:14.494455Z","iopub.execute_input":"2023-03-19T05:20:14.494819Z","iopub.status.idle":"2023-03-19T05:20:14.866398Z","shell.execute_reply.started":"2023-03-19T05:20:14.494786Z","shell.execute_reply":"2023-03-19T05:20:14.865284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building the Neural Network","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout, BatchNormalization","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:54:04.333959Z","iopub.execute_input":"2023-03-19T05:54:04.334390Z","iopub.status.idle":"2023-03-19T05:54:04.340384Z","shell.execute_reply.started":"2023-03-19T05:54:04.334355Z","shell.execute_reply":"2023-03-19T05:54:04.339101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(16, (3,3), 1, activation='relu', input_shape=(28,28,1)))\nmodel.add(MaxPooling2D())\n\nmodel.add(Conv2D(32, (3,3), 1, activation='relu'))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(16, (3,3), 1, activation='relu'))\nmodel.add(MaxPooling2D())\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:16.236830Z","iopub.execute_input":"2023-03-19T05:20:16.237576Z","iopub.status.idle":"2023-03-19T05:20:22.243942Z","shell.execute_reply.started":"2023-03-19T05:20:16.237535Z","shell.execute_reply":"2023-03-19T05:20:22.242786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile('adam', loss=tf.losses.CategoricalCrossentropy(), metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:23.579305Z","iopub.execute_input":"2023-03-19T05:20:23.579815Z","iopub.status.idle":"2023-03-19T05:20:23.608336Z","shell.execute_reply.started":"2023-03-19T05:20:23.579771Z","shell.execute_reply":"2023-03-19T05:20:23.607311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:25.562757Z","iopub.execute_input":"2023-03-19T05:20:25.563125Z","iopub.status.idle":"2023-03-19T05:20:25.596350Z","shell.execute_reply.started":"2023-03-19T05:20:25.563089Z","shell.execute_reply":"2023-03-19T05:20:25.595553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation and Early Stopping","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:27.203629Z","iopub.execute_input":"2023-03-19T05:20:27.204022Z","iopub.status.idle":"2023-03-19T05:20:27.210724Z","shell.execute_reply.started":"2023-03-19T05:20:27.203987Z","shell.execute_reply":"2023-03-19T05:20:27.209563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define data augmentation parameters\ndatagen = ImageDataGenerator(\n    rotation_range=10,  # rotate images randomly within 10 degrees\n    zoom_range=0.1,     # zoom images randomly within 10%\n    width_shift_range=0.1,  # shift images horizontally randomly within 10%\n    height_shift_range=0.1, # shift images vertically randomly within 10%\n    shear_range=0.1,    # shear images randomly within 10 degrees\n    horizontal_flip=False, # flip images horizontally randomly\n    vertical_flip=False   # flip images vertically randomly\n)\n\n# Fit the augmentation method on training set\ndatagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:20:28.580142Z","iopub.execute_input":"2023-03-19T05:20:28.581330Z","iopub.status.idle":"2023-03-19T05:20:28.667955Z","shell.execute_reply.started":"2023-03-19T05:20:28.581283Z","shell.execute_reply":"2023-03-19T05:20:28.666907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define early stopping callback\nearly_stop = EarlyStopping(monitor='val_loss', patience=5)\n\n# Train the model using data augmentation\nhistory = model.fit(\n    datagen.flow(X_train, y_train, batch_size=64),\n    steps_per_epoch=len(X_train)/64, \n    epochs=50, \n    validation_data=(X_val, y_val),\n#     callbacks=[early_stop],\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:27:36.155735Z","iopub.execute_input":"2023-03-19T05:27:36.156780Z","iopub.status.idle":"2023-03-19T05:39:29.584208Z","shell.execute_reply.started":"2023-03-19T05:27:36.156740Z","shell.execute_reply":"2023-03-19T05:39:29.583133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# res = model.predict(x_test)\n# res = np.argmax(res,axis = 1)\n# results_2 = pd.Series(res[0],name=\"Label\")\n# submission_df = pd.DataFrame(results_2)\n# submission_df[\"ImageId\"] = range(1,28001)\n# submission_df = submission_df[[\"ImageId\", \"Label\"]]\n# submission_df","metadata":{"execution":{"iopub.status.busy":"2023-03-19T06:45:36.771511Z","iopub.execute_input":"2023-03-19T06:45:36.772258Z","iopub.status.idle":"2023-03-19T06:45:36.801619Z","shell.execute_reply.started":"2023-03-19T06:45:36.772216Z","shell.execute_reply":"2023-03-19T06:45:36.799954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MNIST CNN NETs Ensemble","metadata":{}},{"cell_type":"code","source":"# BUILD CONVOLUTIONAL NEURAL NETWORKS\nnets = 3\nmodel = [0] *nets\nfor j in range(nets):\n    model[j] = Sequential()\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1)))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation='softmax'))\n    # COMPILE WITH ADAM OPTIMIZER AND CROSS ENTROPY COST\n    model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-03-19T05:54:10.748991Z","iopub.execute_input":"2023-03-19T05:54:10.749651Z","iopub.status.idle":"2023-03-19T05:54:11.333581Z","shell.execute_reply.started":"2023-03-19T05:54:10.749607Z","shell.execute_reply":"2023-03-19T05:54:11.332421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import LearningRateScheduler\n# DECREASE LEARNING RATE EACH EPOCH\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n# TRAIN NETWORKS\nhistory = [0] * nets\nepochs = 30\nfor j in range(nets):\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, y_train, test_size = 0.1)\n    history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),\n        epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,  \n        validation_data = (X_val2,Y_val2), callbacks=[annealer], verbose=1)\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n        j+1,epochs,max(history[j].history['accuracy']),max(history[j].history['val_accuracy']) ))","metadata":{"execution":{"iopub.status.busy":"2023-03-19T06:05:11.151533Z","iopub.execute_input":"2023-03-19T06:05:11.152423Z","iopub.status.idle":"2023-03-19T06:31:11.330740Z","shell.execute_reply.started":"2023-03-19T06:05:11.152382Z","shell.execute_reply":"2023-03-19T06:31:11.328895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ENSEMBLE PREDICTIONS AND SUBMIT\nresults = np.zeros( (x_test.shape[0],10) ) \nfor j in range(nets):\n    resultsi = model[j].predict(x_test)\n    results = results + resultsi\n    \n    resultsi = np.argmax(resultsi,axis = 1)\n    resultsi = pd.Series(resultsi,name=\"Label\")\n    submissioni = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),resultsi],axis = 1)\n    submissioni.to_csv(\"SUBMISSION\"+str(j+1)+\".csv\",index=False)\n    \nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"ENSEMBLE.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-19T06:34:12.618112Z","iopub.execute_input":"2023-03-19T06:34:12.618812Z","iopub.status.idle":"2023-03-19T06:34:27.182759Z","shell.execute_reply.started":"2023-03-19T06:34:12.618776Z","shell.execute_reply":"2023-03-19T06:34:27.181484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}