{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n \"House\n
\n\n","metadata":{}},{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"# Ignore warnings :\nimport warnings \nwarnings.filterwarnings('ignore')\n\n# Handle table-like data and matrices :\nimport numpy as np\nimport pandas as pd\nimport math \nimport itertools\n\n\n# Classification\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier , GradientBoostingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis , QuadraticDiscriminantAnalysis\n\n# Regression\nfrom sklearn.linear_model import LinearRegression,Ridge,Lasso,RidgeCV, ElasticNet\nfrom sklearn.ensemble import RandomForestRegressor,BaggingRegressor,GradientBoostingRegressor,AdaBoostRegressor \nfrom sklearn.svm import SVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.neural_network import MLPRegressor\n\n# Modelling Helpers :\nfrom sklearn.preprocessing import Normalizer , scale\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.feature_selection import RFECV\nfrom sklearn.model_selection import GridSearchCV , KFold , cross_val_score\n\n#preprocessing :\nfrom sklearn.preprocessing import MinMaxScaler , StandardScaler, LabelEncoder\n\n# Classification\nfrom sklearn.metrics import accuracy_score,precision_score,recall_score,f1_score \n\n# Deep Learning Libraries\nfrom keras.models import Sequential, load_model\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization\nfrom keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop\nfrom keras.callbacks import ReduceLROnPlateau, LearningRateScheduler\nfrom keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.impute import SimpleImputer\nimport numpy as np\n\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n\n# Visualisation\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport matplotlib.pylab as pylab\nimport seaborn as sns\nimport missingno as msno\n","metadata":{"execution":{"iopub.status.busy":"2024-08-01T02:40:37.910036Z","iopub.execute_input":"2024-08-01T02:40:37.910449Z","iopub.status.idle":"2024-08-01T02:40:51.104935Z","shell.execute_reply.started":"2024-08-01T02:40:37.910414Z","shell.execute_reply":"2024-08-01T02:40:51.103735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ndf = train.copy()\ndf_test = test.copy()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:22:12.828923Z","iopub.execute_input":"2024-07-29T14:22:12.829424Z","iopub.status.idle":"2024-07-29T14:22:21.373970Z","shell.execute_reply.started":"2024-07-29T14:22:12.829360Z","shell.execute_reply":"2024-07-29T14:22:21.372427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How the Data looks\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:22:28.020644Z","iopub.execute_input":"2024-07-29T14:22:28.021176Z","iopub.status.idle":"2024-07-29T14:22:28.059087Z","shell.execute_reply.started":"2024-07-29T14:22:28.021131Z","shell.execute_reply":"2024-07-29T14:22:28.057781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:26:27.704851Z","iopub.execute_input":"2024-07-29T14:26:27.705278Z","iopub.status.idle":"2024-07-29T14:26:27.722588Z","shell.execute_reply.started":"2024-07-29T14:26:27.705243Z","shell.execute_reply":"2024-07-29T14:26:27.721390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(13, 7))\nplt.hist(x=df['label'], bins=25, edgecolor='black')\nplt.xlabel('Digits')\nplt.ylabel('Frequency')\nplt.title('Distribution of Labels')\n\n# Set the x-axis ticks to show all digits from 0 to 9\nplt.xticks(range(10))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:28:37.809602Z","iopub.execute_input":"2024-07-29T14:28:37.810026Z","iopub.status.idle":"2024-07-29T14:28:38.161625Z","shell.execute_reply.started":"2024-07-29T14:28:37.809994Z","shell.execute_reply":"2024-07-29T14:28:38.160486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train: \", df.shape)\nprint(\"Test: \", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:22:58.761755Z","iopub.execute_input":"2024-07-29T14:22:58.762204Z","iopub.status.idle":"2024-07-29T14:22:58.769098Z","shell.execute_reply.started":"2024-07-29T14:22:58.762170Z","shell.execute_reply":"2024-07-29T14:22:58.767855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualization","metadata":{}},{"cell_type":"code","source":"\nfig, ax = plt.subplots(2, 2, figsize=(10, 5)) # Adjust the figsize to have a smaller height\nfig.set_size_inches(4, 4) # Resize the figure to make it shorter\n\n\nax[0,0].imshow(df.drop('label',axis=1).values[41].reshape(28,28),cmap='gray') #giving random indexes\nax[0,1].imshow(df.drop('label',axis=1).values[243].reshape(28,28),cmap='gray')\nax[1,0].imshow(df.drop('label',axis=1).values[565].reshape(28,28),cmap='gray')\nax[1,1].imshow(df.drop('label',axis=1).values[251].reshape(28,28),cmap='gray')\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T15:18:53.458870Z","iopub.execute_input":"2024-07-29T15:18:53.459451Z","iopub.status.idle":"2024-07-29T15:18:54.498660Z","shell.execute_reply.started":"2024-07-29T15:18:53.459371Z","shell.execute_reply":"2024-07-29T15:18:54.497311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features\n* **Label: ** The Target variable.\n* **Pixels: ** The smallest unit of a Digital Image or Graphic that can be displayed on Digital Display Device.\n Where humans can see the objects due to the Light Receptors in their Eyes which send Signals via the Optic Nerve to the Primary Visual Cortex, where the input is processed ,\n\nComputers on the other hand, see the Image as 2-dimensional arrays of numbers, known as pixels. They Classify Images based on Boundaries and Curvatures of the Object (Represented by pixel values, either RGB or GrayScale) .","metadata":{}},{"cell_type":"markdown","source":"
\n \"CNN\"\n
","metadata":{}},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### Setting random seed","metadata":{}},{"cell_type":"code","source":"seed = 55\nnp.random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:29:08.981764Z","iopub.execute_input":"2024-07-29T14:29:08.982168Z","iopub.status.idle":"2024-07-29T14:29:08.987905Z","shell.execute_reply.started":"2024-07-29T14:29:08.982136Z","shell.execute_reply":"2024-07-29T14:29:08.986567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Splitting data into train and test","metadata":{}},{"cell_type":"code","source":"X = train.iloc[:,1:]\nY = train.iloc[:,0]\nx_train , x_test , y_train , y_test = train_test_split(X, Y , test_size=0.1, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:29:26.725523Z","iopub.execute_input":"2024-07-29T14:29:26.726626Z","iopub.status.idle":"2024-07-29T14:29:26.940890Z","shell.execute_reply.started":"2024-07-29T14:29:26.726587Z","shell.execute_reply":"2024-07-29T14:29:26.939754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reshape the Images\n* Note that we have Images as 1D vector each containing 784 pixels. Before we feed the data to the CNN we must reshape the data into (28x28x1) 3D matrices.\n* This is because Keras wants an Extra Dimension in the end, for channels. If this had been RGB images, there would have been 3 channels, but as MNIST is gray scale it only uses one.","metadata":{}},{"cell_type":"code","source":"# The first parameter in reshape indicates the number of examples.\n# We pass it as -1, which means that it is an unknown dimension and we want numpy to figure it out.\n\n# reshape(examples, height, width, channels)\nx_train = x_train.values.reshape(-1, 28, 28, 1)\nx_test = x_test.values.reshape(-1, 28, 28, 1)\ndf_test=df_test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:29:42.424257Z","iopub.execute_input":"2024-07-29T14:29:42.424949Z","iopub.status.idle":"2024-07-29T14:29:42.432610Z","shell.execute_reply.started":"2024-07-29T14:29:42.424898Z","shell.execute_reply":"2024-07-29T14:29:42.431128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalization\nThe Pixel Values are often stored as **Integer** Numbers in the range 0 to 255, the range that a single 8-bit byte can offer.","metadata":{}},{"cell_type":"code","source":"# You need to make sure that your Image is cast into double/float from int before you do this scaling \n# as you will most likely generate floating point numbers.\n# And had it been int, the values will be truncated to zero.\n\nx_train = x_train.astype(\"float32\")/255\nx_test = x_test.astype(\"float32\")/255\ndf_test = df_test.astype(\"float32\")/255","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:30:02.841133Z","iopub.execute_input":"2024-07-29T14:30:02.841602Z","iopub.status.idle":"2024-07-29T14:30:02.992244Z","shell.execute_reply.started":"2024-07-29T14:30:02.841568Z","shell.execute_reply":"2024-07-29T14:30:02.990996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One Hot Encoding\nThe labels are given as integers between 0-9. We need to one hot encode them , Eg 4 [0, 0, 0, 1, 0, 0, 0, 0, 0, 0] .\n\nThis is done so that we have labels for all the classes, and we can easily carry out the Error/Cost during BackPropogation.\n\nWe have 10 digits [0-9] or classes, therefore we one-hot-encode the target variable with 10 classes","metadata":{}},{"cell_type":"code","source":"y_train = to_categorical(y_train, num_classes=10)\ny_test = to_categorical(y_test, num_classes=10)\n\nprint(y_train[0])","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:30:10.209857Z","iopub.execute_input":"2024-07-29T14:30:10.211852Z","iopub.status.idle":"2024-07-29T14:30:10.224083Z","shell.execute_reply.started":"2024-07-29T14:30:10.211793Z","shell.execute_reply":"2024-07-29T14:30:10.222345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN","metadata":{}},{"cell_type":"markdown","source":"\n\n
\n \"CNN\"\n
\n","metadata":{}},{"cell_type":"markdown","source":"**Steps:**\n\n- **Sequential Keras API**: Start with a linear stack of layers, adding one at a time from input.\n- **Convolutional Layers**: Add layers with independent filters to produce feature maps, considering hyperparameters like filter size, stride, padding, etc.\n- **Output Dimensions**: Calculated by \\(( ( W − F + 2P ) / S ) + 1\\) for height and width; depth equals the number of filters.\n- **Pooling Layers**: Use for downsampling to reduce parameters and computational load, aiding in dimensionality reduction.\n- **Batch Normalization**: Achieves zero mean and unit variance, helping the model generalize better by scaling down outliers.\n- **Dropout**: Apply to randomly drop neurons to avoid overfitting, typically with a dropout rate between 0.2-0.5.\n- **Output Layer**: Use 'sigmoid' for binary classification or 'softmax' for multi-class classification after flattening and adding fully connected layers.\n","metadata":{}},{"cell_type":"code","source":"\nmodel = Sequential()\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', strides=1, padding='same', data_format='channels_last',\n input_shape=(28,28,1)))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', strides=1, padding='same', data_format='channels_last'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='valid' ))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu', strides=1, padding='same', data_format='channels_last'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same', activation='relu', data_format='channels_last'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2), padding='valid', strides=2))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:30:26.192205Z","iopub.execute_input":"2024-07-29T14:30:26.192653Z","iopub.status.idle":"2024-07-29T14:30:26.610704Z","shell.execute_reply.started":"2024-07-29T14:30:26.192621Z","shell.execute_reply":"2024-07-29T14:30:26.609499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimizer\noptimizer = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)\nmodel.compile(optimizer=optimizer, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:33:33.226073Z","iopub.execute_input":"2024-07-29T14:33:33.226601Z","iopub.status.idle":"2024-07-29T14:33:33.241213Z","shell.execute_reply.started":"2024-07-29T14:33:33.226563Z","shell.execute_reply":"2024-07-29T14:33:33.239834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Summary\n","metadata":{}},{"cell_type":"code","source":"model.summary()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:33:52.176610Z","iopub.execute_input":"2024-07-29T14:33:52.177014Z","iopub.status.idle":"2024-07-29T14:33:52.229641Z","shell.execute_reply.started":"2024-07-29T14:33:52.176986Z","shell.execute_reply":"2024-07-29T14:33:52.228435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:35:35.038341Z","iopub.execute_input":"2024-07-29T14:35:35.038762Z","iopub.status.idle":"2024-07-29T14:35:35.044697Z","shell.execute_reply.started":"2024-07-29T14:35:35.038733Z","shell.execute_reply":"2024-07-29T14:35:35.043246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation\n\nData augmentation increases the number of data points by creating new images from existing ones. This helps prevent overfitting and improves model performance.\n\nTechniques include:\n- Rotating images\n- Flipping images\n- Zooming in/out\n- Adjusting brightness\n- Cropping\n\nA CNN with the ability to classify objects correctly, even when they appear in different orientations, has a property called invariance. In real-world applications, images can vary in orientation, location, scale, and lighting. By using data augmentation, we can create diverse training data that helps our model perform well in various conditions.\n","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n featurewise_center=False, # set input mean to 0 over the dataset\n samplewise_center=False, # set each sample mean to 0\n featurewise_std_normalization=False, # divide inputs by std of the dataset\n samplewise_std_normalization=False, # divide each input by its std\n zca_whitening=False, # apply ZCA whitening\n rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180)\n zoom_range = 0.1, # Randomly zoom image \n width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)\n height_shift_range=0.1, # randomly shift images vertically (fraction of total height)\n horizontal_flip=False, # randomly flip images\n vertical_flip=False) # randomly flip images","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:34:14.355357Z","iopub.execute_input":"2024-07-29T14:34:14.355828Z","iopub.status.idle":"2024-07-29T14:34:14.364109Z","shell.execute_reply.started":"2024-07-29T14:34:14.355797Z","shell.execute_reply":"2024-07-29T14:34:14.362314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen.fit(x_train)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:34:21.629291Z","iopub.execute_input":"2024-07-29T14:34:21.629821Z","iopub.status.idle":"2024-07-29T14:34:21.684776Z","shell.execute_reply.started":"2024-07-29T14:34:21.629780Z","shell.execute_reply":"2024-07-29T14:34:21.683435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Fitting","metadata":{}},{"cell_type":"code","source":"batch_size = 64\nepochs = 20\n\n# Fit the Model\nhistory = model.fit(datagen.flow(x_train, y_train, batch_size = batch_size), epochs = epochs, \n validation_data = (x_test, y_test), verbose=2, \n steps_per_epoch=x_train.shape[0] // batch_size,\n callbacks = [reduce_lr])","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:35:38.232925Z","iopub.execute_input":"2024-07-29T14:35:38.233378Z","iopub.status.idle":"2024-07-29T14:54:12.640871Z","shell.execute_reply.started":"2024-07-29T14:35:38.233342Z","shell.execute_reply":"2024-07-29T14:54:12.639236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Evaluation","metadata":{}},{"cell_type":"code","source":"model.evaluate(x_test, y_test)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:55:28.852394Z","iopub.execute_input":"2024-07-29T14:55:28.852817Z","iopub.status.idle":"2024-07-29T14:55:32.554627Z","shell.execute_reply.started":"2024-07-29T14:55:28.852787Z","shell.execute_reply":"2024-07-29T14:55:32.553359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n normalize=False,\n title='Confusion matrix',\n cmap=plt.cm.Blues):\n \"\"\"\n This function prints and plots the confusion matrix.\n Normalization can be applied by setting `normalize=True`.\n \"\"\"\n plt.imshow(cm, interpolation='nearest', cmap=cmap)\n plt.title(title)\n plt.colorbar()\n tick_marks = np.arange(len(classes))\n plt.xticks(tick_marks, classes, rotation=45)\n plt.yticks(tick_marks, classes)\n\n if normalize:\n cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n thresh = cm.max() / 2.\n for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n plt.text(j, i, cm[i, j],\n horizontalalignment=\"center\",\n color=\"white\" if cm[i, j] > thresh else \"black\")\n\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(x_test)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_test,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T14:57:16.415364Z","iopub.execute_input":"2024-07-29T14:57:16.415806Z","iopub.status.idle":"2024-07-29T14:57:20.805422Z","shell.execute_reply.started":"2024-07-29T14:57:16.415773Z","shell.execute_reply":"2024-07-29T14:57:20.804103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_digits_test=np.argmax(model.predict(df_test),axis=1)\n\nimage_id_test=[]\nfor i in range (len(pred_digits_test)):\n image_id_test.append(i+1)\nd={'ImageId':image_id_test,'Label':pred_digits_test}\nanswer=pd.DataFrame(d)\nanswer.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-29T15:39:01.595762Z","iopub.execute_input":"2024-07-29T15:39:01.596166Z","iopub.status.idle":"2024-07-29T15:39:22.309773Z","shell.execute_reply.started":"2024-07-29T15:39:01.596136Z","shell.execute_reply":"2024-07-29T15:39:22.308469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}