{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Installing the scikeras library for GridSearch \n# !pip install scikeras","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:27:48.341851Z","iopub.execute_input":"2024-07-05T19:27:48.342737Z","iopub.status.idle":"2024-07-05T19:27:48.347306Z","shell.execute_reply.started":"2024-07-05T19:27:48.342700Z","shell.execute_reply":"2024-07-05T19:27:48.346396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading required libraries\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score, f1_score\nimport time\nimport matplotlib.pyplot as plt\nimport warnings\nfrom tqdm import tqdm\nwarnings.filterwarnings(\"ignore\")\n# from scikeras.wrappers import KerasClassifier\n\n# Loading tensorflow functions\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.layers import Conv2D,MaxPooling2D, Flatten, Dense, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam, SGD\n# from tensorflow.keras.wrapper.scikit_learn import KerasClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-05T19:27:48.348945Z","iopub.execute_input":"2024-07-05T19:27:48.349291Z","iopub.status.idle":"2024-07-05T19:28:02.500616Z","shell.execute_reply.started":"2024-07-05T19:27:48.349268Z","shell.execute_reply":"2024-07-05T19:28:02.499615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for GPU\ntf.config.experimental.list_physical_devices('GPU')","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:02.501844Z","iopub.execute_input":"2024-07-05T19:28:02.502553Z","iopub.status.idle":"2024-07-05T19:28:02.748372Z","shell.execute_reply.started":"2024-07-05T19:28:02.502517Z","shell.execute_reply":"2024-07-05T19:28:02.747429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading data\ntrain_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:02.752057Z","iopub.execute_input":"2024-07-05T19:28:02.752881Z","iopub.status.idle":"2024-07-05T19:28:08.345742Z","shell.execute_reply.started":"2024-07-05T19:28:02.752850Z","shell.execute_reply":"2024-07-05T19:28:08.344964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digits = []\n# Viewing the digits\nfor i in range(10):\n    digit = train_df[train_df[\"label\"] == i].iloc[:,1:785].values[0].reshape(28,28)\n    plt.imshow(digit,cmap='gray')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:08.346884Z","iopub.execute_input":"2024-07-05T19:28:08.347262Z","iopub.status.idle":"2024-07-05T19:28:10.900218Z","shell.execute_reply.started":"2024-07-05T19:28:08.347228Z","shell.execute_reply":"2024-07-05T19:28:10.899277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyzing data shape\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:10.901687Z","iopub.execute_input":"2024-07-05T19:28:10.902092Z","iopub.status.idle":"2024-07-05T19:28:10.908700Z","shell.execute_reply.started":"2024-07-05T19:28:10.902057Z","shell.execute_reply":"2024-07-05T19:28:10.907708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizing data\ntrain_df.iloc[:,1:785] = train_df.iloc[:,1:785]/255","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:10.910070Z","iopub.execute_input":"2024-07-05T19:28:10.910941Z","iopub.status.idle":"2024-07-05T19:28:12.887913Z","shell.execute_reply.started":"2024-07-05T19:28:10.910915Z","shell.execute_reply":"2024-07-05T19:28:12.887088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data split\nsplit_train, split_test = train_test_split(train_df, train_size = 0.8)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:12.889222Z","iopub.execute_input":"2024-07-05T19:28:12.889881Z","iopub.status.idle":"2024-07-05T19:28:13.438882Z","shell.execute_reply.started":"2024-07-05T19:28:12.889845Z","shell.execute_reply":"2024-07-05T19:28:13.438066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyzing data shape\nsplit_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.440489Z","iopub.execute_input":"2024-07-05T19:28:13.440845Z","iopub.status.idle":"2024-07-05T19:28:13.447515Z","shell.execute_reply.started":"2024-07-05T19:28:13.440811Z","shell.execute_reply":"2024-07-05T19:28:13.446555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyzing data shape\nsplit_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.450990Z","iopub.execute_input":"2024-07-05T19:28:13.451287Z","iopub.status.idle":"2024-07-05T19:28:13.458607Z","shell.execute_reply.started":"2024-07-05T19:28:13.451265Z","shell.execute_reply":"2024-07-05T19:28:13.457645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshaping split train & test data\nsplit_train_2D = split_train.iloc[:,1:785].to_numpy().reshape(33600,28,28,1)\nsplit_test_2D = split_test.iloc[:,1:785].to_numpy().reshape(8400,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.459588Z","iopub.execute_input":"2024-07-05T19:28:13.459853Z","iopub.status.idle":"2024-07-05T19:28:13.833290Z","shell.execute_reply.started":"2024-07-05T19:28:13.459830Z","shell.execute_reply":"2024-07-05T19:28:13.832167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Defining a function which is required by KerasClassifier\ndef create_model(optimizer='adam', activation='relu'):\n    model = Sequential([\n        Conv2D(32, kernel_size=(3, 3), activation=activation, input_shape=(28, 28, 1)),\n        MaxPooling2D(pool_size=(2, 2)),\n        Conv2D(64, kernel_size=(3, 3), activation=activation),\n        MaxPooling2D(pool_size=(2, 2)),\n        Flatten(),\n        Dense(128, activation=activation),\n        Dense(10, activation='softmax')\n    ])\n    model.compile(optimizer=optimizer,\n                  loss='sparse_categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.834520Z","iopub.execute_input":"2024-07-05T19:28:13.834853Z","iopub.status.idle":"2024-07-05T19:28:13.842207Z","shell.execute_reply.started":"2024-07-05T19:28:13.834825Z","shell.execute_reply":"2024-07-05T19:28:13.841211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# Wrap the model using KerasClassifier\nmodel = KerasClassifier(model=create_model, optimizer='adam', activation='relu', verbose=0)\n\n# Define the grid search parameters\nparam_grid = {\n    'optimizer': ['adam', 'sgd'],\n    'activation': ['relu', 'tanh'],\n    'batch_size': [32, 64],\n    'epochs': [10, 25]\n}\n\n# Define custom scoring functions\nscoring = {\n    'accuracy': make_scorer(accuracy_score),\n    'precision': make_scorer(precision_score, average='macro'),\n    'recall': make_scorer(recall_score, average='macro'),\n    'f1': make_scorer(f1_score, average='macro')\n}\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.843599Z","iopub.execute_input":"2024-07-05T19:28:13.844400Z","iopub.status.idle":"2024-07-05T19:28:13.854421Z","shell.execute_reply.started":"2024-07-05T19:28:13.844369Z","shell.execute_reply":"2024-07-05T19:28:13.853508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Commenting out the code because it's been run earlier and we have the best params for our model\n\"\"\"\n# Create GridSearchCV\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3, scoring=scoring, refit='accuracy', return_train_score=True)\n\n# Fit the grid search and track time\nstart_time = time.time()\ngrid_result = grid.fit(split_train_2D, split_train[\"label\"])\nend_time = time.time()\ntotal_time = end_time - start_time\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.855429Z","iopub.execute_input":"2024-07-05T19:28:13.855667Z","iopub.status.idle":"2024-07-05T19:28:13.865313Z","shell.execute_reply.started":"2024-07-05T19:28:13.855647Z","shell.execute_reply":"2024-07-05T19:28:13.864354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Summarize results\nprint(f\"Best: {grid_result.best_score_} using {grid_result.best_params_}\")\nmeans = grid_result.cv_results_['mean_test_accuracy']\nstds = grid_result.cv_results_['std_test_accuracy']\nparams = grid_result.cv_results_['params']\nfit_times = grid_result.cv_results_['mean_fit_time']\nfor mean, std, param, fit_time in zip(means, stds, params, fit_times):\n    print(f\"{mean} ({std}) with: {param}, fit time: {fit_time:.2f} seconds\")\n\n# Print total time taken for grid search\nprint(f\"Total time taken for grid search: {total_time:.2f} seconds\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.866509Z","iopub.execute_input":"2024-07-05T19:28:13.867080Z","iopub.status.idle":"2024-07-05T19:28:13.874513Z","shell.execute_reply.started":"2024-07-05T19:28:13.867049Z","shell.execute_reply":"2024-07-05T19:28:13.873605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Plotting the results\nresults = pd.DataFrame(grid_result.cv_results_)\npivot_table = results.pivot_table(index='param_optimizer', columns='param_epochs', values='mean_test_accuracy')\npivot_table.plot(kind='bar', figsize=(10, 8))\nplt.title('Grid Search Mean Test Scores')\nplt.ylabel('Mean Test Score')\nplt.show()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.875546Z","iopub.execute_input":"2024-07-05T19:28:13.875858Z","iopub.status.idle":"2024-07-05T19:28:13.887302Z","shell.execute_reply.started":"2024-07-05T19:28:13.875828Z","shell.execute_reply":"2024-07-05T19:28:13.886407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Plotting fit times\npivot_table_time = results.pivot_table(index='param_optimizer', columns='param_epochs', values='mean_fit_time')\npivot_table_time.plot(kind='bar', figsize=(10, 8))\nplt.title('Grid Search Mean Fit Times')\nplt.ylabel('Mean Fit Time (seconds)')\nplt.show()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.888540Z","iopub.execute_input":"2024-07-05T19:28:13.888849Z","iopub.status.idle":"2024-07-05T19:28:13.896984Z","shell.execute_reply.started":"2024-07-05T19:28:13.888820Z","shell.execute_reply":"2024-07-05T19:28:13.896143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting train and test data to 2D\ntrain_df_2D = train_df.iloc[:,1:785].to_numpy().reshape(42000,28,28,1)\ntest_df_2D = test_df.to_numpy().reshape(28000,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:13.898336Z","iopub.execute_input":"2024-07-05T19:28:13.898600Z","iopub.status.idle":"2024-07-05T19:28:14.233131Z","shell.execute_reply.started":"2024-07-05T19:28:13.898571Z","shell.execute_reply":"2024-07-05T19:28:14.232328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using the best params for our model\nmodel = Sequential([\n    Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)),\n    MaxPooling2D(pool_size=(2, 2)),\n    Dropout(0.25),\n    Conv2D(64, kernel_size=(3, 3), activation='relu'),\n    MaxPooling2D(pool_size=(2, 2)),\n    Dropout(0.25),\n    Flatten(),\n    Dense(128, activation='relu'),\n    Dropout(0.4),\n    Dense(10, activation='softmax')\n])\n\nmodel.compile(optimizer='sgd',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.fit(train_df_2D, train_df[\"label\"], epochs=25)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:28:14.234339Z","iopub.execute_input":"2024-07-05T19:28:14.234624Z","iopub.status.idle":"2024-07-05T19:29:36.793635Z","shell.execute_reply.started":"2024-07-05T19:28:14.234600Z","shell.execute_reply":"2024-07-05T19:29:36.792678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = model.predict(test_df_2D)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:29:36.794952Z","iopub.execute_input":"2024-07-05T19:29:36.795354Z","iopub.status.idle":"2024-07-05T19:29:39.247868Z","shell.execute_reply.started":"2024-07-05T19:29:36.795320Z","shell.execute_reply":"2024-07-05T19:29:39.246771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df = pd.DataFrame({\n    'ImageId' : np.arange(1,len(predict)+1), \n    'Label' : np.argmax(predict, axis=1)\n})\n\npredict_df.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T19:29:39.249301Z","iopub.execute_input":"2024-07-05T19:29:39.249586Z","iopub.status.idle":"2024-07-05T19:29:39.292769Z","shell.execute_reply.started":"2024-07-05T19:29:39.249561Z","shell.execute_reply":"2024-07-05T19:29:39.292059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}