{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install scikeras","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train,y_train),(X_test,y_test) = keras.datasets.mnist.load_data()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test","metadata":{"execution":{"iopub.status.busy":"2024-06-18T02:06:10.180849Z","iopub.execute_input":"2024-06-18T02:06:10.181299Z","iopub.status.idle":"2024-06-18T02:06:10.189762Z","shell.execute_reply.started":"2024-06-18T02:06:10.181264Z","shell.execute_reply":"2024-06-18T02:06:10.188237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(X_train[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train/255\nX_test = X_test/255\nX_train[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Flatten(input_shape = (28,28)))\nmodel.add(Dense(128, activation=\"relu\"))\nmodel.add(Dense(10, activation=\"softmax\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer = 'Adam')\nfrom keras.callbacks import History \nhistory = History()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train, epochs = 10, validation_split = 0.2, callbacks=[history])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_prob = model.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_prob","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = y_prob.argmax(axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(history.history('loss'))\npd.DataFrame(history.history).plot(figsize=(8,5))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Randomized search CV","metadata":{}},{"cell_type":"code","source":"# from scikeras.wrappers import KerasClassifier\n# # Function to create the model\n# def create_model(optimizer='adam'):\n#     model = Sequential()\n#     model.add(Flatten(input_shape=(28, 28)))\n#     model.add(Dense(128, activation=\"relu\"))\n#     model.add(Dense(10, activation=\"softmax\"))\n#     model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n#     return model\n\n# # Create a KerasClassifier\n# model_2 = KerasClassifier(build_fn=create_model, epochs=10, batch_size=32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_2.get_params().keys()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.keras.backend.clear_session()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import RandomizedSearchCV\n# params={'optimizer':['Adam', 'SGD', 'Adadelta '],  \n#         'epochs':[20, 30, 50],\n#         'loss':[0.1,0.05,0.001],           \n#         } \n\n# # Create RandomizedSearchCV object\n# random_search = RandomizedSearchCV(estimator=model_2, \n#                                    param_distributions=params,\n#                                    n_iter=10, cv=3,\n#                                    verbose=3) \n\n# # Fit to the training \n# random_search.fit(X_train, y_train)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_model = random_search.best_estimator_\n# best_params = random_search.best_params_\n\n# print(best_model)\n# print(\"|||||||||||||||||||||\")\n# print(best_params)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import accuracy_score\n# y_prob_2 = random_search.predict(X_test)\n# accuracy_score(y_test, y_prob_2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_prob_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We can see that Even with RandomizedSearchCV, we could onlt improve our performance by a fraction so we discard this approach and try CNN","metadata":{}},{"cell_type":"markdown","source":"# CNN model","metadata":{}},{"cell_type":"code","source":"from keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model = Sequential([\n    Input(shape=(28,28,1)),\n    Conv2D(64, (3,3), activation='relu', padding='same'),\n    MaxPooling2D((2,2), strides=2),\n    Conv2D(128, (3,3), activation='relu', padding='same'),\n    MaxPooling2D((4,4), strides=2),\n    Flatten(),\n    Dense(128, activation='relu'),\n    Dense(10, activation='softmax')\n])\ncnn_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\n# y_train_encoded = to_categorical(y_train, num_classes=10)\n# y_test_encoded = to_categorical(y_test, num_classes=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x_train = X_train.reshape(X_train.shape[0] , 28,28,1)\n# x_test = X_test.reshape(X_test.shape[0] , 28,28,1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(x_train.shape)\n# print(x_test.shape)\n# print(y_train_encoded.shape)\n# print(y_test_encoded.shape)\n# print(x_train.ndim)\n# print(x_test.ndim)\n# print(y_train_encoded.ndim)\n# print(y_test_encoded.ndim)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\ncnn_model.fit(X_train, y_train, epochs=5, validation_data=(X_test,y_test), callbacks=[history])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction = model.predict(X_test)\nprint(test_prediction)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction_classes = np.argmax(test_prediction, axis=1)\nprint(test_prediction_classes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test, test_prediction_classes)*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction on submission dataset","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:58:36.793275Z","iopub.execute_input":"2024-06-18T01:58:36.793678Z","iopub.status.idle":"2024-06-18T01:58:41.642987Z","shell.execute_reply.started":"2024-06-18T01:58:36.793647Z","shell.execute_reply":"2024-06-18T01:58:41.641796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_test = df_train.label\ndf_train.drop(columns='label', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:58:42.651379Z","iopub.execute_input":"2024-06-18T01:58:42.651842Z","iopub.status.idle":"2024-06-18T01:58:42.775922Z","shell.execute_reply.started":"2024-06-18T01:58:42.651807Z","shell.execute_reply":"2024-06-18T01:58:42.774759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape, b_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:58:55.707543Z","iopub.execute_input":"2024-06-18T01:58:55.708754Z","iopub.status.idle":"2024-06-18T01:58:55.717139Z","shell.execute_reply.started":"2024-06-18T01:58:55.708707Z","shell.execute_reply":"2024-06-18T01:58:55.715901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train/255\ndf_test = df_test/255\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:59:24.752332Z","iopub.execute_input":"2024-06-18T01:59:24.752771Z","iopub.status.idle":"2024-06-18T01:59:24.935987Z","shell.execute_reply.started":"2024-06-18T01:59:24.752735Z","shell.execute_reply":"2024-06-18T01:59:24.934336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nA = df_train.values.reshape(-1, 28, 28, 1)\nA_test = df_test.values.reshape(-1, 28, 28, 1)\n\n\na_train_2, a_test_2, b_train_2, b_test_2 = train_test_split(A, b_test)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:59:28.416227Z","iopub.execute_input":"2024-06-18T01:59:28.416692Z","iopub.status.idle":"2024-06-18T01:59:28.863820Z","shell.execute_reply.started":"2024-06-18T01:59:28.416637Z","shell.execute_reply":"2024-06-18T01:59:28.862876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a_train_2.shape, a_test_2.shape, b_train_2.shape, b_test_2.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-18T01:59:31.145759Z","iopub.execute_input":"2024-06-18T01:59:31.146133Z","iopub.status.idle":"2024-06-18T01:59:31.155242Z","shell.execute_reply.started":"2024-06-18T01:59:31.146104Z","shell.execute_reply":"2024-06-18T01:59:31.153736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model.fit(a_train_2, b_train_2, epochs=10, validation_data=(a_test_2,b_test_2), callbacks=[history])","metadata":{"execution":{"iopub.status.busy":"2024-06-18T02:00:54.322730Z","iopub.execute_input":"2024-06-18T02:00:54.323147Z","iopub.status.idle":"2024-06-18T02:01:45.582820Z","shell.execute_reply.started":"2024-06-18T02:00:54.323113Z","shell.execute_reply":"2024-06-18T02:01:45.581640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_prediction = cnn_model.predict(A_test)\nprint(sub_prediction.shape)\nprint(sub_prediction)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T02:02:03.986810Z","iopub.execute_input":"2024-06-18T02:02:03.988157Z","iopub.status.idle":"2024-06-18T02:02:16.890873Z","shell.execute_reply.started":"2024-06-18T02:02:03.988102Z","shell.execute_reply":"2024-06-18T02:02:16.889331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_prediction_classes = np.argmax(sub_prediction, axis=1)\nprint(sub_prediction_classes)\nsub_prediction_classes.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-18T02:02:51.740683Z","iopub.execute_input":"2024-06-18T02:02:51.741124Z","iopub.status.idle":"2024-06-18T02:02:51.751036Z","shell.execute_reply.started":"2024-06-18T02:02:51.741089Z","shell.execute_reply":"2024-06-18T02:02:51.749720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': np.arange(1, len(sub_prediction_classes) + 1), 'Label': sub_prediction_classes})\nsubmission.to_csv('submission_cnn.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-18T02:10:06.398310Z","iopub.execute_input":"2024-06-18T02:10:06.398764Z","iopub.status.idle":"2024-06-18T02:10:06.446639Z","shell.execute_reply.started":"2024-06-18T02:10:06.398731Z","shell.execute_reply":"2024-06-18T02:10:06.445557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}