{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nimport tensorflow\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Dense,Input\nfrom tensorflow.keras.layers import Conv2D,MaxPooling2D,Dropout,Flatten\nfrom tensorflow.keras.models import Sequential,load_model,Model\nfrom sklearn.model_selection import train_test_split\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-19T01:36:48.729491Z","iopub.execute_input":"2022-03-19T01:36:48.729757Z","iopub.status.idle":"2022-03-19T01:36:49.556987Z","shell.execute_reply.started":"2022-03-19T01:36:48.729726Z","shell.execute_reply":"2022-03-19T01:36:49.556244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:49.558441Z","iopub.execute_input":"2022-03-19T01:36:49.560106Z","iopub.status.idle":"2022-03-19T01:36:54.433417Z","shell.execute_reply.started":"2022-03-19T01:36:49.560064Z","shell.execute_reply":"2022-03-19T01:36:54.432690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:54.434792Z","iopub.execute_input":"2022-03-19T01:36:54.435047Z","iopub.status.idle":"2022-03-19T01:36:54.459202Z","shell.execute_reply.started":"2022-03-19T01:36:54.435011Z","shell.execute_reply":"2022-03-19T01:36:54.458604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_arr  = train.iloc[:,1:].copy()\ntest_arr  = test.iloc[:,:].copy()\ntrain_arr = train_arr.to_numpy().reshape(len(train_arr),28,28)\ntrain_y = train.iloc[:,0].to_numpy()\ntest_arr = test_arr.to_numpy().reshape(len(test_arr),28,28)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:54.461028Z","iopub.execute_input":"2022-03-19T01:36:54.461330Z","iopub.status.idle":"2022-03-19T01:36:54.621924Z","shell.execute_reply.started":"2022-03-19T01:36:54.461294Z","shell.execute_reply":"2022-03-19T01:36:54.621213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(40, 20))\nrows = 4\ncolumns = 4\ngrid = plt.GridSpec(rows, columns, wspace = .25, hspace = .25)\nfor i in range(rows*columns):\n    exec (f\"plt.subplot(grid{[i]})\")\n    plt.imshow(train_arr[i])\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:54.623143Z","iopub.execute_input":"2022-03-19T01:36:54.623369Z","iopub.status.idle":"2022-03-19T01:36:56.350523Z","shell.execute_reply.started":"2022-03-19T01:36:54.623337Z","shell.execute_reply":"2022-03-19T01:36:56.349802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = train[\"label\"]\nlabel.value_counts()\ntrain = StandardScaler().fit_transform(train)\npca = PCA(n_components=2)\npca_res = pca.fit_transform(train)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:56.351776Z","iopub.execute_input":"2022-03-19T01:36:56.352168Z","iopub.status.idle":"2022-03-19T01:36:59.823981Z","shell.execute_reply.started":"2022-03-19T01:36:56.352132Z","shell.execute_reply":"2022-03-19T01:36:59.823216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(20, 10))\nsns.set(rc={'axes.facecolor':'cornflowerblue', 'figure.facecolor':'cornflowerblue'})\nsns.scatterplot(x = pca_res[:,0], y = pca_res[:,1], hue = label, palette = sns.hls_palette(10), legend = 'full');\nplt.title(\"Different Digits features on Tsne\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:36:59.825348Z","iopub.execute_input":"2022-03-19T01:36:59.825624Z","iopub.status.idle":"2022-03-19T01:37:01.482905Z","shell.execute_reply.started":"2022-03-19T01:36:59.825588Z","shell.execute_reply":"2022-03-19T01:37:01.482310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_arr = train_arr.astype(\"float\")/255\nx_test = test_arr.astype(\"float\")/255\ntrain_arr = train_arr.reshape(train_arr.shape[0],28,28,1)\nx_test = x_test.reshape(x_test.shape[0],28,28,1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:37:01.483819Z","iopub.execute_input":"2022-03-19T01:37:01.484066Z","iopub.status.idle":"2022-03-19T01:37:01.679724Z","shell.execute_reply.started":"2022-03-19T01:37:01.484033Z","shell.execute_reply":"2022-03-19T01:37:01.678991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\ntrain_y = to_categorical(train_y)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:37:01.681174Z","iopub.execute_input":"2022-03-19T01:37:01.681470Z","iopub.status.idle":"2022-03-19T01:37:01.686715Z","shell.execute_reply.started":"2022-03-19T01:37:01.681420Z","shell.execute_reply":"2022-03-19T01:37:01.685828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"networks = 10\nmodel = [0] *networks\nfor j in range(networks):\n    model[j] = Sequential()\n\n    model[j].add(Conv2D(100,kernel_size=(3,3),activation=\"relu\", input_shape = (28, 28, 1)))\n    model[j].add(Conv2D(200,kernel_size = (3,3),activation=\"relu\"))\n    model[j].add(MaxPooling2D(pool_size= (3,3)))\n    model[j].add(Flatten())\n    \n    model[j].add(Dense(200,activation=\"relu\"))\n    model[j].add(Dense(100,activation=\"sigmoid\"))\n    model[j].add(Dense(train_y.shape[1],activation=\"softmax\"))\n    model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:46:00.001561Z","iopub.execute_input":"2022-03-19T01:46:00.001817Z","iopub.status.idle":"2022-03-19T01:46:00.428224Z","shell.execute_reply.started":"2022-03-19T01:46:00.001787Z","shell.execute_reply":"2022-03-19T01:46:00.427512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import LearningRateScheduler\ncallback = LearningRateScheduler(lambda i: 1e-3 * 0.97 ** i)\nhistory = [0] * networks\nepochs = 30\nfor i in range(networks):\n    x_train, x_val, y_train, y_val = train_test_split(train_arr,train_y, test_size = 0.1)\n    history[i] = model[i].fit(x_train,y_train, batch_size=1000,\n        epochs = epochs,\n        validation_data = (x_val,y_val), callbacks=[callback], verbose=0)\n    print(f\"Ensemble CNN {i+1}: Epochs: {epochs}, Train accuracy: {max(history[i].history['accuracy'])}, Validation accuracy: {max(history[i].history['val_accuracy']) } \")","metadata":{"execution":{"iopub.status.busy":"2022-03-19T01:46:15.126569Z","iopub.execute_input":"2022-03-19T01:46:15.126829Z","iopub.status.idle":"2022-03-19T02:02:12.317475Z","shell.execute_reply.started":"2022-03-19T01:46:15.126800Z","shell.execute_reply":"2022-03-19T02:02:12.315787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_val = 0\npos = 0\nfor i,_ in enumerate(history):\n    new_val_acc =  max(history[i].history['val_accuracy'])\n    if new_val_acc>max_val:\n        max_val = new_val_acc\n        pos = i\nprint(f\"CNN model {pos} with val Accuracy:{max_val}\")\npred = model[pos+1].predict(x_test)\npredictions = np.argmax(pred,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T02:10:59.839224Z","iopub.execute_input":"2022-03-19T02:10:59.839591Z","iopub.status.idle":"2022-03-19T02:11:01.359674Z","shell.execute_reply.started":"2022-03-19T02:10:59.839553Z","shell.execute_reply":"2022-03-19T02:11:01.358993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use(\"fivethirtyeight\")\nplt.figure(figsize = (20,6))\nplt.subplot(1, 2, 1)\nplt.plot(history[pos].history['loss'])\nplt.plot(history[pos].history['val_loss'])\nplt.title(\"Model Loss\",fontsize=20)\nplt.ylabel('Loss')\nplt.xlabel('Epochs')\nplt.legend(['train loss', 'validation loss'], loc ='best')\n\nplt.subplot(1, 2, 2)\nplt.plot(history[pos].history['accuracy'])\nplt.plot(history[pos].history['val_accuracy'])\nplt.title(\"Model Accuracy\",fontsize=20)\nplt.ylabel('Accuracy')\nplt.xlabel('Epochs')\nplt.legend(['training accuracy', 'validation accuracy'], loc ='best')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T02:11:22.726235Z","iopub.execute_input":"2022-03-19T02:11:22.726909Z","iopub.status.idle":"2022-03-19T02:11:23.134597Z","shell.execute_reply.started":"2022-03-19T02:11:22.726870Z","shell.execute_reply":"2022-03-19T02:11:23.133760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)), \"Label\": predictions})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T02:11:26.266517Z","iopub.execute_input":"2022-03-19T02:11:26.267054Z","iopub.status.idle":"2022-03-19T02:11:26.323499Z","shell.execute_reply.started":"2022-03-19T02:11:26.267018Z","shell.execute_reply":"2022-03-19T02:11:26.322736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-03-19T02:09:23.697889Z","iopub.execute_input":"2022-03-19T02:09:23.698435Z","iopub.status.idle":"2022-03-19T02:09:23.703463Z","shell.execute_reply.started":"2022-03-19T02:09:23.698400Z","shell.execute_reply":"2022-03-19T02:09:23.702628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}