{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Version 30 ","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:40:21.533878Z","iopub.execute_input":"2023-09-02T09:40:21.534877Z","iopub.status.idle":"2023-09-02T09:40:21.559801Z","shell.execute_reply.started":"2023-09-02T09:40:21.534830Z","shell.execute_reply":"2023-09-02T09:40:21.558998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Import\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\n\n#Preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nimport albumentations as A\nfrom sklearn.preprocessing import StandardScaler\n\n#Models\nfrom sklearn.neighbors import KNeighborsClassifier\nimport catboost\nfrom catboost import CatBoost\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.activations import relu, linear\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy\nfrom tensorflow.keras.optimizers import Adam\n\n#Feature Selection\nfrom sklearn.decomposition import PCA\n\n#Tuning\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import accuracy_score\n\n#Visualizations\n\n#Settings\npd.set_option('display.max_column',None)\npd.set_option('display.max_rows',None)\nimport time\n\n#Delete Files\n# for filename in os.listdir('/kaggle/working/images'):\n#     if filename.endswith('.csv') or filename.endswith('.jpg'):\n#         os.remove(os.path.join('/kaggle/working/images', filename))","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:40:21.561600Z","iopub.execute_input":"2023-09-02T09:40:21.562282Z","iopub.status.idle":"2023-09-02T09:40:33.281090Z","shell.execute_reply.started":"2023-09-02T09:40:21.562237Z","shell.execute_reply":"2023-09-02T09:40:33.280247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Import Dataset\ntrain = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntrain_addaugmented = train.copy()\ntest = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:43:21.450107Z","iopub.execute_input":"2023-09-02T09:43:21.450473Z","iopub.status.idle":"2023-09-02T09:43:25.272861Z","shell.execute_reply.started":"2023-09-02T09:43:21.450445Z","shell.execute_reply":"2023-09-02T09:43:25.272049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train and test sets don't have any missing value\n# display(train.head())\n\n# Nmiss_counter = 0\n# for col in test.columns:\n#     if (round(test[col].isnull().sum()/len(train)*100,1)) > 0:\n#         print(test[col])\n#     else:\n#         Nmiss_counter = Nmiss_counter + 1\n# print(Nmiss_counter)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Visualisation\n\nX_train = train.iloc[:,1:].values\ny_train = train.iloc[:,0].values\n\nX_train = X_train.reshape(len(X_train),28,28)\nfig, ax = plt.subplots(nrows=5, ncols=10, figsize=(10,5))\nfor i in range(5):\n    for j in range(10):\n        idx = 3048\n#         idx = np.random.randint(len(X_train))\n        ax[i,j].imshow(X_train[idx],cmap='gray')\n        ax[i,j].set_title(str(y_train[idx]))\n        ax[i,j].axis('off')\nplt.tight_layout()\nplt.show()\n\n# (mnist_Xtrain, mnist_ytrain), (mnist_Xtest, mnist_ytest) = mnist.load_data()\n# num_samples = 20\n# plt.figure(figsize=(num_samples*2,5))\n# for i in range(num_samples):\n#     plt.subplot(1,num_samples,i+1)\n#     plt.imshow(mnist_Xtest[i],cmap='gray')\n#     plt.title(f\"Label: {mnist_ytest[i]}\")\n#     plt.axis('off')\n    \n# plt.tight_layout\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PCA Variance Visualization\n# pca = PCA().fit(train)\n# plt.plot(np.cumsum(pca.explained_variance_ratio_))\n# plt.xlabel('no. of components')\n# plt.ylabel('cumulative explained variance')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PCA Scatter Visualization\n# n_components = 2\n# pca = PCA(n_components=n_components)\n# train_pca = pca.fit_transform(train)\n\n# plt.figure(figsize=(10,10))\n# plt.scatter(train_pca[:,0],train_pca[:,1],c=y,cmap='rainbow')\n# plt.colorbar()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data Augmentation Part I: Saving pixel data into images\nX_train = train.iloc[:,1:].values\ny_train = train.iloc[:,0].values\nchosen_idx = []\nchosen_label = []\nnp.random.seed(0)\nunique_values = set()\n\n#Randomly choose x number of unique samples from train dataset\nwhile len(unique_values) < 1000:\n    idx = np.random.choice(len(X_train),replace=True)\n    if idx not in unique_values:\n        chosen_idx.append(idx)\n        unique_values.add(idx)\n\n#Reshape and change datatype for converting into Image\nX_train = X_train.reshape(len(X_train),28,28)\nX_train_uint8 = X_train.astype(np.uint8)\n\n#Convert data arrays into images\ninput_directory = '/kaggle/working/images/input/'\nos.makedirs(input_directory)\nfor i in chosen_idx:\n    img = Image.fromarray(X_train_uint8[i])\n    img.save(f'/kaggle/working/images/input/image_{i}.jpg', quality=95)\n    chosen_label.append(y_train[i])\n\n#Reshape chosen_labels to be used as y columns in DataFrame later\nchosen_label_reshaped = np.array(chosen_label).reshape(len(chosen_label),-1)\n\nprint(unique_values)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:43:28.500005Z","iopub.execute_input":"2023-09-02T09:43:28.500433Z","iopub.status.idle":"2023-09-02T09:43:28.742836Z","shell.execute_reply.started":"2023-09-02T09:43:28.500367Z","shell.execute_reply":"2023-09-02T09:43:28.741750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data Augmentation Part II: Augmenting Images (USING Albumentations) and Append into Train dataset\n\n#Transformation Rules\ntransform = A.SomeOf([\n    A.HorizontalFlip(p=0.5),\n    A.Rotate(limit=20,p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.0, scale_limit=(-0.20,0.20), rotate_limit=0.0, p=0.5),\n], n=2,p=1)\nnp.random.seed(0)\n\n#Creating augmented output array\naugmented_data = []\nfor filename in os.listdir(input_directory):\n    image = Image.open((os.path.join(input_directory, filename)))\n    augmented = transform(image=np.array(image))['image']\n    augmented_data.append(augmented)\n    if filename == 'image_16384.jpg':\n        augmented_check = augmented\n\n#Reshaping augmented_data array\naugmented_data_reshaped = (np.array(augmented_data)).reshape(len(augmented_data),-1)\nprint(augmented_data_reshaped.shape)\nprint(chosen_label_reshaped.shape)\n\n#Combining augmented_data as X and chosen_label as y into dataframe\ncombined_data = np.concatenate((chosen_label_reshaped,augmented_data_reshaped), axis=1)\nprint(combined_data.shape)\n\n#Append dataframe into train dataset\ntrain_addaugmented = train_addaugmented.append(pd.DataFrame(combined_data,columns=train.columns),ignore_index=True)\ndisplay(train_addaugmented.tail())\n\n#Delete Files\nfor filename in os.listdir('/kaggle/working/images/input'):\n    if filename.endswith('.csv') or filename.endswith('.jpg'):\n        os.remove(os.path.join('/kaggle/working/images/input', filename))\nos.rmdir(input_directory)\nos.rmdir('/kaggle/working/images')","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:43:30.279274Z","iopub.execute_input":"2023-09-02T09:43:30.279685Z","iopub.status.idle":"2023-09-02T09:43:31.321283Z","shell.execute_reply.started":"2023-09-02T09:43:30.279654Z","shell.execute_reply":"2023-09-02T09:43:31.320448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#so albumentations output as pixel data directly so no need convert back to array. \n#Okay great, but why is the image so many nines? is it the shiftscalerotate function?\naugmented_check = augmented_check.reshape(len(augmented),-1)\nprint(augmented_check.shape)\n\nplt.figure(figsize=(5,5))\nplt.imshow(augmented_check,cmap='gray')\nplt.axis('off')\nplt.show()\n\nplt.figure(figsize=(5,5))\nplt.imshow(X_train[16384],cmap='gray')\nplt.axis('off')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Old Data Augmentation\n# ## Data Augmentation Part II: Augmenting Images (USING AUGMENTOR)\n# p = Augmentor.Pipeline('/kaggle/working/images/input','/kaggle/working/images/output/')\n# p.rotate(probability=0.5,max_left_rotation=10,max_right_rotation=10)\n# p.flip_left_right(probability=0.5)\n# p.zoom(probability=0.5,min_factor=0.8,max_factor=1.2)\n# p.sample(2000)\n# p.process()\n\n# output_directory = '/kaggle/working/images/output/'\n# for i, filename in enumerate(os.listdir(output_directory)):\n#     new_filename = f'output_image_{i}.jpg'\n#     os.rename(os.path.join(output_directory, filename), os.path.join(output_directory, new_filename))\n    \n# filelist = os.listdir(output_directory)\n# count = 0\n# for file in filelist:\n#     if file.endswith('jpg'):\n#         count += 1\n# print(count)\n\n# ## Data Augmentation Part III: Converting Output Images back to Pixel\n# augmented_images = []\n# for i in range(20):\n#     img=Image.open(f'/kaggle/working/images/output/output_image_{i}.jpg')\n#     augmented_images.append(img)\n\n# augmented_data = []\n# for img in augmented_images:\n#     data = np.asarray(img)\n#     augmented_data.append(data)\n\n# # for i in range(1000):\n# #     train[chosen_idx[i]] = augmented_data[i]\n# # for i in augmented_data:\n# #     train = train.append(augmented_data[i],ignore_index=True)\n# augmented_data_array = np.array(augmented_data)\n# augmented_data_reshaped = augmented_data_array.reshape(len(augmented_data_array),-1)\n# augmented_data_df = pd.DataFrame(augmented_data_reshaped)\n# display(augmented_data_df.head())\n# print(augmented_data_df.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n## PCA\n#Cross-Validation\nX_subtrain, X_cv = train_test_split(train,test_size=0.2,random_state=0,)\n\n#Creating PCA\nn_components_cv = 629\npca = PCA(n_components=n_components_cv)\ny_subtrain = X_subtrain['label']\ny_cv = X_cv['label'].values\n\n#PCA fitting\nX_subtrain = X_subtrain.drop(columns='label')\nX_cv = X_cv.drop(columns='label')\nX_subtrain_pca = pca.fit_transform(X_subtrain)\nX_cv_pca = pca.fit_transform(X_cv)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:43:34.141231Z","iopub.execute_input":"2023-09-02T09:43:34.142445Z","iopub.status.idle":"2023-09-02T09:43:41.376512Z","shell.execute_reply.started":"2023-09-02T09:43:34.142402Z","shell.execute_reply":"2023-09-02T09:43:41.375041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# ## Scaling\n# std_scaler = StandardScaler()\n# X_subtrain_pca = std_scaler.fit_transform(X_subtrain_pca)\n# X_cv_pca = std_scaler.fit_transform(X_cv_pca)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T06:18:17.378969Z","iopub.execute_input":"2023-08-31T06:18:17.379529Z","iopub.status.idle":"2023-08-31T06:18:17.886554Z","shell.execute_reply.started":"2023-08-31T06:18:17.379486Z","shell.execute_reply":"2023-08-31T06:18:17.885421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n## CV Modelling\n#Model: KNN\nmodel_cv = KNeighborsClassifier(n_neighbors=5)\nmodel_cv.fit(X_subtrain,y_subtrain)\ny_pred = model_cv.predict(X_cv)\n\n#Model: Catboost\n# model_cv = catboost.CatBoostClassifier(loss_function='MultiClass',one_hot_max_size=4,iterations=100,max_depth=10,learning_rate=0.1,l2_leaf_reg=1,\\\n#                                        random_state=0)\n# model_cv.fit(X_subtrain,y_subtrain,verbose=False)\n# y_pred = model_cv.predict(X_cv)\n\n#Model: LogReg Elastic Net\n# model_cv = LogisticRegression(penalty='elasticnet',solver='saga',l1_ratio=0.5,multi_class='multinomial')\n# model_cv.fit(X_subtrain_pca,y_subtrain)\n# y_pred = model_cv.predict(X_cv_pca)\n\n#Model: SVC\n# model_cv = SVC(kernel='linear',C=1,decision_function_shape='ovo')\n# model_cv.fit(X_subtrain_pca,y_subtrain)\n# y_pred = model_cv.predict(X_cv_pca)\n\n#Model: Tensorflow Neural Network\n# tf.random.set_seed(0)\n# model_cv = Sequential(\n#     [\n#     Dense(10,activation='relu'),\n#     Dense(10,activation='linear')\n#     ], name = 'First')\n\n# model_cv.compile(\n#     loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#     optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),\n# )\n\n# model_cv.fit(X_subtrain_pca,y_subtrain,epochs=100)\n# y_pred = model_cv.predict(X_cv_pca)\n\n#One-hot encoding of y for metrics\nn_classes = len(np.unique(train['label']))\ny_cv_reshaped = np.zeros((len(y_cv), n_classes))\ny_cv_reshaped[np.arange(len(y_cv)),y_cv] = 1\ny_pred = y_pred.astype(int).flatten()\ny_pred_reshaped = np.zeros((len(y_pred), n_classes))\ny_pred_reshaped[np.arange(len(y_pred)), y_pred] = 1\nprint(y_pred_reshaped)\n\n#Metrics\naccu = accuracy_score(y_cv_reshaped,y_pred_reshaped)\nroc = roc_auc_score(y_cv_reshaped,y_pred_reshaped,average='weighted',multi_class='ovo')\nf1 = f1_score(y_cv_reshaped,y_pred_reshaped,average='weighted')\n\nprint('Accuracy score is ' + str(format(accu,'.5f')))\nprint('ROC AUC score is ' + str(format(roc,'.5f')))\nprint('F1 score is ' + str(format(f1,'.5f')))","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:43:46.677171Z","iopub.execute_input":"2023-09-02T09:43:46.677578Z","iopub.status.idle":"2023-09-02T09:44:13.488446Z","shell.execute_reply.started":"2023-09-02T09:43:46.677543Z","shell.execute_reply":"2023-09-02T09:44:13.487164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## PCA for train and test sets\nX = train\ny = train['label']\nX = X.drop(columns='label')\n\nn_components = 629\npca = PCA(n_components=n_components)\ntrain = train.drop(columns='label')\nX_pca = pd.DataFrame(pca.fit_transform(train))\ntest_pca = pd.DataFrame(pca.fit_transform(test))\nid = test['id']\ntest = test.drop(columns='id')","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:41:39.145456Z","iopub.execute_input":"2023-09-02T09:41:39.145828Z","iopub.status.idle":"2023-09-02T09:41:47.758895Z","shell.execute_reply.started":"2023-09-02T09:41:39.145800Z","shell.execute_reply":"2023-09-02T09:41:47.757610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n## Modelling\n# model = catboost.CatBoostClassifier(loss_function='MultiClass',one_hot_max_size=4,iterations=100,max_depth=10,learning_rate=0.1,l2_leaf_reg=1,\\\n#                                     random_state=0,verbose=False)\nmodel = KNeighborsClassifier(n_neighbors=5)\nmodel.fit(X,y)\npredictions = model.predict(test)\noutput = pd.concat([id,pd.DataFrame(predictions, columns=['label'])],axis=1)\noutput.to_csv('/kaggle/working/submission.csv',index=False)\nprint('Your submission was successfully saved!')","metadata":{"execution":{"iopub.status.busy":"2023-09-02T09:41:50.550861Z","iopub.execute_input":"2023-09-02T09:41:50.551264Z","iopub.status.idle":"2023-09-02T09:42:03.712252Z","shell.execute_reply.started":"2023-09-02T09:41:50.551232Z","shell.execute_reply":"2023-09-02T09:42:03.710470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## v2, KNN. PCA of n=2.\n# Accuracy score is 6964.00000\n# ROC AUC score is 0.76685\n# F1 score is 0.57634\n\n## v2.5, KNN. PCA of n=500.\n# Accuracy score is 8364.00000\n# ROC AUC score is 0.83170\n# F1 score is 0.68594\n\n## v3, KNN. PCA of n=629.\n# Accuracy score is 8369.00000\n# ROC AUC score is 0.83193\n# F1 score is 0.68663\n\n## v4, KNN. Added 1000 Albumentation data.\n# Accuracy score is 4250.00000\n# ROC AUC score is 0.63804\n# F1 score is 0.33896\n\n## v14, Catboost. WRONG INPUT. One_hot_max_size=4, iterations=100. No augmented data. \n# Accuracy score is 0.00000\n# ROC AUC score is 0.50000\n# F1 score is 0.18188\n\n## v15, Catboost. WRONG INPUT. With augmented data.\n# Accuracy score is 0.00000\n# ROC AUC score is 0.50000\n# F1 score is 0.18192\n\n## v16, LogReg with Elastic Net. (penalty='elasticnet',solver='saga',l1_ratio=0.5,multi_class='multinomial'). With augmented data. \n# Accuracy score is 0.38287\n# ROC AUC score is 0.65737\n# F1 score is 0.37030\n# 10min\n\n## v17, LogReg with Elastic Net. (penalty='elasticnet',solver='saga',l1_ratio=0.5,multi_class='multinomial'). Without augmented data. \n# Accuracy score is 0.53967\n# ROC AUC score is 0.74440\n# F1 score is 0.52992\n\n## v20, Catboost. (loss_function='MultiClass',one_hot_max_size=4,iterations=100,max_depth=10,learning_rate=0.1,l2_leaf_reg=1,random_state=0). \n# No augmented data.\n# Accuracy score is 0.68183\n# ROC AUC score is 0.82328\n# F1 score is 0.66741\n#1hr 38min\n\n## v22, Catboost. (loss_function='MultiClass',one_hot_max_size=4,iterations=100,max_depth=10,learning_rate=0.1,l2_leaf_reg=1,random_state=0). \n# With augmented data.\n# Accuracy score is 0.44025\n# ROC AUC score is 0.68923\n# F1 score is 0.43788\n#1hr 47min\n\n## v26, Catboost. Removed PCA.\n# Accuracy score is 0.97467\n# ROC AUC score is 0.98593\n# F1 score is 0.97465\n#1hr 47min\n\n## v28, Catboost. With augmented data.\n# Accuracy score is 0.95680\n# ROC AUC score is 0.97600\n# F1 score is 0.95680\n\n## v29, KNN. No PCA, With augmented data.\n# Accuracy score is 0.96328\n# ROC AUC score is 0.97959\n# F1 score is 0.96337\n\n## v30, KNN. No PCA. No augmented data.\n# Accuracy score is 0.98333\n# ROC AUC score is 0.99073\n# F1 score is 0.98335","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}