{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phần 1: Nhập các thư viện cần thiết\n------","metadata":{"_uuid":"ebd546a2-326f-4a93-9042-31cb470d1cc4","_cell_guid":"fe85bd07-ce02-43ce-933f-2d24429c163b","trusted":true}},{"cell_type":"code","source":"## IMPORT THE NECESSARY PYTHON LIBRARIES\n\n# Read & Explore Data\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import MinMaxScaler\n\n# Principal Component Analysis (PCA)\nfrom sklearn.decomposition import PCA\nfrom sklearn.pipeline import Pipeline\nimport plotly.express as px\n\n# K-Nearest Neighbor Classifier (KNN)\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, \\\n    confusion_matrix, ConfusionMatrixDisplay \n\n# VHT\nfrom scipy.ndimage import rotate","metadata":{"_uuid":"c9f6edfd-588d-4af2-97d8-1073e4883692","_cell_guid":"8d8d769c-d2a0-4558-b63f-86352b4bafb8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:02:50.683988Z","iopub.execute_input":"2024-05-24T08:02:50.684552Z","iopub.status.idle":"2024-05-24T08:02:54.326474Z","shell.execute_reply.started":"2024-05-24T08:02:50.684510Z","shell.execute_reply":"2024-05-24T08:02:54.325078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def func_array1d_rotate2d(Narray1d,rotate_angle,sizePixel):\n\n    N = Narray1d.shape[0]\n\n    Narray1d_rotate = np.zeros(Narray1d.shape)\n    \n    for i in range(N):\n        image = np.reshape(Narray1d[i,:],(sizePixel,sizePixel))\n        rotated_image = rotate(image,rotate_angle, reshape=False)\n        rotated_array1d = np.reshape(rotated_image,(1,-1))\n        Narray1d_rotate[i,:] = rotated_array1d\n        \n    return Narray1d_rotate","metadata":{"_uuid":"76ecec72-7aaa-4e9c-9366-c53a6fb1d164","_cell_guid":"f51ca997-1fbb-4259-94ee-5b05b70a035a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:02:54.331056Z","iopub.execute_input":"2024-05-24T08:02:54.331674Z","iopub.status.idle":"2024-05-24T08:02:54.340428Z","shell.execute_reply.started":"2024-05-24T08:02:54.331630Z","shell.execute_reply":"2024-05-24T08:02:54.339038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n# Phần 2: Đọc dữ liệu và khai thác dữ liệu","metadata":{"_uuid":"0ff27da2-835c-4122-aca0-9e731f3dc7ea","_cell_guid":"6248d1ca-53a8-45d0-9ab6-9003187ef756","trusted":true}},{"cell_type":"code","source":"# Read the data files using Pandas\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test =  pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"_uuid":"078702a8-cfea-4f1c-bb0b-1018fea19c2c","_cell_guid":"8b4f5e4b-fc80-473b-bd7a-7e620b412c90","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:02:54.342396Z","iopub.execute_input":"2024-05-24T08:02:54.342886Z","iopub.status.idle":"2024-05-24T08:03:01.580328Z","shell.execute_reply.started":"2024-05-24T08:02:54.342843Z","shell.execute_reply":"2024-05-24T08:03:01.578389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the size of the train and test DataFrames\nprint(\"df_train shape:\", df_train.shape)  # 70%\nprint(\"df_test shape: \", df_test.shape)  # 30%","metadata":{"_uuid":"ca8aa9af-b761-403d-aa85-a3642ea77c5c","_cell_guid":"ba38193d-5d27-48f6-a7b3-f505b93dfe84","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:01.583620Z","iopub.execute_input":"2024-05-24T08:03:01.584114Z","iopub.status.idle":"2024-05-24T08:03:01.591460Z","shell.execute_reply.started":"2024-05-24T08:03:01.584075Z","shell.execute_reply":"2024-05-24T08:03:01.589629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fortunately there are no missing values in our data\nprint(\n    \"Missing values in df_train: \", \\\n    df_train.isna().sum().sum()\n)\nprint(\n    \"Missing values in df_test: \", \\\n    df_test.isna().sum().sum()\n)","metadata":{"_uuid":"a10fb707-86d3-4a6b-98f0-ace8a866768f","_cell_guid":"7cede701-9df4-4ac1-a358-2c567d72cd67","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:01.593316Z","iopub.execute_input":"2024-05-24T08:03:01.593707Z","iopub.status.idle":"2024-05-24T08:03:01.655323Z","shell.execute_reply.started":"2024-05-24T08:03:01.593665Z","shell.execute_reply":"2024-05-24T08:03:01.654049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's have a look at the first five training rows\ndf_train.head()","metadata":{"_uuid":"087da919-79ff-4a00-a357-fdb65381edda","_cell_guid":"cbfdc56f-8ade-4af2-b159-a2f526b273a5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:01.657091Z","iopub.execute_input":"2024-05-24T08:03:01.657452Z","iopub.status.idle":"2024-05-24T08:03:01.692666Z","shell.execute_reply.started":"2024-05-24T08:03:01.657421Z","shell.execute_reply":"2024-05-24T08:03:01.690918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can see that the first colum is the label,\n# and the rest of the columns are the 784 pixels\n# that represent 28x28 gray-scale images.\n# Let's separate X and Y\n\nX_train = df_train.drop(columns=['label']).values\ny_train = df_train['label'].values","metadata":{"_uuid":"2aefa02a-2611-42d1-9764-6ab390b73c0b","_cell_guid":"ccfff756-d687-424c-877d-028bb3180a59","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:01.694247Z","iopub.execute_input":"2024-05-24T08:03:01.695155Z","iopub.status.idle":"2024-05-24T08:03:01.820605Z","shell.execute_reply.started":"2024-05-24T08:03:01.695107Z","shell.execute_reply":"2024-05-24T08:03:01.819579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's visualize the distribution of the pixel values\n\nax = sns.histplot(x=X_train.flatten(), color='#4895EE')\n_ = ax.set(title='Pixel Value Histogram', xlabel='Pixel Value')\n\n# As we can see, the 0 value represents the background\n# and the >0 values represent the digits in the images","metadata":{"_uuid":"7a976629-a208-4444-970a-6f990f01b730","_cell_guid":"f8a9174d-c11e-4467-8dcd-bba2e0d51ac3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:01.821685Z","iopub.execute_input":"2024-05-24T08:03:01.822030Z","iopub.status.idle":"2024-05-24T08:03:19.973509Z","shell.execute_reply.started":"2024-05-24T08:03:01.821999Z","shell.execute_reply":"2024-05-24T08:03:19.972287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check that the target is balanced.\n# It means that the ten categories in the target\n# have roughly the same representation in the data.\n\nax = sns.countplot(x=y_train, color='#4895EE')\nax.set(\n    title='Target distribution is balanced? Yes!',\n    xlabel='Label'\n)\n# The expected number of samples for a balanced \n# class distribution is 42000/10 = 4200\n_ = ax.axhline(y=4200, color='#12086F', linestyle='--')\n\n# Since the target classes are balances, we don't have\n# worry about sampling methods and biases in our models.","metadata":{"_uuid":"56cd7484-96d5-4052-a732-31184b3010be","_cell_guid":"31dd96d2-e9fc-4ec8-8027-6182b59bbb56","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:19.975503Z","iopub.execute_input":"2024-05-24T08:03:19.975874Z","iopub.status.idle":"2024-05-24T08:03:20.272720Z","shell.execute_reply.started":"2024-05-24T08:03:19.975842Z","shell.execute_reply":"2024-05-24T08:03:20.271308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Next, let's visualize one image from each class.\n# We will use the matplotlib 'imshow' function.\n\nfig, axs = plt.subplots(nrows=2, ncols=5, figsize=(10,5))\n\nfor i in range(10):\n    idx = np.where(y_train==i)[0][2]\n    img = axs[int(i >= 5), i%5].imshow(\n        X=X_train[idx].reshape(28, 28),\n        cmap='Blues'\n    )\n    img.axes.get_yaxis().set_visible(False)\n    img.axes.get_xaxis().set_visible(False)\n    img.axes.set_title(f'Label: {i}')","metadata":{"_uuid":"6e57247a-308f-4d8c-b860-3fcca06504e8","_cell_guid":"85fb1afc-211b-4d78-a121-385e208ece49","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:20.278688Z","iopub.execute_input":"2024-05-24T08:03:20.279464Z","iopub.status.idle":"2024-05-24T08:03:21.009653Z","shell.execute_reply.started":"2024-05-24T08:03:20.279412Z","shell.execute_reply":"2024-05-24T08:03:21.008071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"transform data","metadata":{"_uuid":"9d4cd46f-3405-4ee3-a2e3-758b928024b1","_cell_guid":"abe3ee94-93c7-4941-9358-150e526414ec","trusted":true}},{"cell_type":"code","source":"# -------------------------------------------------- declare\nnumRotate_prior = 10\n\nangleOffset_prior = -45\nangleRange_prior = 90\n\n\n#2000-2099\nX_train_pick2rotate = {\n    'index': np.array([2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, \n                       2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020,\n                        2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030,\n                      2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040,\n                      2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050,\n                      2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060,\n                      2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 200,\n                      2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080,\n                      2081, 2082, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090,\n                      2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, 2099]),\n    'offset': np.array([\n        #0\n        [0,0,0,-15,0, \n         20,0,20,0,0],\n        #1\n        [0,0,0,-15,0, \n         20,-65,0,-15,0],\n        #2\n        [0,20,0,-20,15, \n         0,-55,0,0,-20],\n        #3\n        [0,30,0,0,10, \n         15,-65,0,0,0],\n        #4\n        [-90,10,0,-20,0, \n         15,-60,0,-5,10],\n        #5\n        [0,30,-20,0,20, \n         0,-20,0,-30,-30],\n        #6\n        [0,-11,0,0,22, \n         0,-40,0,10,0],\n        #7\n        [0,0,0,0,0, \n         15,-70,-10,0,20],\n        #8\n        [0,24,0,0,7, \n         0,-70,-12,0,10],\n        #9\n        [20,20,-5,0,0, \n        7,-16,0,10,0],\n        #10\n        [30,10,0,0,20, \n        5,-10,0,20,0],\n        #11\n        [0,0,0,0,0, \n        0,5,0,0,10],\n        #12\n        [20,30,0,0,20, \n        10,20,-5,-5,10],\n        #13\n        [20,30,0,15,15, \n        15,-5,0,25,10],\n        #14\n        [5,20,10,0,0, \n        15,0,0,20,0],\n        #15\n        [30,40,0,0,0, \n        0,-10,10,-5,10],\n        #16\n        [10,-10,10,20,10, \n        0,0,0,20,0],\n        #17\n        [10,30,-10,0,15, \n        20,8,15,10,0],\n        #18\n        [10,10,0,0,15, \n        20,0,0,-10,20],\n        #19\n        [10,40,0,-10,0, \n        10,0,0,-10,0],\n        #20\n        [20,0,0,20,0, \n        0,20,0,10,20],\n        #21\n        [20,13,0,11,25, \n        25,-20,0,0,10],\n        #22\n        [20,-12,0,0,-5, \n        0,0,-7,-5,12],\n        #23\n        [5,-7,5,-7,0, \n        2,0,15,-3,3],\n        #24\n        [50,5,-5,3,0, \n        0,0,0,0,5],\n        #25\n        [20,5,0,-7,0, \n        7,-10,-7,8,0],\n        #26\n        [0,20,0,-15,0, \n        10,-40,0,5,5],\n        #27\n        [25,-5,-4,-15,5, \n        15,8,4,5,4],\n        #28\n        [0,0,0,15,0, \n        15,-10,12,10,0],\n        #29\n        [0,25,0,13,30, \n        19,-90,7,0,-6],\n        #30\n        [0,43,0,17,0, \n        27,-23,0,5,0],\n        #31\n        [20,10,0,0,10, \n        10,-90,-10,10,30,],\n        #32\n        [10,20,0,0,10, \n        20,30,0,20,0,],\n        #33\n        [-10,0,0,10,0, \n        10,0,15,20,-10,],\n        #34\n        [10,-20,10,-10,15, \n        0,25,0,0,30,],\n        #35\n        [0,10,10,-20,30, \n        10,0,0,20,10,],\n        #36\n        [0,25,10,0,10, \n        10,0,15,10,10,],\n        #37\n        [20,8,8,0,0, \n        10,10,0,25,0,],\n        #38\n        [0,30,-10,10,-5, \n        15,-20,-8,40,10,],\n        #39\n        [5,25,10,0,0, \n        0,-90,0,0,15,],\n        #40\n        [35,7,0,20,20, \n        10,0,-10,8,15,],\n        #41\n        [0,20,0,0,15, \n        20,-105,0,3,0,],\n        #42\n        [-15,25,-15,-7,15, \n        -17,-100,15,15,25,],\n        #43\n        [-10,25,0,20,10, \n        -5,-100,-7,-10,10,],\n        #44\n        [-10,43,0,-3,20, \n        17,-90,0,0,30],\n        #45\n        [10,8,0,-10,0, \n        0,-90,-10,15,15,],\n        #46\n        [0,20,20,13,20, \n        0,0,0,-15,10,],\n        #47\n        [-3,7,0,0,0, \n        0,0,0,0,25,],\n        #48\n        [0,0,0,5,17, \n        13,-30,0,-10,15,],\n        #49\n        [20,-8,-10,-5,0, \n        25,0,7,13,-8,],\n        #50\n        [-90,25,-13,0,-4, \n        10,0,13,0,10,],\n        #51\n        [-90,25,-3,0,25, \n        25,-90,-10,0,30,],\n        #52\n        [-7,-7,10,-5,33, \n        19,-90,0,16,14,],\n        #53\n        [13,-5,0,0,30, \n        7,-24,0,5,15,],\n        #54\n        [25,0,-5,7,5, \n        0,0,-4,-4,30],\n        #55\n        [40,-5,20,10,0, \n        0,0,0,-7,10,],\n        #56\n        [-40,25,0,-5,10, \n        20,0,-30,10,25,],\n        #57\n        [0,10,10,-5,0, \n        5,30,0,30,0,],\n        #58\n        [35,-10,-5,0,-15, \n        20,0,0,-10,10,],\n        #59\n        [15,5,0,17,16, \n        20,-20,0,0,20,],\n        #60\n        [-20,40,0,-15,10, \n        10,0,0,6,10,],\n        #61\n        [20,30,-5,15,20, \n        -3,-10,0,-4,20,],\n        #62\n        [4,10,10,10,20, \n        0,10,0,17,0,],\n        #63\n        [20,30,10,-10,0, \n        10,-5,0,-10,20,],\n        #64\n        [0,30,0,0,20, \n        10,0,0,0,15,],\n        #65\n        [0,20,0,0,0, \n        0,0,0,-5,10,],\n        #66\n        [0,25,10,-10,5, \n        4,5,0,10,0,],\n        #67\n        [5,15,0,7,10, \n        0,0,5,5,0,],\n        #68\n        [13,2,-3,0,20, \n        10,5,5,10,0,],\n        #69\n        [10,7,0,5,15, \n        0,30,-5,5,10,],\n        #70\n        [0,5,10,0,20, \n        13,0,0,5,-3,],\n        #71\n        [20,3,2,-4,-10, \n        0,0,0,5,0,],\n        #72\n        [20,15,8,0,10, \n        7,5,-3,0,5,],\n        #73\n        [0,0,20,5,10, \n        0,0,0,5,25,],\n        #74\n        [10,5,0,3,0, \n        10,0,0,5,0,],\n        #75\n        [10,-10,3,0,30, \n        0,-20,10,10,-5,],\n        #76\n        [25,-3,4,-5,20, \n        -10,10,3,30,20,],\n        #77\n        [20,2,6,0,-3, \n        15,0,0,-5,20,],\n        #78\n        [45,0,0,5,8, \n        0,0,0,5,15,],\n        #79\n        [7,4,5,-5,10, \n        -5,0,0,15,10,],\n        #80\n        [-5,3,0,2,2, \n        7,-10,-59,2,15,],\n        #81\n        [5,5,5,3,-10, \n        10,-6,0,-5,10,],\n        #82\n        [0,30,0,5,10, \n        7,0,0,20,10,],\n        #83\n        [20,0,20,0,10, \n        0,0,0,5,0,],\n        #84\n        [10,0,-7,10,15, \n        10,-10,10,5,0,],\n        #85\n        [0,-5,0,0,-25, \n        10,15,15,10,5,],\n        #86\n        [30,-20,-10,0,5, \n        10,0,0,0,0,],\n        #87\n        [0,4,0,0,30, \n        0,20,0,30,-8,],\n        #88\n        [20,-6,0,0,6, \n        4,0,0,3,15,],\n        #89\n        [20,3,-5,3,10, \n        10,5,0,3,-10,],\n        #90\n        [15,-12,0,6,20, \n        4,0,0,4,15,],\n        #91\n        [15,-3,5,-3,0, \n        15,0,0,-5,0,],\n        #92\n        [7,3,-5,-5,20, \n        50,-10,-7,10,0,],\n        #93\n        [10,7,0,0,-7, \n        10,-70,-7,8,9,],\n        #94\n        [32,20,-10,0,25, \n        27,-70,0,8,-15,],\n        #95\n        [30,-8,0,4,4, \n        5,0,0,8,12,],\n        #96\n        [20,20,0,0,0, \n        0,-60,0,10,0,],\n        #97\n        [0,7,-5,-5,0, \n        10,0,0,0,5,],\n        #98\n        [20,-7,0,0,0, \n        -30,-70,7,-5,0,],\n        #99\n        [0,0,-3,0,0, \n        0,0,-5,0,0,],\n        \n                \n    ])\n}\n\nnumPickRotate = len(X_train_pick2rotate['index'])\n\nprint('numPickRotate = ', numPickRotate)\n\n# -------------------------------------------------- rotate\n\nX_sample_rotate = np.zeros((10,numRotate_prior,numPickRotate,X_train.shape[1]))\ny_sample_rotate = np.zeros((10,numRotate_prior,numPickRotate))\n\nfor k0 in range(10):\n    for k1 in range(numRotate_prior):\n        for k2 in range(numPickRotate):\n            \n                idxPick_X_train = X_train_pick2rotate['index'][k2]\n                \n                idx = np.where(y_train==k0)[0][idxPick_X_train]\n                \n                y_sample_rotate[k0,k1,k2] = k0\n                \n                angleRotate = (k1/numRotate_prior * angleRange_prior) + angleOffset_prior + X_train_pick2rotate['offset'][k2][k0]\n                \n                X_sample_rotate[k0,k1,k2,:] = func_array1d_rotate2d(X_train[idx][np.newaxis,:],angleRotate, 28)\n\n                \nX_sample_rotate[X_sample_rotate < 10] = 0\n#X_sample_rotate[X_sample_rotate > 300] = 0\n\n# Let's normalize the pixel values to the interval [0,1]\n# Formula: MinMaxscale(X) = (X - X_min) / (X_max - X_min)\n\nprint(\"Before applying MinMax Scaler:\")\nprint(\"  - Minimum value:\", X_sample_rotate.min())\nprint(\"  - Maximum value:\", X_sample_rotate.max())\n\nminmax_scaler = MinMaxScaler()\n\nfor k0 in range(10):\n    for k1 in range(numRotate_prior):\n        X_sample_rotate[k0,k1,:,:] = minmax_scaler.fit_transform(X_sample_rotate[k0,k1,:,:]) * 255\n\nprint(\"\\nAfter applying MinMax Scaler:\")\nprint(\"  - Minimum value:\", X_sample_rotate.min())\nprint(\"  - Maximum value:\", X_sample_rotate.max())\n\n\n\n# Next, let's visualize one image from each class.\n# We will use the matplotlib 'imshow' function.\n\n# for k2 in range(numPickRotate):\nfor k2 in range(90,100):\n    k1 = 0\n    # k1 = numRotate_prior - 1\n\n    fig, axs = plt.subplots(nrows=2, ncols=5, figsize=(10, 5))\n    \n    for k0, ax in enumerate(axs.flat):\n        img = ax.imshow(\n            X=X_sample_rotate[k0, k1, k2, :].reshape(28, 28),\n            cmap='Blues'\n        )      \n        ax.get_yaxis().set_visible(False)\n        ax.get_xaxis().set_visible(False)\n        ax.set_title(f'Label: {k0}')\n\n    # Hiển thị số lưới ảnh trên hình\n    fig.suptitle(f'Lưới ảnh số {k2}', fontsize=14)\n    plt.show()\nX_train_rotate = np.reshape(X_sample_rotate, (-1,784))\ny_train_rotate = np.reshape(y_sample_rotate, (-1,1)).flatten()","metadata":{"_uuid":"e87363e6-d051-4009-be4f-a825679930e7","_cell_guid":"db89b71d-4901-4716-9507-e9497e268d74","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:08:45.149734Z","iopub.execute_input":"2024-05-24T08:08:45.150186Z","iopub.status.idle":"2024-05-24T08:08:54.529672Z","shell.execute_reply.started":"2024-05-24T08:08:45.150151Z","shell.execute_reply":"2024-05-24T08:08:54.528166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---------\n# Phần 3: Phân tích thành phần chính","metadata":{"_uuid":"8794e86b-0a44-46a2-8d06-c6186454767d","_cell_guid":"d404fd37-ba51-4a9f-9de3-878c4bb0ad21","trusted":true}},{"cell_type":"code","source":"# Let's normalize the pixel values to the interval [0,1]\n# Formula: MinMaxscale(X) = (X - X_min) / (X_max - X_min)\n\nprint(\"Before applying MinMax Scaler:\")\nprint(\"  - Minimum value:\", X_train.min())\nprint(\"  - Maximum value:\", X_train.max())\n\nminmax_scaler = MinMaxScaler()\nX_train_rotate_scaled = minmax_scaler.fit_transform(X_train_rotate)\n\nprint(\"\\nAfter applying MinMax Scaler:\")\nprint(\"  - Minimum value:\", X_train_rotate_scaled.min())\nprint(\"  - Maximum value:\", X_train_rotate_scaled.max())","metadata":{"_uuid":"2f018749-1981-43d6-b1b8-b1716a72e3fb","_cell_guid":"7c68f163-b747-4392-9a1e-8792cb07ccde","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:25.876299Z","iopub.execute_input":"2024-05-24T08:03:25.877014Z","iopub.status.idle":"2024-05-24T08:03:26.017236Z","shell.execute_reply.started":"2024-05-24T08:03:25.876968Z","shell.execute_reply":"2024-05-24T08:03:26.015923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Actually, we will embed the MinMaxScaler and the PCA\n# modules into a Sklearn Pipeline.\n\npca = Pipeline(steps=[\n    ('scaler', MinMaxScaler()),\n    ('pca', PCA(n_components=40))\n])\n\nX_train_rotate_pca = pca.fit_transform(X=X_train_rotate)\n\nprint('X_train_pca shape:', X_train_rotate_pca.shape)","metadata":{"_uuid":"4bfc3273-16ce-4a37-93ae-9cf46373e2c4","_cell_guid":"2d2d1d92-8b2d-4b97-af06-40a19024b09e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:26.021450Z","iopub.execute_input":"2024-05-24T08:03:26.021854Z","iopub.status.idle":"2024-05-24T08:03:27.696263Z","shell.execute_reply.started":"2024-05-24T08:03:26.021819Z","shell.execute_reply":"2024-05-24T08:03:27.694471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's extract the explained variance ratio\n# And let's compute their cumulative sum\n\npca_object = pca.named_steps['pca']\nvar_ratio = pd.Series(\n    data=pca_object.explained_variance_ratio_,\n    index=range(1, 41)\n)\n\ncumsum_var_ratio = var_ratio.cumsum()","metadata":{"_uuid":"e551b2e2-ce61-4b14-abca-472ab8f8fa6f","_cell_guid":"8fe76194-de6d-44ef-89b7-489a936a3a12","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:27.705410Z","iopub.execute_input":"2024-05-24T08:03:27.712425Z","iopub.status.idle":"2024-05-24T08:03:27.733114Z","shell.execute_reply.started":"2024-05-24T08:03:27.712344Z","shell.execute_reply":"2024-05-24T08:03:27.731053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the explained variance ratio for each component (left)\n# And plot their cumulative sum (right)\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,5))\n\nsns.barplot(\n    x=var_ratio.index, \n    y=var_ratio.values, \n    color='#4895EE', \n    ax=ax1\n)\nax1.set(\n    title='Explained Variance Ratio PCA',\n    xlabel='Component',\n    ylabel='Var Ratio',\n)\nax1.set_ylim(0, 0.5)\n\nsns.barplot(\n    x=cumsum_var_ratio.index, \n    y=cumsum_var_ratio.values, \n    color='#4895EE', \n    ax=ax2\n)\nax2.set(\n    title='Cumulative Explained Variance Ratio PCA',\n    xlabel='Component',\n    ylabel='Cumulative Var Ratio',\n)\n_ = ax2.set_ylim(0, 0.5)","metadata":{"_uuid":"34ab9680-e976-4c81-bf92-6e9d8a0236bb","_cell_guid":"5e8c9489-629b-45b3-a577-b855fe33869f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:27.744156Z","iopub.execute_input":"2024-05-24T08:03:27.745376Z","iopub.status.idle":"2024-05-24T08:03:29.310930Z","shell.execute_reply.started":"2024-05-24T08:03:27.745334Z","shell.execute_reply":"2024-05-24T08:03:29.309325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's put the components in Pandas DataFrame format\n# This way we can use the seaborn and plotly libraries\n# to visualize the data.\n# (these packages work better with Pandas DataFrames)\n\ndf_train_pca = pd.DataFrame(\n    data=X_train_rotate_pca[:,:3], \n    columns=['x', 'y', 'z']\n)\n# Let's also add the target to set the colors\ndf_train_pca['target'] = y_train_rotate\n\nprint(df_train_pca.shape)\ndf_train_pca.head()","metadata":{"_uuid":"b779e0bc-6938-424a-9c08-3579404299da","_cell_guid":"8d64a105-5b85-4924-9d4c-d5bc95d474e8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:29.312358Z","iopub.execute_input":"2024-05-24T08:03:29.312705Z","iopub.status.idle":"2024-05-24T08:03:29.329801Z","shell.execute_reply.started":"2024-05-24T08:03:29.312675Z","shell.execute_reply":"2024-05-24T08:03:29.328900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two-dimensional visualization of the data\n# Here the Top2 principal components are plotted\n\nsns.set_style(\"dark\")\nax = sns.scatterplot(\n    data=df_train_pca,\n    x='x', \n    y='y', \n    hue='target', \n    palette='Blues',\n)\nax.set(\n    title='PCA Two-Dimensional Projection',\n    xlabel='1st Comp.',\n    ylabel='2nd Comp.'\n)\nax.axhline(y=0, linestyle='--', color='#FFBF00', zorder=-1)\nax.axvline(x=0, linestyle='--', color='#FFBF00', zorder=-1)","metadata":{"_uuid":"90d9d1e3-fbcb-400a-b8d5-68d2a5f4f646","_cell_guid":"c05ac3f3-ec1c-4fd0-8e88-cd0991afb146","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:29.330920Z","iopub.execute_input":"2024-05-24T08:03:29.331792Z","iopub.status.idle":"2024-05-24T08:03:31.631290Z","shell.execute_reply.started":"2024-05-24T08:03:29.331755Z","shell.execute_reply":"2024-05-24T08:03:31.630057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two-dimensional visualization of the data\n# Here the Top3 principal components are plotted\n\nsns.set_style(\"white\")\nfig = px.scatter_3d(\n    data_frame=df_train_pca,\n    x='x',\n    y='y',\n    z='z',\n    labels={\n        'x': '1st Comp.',\n        'y': '2nd Comp.',\n        'z': '3rd Comp.'\n    },\n    color='target',\n    title='PCA Three-Dimensional Projection',\n    color_continuous_scale='blues',\n    template='ggplot2'\n)\n\nfig.show()","metadata":{"_uuid":"1fb19f51-6852-4f02-8660-90a8a537fa30","_cell_guid":"b76b9b11-2a7e-4cc9-983c-0ae311d92428","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:31.632869Z","iopub.execute_input":"2024-05-24T08:03:31.633996Z","iopub.status.idle":"2024-05-24T08:03:33.785531Z","shell.execute_reply.started":"2024-05-24T08:03:31.633930Z","shell.execute_reply":"2024-05-24T08:03:33.784426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now, let's apply the inverse_transform to compute\n# the approximation of images based on the 10 principal\n# compontents.\n# We can see that with just 10 components we convey a large\n# part of the information of each image.\n\npca_object = pca.named_steps['pca']\n\nfig, axs = plt.subplots(nrows=10, ncols=2, figsize=(6,20))\n\nfor i in range(10):\n    idx = np.where(y_train_rotate==i)[0][0]\n    original_img = axs[i, 0].imshow(\n        X=X_train_rotate[idx].reshape(28, 28),\n        cmap='Blues'\n    )\n    original_img.axes.get_yaxis().set_visible(False)\n    original_img.axes.get_xaxis().set_visible(False)\n    original_img.axes.set_title(f'Original [{i}]:')\n    \n    approx_img_ = pca_object.inverse_transform(X=X_train_rotate_pca[idx])\n    approx_img = axs[i, 1].imshow(\n        X=approx_img_.reshape(28, 28),\n        cmap='Blues'\n    )\n    approx_img.axes.get_yaxis().set_visible(False)\n    approx_img.axes.get_xaxis().set_visible(False)\n    approx_img.axes.set_title(f'Approximation [{i}]:')","metadata":{"_uuid":"0d814515-0bec-4c78-af57-09ed84b2dfc4","_cell_guid":"7e42f02f-c973-4898-8c10-80b0ca2d625b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:33.786988Z","iopub.execute_input":"2024-05-24T08:03:33.787360Z","iopub.status.idle":"2024-05-24T08:03:35.046180Z","shell.execute_reply.started":"2024-05-24T08:03:33.787329Z","shell.execute_reply":"2024-05-24T08:03:35.044745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show that the PCA components are uncorrelated\n# (This statement is true by definition and construction\n# of the Principal Component Analysis algorithm)\n\ncorr = np.corrcoef(x=X_train_rotate_pca, rowvar=False)\ncorr = np.around(corr, decimals=2)\n\nax = sns.heatmap(\n    data=corr, \n    vmax=1, \n    vmin=-1,\n    square=True,\n    linewidths=.5,\n    annot=True,\n    cmap='Blues',\n)\n\n_ = ax.set(\n    title='Correlation PCA Components [Uncorrelated]',\n    xlabel='Components',\n    ylabel='Components'\n)","metadata":{"_uuid":"8a21533b-a961-49b1-b24c-d4e7050cc806","_cell_guid":"32e1e369-193f-4071-9914-b2f03cce2893","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:35.048250Z","iopub.execute_input":"2024-05-24T08:03:35.048768Z","iopub.status.idle":"2024-05-24T08:03:39.778867Z","shell.execute_reply.started":"2024-05-24T08:03:35.048723Z","shell.execute_reply":"2024-05-24T08:03:39.777532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n# Phần 4: Sử dụng K-Nearest Neighbors Classifier phân loại 10 thành phần chính hàng đầu từ PCA.","metadata":{"_uuid":"370f38cc-e5fb-4b4e-8dd3-6b9e303ee060","_cell_guid":"43a48481-ea7f-470e-b826-8fd2fcc9e37f","trusted":true}},{"cell_type":"markdown","source":"---\nmô hình học máy trên tập dữ liệu Train","metadata":{"_uuid":"7c0a4822-eedd-4aa0-bac6-a2cf5a7b89bf","_cell_guid":"39e3a7f5-5363-4914-b78d-295251352529","trusted":true}},{"cell_type":"code","source":"knn_clf = Pipeline(steps=[\n    ('scaler', MinMaxScaler()),\n    ('pca', PCA(n_components=40)),\n    ('knn', KNeighborsClassifier())\n])\n\nknn_clf_params = {\n    'knn__n_neighbors': [2, 4, 8, 16, 32]\n}\n\nknn_clf_gs = GridSearchCV(\n    estimator=knn_clf, \n    param_grid=knn_clf_params,\n    scoring='accuracy',\n    n_jobs=-1,\n    cv=5,\n    verbose=1,\n    return_train_score=True\n)\n\nknn_clf_gs.fit(X=X_train_rotate_pca, y=y_train_rotate)\n\nprint('best PCA+KNN params:', knn_clf_gs.best_params_)\nprint('best PCA+KNN cv accuracy:', knn_clf_gs.best_score_)","metadata":{"_uuid":"2d7cd392-e51f-420a-aad2-0eb5c72ff742","_cell_guid":"797ae756-d6fc-4456-83e5-447d3cc034e2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:39.780298Z","iopub.execute_input":"2024-05-24T08:03:39.780680Z","iopub.status.idle":"2024-05-24T08:03:56.514792Z","shell.execute_reply.started":"2024-05-24T08:03:39.780648Z","shell.execute_reply":"2024-05-24T08:03:56.512941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the Training and Validation Accuracies\n# for each Hyperparamter value tested during the\n# Grid Search.\n# This plot allows us to check for overfitting.\n\nfig, ax = plt.subplots()\n\nax.plot(\n    range(len(knn_clf_params['knn__n_neighbors'])), \n    knn_clf_gs.cv_results_['mean_train_score'],\n    label='train',\n    color='blue'\n)\nax.plot(\n    range(len(knn_clf_params['knn__n_neighbors'])),\n    knn_clf_gs.cv_results_['mean_test_score'],\n    label='test',\n    color='orange'\n)\nax.set_xticks(\n    ticks=range(len(knn_clf_params['knn__n_neighbors'])),\n    labels=knn_clf_params['knn__n_neighbors']\n)\n\nii = knn_clf_params['knn__n_neighbors'].index(\n    knn_clf_gs.best_params_['knn__n_neighbors']\n)\n\nplt.axvline(x=ii, linestyle='--', color='red', label='best', alpha=0.3)\nplt.legend()\nplt.xlabel(\"'n_neighbors' vals\")\nplt.ylabel('Accuracy')\nplt.title(\"PCA+KNN GridSearchCV Results\")\nplt.show()","metadata":{"_uuid":"2abf36ed-c1a2-4581-8a67-414fbfe556f5","_cell_guid":"38a69c14-7bf2-4328-abc9-45a9ef2c1b11","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:56.525623Z","iopub.execute_input":"2024-05-24T08:03:56.532060Z","iopub.status.idle":"2024-05-24T08:03:56.962934Z","shell.execute_reply.started":"2024-05-24T08:03:56.531945Z","shell.execute_reply":"2024-05-24T08:03:56.961757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Phần 5: Tạo một tệp tin chứa các dự đoán của mô hình học máy trên tập dữ liệu Test","metadata":{"_uuid":"2254c327-a8ec-41ca-9fa2-c41a850f4c75","_cell_guid":"adcdd26f-00ba-479f-8e33-e1bc6d65eafb","trusted":true}},{"cell_type":"code","source":"# Compute and display the Confusion Matrix\n# Note: the entire training dataset is used here\n\nbest_cv_clf = knn_clf_gs.best_estimator_\ny_pred = best_cv_clf.predict(X=X_train_rotate_pca)\nconf_matrix = confusion_matrix(y_true=y_train_rotate, y_pred=y_pred)\n\nconf_matrix_disp = ConfusionMatrixDisplay(conf_matrix)\nconf_matrix_disp.plot(cmap='Blues')","metadata":{"_uuid":"c3ac8535-57d6-4c40-9146-3b229c95284e","_cell_guid":"aa178b6c-9094-48f3-a36e-5cc5da6e3c50","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:56.964772Z","iopub.execute_input":"2024-05-24T08:03:56.966757Z","iopub.status.idle":"2024-05-24T08:03:58.901542Z","shell.execute_reply.started":"2024-05-24T08:03:56.966695Z","shell.execute_reply":"2024-05-24T08:03:58.900140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute the classification report using sklearn\n# Note: the entire training dataset is used here\n\nclf_report = classification_report(\n    y_true=y_train_rotate,\n    y_pred=y_pred,\n    labels=range(10),\n)\n\n# Print the classification report\nprint(clf_report)","metadata":{"_uuid":"80ad0fa3-460a-4b42-aaf6-52edfa03afe9","_cell_guid":"04c9e15a-0874-4e08-b997-858cbcc9a37e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-05-24T08:03:58.903342Z","iopub.execute_input":"2024-05-24T08:03:58.903859Z","iopub.status.idle":"2024-05-24T08:03:58.937136Z","shell.execute_reply.started":"2024-05-24T08:03:58.903804Z","shell.execute_reply":"2024-05-24T08:03:58.935476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute and display the Confusion Matrix\n# Note: the entire training dataset is used here\n\nbest_cv_clf = knn_clf_gs.best_estimator_\ny_pred = best_cv_clf.predict(X=pca_object.transform(X_train))\nconf_matrix = confusion_matrix(y_true=y_train, y_pred=y_pred)\n\nconf_matrix_disp = ConfusionMatrixDisplay(conf_matrix)\nconf_matrix_disp.plot(cmap='Blues')\n\n# Compute the classification report using sklearn\n# Note: the entire training dataset is used here\n\nclf_report = classification_report(\n    y_true=y_train,\n    y_pred=y_pred,\n    labels=range(10),\n)\n\n# Print the classification report\nprint(clf_report)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T08:03:58.938741Z","iopub.execute_input":"2024-05-24T08:03:58.939280Z","iopub.status.idle":"2024-05-24T08:04:04.649515Z","shell.execute_reply.started":"2024-05-24T08:03:58.939244Z","shell.execute_reply":"2024-05-24T08:04:04.648273Z"},"trusted":true},"execution_count":null,"outputs":[]}]}