{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport tensorflow as tf\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom keras.layers import Dense, Conv2D, MaxPool2D , Flatten,Dropout\nimport seaborn as sns\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-21T19:23:47.853007Z","iopub.execute_input":"2021-12-21T19:23:47.853333Z","iopub.status.idle":"2021-12-21T19:23:54.904294Z","shell.execute_reply.started":"2021-12-21T19:23:47.853249Z","shell.execute_reply":"2021-12-21T19:23:54.903590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Load the data\ntrain = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntrain.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:24:10.086331Z","iopub.execute_input":"2021-12-21T19:24:10.086633Z","iopub.status.idle":"2021-12-21T19:24:15.166001Z","shell.execute_reply.started":"2021-12-21T19:24:10.086599Z","shell.execute_reply":"2021-12-21T19:24:15.164982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.label\nX = train.loc[:,'pixel0':]\nX.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:24:32.472055Z","iopub.execute_input":"2021-12-21T19:24:32.472879Z","iopub.status.idle":"2021-12-21T19:24:32.493094Z","shell.execute_reply.started":"2021-12-21T19:24:32.472791Z","shell.execute_reply":"2021-12-21T19:24:32.492189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the data\nX.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:24:58.066728Z","iopub.execute_input":"2021-12-21T19:24:58.067075Z","iopub.status.idle":"2021-12-21T19:24:58.088578Z","shell.execute_reply.started":"2021-12-21T19:24:58.067041Z","shell.execute_reply":"2021-12-21T19:24:58.087607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:25:18.794718Z","iopub.execute_input":"2021-12-21T19:25:18.795002Z","iopub.status.idle":"2021-12-21T19:25:18.810721Z","shell.execute_reply.started":"2021-12-21T19:25:18.794973Z","shell.execute_reply":"2021-12-21T19:25:18.809987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of rows: {train.shape[0]};  Number of columns: {train.shape[1]}; No of missing values: {sum(train.isna().sum())}')","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:25:32.539477Z","iopub.execute_input":"2021-12-21T19:25:32.540059Z","iopub.status.idle":"2021-12-21T19:25:32.587486Z","shell.execute_reply.started":"2021-12-21T19:25:32.540022Z","shell.execute_reply":"2021-12-21T19:25:32.586590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize the data","metadata":{}},{"cell_type":"code","source":"X= X / 255.0\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:26:07.466325Z","iopub.execute_input":"2021-12-21T19:26:07.466607Z","iopub.status.idle":"2021-12-21T19:26:07.587783Z","shell.execute_reply.started":"2021-12-21T19:26:07.466575Z","shell.execute_reply":"2021-12-21T19:26:07.587062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Split training ","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=42, stratify = y)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:26:34.219072Z","iopub.execute_input":"2021-12-21T19:26:34.219348Z","iopub.status.idle":"2021-12-21T19:26:34.498795Z","shell.execute_reply.started":"2021-12-21T19:26:34.219322Z","shell.execute_reply":"2021-12-21T19:26:34.498076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### It will zero variance features\nfrom sklearn.feature_selection import VarianceThreshold\nvar_thres=VarianceThreshold(threshold=0)\nvar_thres.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:27:09.160155Z","iopub.execute_input":"2021-12-21T19:27:09.160863Z","iopub.status.idle":"2021-12-21T19:27:09.581359Z","shell.execute_reply.started":"2021-12-21T19:27:09.160824Z","shell.execute_reply":"2021-12-21T19:27:09.580522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var_thres.get_support()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:27:22.786603Z","iopub.execute_input":"2021-12-21T19:27:22.787443Z","iopub.status.idle":"2021-12-21T19:27:22.795829Z","shell.execute_reply.started":"2021-12-21T19:27:22.787387Z","shell.execute_reply":"2021-12-21T19:27:22.794807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Finding non constant features\nsum(var_thres.get_support())","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:27:39.932847Z","iopub.execute_input":"2021-12-21T19:27:39.933738Z","iopub.status.idle":"2021-12-21T19:27:39.942002Z","shell.execute_reply.started":"2021-12-21T19:27:39.933694Z","shell.execute_reply":"2021-12-21T19:27:39.940951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets Find non-constant features \nlen(X.columns[var_thres.get_support()])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:27:52.253531Z","iopub.execute_input":"2021-12-21T19:27:52.254102Z","iopub.status.idle":"2021-12-21T19:27:52.260128Z","shell.execute_reply.started":"2021-12-21T19:27:52.254050Z","shell.execute_reply":"2021-12-21T19:27:52.259142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"constant_columns = [column for column in X_train.columns\n                    if column not in X_train.columns[var_thres.get_support()]]\n\nprint(len(constant_columns))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:28:04.753017Z","iopub.execute_input":"2021-12-21T19:28:04.753979Z","iopub.status.idle":"2021-12-21T19:28:04.808439Z","shell.execute_reply.started":"2021-12-21T19:28:04.753933Z","shell.execute_reply":"2021-12-21T19:28:04.807391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in constant_columns:\n    print(column)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:28:16.704928Z","iopub.execute_input":"2021-12-21T19:28:16.705202Z","iopub.status.idle":"2021-12-21T19:28:16.718985Z","shell.execute_reply.started":"2021-12-21T19:28:16.705175Z","shell.execute_reply":"2021-12-21T19:28:16.716942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Feature Selection- With Correlation\nIn this step we will be removing the features which are highly correlated","metadata":{}},{"cell_type":"code","source":"from sklearn.datasets import load_boston","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:28:54.519583Z","iopub.execute_input":"2021-12-21T19:28:54.519894Z","iopub.status.idle":"2021-12-21T19:28:54.595431Z","shell.execute_reply.started":"2021-12-21T19:28:54.519861Z","shell.execute_reply":"2021-12-21T19:28:54.594524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.corr()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:29:06.228847Z","iopub.execute_input":"2021-12-21T19:29:06.229153Z","iopub.status.idle":"2021-12-21T19:29:52.435018Z","shell.execute_reply.started":"2021-12-21T19:29:06.229121Z","shell.execute_reply":"2021-12-21T19:29:52.434187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with the following function we can select highly correlated features\n# it will remove the first feature that is correlated with anything other feature\n\ndef correlation(dataset, threshold):\n    col_corr = set()  # Set of all the names of correlated columns\n    corr_matrix = dataset.corr()\n    for i in range(len(corr_matrix.columns)):\n        for j in range(i):\n            if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value\n                colname = corr_matrix.columns[i]  # getting the name of column\n                col_corr.add(colname)\n    return col_corr","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:30:13.046558Z","iopub.execute_input":"2021-12-21T19:30:13.046816Z","iopub.status.idle":"2021-12-21T19:30:13.052653Z","shell.execute_reply.started":"2021-12-21T19:30:13.046789Z","shell.execute_reply":"2021-12-21T19:30:13.051696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_features = correlation(X_train, 0.7)\nlen(set(corr_features))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:30:17.160416Z","iopub.execute_input":"2021-12-21T19:30:17.161194Z","iopub.status.idle":"2021-12-21T19:31:11.261045Z","shell.execute_reply.started":"2021-12-21T19:30:17.161155Z","shell.execute_reply":"2021-12-21T19:31:11.260188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_features","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:31:34.657459Z","iopub.execute_input":"2021-12-21T19:31:34.658035Z","iopub.status.idle":"2021-12-21T19:31:34.671131Z","shell.execute_reply.started":"2021-12-21T19:31:34.657985Z","shell.execute_reply":"2021-12-21T19:31:34.670511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.Feature Selection-Information gain - mutual information In Classification Problem Statements","metadata":{}},{"cell_type":"markdown","source":"Mutual Information MI Estimate mutual information for a discrete target variable.\n\nMutual information (MI) between two random variables is a non-negative value, which measures the dependency between the variables. It is equal to zero if and only if two random variables are independent, and higher values mean higher dependency.\n\nThe function relies on nonparametric methods based on entropy estimation from k-nearest neighbors distances.\n\nInshort\n\nA quantity called mutual information measures the amount of information one can obtain from one random variable given another.\n\nThe mutual information between two random variables X and Y can be stated formally as follows:\n\nI(X ; Y) = H(X) – H(X | Y) Where I(X ; Y) is the mutual information for X and Y, H(X) is the entropy for X and H(X | Y) is the conditional entropy for X given Y. The result has the units of bits.","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_classif\n# determine the mutual information\nmutual_info = mutual_info_classif(X_train, y_train)\nmutual_info","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:31:59.328047Z","iopub.execute_input":"2021-12-21T19:31:59.328298Z","iopub.status.idle":"2021-12-21T19:35:15.937563Z","shell.execute_reply.started":"2021-12-21T19:31:59.328272Z","shell.execute_reply":"2021-12-21T19:35:15.936692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mutual_info = pd.Series(mutual_info)\nmutual_info.index = X_train.columns\nmutual_info.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:35:43.977676Z","iopub.execute_input":"2021-12-21T19:35:43.978312Z","iopub.status.idle":"2021-12-21T19:35:43.991537Z","shell.execute_reply.started":"2021-12-21T19:35:43.978266Z","shell.execute_reply":"2021-12-21T19:35:43.990499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train.drop(corr_features,axis=1)","metadata":{}},{"cell_type":"code","source":"#let's plot the ordered mutual_info values per feature\nmutual_info.sort_values(ascending=False).plot.bar(figsize=(20, 8))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:35:48.146803Z","iopub.execute_input":"2021-12-21T19:35:48.147143Z","iopub.status.idle":"2021-12-21T19:35:58.839097Z","shell.execute_reply.started":"2021-12-21T19:35:48.147106Z","shell.execute_reply":"2021-12-21T19:35:58.838082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:37:31.002280Z","iopub.execute_input":"2021-12-21T19:37:31.002644Z","iopub.status.idle":"2021-12-21T19:37:31.007282Z","shell.execute_reply.started":"2021-12-21T19:37:31.002606Z","shell.execute_reply":"2021-12-21T19:37:31.006380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#No we Will select the  top 5 important features\nsel_ten_cols = SelectKBest(mutual_info_classif, k=10)\nsel_ten_cols.fit(X_train, y_train)\nX_train.columns[sel_ten_cols.get_support()]","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:37:33.761970Z","iopub.execute_input":"2021-12-21T19:37:33.762643Z","iopub.status.idle":"2021-12-21T19:40:45.976772Z","shell.execute_reply.started":"2021-12-21T19:37:33.762597Z","shell.execute_reply":"2021-12-21T19:40:45.975862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.Fisher Score- Chisquare  Test For Feature Selection","metadata":{}},{"cell_type":"markdown","source":"Compute chi-squared stats between each non-negative feature and class.\n\nThis score should be used to evaluate categorical variables in a classification task. This score can be used to select the n_features features with the highest values for the test chi-squared statistic from X, which must contain only non-negative features such as booleans or frequencies (e.g., term counts in document classification), relative to the classes.\n\nRecall that the chi-square test measures dependence between stochastic variables, so using this function “weeds out” the features that are the most likely to be independent of class and therefore irrelevant for classification. The Chi Square statistic is commonly used for testing relationships between categorical variables.\n\nIt compares the observed distribution of the different classes of target Y among the different categories of the feature, against the expected distribution of the target classes, regardless of the feature categories.","metadata":{}},{"cell_type":"code","source":"## Perform chi2 test\n### chi2 returns 2 values\n### Fscore and the pvalue\nfrom sklearn.feature_selection import chi2\nf_p_values=chi2(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:43:40.800595Z","iopub.execute_input":"2021-12-21T19:43:40.801479Z","iopub.status.idle":"2021-12-21T19:43:40.926744Z","shell.execute_reply.started":"2021-12-21T19:43:40.801436Z","shell.execute_reply":"2021-12-21T19:43:40.925650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_p_values","metadata":{"execution":{"iopub.status.busy":"2021-12-21T19:45:04.295034Z","iopub.execute_input":"2021-12-21T19:45:04.295354Z","iopub.status.idle":"2021-12-21T19:45:04.318961Z","shell.execute_reply.started":"2021-12-21T19:45:04.295324Z","shell.execute_reply":"2021-12-21T19:45:04.318246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}