{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-30T14:40:18.522932Z","iopub.execute_input":"2021-08-30T14:40:18.52359Z","iopub.status.idle":"2021-08-30T14:40:18.543039Z","shell.execute_reply.started":"2021-08-30T14:40:18.523467Z","shell.execute_reply":"2021-08-30T14:40:18.541738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:18.544678Z","iopub.execute_input":"2021-08-30T14:40:18.545105Z","iopub.status.idle":"2021-08-30T14:40:22.992711Z","shell.execute_reply.started":"2021-08-30T14:40:18.545063Z","shell.execute_reply":"2021-08-30T14:40:22.991722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:22.994845Z","iopub.execute_input":"2021-08-30T14:40:22.995296Z","iopub.status.idle":"2021-08-30T14:40:23.974983Z","shell.execute_reply.started":"2021-08-30T14:40:22.995249Z","shell.execute_reply":"2021-08-30T14:40:23.973962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"char = df.select_dtypes(include='object') \nnumeric = df.select_dtypes(exclude='object') \n","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:23.976889Z","iopub.execute_input":"2021-08-30T14:40:23.977529Z","iopub.status.idle":"2021-08-30T14:40:24.076362Z","shell.execute_reply.started":"2021-08-30T14:40:23.977477Z","shell.execute_reply":"2021-08-30T14:40:24.07556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Lets seprate the data types into ordinal, nominal,binary,date\n\nbinary = df.loc[:,\"bin_0\":\"bin_4\"]\nnominal = df.loc[:,\"nom_0\":\"nom_9\"]\nordinal = df.loc[:,\"ord_0\":\"ord_5\"]\ndate = df.loc[:,\"day\":\"month\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:24.077666Z","iopub.execute_input":"2021-08-30T14:40:24.078227Z","iopub.status.idle":"2021-08-30T14:40:24.196896Z","shell.execute_reply.started":"2021-08-30T14:40:24.078189Z","shell.execute_reply":"2021-08-30T14:40:24.195699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##lets display null values\nnull_cols = []\nfor cols in df.columns:\n    if df[cols].isnull().sum() > 0:\n        print(cols, \"have\", df[cols].isnull().sum() ,\"null values\")\n        ##null_cols.append(cols)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:24.198489Z","iopub.execute_input":"2021-08-30T14:40:24.198931Z","iopub.status.idle":"2021-08-30T14:40:26.224337Z","shell.execute_reply.started":"2021-08-30T14:40:24.198879Z","shell.execute_reply":"2021-08-30T14:40:26.223384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Lets loook at binary values\nfor cols in binary:\n    print(cols, \":\\n\",df[cols].value_counts(),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:26.225592Z","iopub.execute_input":"2021-08-30T14:40:26.225886Z","iopub.status.idle":"2021-08-30T14:40:26.520981Z","shell.execute_reply.started":"2021-08-30T14:40:26.225856Z","shell.execute_reply":"2021-08-30T14:40:26.519898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[[\"bin_0\",\"bin_1\",\"bin_2\"]] = df[[\"bin_0\",\"bin_1\",\"bin_2\"]].replace(np.nan,0.0)\ndf[\"bin_3\"] = df[\"bin_3\"].replace(np.nan,\"N\")\ndf[\"bin_4\"] = df[\"bin_4\"].replace(np.nan,\"Y\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:26.523472Z","iopub.execute_input":"2021-08-30T14:40:26.523762Z","iopub.status.idle":"2021-08-30T14:40:26.703504Z","shell.execute_reply.started":"2021-08-30T14:40:26.523735Z","shell.execute_reply":"2021-08-30T14:40:26.702369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets take a look at the ordinal variables\nfor col in ordinal:\n    print(col, \":\",df[col].value_counts(),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:26.70567Z","iopub.execute_input":"2021-08-30T14:40:26.706106Z","iopub.status.idle":"2021-08-30T14:40:27.419421Z","shell.execute_reply.started":"2021-08-30T14:40:26.706061Z","shell.execute_reply":"2021-08-30T14:40:27.41841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['ord_0','ord_1','ord_2','ord_3','ord_4','ord_5']] = df[['ord_0','ord_1','ord_2','ord_3','ord_4','ord_5']].replace(np.nan,\"missing\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:27.420662Z","iopub.execute_input":"2021-08-30T14:40:27.420969Z","iopub.status.idle":"2021-08-30T14:40:27.871353Z","shell.execute_reply.started":"2021-08-30T14:40:27.420927Z","shell.execute_reply":"2021-08-30T14:40:27.870393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##lets look at variable\nfor col in nominal:\n    print(col, \":\",df[col].value_counts(),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:27.872675Z","iopub.execute_input":"2021-08-30T14:40:27.873138Z","iopub.status.idle":"2021-08-30T14:40:29.345856Z","shell.execute_reply.started":"2021-08-30T14:40:27.873094Z","shell.execute_reply":"2021-08-30T14:40:29.344905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"Red\" is most frequent value for nom_0 variable, lets replace missing values with \"Red\"\ndf['nom_0'] = df['nom_0'].replace(np.nan,\"Red\")\n\n# \"Theremin\" is most frequent value for nom_0 variable, lets replace missing values with \"Theremin\"\ndf['nom_4'] = df['nom_4'].replace(np.nan,\"Theremin\")\n\n# its difficult to make a call for other variables, so, let replace with \"missing\"\ndf[['nom_1','nom_2','nom_3','nom_5','nom_6','nom_7','nom_8','nom_9']] = df[['nom_1','nom_2','nom_3','nom_5','nom_6','nom_7','nom_8','nom_9']].replace(np.nan,\"missing\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:29.347196Z","iopub.execute_input":"2021-08-30T14:40:29.347502Z","iopub.status.idle":"2021-08-30T14:40:30.388575Z","shell.execute_reply.started":"2021-08-30T14:40:29.347469Z","shell.execute_reply":"2021-08-30T14:40:30.387811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets take a look at the day variables\nfor col in date:\n    print(col, \":\",df[col].value_counts(),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:30.38968Z","iopub.execute_input":"2021-08-30T14:40:30.390148Z","iopub.status.idle":"2021-08-30T14:40:30.418422Z","shell.execute_reply.started":"2021-08-30T14:40:30.390103Z","shell.execute_reply":"2021-08-30T14:40:30.41736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['day','month']] = df[['day','month']].replace(np.nan,0.0)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:30.419966Z","iopub.execute_input":"2021-08-30T14:40:30.42039Z","iopub.status.idle":"2021-08-30T14:40:30.44152Z","shell.execute_reply.started":"2021-08-30T14:40:30.420344Z","shell.execute_reply":"2021-08-30T14:40:30.440643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:30.442791Z","iopub.execute_input":"2021-08-30T14:40:30.443407Z","iopub.status.idle":"2021-08-30T14:40:31.468367Z","shell.execute_reply.started":"2021-08-30T14:40:30.443363Z","shell.execute_reply":"2021-08-30T14:40:31.467371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##CAHR TO NUMERIC\n##Binary Variables - will be replaced by 0 and 1\n##Ordinal Variables (low cardinality) - will apply Label Encoding\n##Ordinal Variables (high cardinality) - will apply Label Encoding\n##Nominal Variables (low cardinality) - will apply One Hot Encodihng\n##Nominal Variables (high cardinality) - will apply Hashing Encoding\ndf['bin_3'] = df['bin_3'].map({'F': 1, 'T': 0})\n\ndf['bin_4'] = df['bin_4'].map({'Y': 1, 'N': 0})\n\n##or\n\nfrom sklearn import preprocessing\nlbl = preprocessing.LabelEncoder()\nfor col in df:\n   df[col] = lbl.fit_transform(df[col].astype(str).values)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:31.469561Z","iopub.execute_input":"2021-08-30T14:40:31.470108Z","iopub.status.idle":"2021-08-30T14:40:45.313507Z","shell.execute_reply.started":"2021-08-30T14:40:31.470068Z","shell.execute_reply":"2021-08-30T14:40:45.312631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets divide nominal variable into high and low cardinality\n#nunique() function return number of unique elements in the object\nlow_card = []\nhigh_card = []\n\nfor cols in df:\n    if df[cols].nunique()>10:\n        high_card.append(cols)\n    else:\n        low_card.append(cols)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.314618Z","iopub.execute_input":"2021-08-30T14:40:45.315041Z","iopub.status.idle":"2021-08-30T14:40:45.524803Z","shell.execute_reply.started":"2021-08-30T14:40:45.315013Z","shell.execute_reply":"2021-08-30T14:40:45.524044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"low_card , high_card","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.525782Z","iopub.execute_input":"2021-08-30T14:40:45.526109Z","iopub.status.idle":"2021-08-30T14:40:45.532182Z","shell.execute_reply.started":"2021-08-30T14:40:45.526078Z","shell.execute_reply":"2021-08-30T14:40:45.531077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.533456Z","iopub.execute_input":"2021-08-30T14:40:45.533793Z","iopub.status.idle":"2021-08-30T14:40:45.548117Z","shell.execute_reply.started":"2021-08-30T14:40:45.53376Z","shell.execute_reply":"2021-08-30T14:40:45.546998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Feature Selection\nx = df.drop('target', axis=1)\ny = df['target']","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.549706Z","iopub.execute_input":"2021-08-30T14:40:45.55018Z","iopub.status.idle":"2021-08-30T14:40:45.691357Z","shell.execute_reply.started":"2021-08-30T14:40:45.550132Z","shell.execute_reply":"2021-08-30T14:40:45.690357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrainx,testx,trainy,testy =  train_test_split(x,y, test_size=0.33)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.692552Z","iopub.execute_input":"2021-08-30T14:40:45.692855Z","iopub.status.idle":"2021-08-30T14:40:45.956546Z","shell.execute_reply.started":"2021-08-30T14:40:45.692823Z","shell.execute_reply":"2021-08-30T14:40:45.955612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx.shape , trainy.shape, testx.shape , testy.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.957853Z","iopub.execute_input":"2021-08-30T14:40:45.958159Z","iopub.status.idle":"2021-08-30T14:40:45.964459Z","shell.execute_reply.started":"2021-08-30T14:40:45.95813Z","shell.execute_reply":"2021-08-30T14:40:45.963408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import VarianceThreshold\nvar = VarianceThreshold(threshold = 0)\nvar.fit(trainx)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:45.968606Z","iopub.execute_input":"2021-08-30T14:40:45.969267Z","iopub.status.idle":"2021-08-30T14:40:46.300792Z","shell.execute_reply.started":"2021-08-30T14:40:45.969218Z","shell.execute_reply":"2021-08-30T14:40:46.299867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var.get_support()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:46.302222Z","iopub.execute_input":"2021-08-30T14:40:46.302497Z","iopub.status.idle":"2021-08-30T14:40:46.308607Z","shell.execute_reply.started":"2021-08-30T14:40:46.302471Z","shell.execute_reply":"2021-08-30T14:40:46.307537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Quasi_Constant\n\nQfiltter = VarianceThreshold(threshold = 0.01)\nQfiltter.fit(trainx)","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:46.309884Z","iopub.execute_input":"2021-08-30T14:40:46.310312Z","iopub.status.idle":"2021-08-30T14:40:46.446641Z","shell.execute_reply.started":"2021-08-30T14:40:46.31028Z","shell.execute_reply":"2021-08-30T14:40:46.44554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Qfiltter.get_support()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:46.447879Z","iopub.execute_input":"2021-08-30T14:40:46.448185Z","iopub.status.idle":"2021-08-30T14:40:46.45516Z","shell.execute_reply.started":"2021-08-30T14:40:46.448156Z","shell.execute_reply":"2021-08-30T14:40:46.45389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx.columns[Qfiltter.get_support()]","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:46.456892Z","iopub.execute_input":"2021-08-30T14:40:46.457412Z","iopub.status.idle":"2021-08-30T14:40:46.47127Z","shell.execute_reply.started":"2021-08-30T14:40:46.457363Z","shell.execute_reply":"2021-08-30T14:40:46.47019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Dublicate \ntrainx_T = trainx.T\ntrainx_T.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:40:46.472633Z","iopub.execute_input":"2021-08-30T14:40:46.473102Z","iopub.status.idle":"2021-08-30T14:40:46.484503Z","shell.execute_reply.started":"2021-08-30T14:40:46.473068Z","shell.execute_reply":"2021-08-30T14:40:46.483608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx_T.duplicated()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:45:03.455995Z","iopub.execute_input":"2021-08-30T14:45:03.456369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx_T.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:44:57.493699Z","iopub.status.idle":"2021-08-30T14:44:57.494188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx = trainx_T.drop_duplicated(keep='first').T ","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:44:57.498144Z","iopub.status.idle":"2021-08-30T14:44:57.498567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainx.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-30T14:44:57.499485Z","iopub.status.idle":"2021-08-30T14:44:57.499897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrainx = scaler.fit_transform(trainx)\ntestx =  scaler.fit_transform(testx)\n","metadata":{},"execution_count":null,"outputs":[]}]}