{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Clean the data   "},{"metadata":{"trusted":true},"cell_type":"code","source":"#find and handle missing data.\nprint(test.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a=test.fillna(0)\na.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#verify no data is missing and lenghts match\na.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt \nimport numpy as np \nimport seaborn as sns\n\nsns.countplot(a['nom_3']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\ntemperature_count = a['ord_2'].value_counts()\nsns.set(style=\"darkgrid\")\nsns.barplot(temperature_count.index, temperature_count.values, alpha=0.9)\nplt.title('Frequency of temperatures')\nplt.ylabel('Occurrences', fontsize=12)\nplt.xlabel('Temperature', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Three types of preprocessing available**:   \nDrop, LabelEncoding, One-hot encoding   \nIn general, one-hot encoding  will typically perform best, and dropping the categorical columns typically performs worst, but it varies on a case-by-case basis.   "},{"metadata":{"trusted":true},"cell_type":"code","source":"#Drop float columns for logistic regression\n\ndropd=a.drop(['bin_0','bin_1','bin_2','ord_0','day', 'month'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#how many categories in each column\n#id is still numerical, others are \ndropd.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#visualize some columns\ncategorical_features = ['bin_4',\"nom_1\", \"ord_3\"]\nfig, ax = plt.subplots(1, len(categorical_features))\nfor i, categorical_feature in enumerate(dropd[categorical_features]):\n    dropd[categorical_feature].value_counts().plot(\"bar\", ax=ax[i]).set_title(categorical_feature)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In dropping all numerical data (except Id), I could be missing relevant information.   \nThere are better data processing methods."},{"metadata":{"trusted":true},"cell_type":"code","source":"##labelEncoding\n#It convert the data in machine-readable data, but it assigns a unique number(from 0 on) to each class of data. \n#This may lead to priority issues in training of data sets:\n#that is, a label with high value may be considered to have high priority than a label having lower value.\n\nfrom sklearn.preprocessing import LabelEncoder\n\n#It requires the category column to be of ‘category’ datatype. By default, a non-numerical column is of ‘object’ type.\n#So, it must be changed to ‘category’ type before running it.\n\ndropd['bin_3'] = dropd['bin_3'].astype('category')# Assigning numerical values and storing in another column\ndropd['bin_3_Cat'] = dropd['bin_3'].cat.codes\ndropd.head(2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dropd['bin_3'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plot\ndropd.hist(column='bin_3_Cat', color='green')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**One-hot encoding**\nIt takes a categorical column and splits its 'labels,' and then splits the column into multiple columns. \nThe numbers are replaced by 1s and 0s, depending on which column has what value.\nAlso, it requires numerical labels as inputs, so we need to either GetDummies or LabelEncode first."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Go back to dataframe: a\n#Label-encode the rest of the columns.\na['bin_3'] = a['bin_3'].astype('category')# Assigning numerical values and storing in another column\na['bin_3_Cat'] = a['bin_3'].cat.codes\n\na['bin_4'] = a['bin_4'].astype('category')# Assigning numerical values and storing in another column\na['bin_4_Cat'] = a['bin_4'].cat.codes\n\na['nom_0'] = a['nom_0'].astype('category')\na['nom_0_Cat'] = a['nom_0'].cat.codes\n\na['nom_1'] = a['nom_1'].astype('category')\na['nom_1_Cat'] = a['nom_1'].cat.codes\n\na['nom_2'] = a['nom_2'].astype('category')\na['nom_2_Cat'] = a['nom_2'].cat.codes\n\na['nom_3'] = a['nom_3'].astype('category')\na['nom_3_Cat'] = a['nom_3'].cat.codes\na['nom_4'] = a['nom_4'].astype('category')\na['nom_4_Cat'] = a['nom_4'].cat.codes\na['nom_5'] = a['nom_5'].astype('category')\na['nom_5_Cat'] = a['nom_5'].cat.codes\na['nom_6'] = a['nom_6'].astype('category')\na['nom_6_Cat'] = a['nom_6'].cat.codes\na['nom_7'] = a['nom_7'].astype('category')\na['nom_7_Cat'] = a['nom_7'].cat.codes\na['nom_8'] = a['nom_8'].astype('category')\na['nom_8_Cat'] = a['nom_8'].cat.codes\na['nom_9'] = a['nom_9'].astype('category')\na['nom_9_Cat'] = a['nom_9'].cat.codes\n\n\na['ord_1'] = a['ord_1'].astype('category')\na['ord_1_Cat'] = a['ord_1'].cat.codes\na['ord_2'] = a['ord_2'].astype('category')\na['ord_2_Cat'] = a['ord_2'].cat.codes\na['ord_3'] = a['ord_3'].astype('category')\na['ord_3_Cat'] = a['ord_3'].cat.codes\na['ord_4'] = a['ord_4'].astype('category')\na['ord_4_Cat'] = a['ord_4'].cat.codes\na['ord_5'] = a['ord_5'].astype('category')\na['ord_5_Cat'] = a['ord_5'].cat.codes\n\na.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**One-Hot encoding** should not be performed if the number of categories are high.       \n#This results in a sparse data.      \n#Depending on the project, you need to explore the data and do some feature engineering like grouping categories     \n#or assign integer values to match the relation of the variable with the output.   \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"##Data cleaning\n#remove the old, non-numerical columns.\nInts=a.drop(['bin_3','bin_4','nom_0','nom_1','nom_2','nom_3','nom_4','nom_5','nom_6','nom_7','nom_8','nom_9','ord_1', 'ord_2', 'ord_3', 'ord_4','ord_5' ], axis=1)\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Data cleaning\n\n#convert the data to int\nInts.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##One-hot encode categories with less than 10 unique entries.\n#these are:\n#'bin_3_Cat','bin_4_Cat','nom_0_Cat','nom_1_Cat','nom_2_Cat','nom_3_Cat','nom_4_Cat','ord_1_Cat','ord_2_Cat'\n\nfrom sklearn.preprocessing import OneHotEncoder\n \n# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['bin_3_Cat']]).toarray()) \n\n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The encoding of one column above shows how bin_3_Cat had three choices.    \nThose are now new columns: 0,1,2   \nSince I will encode several columns, this alone will not work well.   \nI need to differentiate the new columns per original column."},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"b3_zero\", 1: \"b3_False\", 2: 'b3_True'}) #the original categories 0,F,T\n#drop old column\nInts=Ints.drop(['bin_3_Cat'], axis=1)\nInts.head(2)   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"repeat for:\n#'bin_4_Cat','nom_0_Cat','nom_1_Cat','nom_2_Cat','nom_3_Cat','nom_4_Cat','ord_1_Cat','ord_2_Cat'\nbut need to find their category-labels.\n\n1st, I found the labels of their original categories using the code like:\n**a['bin_4'].unique()** "},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['bin_4_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"b4_zero\", 1: \"b4_NO\", 2: 'b4_YES'}) #the original categories 0,N,Y\n#drop old columns\nInts=Ints.drop(['bin_4_Cat' ], axis=1)\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['nom_0_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"n0_zero\", 1: \"n0_Blue\", 2: 'n0_Green', 3: 'n0Red'}) #the original categories Blue, Red, Green, 0\n#drop old column\nInts=Ints.drop(['nom_0_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['nom_1_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"n1_zero\", 1: \"n1_Circl\", 2: 'n1_PolyG', 3: 'n1_Sqr', 4:'n1_Star', 5: 'n1_TrapZd', 6: 'n1_Triangl'}) # Polygon, Circle, Star, Trapezoid, Triangle, Square, 0 \n#drop old column\nInts=Ints.drop(['nom_1_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['nom_2_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"n2_zero\", 1: \"n2_Axolotl\", 2: 'n2_CAT', 3: 'n2_Dog', 4:'n2_Hamster', 5: 'n2_Lion', 6: 'n2_Snake'}) # Axolotl, Lion, 0, Dog, Hamster, Snake, Cat \n#drop old column\nInts=Ints.drop(['nom_2_Cat' ], axis=1)\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['nom_3_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_3   #results in: Finland, Russia, Costa Rica, India, China, Canada, 0\nInts=Ints.rename(columns={0: \"n3_zero\", 1: \"n3_Canada\", 2: 'n3_China', 3: 'n3_Costa Rica', 4:'n3_Finland', 5: 'n3_India', 6: 'n3_Russia'})  \n#drop old column\nInts=Ints.drop(['nom_3_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['nom_4_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_4   #results in: Piano, Bassoon, Theremin, Oboe, 0\nInts=Ints.rename(columns={0: \"n4_zero\", 1: \"n4_Bassoon\", 2: 'n4_Oboe', 3: 'n4_Piano', 4:'n4_Theremin'})  \n#drop old column\nInts=Ints.drop(['nom_4_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['ord_1_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # Novice, Expert, Contributor, Grandmaster, Master, 0\nInts=Ints.rename(columns={0: \"o1_zero\", 1: \"o1_Contribtr\", 2: 'o1_Exprt', 3: 'o1_GrndMstr', 4:'o1_Master', 5:'o1_Novice'})  \n#drop old column\nInts=Ints.drop(['ord_1_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['ord_2_Cat']]).toarray()) \n\n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Boiling Hot, Cold, Warm, Hot, Lava Hot, Freezing, 0\nInts=Ints.rename(columns={0: \"o2_zero\", 1: \"o2_Boiling Hot\", 2: 'o2_Cold', 3: 'o2_Freezing', 4:'o2_Hot', 5:'o2_Lava Hot', 6: 'o2_Warm'})  \n#drop old column\nInts=Ints.drop(['ord_2_Cat' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"The above was manual and tedious. But I did it that way to ensure each category and new column name matched the encoding.    \nBut with larger data, like the rest of the columns, it would take too long.   \nBelow, I explore the data for options."},{"metadata":{"trusted":true},"cell_type":"code","source":"#there dont seem to be correlation among the large set of columns.\nInts.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a['nom_5'].value_counts()\n#I repeated this for the rest of the columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Binning\n#For continuous variables, like 'id', you can use optional “bins” argument to separate the values\nexample=Ints['id'].value_counts(bins=5)\nexample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Add a columns based on other column\n\n# it’s each value will be calculated based on other columns in each row i.e.\nInts = Ints.assign(o5_duos_zeros = lambda x: (x['ord_5_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(o5_duos1 = lambda x: (x['ord_5_Cat']>0 )<= 38)\nInts = Ints.assign(o5_duos2 = lambda x: (x['ord_5_Cat']>39 )<= 76)\nInts = Ints.assign(o5_duos3 = lambda x: (x['ord_5_Cat']>77 )<= 114)\nInts = Ints.assign(o5_duos4 = lambda x: (x['ord_5_Cat']>115 )<= 152)\nInts = Ints.assign(o5_duos5 = lambda x: (x['ord_5_Cat']>153 )<= 190)\n#drop old column\nInts=Ints.drop(['ord_5_Cat'], axis=1)\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert boolean to int\nInts['o5_duos_zeros'] = (Ints['o5_duos_zeros']).astype(int)\nInts['o5_duos1'] = (Ints['o5_duos1']).astype(int)\nInts['o5_duos2'] = (Ints['o5_duos2']).astype(int)\nInts['o5_duos3'] = (Ints['o5_duos3']).astype(int)\nInts['o5_duos4'] = (Ints['o5_duos4']).astype(int)\nInts['o5_duos5'] = (Ints['o5_duos5']).astype(int)\n#verify\nInts.head(2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#repeat with ord_3_Cat and ord_4_Cat\nInts = Ints.assign(o3_zeros = lambda x: (x['ord_3_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(o3_lowCase1 = lambda x: (x['ord_3_Cat']>0 )<= 5)\nInts = Ints.assign(o3_lowCase2 = lambda x: (x['ord_3_Cat']>6 )<= 10)\nInts = Ints.assign(o3_lowCase3 = lambda x: (x['ord_3_Cat']>11 )<= 15)\nInts = Ints.assign(o3_lowCase4 = lambda x: (x['ord_3_Cat']>16 )<= 25)\n#drop old column\nInts=Ints.drop(['ord_3_Cat'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert boolean to int\nInts['o3_zeros'] = (Ints['o3_zeros']).astype(int)\nInts['o3_lowCase1'] = (Ints['o3_lowCase1']).astype(int)\nInts['o3_lowCase2'] = (Ints['o3_lowCase2']).astype(int)\nInts['o3_lowCase3'] = (Ints['o3_lowCase3']).astype(int)\nInts['o3_lowCase4'] = (Ints['o3_lowCase4']).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints = Ints.assign(o4_zeros = lambda x: (x['ord_4_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(o4_UprCase1 = lambda x: (x['ord_4_Cat']>0 )<= 5)\nInts = Ints.assign(o4_UprCase2 = lambda x: (x['ord_4_Cat']>6 )<= 10)\nInts = Ints.assign(o4_UprCase3 = lambda x: (x['ord_4_Cat']>11 )<= 15)\nInts = Ints.assign(o4_UprCase4 = lambda x: (x['ord_4_Cat']>16 )<= 20)\nInts = Ints.assign(o4_UprCase5 = lambda x: (x['ord_4_Cat']>21 )<= 25)\n#drop old column\nInts=Ints.drop(['ord_4_Cat'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert boolean to int\nInts['o4_zeros'] = (Ints['o4_zeros']).astype(int)\nInts['o4_UprCase1'] = (Ints['o4_UprCase1']).astype(int)\nInts['o4_UprCase2'] = (Ints['o4_UprCase2']).astype(int)\nInts['o4_UprCase3'] = (Ints['o4_UprCase3']).astype(int)\nInts['o4_UprCase4'] = (Ints['o4_UprCase4']).astype(int)\nInts['o4_UprCase5'] = (Ints['o4_UprCase5']).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Repeat for num_5 to 9, which are random strings series with no obvious distinction among the values\n#these are large\n#num_5 has 1219 labels\nInts = Ints.assign(n5_zeros = lambda x: (x['nom_5_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(n5_serie1 = lambda x: (x['nom_5_Cat']>0 )<= 243)\nInts = Ints.assign(n5_serie2 = lambda x: (x['nom_5_Cat']>244 )<= 486)\nInts = Ints.assign(n5_serie3 = lambda x: (x['nom_5_Cat']>487 )<= 730)\nInts = Ints.assign(n5_serie4 = lambda x: (x['nom_5_Cat']>731 )<= 975)\nInts = Ints.assign(n5_serie5 = lambda x: (x['nom_5_Cat']>976 )<= 1219)\n#drop old column\nInts=Ints.drop(['nom_5_Cat'], axis=1)\n#convert boolean to int\nInts['n5_zeros'] = (Ints['n5_zeros']).astype(int)\nInts['n5_serie1'] = (Ints['n5_serie1']).astype(int)\nInts['n5_serie2'] = (Ints['n5_serie2']).astype(int)\nInts['n5_serie3'] = (Ints['n5_serie3']).astype(int)\nInts['n5_serie4'] = (Ints['n5_serie4']).astype(int)\nInts['n5_serie5'] = (Ints['n5_serie5']).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_6 has 1517 labels\nInts = Ints.assign(n6_zeros = lambda x: (x['nom_6_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(n6_serie1 = lambda x: (x['nom_6_Cat']>0 )<= 303)\nInts = Ints.assign(n6_serie2 = lambda x: (x['nom_6_Cat']>304 )<= 607)\nInts = Ints.assign(n6_serie3 = lambda x: (x['nom_6_Cat']>608 )<= 909)\nInts = Ints.assign(n6_serie4 = lambda x: (x['nom_6_Cat']>910 )<= 1212)\nInts = Ints.assign(n6_serie5 = lambda x: (x['nom_6_Cat']>1213 )<= 1517)\n#drop old column\nInts=Ints.drop(['nom_6_Cat'], axis=1)\n#convert boolean to int\nInts['n6_zeros'] = (Ints['n6_zeros']).astype(int)\nInts['n6_serie1'] = (Ints['n6_serie1']).astype(int)\nInts['n6_serie2'] = (Ints['n6_serie2']).astype(int)\nInts['n6_serie3'] = (Ints['n6_serie3']).astype(int)\nInts['n6_serie4'] = (Ints['n6_serie4']).astype(int)\nInts['n6_serie5'] = (Ints['n6_serie5']).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_7 and nom_8 have 222 labels each\nInts = Ints.assign(n7_zeros = lambda x: (x['nom_7_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(n7_serie1 = lambda x: (x['nom_7_Cat']>0 )<= 74)\nInts = Ints.assign(n7_serie2 = lambda x: (x['nom_7_Cat']>75 )<= 147)\nInts = Ints.assign(n7_serie3 = lambda x: (x['nom_7_Cat']>148 )<= 222)\n\n#drop old column\nInts=Ints.drop(['nom_7_Cat'], axis=1)\n#convert boolean to int\nInts['n7_zeros'] = (Ints['n7_zeros']).astype(int)\nInts['n7_serie1'] = (Ints['n7_serie1']).astype(int)\nInts['n7_serie2'] = (Ints['n7_serie2']).astype(int)\nInts['n7_serie3'] = (Ints['n7_serie3']).astype(int)\n\n#for nom_8\nInts = Ints.assign(n8_zeros = lambda x: (x['nom_8_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(n8_serie1 = lambda x: (x['nom_8_Cat']>0 )<= 74)\nInts = Ints.assign(n8_serie2 = lambda x: (x['nom_8_Cat']>75 )<= 147)\nInts = Ints.assign(n8_serie3 = lambda x: (x['nom_8_Cat']>148 )<= 222)\n\n#drop old column\nInts=Ints.drop(['nom_8_Cat'], axis=1)\n#convert boolean to int\nInts['n8_zeros'] = (Ints['n8_zeros']).astype(int)\nInts['n8_serie1'] = (Ints['n8_serie1']).astype(int)\nInts['n8_serie2'] = (Ints['n8_serie2']).astype(int)\nInts['n8_serie3'] = (Ints['n8_serie3']).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_9 has 2216\n#for nom_9\nInts = Ints.assign(n9_zeros = lambda x: (x['nom_9_Cat']==0 )) #kept 0s separate\nInts = Ints.assign(n9_serie1 = lambda x: (x['nom_9_Cat']>0 )<= 554)\nInts = Ints.assign(n9_serie2 = lambda x: (x['nom_9_Cat']>555 )<= 1108)\nInts = Ints.assign(n9_serie3 = lambda x: (x['nom_9_Cat']>148 )<= 1662)\nInts = Ints.assign(n9_serie4 = lambda x: (x['nom_9_Cat']>1663 )<= 2216)\n\n#drop old column\nInts=Ints.drop(['nom_9_Cat'], axis=1)\n#convert boolean to int\nInts['n9_zeros'] = (Ints['n9_zeros']).astype(int)\nInts['n9_serie1'] = (Ints['n9_serie1']).astype(int)\nInts['n9_serie2'] = (Ints['n9_serie2']).astype(int)\nInts['n9_serie3'] = (Ints['n9_serie3']).astype(int)\nInts['n9_serie4'] = (Ints['n9_serie4']).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating instance of one-hot-encoder\nenc = OneHotEncoder(handle_unknown='ignore')\n# passing the label-encoded values \n\nenc_df = pd.DataFrame(enc.fit_transform(Ints[['ord_0']]).toarray()) \n# merge with the dataframe on key values\nInts = Ints.join(enc_df)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ints=Ints.rename(columns={0: \"o0_zero\", 1: \"o0_1\", 2: 'o0_2', 3: 'o0_3'}) # 1,2,3,0 \n#drop old column\nInts=Ints.drop(['ord_0'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#day had 0-7 and month and 0-12\n\n#for day, I separate weekday (Monday-Thurd) and weekends (Fri-Sun)\nInts = Ints.assign(day_zeros = lambda x: (x['day']==0 )) #kept 0s separate\nInts = Ints.assign(Weekday = lambda x: (x['day']>0 )<= 4)\nInts = Ints.assign(WkEnd = lambda x: (x['day']>5 )<= 7)\n\n#drop old column\nInts=Ints.drop(['day'], axis=1)\n#convert boolean to int\nInts['day_zeros'] = (Ints['day_zeros']).astype(float)\nInts['Weekday'] = (Ints['Weekday']).astype(float)\nInts['WkEnd'] = (Ints['WkEnd']).astype(float)\n\n#verify\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for month, I separated as winter spring summer and fall\nInts = Ints.assign(month_zeros = lambda x: (x['month']==0 )) #kept 0s separate\nInts = Ints.assign(wintr = lambda x: (x['month']>0 )<= 3)\nInts = Ints.assign(sprg = lambda x: (x['month']>4 )<= 6)\nInts = Ints.assign(summr = lambda x: (x['month']>7 )<= 9)\nInts = Ints.assign(fall = lambda x: (x['month']>10 )<= 12)\n\n#drop old column\nInts=Ints.drop(['month'], axis=1)\n#convert boolean to int\nInts['month_zeros'] = (Ints['month_zeros']).astype(int)\nInts['wintr'] = (Ints['wintr']).astype(int)\nInts['sprg'] = (Ints['sprg']).astype(int)\nInts['summr'] = (Ints['summr']).astype(int)\nInts['fall'] = (Ints['fall']).astype(int)\n\n#verify\nInts.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Pepare a new dataframe without 'zero' columns (these represent NaN values in the original data)\nBinaries=Ints.drop(['b4_zero', 'day_zeros', 'b3_zero', 'month_zeros', 'n5_zeros', 'n6_zeros', 'n7_zeros', 'n8_zeros', 'n9_zeros' ], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#drop the rest of zeros to 'clean' data \nBinary1=Binaries.drop(['n0_zero', 'n1_zero', 'n2_zero', 'n3_zero', 'n4_zero', 'o1_zero', 'o2_zero', 'o5_duos_zeros', 'o3_zeros', 'o4_zeros', 'o0_zero' ], axis=1)\nBinary1.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Data cleaning\n#even the columns to int\nBinary1.astype('int')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Binary classification\nROC is a graphic plot for diagnostics of a binary classifier system.   \n**ROC** is a probability curve for different classes. ROC tells us how good the model is for distinguishing the given classes, in terms of the predicted probability.   \n\nA typical ROC curve has False Positive Rate (FPR) on the X-axis and True Positive Rate (TPR) on the Y-axis.   \nThe area covered by the curve is the area between the line (ROC) and the axis. This area covered is AUC. The bigger the area covered, the better the machine learning models is at distinguishing the given classes. Ideal value for AUC is 1.   \n\n**AUC-ROC** (Area Under Curve - Receiver Operating Characteristics)=the most commonly used metrics.   "},{"metadata":{"trusted":true},"cell_type":"code","source":"#the 'id' column remains a series of continuous data from 600000 to 999999 as identifying rows for target and prediction\n#the rest is BINARY (0/1)\n\nfrom sklearn.linear_model import *\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n#Rename the column \nBinary1=Binary1.rename(columns={'n2_CAT': \"CAT\"})\nBinary1=Binary1.rename(columns={'b4_YES': \"YES\"})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# roc curve\nfrom sklearn.datasets import make_classification\nfrom sklearn.metrics import roc_curve\nfrom matplotlib import pyplot\n\nX=Binary1['YES']\ny=Binary1['CAT']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate 2 class dataset\nX, y = make_classification(n_samples=2000, n_classes=2, random_state=1)\n\n# split into train/test sets\ntrainX, testX, trainy, testy = train_test_split(X, y, test_size=0.5, random_state=2)\n# fit a model\nmodel = LogisticRegression()\nmodel.fit(trainX, trainy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.predict(testX)) ##array\n\ndf = pd.DataFrame(model.predict(testX))\nprint(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# predict probabilities\nprobs = model.predict_proba(testX)\n# keep probabilities for the positive outcome only\nprobs1 = probs[:, 1]\n# calculate roc curve\nfpr, tpr, thresholds = roc_curve(testy, probs1)\n# plot no skill\npyplot.plot([0, 1], [0, 1], linestyle='--')\n# plot the roc curve for the model\npyplot.plot(fpr, tpr)\n# show the plot\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#The integrated area under the ROC curve(AUC) measures the skill of the model across all evaluated thresholds.\nfrom sklearn.metrics import roc_auc_score\n# calculate log-loss/AUC\nauc = roc_auc_score(testy, probs1)\nprint(auc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Deploy model** ###"},{"metadata":{"trusted":true},"cell_type":"code","source":"##Run model on full dataset for prediction\n# generate 2 class dataset\nX1=Binary1['YES']\ny1=Binary1['CAT']\n\nX1, y1 = make_classification(n_samples=400000, n_classes=2, random_state=1)\n\n# fit the defined model\nmodel.fit(X1, y1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = pd.DataFrame(model.predict(X1))\ndf1.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = pd.DataFrame(model.predict_proba(X1))\ndf2.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3 = model.predict_proba(X1)\n# keep probabilities for the positive outcome only\ndf4 = df3[:, 1]\n\ndf4 = pd.DataFrame(df4)\ndf4.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids=Binary1[['id']]\nids.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submissions are evaluated on area under the ROC curve between the predicted probability and the observed target.   \n \nFor each id in the test set, you must predict a probability for the target variable.   \nThe file should contain a header and have the following format:   \n\nid,target   \n600000,0.5   \n600001,0.5   \n600002,0.5   \n...   \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#join two dataframes\nanswerSubm = pd.concat([ids, df4], axis=1)\n#rename 0 to target\n\nanswerSubm=answerSubm.rename(columns={0: \"target\"})\n\nanswerSubm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answerSubm.to_csv('answerSubm.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}