{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Data","metadata":{}},{"cell_type":"code","source":"# Importing the libraries\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-23T01:48:57.738131Z","iopub.execute_input":"2021-11-23T01:48:57.739477Z","iopub.status.idle":"2021-11-23T01:48:57.743326Z","shell.execute_reply.started":"2021-11-23T01:48:57.739384Z","shell.execute_reply":"2021-11-23T01:48:57.742555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the dataset\n\ntrain_df = pd.read_csv('../input/titanic/train.csv')\ntest_df = pd.read_csv('../input/titanic/test.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.744536Z","iopub.execute_input":"2021-11-23T01:48:57.744778Z","iopub.status.idle":"2021-11-23T01:48:57.773669Z","shell.execute_reply.started":"2021-11-23T01:48:57.744755Z","shell.execute_reply":"2021-11-23T01:48:57.772888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# Lets look at the top 5 rows\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.774837Z","iopub.execute_input":"2021-11-23T01:48:57.775086Z","iopub.status.idle":"2021-11-23T01:48:57.788248Z","shell.execute_reply.started":"2021-11-23T01:48:57.775063Z","shell.execute_reply":"2021-11-23T01:48:57.78777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for null values\ntrain_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.789014Z","iopub.execute_input":"2021-11-23T01:48:57.789814Z","iopub.status.idle":"2021-11-23T01:48:57.798256Z","shell.execute_reply.started":"2021-11-23T01:48:57.789791Z","shell.execute_reply":"2021-11-23T01:48:57.797092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Embarked'].unique())\nprint(train_df['Cabin'].unique())\nprint(train_df['Ticket'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.799794Z","iopub.execute_input":"2021-11-23T01:48:57.8002Z","iopub.status.idle":"2021-11-23T01:48:57.818083Z","shell.execute_reply.started":"2021-11-23T01:48:57.800176Z","shell.execute_reply":"2021-11-23T01:48:57.817333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Processing**\n\n1. One-Hot encoding\n2. Mean encoding","metadata":{}},{"cell_type":"code","source":"#Remove unnecessary features\ntrain_df = train_df.drop(['Name', 'Ticket', 'PassengerId'], axis = 1 )\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.819307Z","iopub.execute_input":"2021-11-23T01:48:57.819984Z","iopub.status.idle":"2021-11-23T01:48:57.846726Z","shell.execute_reply.started":"2021-11-23T01:48:57.81995Z","shell.execute_reply":"2021-11-23T01:48:57.846096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*one-hot encoding*","metadata":{}},{"cell_type":"code","source":"Sex_encoding = pd.get_dummies(train_df['Sex'])","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.84777Z","iopub.execute_input":"2021-11-23T01:48:57.848748Z","iopub.status.idle":"2021-11-23T01:48:57.85657Z","shell.execute_reply.started":"2021-11-23T01:48:57.848671Z","shell.execute_reply":"2021-11-23T01:48:57.85465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_df, Sex_encoding],axis =1)\ntrain_df = train_df.drop(['Sex'],axis=1)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.858669Z","iopub.execute_input":"2021-11-23T01:48:57.859007Z","iopub.status.idle":"2021-11-23T01:48:57.891057Z","shell.execute_reply.started":"2021-11-23T01:48:57.858966Z","shell.execute_reply":"2021-11-23T01:48:57.889828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Mean encoding*","metadata":{}},{"cell_type":"code","source":"#Embarked_mean\n\ntarget = 'Survived'\nEmbarked_mean = train_df.groupby('Embarked')[target].mean()\n\ntrain_df['Embarked_mean'] = train_df['Embarked'].map(Embarked_mean)\ntrain_df = train_df.drop(['Embarked'] , axis = 1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.892957Z","iopub.execute_input":"2021-11-23T01:48:57.893563Z","iopub.status.idle":"2021-11-23T01:48:57.914558Z","shell.execute_reply.started":"2021-11-23T01:48:57.893529Z","shell.execute_reply":"2021-11-23T01:48:57.913853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cabin_mean\n\nCabin_mean = train_df.groupby('Cabin')[target].mean()\n\ntrain_df['Cabin_mean'] = train_df['Cabin'].map(Cabin_mean)\ntrain_df = train_df.drop(['Cabin'] , axis = 1)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:48:57.915607Z","iopub.execute_input":"2021-11-23T01:48:57.916376Z","iopub.status.idle":"2021-11-23T01:48:57.935668Z","shell.execute_reply.started":"2021-11-23T01:48:57.916343Z","shell.execute_reply":"2021-11-23T01:48:57.934688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Mean imputation of null values**","metadata":{}},{"cell_type":"code","source":"# null값이 반 이상 차지하는 Cabin_mean은 제거하고 Age와 Embarked_mean의 null 값을 평균으로 대치한다.\n# Cabin_mean, which occupies more than half of null values, is removed and the null values of Age and Embarked_mean are replaced with the average.\ntrain_df.isnull().sum()\ntrain_df = train_df.drop(['Cabin_mean'], axis = 1)\ntrain_df['Age_not_null'] = train_df['Age'].fillna(train_df['Age'].mean())\n\ntrain_df['Embarked_not_null'] = train_df['Embarked_mean'].fillna(train_df['Embarked_mean'].mean())\ntrain_df = train_df.drop(['Age', 'Embarked_mean'],axis=1)\n\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:56:40.470235Z","iopub.execute_input":"2021-11-23T01:56:40.471296Z","iopub.status.idle":"2021-11-23T01:56:40.491128Z","shell.execute_reply.started":"2021-11-23T01:56:40.471238Z","shell.execute_reply":"2021-11-23T01:56:40.489845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:00:08.760402Z","iopub.execute_input":"2021-11-23T02:00:08.760712Z","iopub.status.idle":"2021-11-23T02:00:08.784396Z","shell.execute_reply.started":"2021-11-23T02:00:08.760681Z","shell.execute_reply":"2021-11-23T02:00:08.783156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize']=15,6 \nsns.set_style(\"darkgrid\")\n\nx = train_df.drop(['Survived'],axis=1)\ny = train_df['Survived']\n\nfrom sklearn.ensemble import ExtraTreesClassifier\nimport matplotlib.pyplot as plt\nmodel = ExtraTreesClassifier()\nmodel.fit(x,y)\nprint(model.feature_importances_) \nfeat_importances = pd.Series(model.feature_importances_, index=x.columns)\nfeat_importances.nlargest(12).plot(kind='barh')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:57:35.781534Z","iopub.execute_input":"2021-11-23T01:57:35.781953Z","iopub.status.idle":"2021-11-23T01:57:36.151398Z","shell.execute_reply.started":"2021-11-23T01:57:35.781926Z","shell.execute_reply":"2021-11-23T01:57:36.150485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"markdown","source":"*Finding outliers*","metadata":{}},{"cell_type":"code","source":"# Boxplot for ejection_fraction\n\nsns.boxplot(x = train_df.Fare, color = 'teal')\nplt.show()\n\n\n# We can see there are a lot of outliers.\n# train_df[train_df['Fare']>=100]","metadata":{"execution":{"iopub.status.busy":"2021-11-23T01:57:48.375911Z","iopub.execute_input":"2021-11-23T01:57:48.376177Z","iopub.status.idle":"2021-11-23T01:57:48.561995Z","shell.execute_reply.started":"2021-11-23T01:57:48.376153Z","shell.execute_reply":"2021-11-23T01:57:48.560241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=train_df.Embarked_not_null, color = 'teal')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:00:14.824351Z","iopub.execute_input":"2021-11-23T02:00:14.825303Z","iopub.status.idle":"2021-11-23T02:00:15.018034Z","shell.execute_reply.started":"2021-11-23T02:00:14.825226Z","shell.execute_reply":"2021-11-23T02:00:15.0166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=train_df.Age_not_null, color = 'teal')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:00:52.164959Z","iopub.execute_input":"2021-11-23T02:00:52.165993Z","iopub.status.idle":"2021-11-23T02:00:52.365138Z","shell.execute_reply.started":"2021-11-23T02:00:52.165966Z","shell.execute_reply":"2021-11-23T02:00:52.364015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of Age\n\nimport plotly.graph_objects as go\n\nfig = go.Figure()\nfig.add_trace(go.Histogram(\n    x = train_df['Age_not_null'],\n    xbins=dict( # bins used for histogram\n        start=0,\n        end=95,\n        size=2\n    ),\n    marker_color='#e8ab60',\n    opacity=1\n))\n\nfig.update_layout(\n    title_text='AGE DISTRIBUTION',\n    xaxis_title_text='AGE',\n    yaxis_title_text='COUNT', \n    bargap=0.05, # gap between bars of adjacent location coordinates\n    xaxis =  {'showgrid': False },\n    yaxis = {'showgrid': False },\n    template = 'plotly_dark'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:03:31.590875Z","iopub.execute_input":"2021-11-23T02:03:31.591237Z","iopub.status.idle":"2021-11-23T02:03:31.637753Z","shell.execute_reply.started":"2021-11-23T02:03:31.591212Z","shell.execute_reply":"2021-11-23T02:03:31.636702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now lets categorize the above histogram by Survived\n\nimport plotly.express as px\nfig = px.histogram(train_df, x=\"Age_not_null\", color=\"Survived\", marginal=\"violin\", hover_data=train_df.columns, \n                   title =\"Distribution of AGE Vs Survived\", \n                   labels={\"age\": \"AGE\"},\n                   template=\"plotly_dark\",\n                   color_discrete_map={\"0\": \"RebeccaPurple\", \"1\": \"MediumPurple\"}\n                  )\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:09:43.748729Z","iopub.execute_input":"2021-11-23T02:09:43.748989Z","iopub.status.idle":"2021-11-23T02:09:43.8433Z","shell.execute_reply.started":"2021-11-23T02:09:43.748962Z","shell.execute_reply":"2021-11-23T02:09:43.842345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Histogram(\n    x = train_df['Fare'],\n    xbins=dict( # bins used for histogram\n        start=0,\n        end=582,\n        size=15\n    ),\n    marker_color='#FE6F5E',\n    opacity=1\n))\n\nfig.update_layout(\n    title_text='FARE DISTRIBUTION',\n    xaxis_title_text='FARE',\n    yaxis_title_text='COUNT', \n    bargap=0.05, # gap between bars of adjacent location coordinates\n    xaxis =  {'showgrid': False },\n    yaxis = {'showgrid': False },\n    template = 'plotly_dark'\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:11:37.986345Z","iopub.execute_input":"2021-11-23T02:11:37.987716Z","iopub.status.idle":"2021-11-23T02:11:38.039752Z","shell.execute_reply.started":"2021-11-23T02:11:37.987611Z","shell.execute_reply":"2021-11-23T02:11:38.038672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now lets categorize the above histogram by Survived\n\nimport plotly.express as px\nfig = px.histogram(train_df, x=\"Fare\", color=\"Survived\", marginal=\"violin\", hover_data=train_df.columns,\n                   title =\"Distribution of FARE Vs Survived\", \n                   labels={\"fare\": \"FARE\"},\n                   template=\"plotly_dark\",\n                   color_discrete_map={\"0\": \"RebeccaPurple\", \"1\": \"MediumPurple\"})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:12:15.275737Z","iopub.execute_input":"2021-11-23T02:12:15.276273Z","iopub.status.idle":"2021-11-23T02:12:15.41415Z","shell.execute_reply.started":"2021-11-23T02:12:15.276238Z","shell.execute_reply":"2021-11-23T02:12:15.41364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine Learning Modeling","metadata":{}},{"cell_type":"code","source":"train_df['female'] = train_df['female'].astype('int64')\ntrain_df['male'] = train_df['male'].astype('int64')\ntrain_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:23:58.949064Z","iopub.execute_input":"2021-11-23T02:23:58.949317Z","iopub.status.idle":"2021-11-23T02:23:58.9596Z","shell.execute_reply.started":"2021-11-23T02:23:58.949294Z","shell.execute_reply":"2021-11-23T02:23:58.95823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n# data segmentation\nX = train_df.drop('Survived', axis=1)\ny = train_df['Survived']\n\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state =0)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:31:16.734634Z","iopub.execute_input":"2021-11-23T02:31:16.734997Z","iopub.status.idle":"2021-11-23T02:31:16.743735Z","shell.execute_reply.started":"2021-11-23T02:31:16.734966Z","shell.execute_reply":"2021-11-23T02:31:16.742848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Scaling\n\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\nx_train = sc.fit_transform(x_train)\nx_test = sc.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:31:54.227489Z","iopub.execute_input":"2021-11-23T02:31:54.227747Z","iopub.status.idle":"2021-11-23T02:31:54.235845Z","shell.execute_reply.started":"2021-11-23T02:31:54.227723Z","shell.execute_reply":"2021-11-23T02:31:54.234436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*LOGISTIC REGRESSION*","metadata":{}},{"cell_type":"code","source":"# Applying logistic regression on the training set\n\nfrom sklearn.linear_model import LogisticRegression\nclassifier = LogisticRegression()\nclassifier.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:32:04.655035Z","iopub.execute_input":"2021-11-23T02:32:04.655277Z","iopub.status.idle":"2021-11-23T02:32:04.668538Z","shell.execute_reply.started":"2021-11-23T02:32:04.655251Z","shell.execute_reply":"2021-11-23T02:32:04.667525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:40:53.965319Z","iopub.execute_input":"2021-11-23T02:40:53.966707Z","iopub.status.idle":"2021-11-23T02:40:53.987582Z","shell.execute_reply.started":"2021-11-23T02:40:53.966661Z","shell.execute_reply":"2021-11-23T02:40:53.985399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making Confusion Matrix and calculating accuracy score\n\nmylist = []\nfrom sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nmylist.append(ac)\nprint(cm)\nprint(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:41:09.287933Z","iopub.execute_input":"2021-11-23T02:41:09.288385Z","iopub.status.idle":"2021-11-23T02:41:09.297065Z","shell.execute_reply.started":"2021-11-23T02:41:09.288355Z","shell.execute_reply":"2021-11-23T02:41:09.296081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding the optimum number of neighbors \n\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nlist1 = []\nfor neighbors in range(3,10):\n    classifier = KNeighborsClassifier(n_neighbors=neighbors, metric='minkowski')\n    classifier.fit(x_train, y_train)\n    y_pred = classifier.predict(x_test)\n    list1.append(accuracy_score(y_test,y_pred))\nplt.plot(list(range(3,10)), list1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:40:55.048489Z","iopub.execute_input":"2021-11-23T02:40:55.049366Z","iopub.status.idle":"2021-11-23T02:40:55.388619Z","shell.execute_reply.started":"2021-11-23T02:40:55.049327Z","shell.execute_reply":"2021-11-23T02:40:55.387508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*K NEAREST NEIGHBOR*","metadata":{}},{"cell_type":"code","source":"# Training the K Nearest Neighbor Classifier on the Training set\n\nclassifier = KNeighborsClassifier(n_neighbors=6)\nclassifier.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:32:14.618017Z","iopub.execute_input":"2021-11-23T02:32:14.618255Z","iopub.status.idle":"2021-11-23T02:32:14.627998Z","shell.execute_reply.started":"2021-11-23T02:32:14.618231Z","shell.execute_reply":"2021-11-23T02:32:14.627367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predicting the Test set results\n\ny_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:41:36.738148Z","iopub.execute_input":"2021-11-23T02:41:36.738584Z","iopub.status.idle":"2021-11-23T02:41:36.755635Z","shell.execute_reply.started":"2021-11-23T02:41:36.738553Z","shell.execute_reply":"2021-11-23T02:41:36.754151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nmylist.append(ac)\nprint(cm)\nprint(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:41:44.495569Z","iopub.execute_input":"2021-11-23T02:41:44.49598Z","iopub.status.idle":"2021-11-23T02:41:44.505031Z","shell.execute_reply.started":"2021-11-23T02:41:44.495954Z","shell.execute_reply":"2021-11-23T02:41:44.50376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*SUPPORT VECTOR CLASSIFIER*","metadata":{}},{"cell_type":"code","source":"#Finding the optimum number of n_estimators\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nlist1 = []\nfor c in [0.5,0.6,0.7,0.8,0.9,1.0]:\n    classifier = SVC(C = c, random_state=0, kernel = 'rbf')\n    classifier.fit(x_train, y_train)\n    y_pred = classifier.predict(x_test)\n    list1.append(accuracy_score(y_test,y_pred))\nplt.plot([0.5,0.6,0.7,0.8,0.9,1.0], list1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:00.581898Z","iopub.execute_input":"2021-11-23T02:42:00.582197Z","iopub.status.idle":"2021-11-23T02:42:00.895198Z","shell.execute_reply.started":"2021-11-23T02:42:00.582167Z","shell.execute_reply":"2021-11-23T02:42:00.893599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = SVC(C = 0.6, random_state=0, kernel = 'rbf')\nclassifier.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:13.037741Z","iopub.execute_input":"2021-11-23T02:42:13.03803Z","iopub.status.idle":"2021-11-23T02:42:13.073325Z","shell.execute_reply.started":"2021-11-23T02:42:13.038001Z","shell.execute_reply":"2021-11-23T02:42:13.072529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:19.081724Z","iopub.execute_input":"2021-11-23T02:42:19.082335Z","iopub.status.idle":"2021-11-23T02:42:19.093097Z","shell.execute_reply.started":"2021-11-23T02:42:19.0823Z","shell.execute_reply":"2021-11-23T02:42:19.091988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nprint(cm)\nprint(ac)\nmylist.append(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:23.307303Z","iopub.execute_input":"2021-11-23T02:42:23.307621Z","iopub.status.idle":"2021-11-23T02:42:23.31721Z","shell.execute_reply.started":"2021-11-23T02:42:23.30759Z","shell.execute_reply":"2021-11-23T02:42:23.316475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*DECISION TREE CLASSIFIER*","metadata":{}},{"cell_type":"code","source":"#Finding the optimum number of n_estimators\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nlist1 = []\nfor leaves in range(2,10):\n    classifier = DecisionTreeClassifier(max_leaf_nodes = leaves, random_state=0, criterion='entropy')\n    classifier.fit(x_train, y_train)\n    y_pred = classifier.predict(x_test)\n    list1.append(accuracy_score(y_test,y_pred))\n#print(mylist)\nplt.plot(list(range(2,10)), list1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:54.218383Z","iopub.execute_input":"2021-11-23T02:42:54.218688Z","iopub.status.idle":"2021-11-23T02:42:54.475184Z","shell.execute_reply.started":"2021-11-23T02:42:54.218661Z","shell.execute_reply":"2021-11-23T02:42:54.473754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = DecisionTreeClassifier(max_leaf_nodes = 3, random_state=0, criterion='entropy')\nclassifier.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:43:12.99587Z","iopub.execute_input":"2021-11-23T02:43:12.996155Z","iopub.status.idle":"2021-11-23T02:43:13.010642Z","shell.execute_reply.started":"2021-11-23T02:43:12.996132Z","shell.execute_reply":"2021-11-23T02:43:13.009836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:43:24.095312Z","iopub.execute_input":"2021-11-23T02:43:24.095653Z","iopub.status.idle":"2021-11-23T02:43:24.103299Z","shell.execute_reply.started":"2021-11-23T02:43:24.095622Z","shell.execute_reply":"2021-11-23T02:43:24.102596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nprint(cm)\nprint(ac)\nmylist.append(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:43:31.475489Z","iopub.execute_input":"2021-11-23T02:43:31.47573Z","iopub.status.idle":"2021-11-23T02:43:31.485649Z","shell.execute_reply.started":"2021-11-23T02:43:31.475707Z","shell.execute_reply":"2021-11-23T02:43:31.484061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*RANDOM FOREST CLASSIFCATION*","metadata":{}},{"cell_type":"code","source":"#Finding the optimum number of n_estimators\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nlist1 = []\nfor estimators in range(10,30):\n    classifier = RandomForestClassifier(n_estimators = estimators, random_state=0, criterion='entropy')\n    classifier.fit(x_train, y_train)\n    y_pred = classifier.predict(x_test)\n    list1.append(accuracy_score(y_test,y_pred))\n#print(mylist)\nplt.plot(list(range(10,30)), list1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:43:47.852722Z","iopub.execute_input":"2021-11-23T02:43:47.853043Z","iopub.status.idle":"2021-11-23T02:43:49.27611Z","shell.execute_reply.started":"2021-11-23T02:43:47.853003Z","shell.execute_reply":"2021-11-23T02:43:49.275281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nclassifier = RandomForestClassifier(n_estimators = 11, criterion='entropy', random_state=0)\nclassifier.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:43:54.540474Z","iopub.execute_input":"2021-11-23T02:43:54.541534Z","iopub.status.idle":"2021-11-23T02:43:54.572189Z","shell.execute_reply.started":"2021-11-23T02:43:54.541474Z","shell.execute_reply":"2021-11-23T02:43:54.571315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:24.634589Z","iopub.execute_input":"2021-11-23T02:44:24.634911Z","iopub.status.idle":"2021-11-23T02:44:24.647032Z","shell.execute_reply.started":"2021-11-23T02:44:24.634871Z","shell.execute_reply":"2021-11-23T02:44:24.644966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nmylist.append(ac)\nprint(cm)\nprint(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:29.285271Z","iopub.execute_input":"2021-11-23T02:44:29.28559Z","iopub.status.idle":"2021-11-23T02:44:29.293216Z","shell.execute_reply.started":"2021-11-23T02:44:29.285558Z","shell.execute_reply":"2021-11-23T02:44:29.292464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nlist1 = []\nfor estimators in range(10,30,1):\n    classifier = XGBClassifier(n_estimators = estimators, max_depth=12, subsample=0.7)\n    classifier.fit(x_train, y_train)\n    y_pred = classifier.predict(x_test)\n    list1.append(accuracy_score(y_test,y_pred))\n#print(mylist)\nplt.plot(list(range(10,30,1)), list1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:49:47.207933Z","iopub.execute_input":"2021-11-23T02:49:47.208974Z","iopub.status.idle":"2021-11-23T02:49:49.402198Z","shell.execute_reply.started":"2021-11-23T02:49:47.208845Z","shell.execute_reply":"2021-11-23T02:49:49.400875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nclassifier = XGBClassifier(n_estimators = 10, max_depth=12, subsample=0.7)\nclassifier.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:07.955919Z","iopub.execute_input":"2021-11-23T02:50:07.956247Z","iopub.status.idle":"2021-11-23T02:50:08.031406Z","shell.execute_reply.started":"2021-11-23T02:50:07.956212Z","shell.execute_reply":"2021-11-23T02:50:08.029501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:14.106018Z","iopub.execute_input":"2021-11-23T02:50:14.106305Z","iopub.status.idle":"2021-11-23T02:50:14.119127Z","shell.execute_reply.started":"2021-11-23T02:50:14.106275Z","shell.execute_reply":"2021-11-23T02:50:14.118596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nmylist.append(ac)\nprint(cm)\nprint(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:19.838356Z","iopub.execute_input":"2021-11-23T02:50:19.838701Z","iopub.status.idle":"2021-11-23T02:50:19.848124Z","shell.execute_reply.started":"2021-11-23T02:50:19.838668Z","shell.execute_reply":"2021-11-23T02:50:19.847607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nclassifier = CatBoostClassifier()\nclassifier.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:26.01571Z","iopub.execute_input":"2021-11-23T02:50:26.016077Z","iopub.status.idle":"2021-11-23T02:50:27.523313Z","shell.execute_reply.started":"2021-11-23T02:50:26.016052Z","shell.execute_reply":"2021-11-23T02:50:27.522123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(x_test)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:47.388387Z","iopub.execute_input":"2021-11-23T02:50:47.389462Z","iopub.status.idle":"2021-11-23T02:50:47.400797Z","shell.execute_reply.started":"2021-11-23T02:50:47.389398Z","shell.execute_reply":"2021-11-23T02:50:47.398744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\ncm = confusion_matrix(y_test, y_pred)\nac = accuracy_score(y_test, y_pred)\nmylist.append(ac)\nprint(cm)\nprint(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:50:52.482647Z","iopub.execute_input":"2021-11-23T02:50:52.482976Z","iopub.status.idle":"2021-11-23T02:50:52.49212Z","shell.execute_reply.started":"2021-11-23T02:50:52.482932Z","shell.execute_reply":"2021-11-23T02:50:52.490697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep Learning Modeling","metadata":{}},{"cell_type":"markdown","source":"### ANN","metadata":{}},{"cell_type":"code","source":"np.random.seed(0)\nimport tensorflow as tf\n\n# Initialising the ANN\n\nann = tf.keras.models.Sequential()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:47:05.526257Z","iopub.execute_input":"2021-11-23T02:47:05.526561Z","iopub.status.idle":"2021-11-23T02:47:11.671282Z","shell.execute_reply.started":"2021-11-23T02:47:05.526527Z","shell.execute_reply":"2021-11-23T02:47:11.670406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adding the input layer and the first hidden layer\n\nann.add(tf.keras.layers.Dense(units = 7, activation = 'relu'))\nann.add(tf.keras.layers.Dense(units = 7, activation = 'relu'))\nann.add(tf.keras.layers.Dense(units = 7, activation = 'relu'))\nann.add(tf.keras.layers.Dense(units = 7, activation = 'relu'))\nann.add(tf.keras.layers.Dense(units = 1, activation = 'sigmoid'))\nann.compile(optimizer = 'adam', loss = 'binary_crossentropy' , metrics = ['accuracy'] )","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:47:11.672855Z","iopub.execute_input":"2021-11-23T02:47:11.673201Z","iopub.status.idle":"2021-11-23T02:47:11.711821Z","shell.execute_reply.started":"2021-11-23T02:47:11.673174Z","shell.execute_reply":"2021-11-23T02:47:11.711121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ann.fit(x_train, y_train, batch_size = 32, epochs = 100)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:47:17.847096Z","iopub.execute_input":"2021-11-23T02:47:17.847886Z","iopub.status.idle":"2021-11-23T02:47:25.446003Z","shell.execute_reply.started":"2021-11-23T02:47:17.847841Z","shell.execute_reply":"2021-11-23T02:47:25.445467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = ann.predict(x_test)\ny_pred = (y_pred > 0.5)\nnp.set_printoptions()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:48:13.884209Z","iopub.execute_input":"2021-11-23T02:48:13.884831Z","iopub.status.idle":"2021-11-23T02:48:13.98792Z","shell.execute_reply.started":"2021-11-23T02:48:13.884803Z","shell.execute_reply":"2021-11-23T02:48:13.986594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, accuracy_score\n\n# confusion matrix\ncm = confusion_matrix(y_test,y_pred)\nprint(\"Confusion Matrix\")\nprint(cm)\nprint()\n\n# accuracy\nac = accuracy_score(y_test,y_pred)\nprint(\"Accuracy\")\nprint(ac)\nmylist.append(ac)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:49:27.270027Z","iopub.execute_input":"2021-11-23T02:49:27.270325Z","iopub.status.idle":"2021-11-23T02:49:27.283184Z","shell.execute_reply.started":"2021-11-23T02:49:27.270295Z","shell.execute_reply":"2021-11-23T02:49:27.281656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"code","source":"mylist2 = [\"Logistic Regression\", \"KNearestNeighbours\",\"SupportVector\",\"DecisionTree\",\"RandomForest\",\"ANN\", \"XGBOOST\",\"CATBOOST\"]","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:53:41.422235Z","iopub.execute_input":"2021-11-23T02:53:41.422544Z","iopub.status.idle":"2021-11-23T02:53:41.428396Z","shell.execute_reply.started":"2021-11-23T02:53:41.422516Z","shell.execute_reply":"2021-11-23T02:53:41.427074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize']=15,6 \nsns.set_style(\"darkgrid\")\nax = sns.barplot(x=mylist2, y=mylist, palette = \"rocket\", saturation =1.5)\nplt.xlabel(\"Classifier Models\", fontsize = 20 )\nplt.ylabel(\"% of Accuracy\", fontsize = 20)\nplt.title(\"Accuracy of different Classifier Models\", fontsize = 20)\nplt.xticks(fontsize = 12, horizontalalignment = 'center', rotation = 8)\nplt.yticks(fontsize = 13)\nfor p in ax.patches:\n    width, height = p.get_width(), p.get_height()\n    x, y = p.get_xy() \n    ax.annotate(f'{height:.2%}', (x + width/2, y + height*1.02), ha='center', fontsize = 'x-large')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:53:42.704504Z","iopub.execute_input":"2021-11-23T02:53:42.704808Z","iopub.status.idle":"2021-11-23T02:53:42.990984Z","shell.execute_reply.started":"2021-11-23T02:53:42.704781Z","shell.execute_reply":"2021-11-23T02:53:42.989739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**I gave you the basics of machine learning and deep learning. You should try things like generating derived variables, engineering features, and tuning hyperparameters to improve performance. Good luck!** 👍💥👍 ","metadata":{}}]}