{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport copy\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold \nfrom sklearn.decomposition import PCA\nfrom sklearn.linear_model import LogisticRegression, SGDClassifier\nfrom sklearn.metrics import accuracy_score, confusion_matrix, plot_confusion_matrix, classification_report,ConfusionMatrixDisplay\nfrom sklearn.utils import compute_class_weight, class_weight\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\n\nimport lightgbm as lgb\n\nfrom imblearn.over_sampling import SMOTE, BorderlineSMOTE, ADASYN, SVMSMOTE\nfrom imblearn.under_sampling import RandomUnderSampler, NearMiss, AllKNN\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:12:54.021380Z","iopub.execute_input":"2022-12-17T11:12:54.021722Z","iopub.status.idle":"2022-12-17T11:12:55.066143Z","shell.execute_reply.started":"2022-12-17T11:12:54.021655Z","shell.execute_reply":"2022-12-17T11:12:55.065531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import dataframes\ndf_train = pd.read_csv(\"/kaggle/input/2022-ucs757-lab-1-exam-new/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/2022-ucs757-lab-1-exam-new/test.csv\")\ndf_sub = pd.read_csv(\"/kaggle/input/2022-ucs757-lab-1-exam-new/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:12:56.283857Z","iopub.execute_input":"2022-12-17T11:12:56.285602Z","iopub.status.idle":"2022-12-17T11:13:04.127130Z","shell.execute_reply.started":"2022-12-17T11:12:56.285572Z","shell.execute_reply":"2022-12-17T11:13:04.126315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy dataframes to worksets\ntrain = df_train.copy()\ntest = df_test.copy()\nsub = df_sub.copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.129768Z","iopub.execute_input":"2022-12-17T11:13:04.130149Z","iopub.status.idle":"2022-12-17T11:13:04.240143Z","shell.execute_reply.started":"2022-12-17T11:13:04.130114Z","shell.execute_reply":"2022-12-17T11:13:04.239300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View train dataframe\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.241346Z","iopub.execute_input":"2022-12-17T11:13:04.241669Z","iopub.status.idle":"2022-12-17T11:13:04.262047Z","shell.execute_reply.started":"2022-12-17T11:13:04.241637Z","shell.execute_reply":"2022-12-17T11:13:04.261290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View test dataframe\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.264521Z","iopub.execute_input":"2022-12-17T11:13:04.264788Z","iopub.status.idle":"2022-12-17T11:13:04.283028Z","shell.execute_reply.started":"2022-12-17T11:13:04.264764Z","shell.execute_reply":"2022-12-17T11:13:04.282148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for types\ntrain.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.284236Z","iopub.execute_input":"2022-12-17T11:13:04.284641Z","iopub.status.idle":"2022-12-17T11:13:04.292629Z","shell.execute_reply.started":"2022-12-17T11:13:04.284611Z","shell.execute_reply":"2022-12-17T11:13:04.291769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for missing values\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.293676Z","iopub.execute_input":"2022-12-17T11:13:04.294072Z","iopub.status.idle":"2022-12-17T11:13:04.338715Z","shell.execute_reply.started":"2022-12-17T11:13:04.294049Z","shell.execute_reply":"2022-12-17T11:13:04.337848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check label distribution\nsns.countplot(data=train,x=\"target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.339714Z","iopub.execute_input":"2022-12-17T11:13:04.340004Z","iopub.status.idle":"2022-12-17T11:13:04.462926Z","shell.execute_reply.started":"2022-12-17T11:13:04.339971Z","shell.execute_reply":"2022-12-17T11:13:04.462159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create data and labels\nX = train.drop(\"target\",axis=1)\ny = train[\"target\"]\nX_test = test.drop(\"id\",axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:04.463938Z","iopub.execute_input":"2022-12-17T11:13:04.464218Z","iopub.status.idle":"2022-12-17T11:13:04.572070Z","shell.execute_reply.started":"2022-12-17T11:13:04.464187Z","shell.execute_reply":"2022-12-17T11:13:04.571163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Imbalanced labels need to be undersampled using the imblearn-library\n# undersample = NearMiss()\n# X, y = undersample.fit_resample(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:16.134124Z","iopub.execute_input":"2022-12-17T11:13:16.134448Z","iopub.status.idle":"2022-12-17T11:13:16.137790Z","shell.execute_reply.started":"2022-12-17T11:13:16.134422Z","shell.execute_reply":"2022-12-17T11:13:16.136973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Check labels\n# sns.countplot(x=y)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:17.927385Z","iopub.execute_input":"2022-12-17T11:13:17.927821Z","iopub.status.idle":"2022-12-17T11:13:17.930814Z","shell.execute_reply.started":"2022-12-17T11:13:17.927796Z","shell.execute_reply":"2022-12-17T11:13:17.930259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Weight distribution of the label.\nclasses = [0,1,2,3,4]\n\nweights = class_weight.compute_class_weight(class_weight='balanced',y=y,classes=classes)\n\nweights_dict = {}\nfor idx,clas in enumerate(classes):\n    weights_dict[clas] = weights[idx]\n\nweights_dict","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:19.266092Z","iopub.execute_input":"2022-12-17T11:13:19.266526Z","iopub.status.idle":"2022-12-17T11:13:19.282111Z","shell.execute_reply.started":"2022-12-17T11:13:19.266504Z","shell.execute_reply":"2022-12-17T11:13:19.281238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X.shape: {X.shape}\")\nprint(f\"y.shape: {y.shape}\")\nprint(f\"X_test.shape: {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:22.119519Z","iopub.execute_input":"2022-12-17T11:13:22.120580Z","iopub.status.idle":"2022-12-17T11:13:22.125485Z","shell.execute_reply.started":"2022-12-17T11:13:22.120544Z","shell.execute_reply":"2022-12-17T11:13:22.124661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scale the data\nscaler = StandardScaler()\n\nX = scaler.fit_transform(X)\nX_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:24.899734Z","iopub.execute_input":"2022-12-17T11:13:24.900243Z","iopub.status.idle":"2022-12-17T11:13:25.444343Z","shell.execute_reply.started":"2022-12-17T11:13:24.900211Z","shell.execute_reply":"2022-12-17T11:13:25.443528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use PCA to check for an 85% explained variance ratio.\npca = PCA()\n\nX_pca = pca.fit_transform(X)\n\n# Determine explained variance using explained_variance_ration_ attribute\nexp_var_pca = pca.explained_variance_ratio_\n\n# Cumulative sum of eigenvalues\ncum_sum_eigenvalues = np.cumsum(exp_var_pca)\n\n# Create the visualization plot\nplt.bar(range(0,len(exp_var_pca)), exp_var_pca, alpha=0.5, align='center', label='Individual explained variance')\nplt.step(range(0,len(cum_sum_eigenvalues)), cum_sum_eigenvalues, where='mid',label='Cumulative explained variance')\nplt.axhline(0.85,color=\"red\",linestyle=\"--\",label=\"85%\")\nplt.ylabel('Explained variance ratio')\nplt.xlabel('Principal component index')\nplt.legend(loc='best')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:13:27.639689Z","iopub.execute_input":"2022-12-17T11:13:27.640174Z","iopub.status.idle":"2022-12-17T11:13:36.444729Z","shell.execute_reply.started":"2022-12-17T11:13:27.640146Z","shell.execute_reply":"2022-12-17T11:13:36.443946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data in train and development set\nX_train, X_dev, y_train, y_dev = train_test_split(X, y, test_size=0.33, random_state=49,stratify=y)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:14:44.816483Z","iopub.execute_input":"2022-12-17T11:14:44.817217Z","iopub.status.idle":"2022-12-17T11:14:45.134680Z","shell.execute_reply.started":"2022-12-17T11:14:44.817189Z","shell.execute_reply":"2022-12-17T11:14:45.134038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply PCA and reduce to 200 features.\npca_200 = PCA(n_components=200)\n\nX_train = pca_200.fit_transform(X_train)\nX_dev = pca_200.transform(X_dev)\nX_test = pca_200.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:14:46.909100Z","iopub.execute_input":"2022-12-17T11:14:46.909503Z","iopub.status.idle":"2022-12-17T11:14:51.161074Z","shell.execute_reply.started":"2022-12-17T11:14:46.909476Z","shell.execute_reply":"2022-12-17T11:14:51.160332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X_train.shape: {X_train.shape}\")\nprint(f\"X_dev.shape: {X_dev.shape}\")\nprint(f\"X_test.shape: {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:16:00.698675Z","iopub.execute_input":"2022-12-17T11:16:00.698977Z","iopub.status.idle":"2022-12-17T11:16:00.703981Z","shell.execute_reply.started":"2022-12-17T11:16:00.698951Z","shell.execute_reply":"2022-12-17T11:16:00.703054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use various models from the sklearn-library.\nlr = LogisticRegression(penalty=\"l1\",C=0.001,solver=\"saga\",class_weight=weights_dict,random_state=49)\n# Gridsearch: ''C': 0.009, 'l1_ratio': 0.01, 'penalty': 'l1'\n# Accuracy: 0.38\n\ndtc = DecisionTreeClassifier(class_weight=weights_dict,random_state=49)\n# Gridsearch: 'max_depth': None, 'max_features': None\n# Accuracy: 0.35\n\nlightgb = lgb.LGBMClassifier(learning_rate=0.1,max_depth=15,num_leaves=60,class_weight=weights_dict,random_state=49)\n# Gridsearch: 'learning_rate': 0.1, 'max_depth': 15, 'num_leaves': 60\n# Accuracy: 0.48\n\nsvc = SVC(C=2,degree=2,class_weight=weights_dict,random_state=49)\n# Gridsearch: {'C': 2, 'degree': 2}\n# Accuracy: 0.35\n\nsgd = SGDClassifier(class_weight=weights_dict,random_state=49)\n# Gridsearch: 'l1_ratio': 0.4, 'loss': 'log', 'penalty': 'elasticnet'\n# Accuracy: 0.36\n\n\nestimators = [\n    (\"lr\",lr),\n    (\"dtc\",dtc),\n    (\"lightgb\",lightgb),\n    (\"svc\",svc),\n    (\"sgd\",sgd)\n]\n\nvoting = VotingClassifier(estimators=estimators,voting=\"hard\")\n# Gridsearch: 'voting': 'hard'\n\ncv = StratifiedKFold(n_splits=5,shuffle=True,random_state=49)\n\nparams = {\n}\n\ngrid = GridSearchCV(estimator=voting,param_grid=params,scoring=\"accuracy\",verbose=2,cv=cv)\ngrid.fit(X_train,y_train)\n\nprint(grid.best_params_)\nbest_model = grid.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:25:35.975667Z","iopub.execute_input":"2022-12-17T11:25:35.975973Z","iopub.status.idle":"2022-12-17T11:42:34.451104Z","shell.execute_reply.started":"2022-12-17T11:25:35.975948Z","shell.execute_reply":"2022-12-17T11:42:34.450204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on development set\ny_pred = best_model.predict(X_dev)\n\n# Accuracy score\nprint(f\"Accuracy: {classification_report(y_dev,y_pred)}\")\n\ncm = confusion_matrix(y_dev,y_pred)\nConfusionMatrixDisplay(confusion_matrix=cm).plot();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:43:25.129612Z","iopub.execute_input":"2022-12-17T11:43:25.129947Z","iopub.status.idle":"2022-12-17T11:43:25.432019Z","shell.execute_reply.started":"2022-12-17T11:43:25.129919Z","shell.execute_reply":"2022-12-17T11:43:25.431251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on the test set\npredictions = best_model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:43:35.103771Z","iopub.execute_input":"2022-12-17T11:43:35.104078Z","iopub.status.idle":"2022-12-17T11:43:35.124410Z","shell.execute_reply.started":"2022-12-17T11:43:35.104055Z","shell.execute_reply":"2022-12-17T11:43:35.123657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dataframe\nsub[\"target\"] = predictions\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:43:36.356009Z","iopub.execute_input":"2022-12-17T11:43:36.356350Z","iopub.status.idle":"2022-12-17T11:43:36.365881Z","shell.execute_reply.started":"2022-12-17T11:43:36.356321Z","shell.execute_reply":"2022-12-17T11:43:36.365104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of labels\nsns.countplot(data=sub,x=\"target\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T11:43:39.240722Z","iopub.execute_input":"2022-12-17T11:43:39.241018Z","iopub.status.idle":"2022-12-17T11:43:39.346143Z","shell.execute_reply.started":"2022-12-17T11:43:39.240994Z","shell.execute_reply":"2022-12-17T11:43:39.345499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit\nsub.to_csv(\"submissionBBG007.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T10:05:10.996201Z","iopub.execute_input":"2022-12-15T10:05:10.996701Z","iopub.status.idle":"2022-12-15T10:05:11.058707Z","shell.execute_reply.started":"2022-12-15T10:05:10.996674Z","shell.execute_reply":"2022-12-15T10:05:11.057819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}