{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Feature Encoding Challenge II\nBinary classification, with every feature a categorical (and i\na dataset that contains only categorical features, and includes:\n\n* binary features\n* low- and high-cardinality nominal features\n* low- and high-cardinality ordinal features\n* (potentially) cyclical features\n\nThis challenge adds the additional complexity of feature interactions, as well as missing data.\n\nIn this competition, you will be predicting the probability [0, 1] of a binary target column. Submissions are evaluated on area under the ROC curve between the predicted probability and the observed target.\n\nThe data contains binary features (bin_), nominal features (nom_), ordinal features (ord_) as well as (potentially cyclical) day (of the week) and month features. The string ordinal features ord_{3-5} are lexically ordered according to string.ascii_letters.\n\nFinal submission deadline: March 31, 2020"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n!pip install --upgrade wandb\nimport wandb\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport scipy\nimport pandas as pd\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import Ridge\nfrom sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingClassifier\n!pip install cuml\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn import naive_bayes\nfrom sklearn.model_selection import train_test_split\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\", index_col=\"id\")\ndf_test = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\", index_col=\"id\")\n\ny = df[\"target\"]\nD = df.drop(columns=\"target\")\nfeatures = D.columns\ntest_ids = df_test.index\n\nD_all = pd.concat([D, df_test])\nnum_train = len(D)\n\nprint(f\"D_all.shape = {D_all.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Map value in train xor test\nfor col in D.columns.difference([\"id\"]):\n    train_vals = set(D[col].dropna().unique())\n    test_vals = set(df_test[col].dropna().unique())\n\n    xor_cat_vals = train_vals ^ test_vals\n    if xor_cat_vals:\n        print(f\"Replacing {len(xor_cat_vals)} values in {col}, {xor_cat_vals}\")\n        D_all.loc[D_all[col].isin(xor_cat_vals), col] = \"xor\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ordinal encoding\nord_maps = {\n    \"ord_0\": {val: i for i, val in enumerate([1, 2, 3])},\n    \"ord_1\": {\n        val: i\n        for i, val in enumerate(\n            [\"Novice\", \"Contributor\", \"Expert\", \"Master\", \"Grandmaster\"]\n        )\n    },\n    \"ord_2\": {\n        val: i\n        for i, val in enumerate(\n            [\"Freezing\", \"Cold\", \"Warm\", \"Hot\", \"Boiling Hot\", \"Lava Hot\"]\n        )\n    },\n    **{col: {val: i for i, val in enumerate(sorted(D_all[col].dropna().unique()))} for col in [\"ord_3\", \"ord_4\", \"ord_5\", \"day\", \"month\"]},\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# OneHot encoding\noh_cols = D_all.columns.difference(ord_maps.keys() - {\"day\", \"month\"})\n\nprint(f\"OneHot encoding {len(oh_cols)} columns\")\n\none_hot = pd.get_dummies(\n    D_all[oh_cols],\n    columns=oh_cols,\n    drop_first=True,\n    dummy_na=True,\n    sparse=True,\n    dtype=\"int8\",\n).sparse.to_coo()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ordinal encoding\nord_cols = pd.concat([D_all[col].map(ord_map).fillna(max(ord_map.values())//2).astype(\"float32\") for col, ord_map in ord_maps.items()], axis=1)\nord_cols /= ord_cols.max()  # for convergence\n\nord_cols_sqr = 4*(ord_cols - 0.5)**2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine data\nX = scipy.sparse.hstack([one_hot, ord_cols, ord_cols_sqr]).tocsr()\nprint(f\"X.shape = {X.shape}\")\n\n# Split into training and validation sets\nX_train, X_test, y_train, y_test = train_test_split(X[:num_train], y, test_size=0.1, random_state=42, shuffle=False)\nX_train = X_train[:10000]\ny_train = y_train[:10000]\nX_test = X_test[:2000]\ny_test = y_test[:2000]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train models, visualize in sklearn"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Classification - predict pulsar\n# Train a model, get predictions\nlog = LogisticRegression(C=0.05, solver=\"lbfgs\", max_iter=5000)\ndtree = DecisionTreeClassifier(random_state=4)\nrtree = RandomForestClassifier(n_estimators=100, random_state=4)\nsvm = SVC(random_state=4, probability=True)\nnb = GaussianNB()\ngbc = GradientBoostingClassifier()\nknn = KNeighborsClassifier(n_neighbors=400)\nadaboost = AdaBoostClassifier(n_estimators=500, learning_rate=0.01, random_state=42,\n                             base_estimator=DecisionTreeClassifier(max_depth=8,\n                             min_samples_leaf=10, random_state=42))\nlabels = [0,1]\n\ndef model_algorithm(clf, X_train, y_train, X_test, y_test, name, labels, features):\n    clf.fit(X_train, y_train)\n    y_probas = clf.predict_proba(X_test)\n    y_pred = clf.predict(X_test)\n    wandb.init(anonymous='allow', project=\"kaggle-feature-encoding\", name=name, reinit=True)\n    # wandb.sklearn.plot_roc(y_test, y_probas, labels, reinit = True)\n    wandb.termlog('\\nPlotting %s.'%name)\n    wandb.sklearn.plot_learning_curve(clf, X_train, y_train)\n    wandb.termlog('Logged learning curve.')\n    wandb.sklearn.plot_confusion_matrix(y_test, y_pred, labels)\n    wandb.termlog('Logged confusion matrix.')\n    wandb.sklearn.plot_summary_metrics(clf, X=X_train, y=y_train, X_test=X_test, y_test=y_test)\n    wandb.termlog('Logged summary metrics.')\n    wandb.sklearn.plot_class_proportions(y_train, y_test, labels)\n    wandb.termlog('Logged class proportions.')\n    if(not isinstance(clf, naive_bayes.MultinomialNB)):\n        wandb.sklearn.plot_calibration_curve(clf, X_train, y_train, name)\n    wandb.termlog('Logged calibration curve.')\n    wandb.sklearn.plot_roc(y_test, y_probas, labels)\n    wandb.termlog('Logged roc curve.')\n    wandb.sklearn.plot_precision_recall(y_test, y_probas, labels)\n    wandb.termlog('Logged precision recall curve.')\n    csv_name = \"submission_\"+name+\".csv\"\n    # Create submission file\n    # pd.DataFrame({\"id\": test_ids, \"target\": y_pred}).to_csv(csv_name, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(log, X_train, y_train, X_test, y_test, 'LogisticRegression', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(svm, X_train, y_train, X_test, y_test, 'SVM', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(knn, X_train, y_train, X_test, y_test, 'KNearestNeighbor', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(adaboost, X_train, y_train, X_test, y_test, 'AdaBoost', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(gbc, X_train, y_train, X_test, y_test, 'GradientBoosting', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(dtree, X_train, y_train, X_test, y_test, 'DecisionTree', labels, None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_algorithm(rtree, X_train, y_train, X_test, y_test, 'RandomForest', labels, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf=LogisticRegression(C=0.05, solver=\"lbfgs\", max_iter=5000)\nclf.fit(X_train, y_train)\npred = clf.predict_proba(X_test)[:, 1]\npd.DataFrame({\"id\": test_ids, \"target\": pred}).to_csv(\"submission_lr.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Add Sweep"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Add Stacking & Blending"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Report"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}