{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import plot_roc_curve\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import make_scorer\n\npd.options.display.max_columns = None","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"submission_test = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")\ntrain = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train.columns:\n    print(col)\n    print(train[col].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = OneHotEncoder()\ndef change_column_order(df, col_number, number_of_columns):\n    cols = df.columns.tolist()\n    cols = cols[:col_number] + cols[-number_of_columns:] + cols[col_number:-number_of_columns]\n    df = df[cols]\n    return df\n\ndef one_hot_encoding(df, col_name):\n    df[col_name] = df[col_name].fillna(\"NaN\")\n    unique_vals = sorted(df[col_name].unique().tolist())\n    columns = {v: f\"{col_name}_{k}\" for v, k in enumerate(unique_vals)}\n    enc_df = pd.DataFrame(encoder.fit_transform(df[[col_name]]).toarray()).rename(columns=columns)\n    df = df.join(enc_df)\n    df = change_column_order(df, df.columns.get_loc(col_name), len(unique_vals))\n    del df[col_name]\n    del df[f\"{col_name}_NaN\"]\n    return df\n\ndef hexstr2int(df, col_name):\n    df[col_name] = df[col_name].map(lambda x: int(x, 16) if isinstance(x, str) else x)\n    df[col_name] = df[col_name].fillna(df[col_name].median())\n    return df\n\ndef preprocessing(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.copy()\n    \n    df[\"bin_0\"] = df[\"bin_0\"].fillna(df[\"bin_0\"].median())\n    df[\"bin_1\"] = df[\"bin_1\"].fillna(df[\"bin_1\"].median())\n    df[\"bin_2\"] = df[\"bin_2\"].fillna(df[\"bin_2\"].median())\n    \n    df[\"bin_3\"] = df[\"bin_3\"].map({\"F\": 0, \"T\": 1})\n    df[\"bin_3\"] = df[\"bin_3\"].fillna(df[\"bin_3\"].median())\n    \n    df[\"bin_4\"] = df[\"bin_4\"].map({\"N\": 0, \"Y\": 1})\n    df[\"bin_4\"] = df[\"bin_4\"].fillna(df[\"bin_4\"].median())\n    \n    for i in range(0, 5):\n        df = one_hot_encoding(df, f\"nom_{i}\")\n    for i in range(5, 10):\n        df = hexstr2int(df, f\"nom_{i}\")\n    \n    df[\"ord_0\"] = df[\"ord_0\"].fillna(df[\"ord_0\"].median())\n    \n    df[\"ord_1\"] = df[\"ord_1\"].map({\"Novice\": 0, \"Contributor\": 1, \"Expert\": 2, \"Master\": 3, \"Grandmaster\": 4})\n    df[\"ord_1\"] = df[\"ord_1\"].fillna(df[\"ord_1\"].median())\n    \n    df[\"ord_2\"] = df[\"ord_2\"].map({\"Freezing\": 0, \"Cold\": 1, \"Warm\": 2, \"Hot\": 3, \"Boiling Hot\": 4, \"Lava Hot\": 5})\n    df[\"ord_2\"] = df[\"ord_2\"].fillna(df[\"ord_2\"].median())\n    \n    df[\"ord_3\"] = df[\"ord_3\"].map(lambda x: ord(x) if isinstance(x, str) else x)\n    df[\"ord_3\"] = df[\"ord_3\"].fillna(df[\"ord_3\"].median())\n    \n    df[\"ord_4\"] = df[\"ord_4\"].map(lambda x: ord(x) if isinstance(x, str) else x)\n    df[\"ord_4\"] = df[\"ord_4\"].fillna(df[\"ord_4\"].median())\n    \n    df[\"ord_5\"] = df[\"ord_5\"].map(lambda x: sum(ord(ch) for ch in x) if isinstance(x, str) else x)\n    df[\"ord_5\"] = df[\"ord_5\"].fillna(df[\"ord_5\"].median())\n    \n    df[\"day\"] = df[\"day\"].fillna(df[\"day\"].median())\n    df[\"month\"] = df[\"month\"].fillna(df[\"month\"].median())\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Normalizer:\n    def __init__(self, X_train: pd.DataFrame):\n        self.mins = [X_train[col].min() for col in X_train]\n        self.maxes = [X_train[col].max() for col in X_train]\n\n    def __call__(self, df: pd.DataFrame):\n        for idx, col in enumerate(df):\n            df[col] = (df[col] - self.mins[idx]) / (self.maxes[idx] - self.mins[idx])\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = train.columns.tolist()\ncols.remove(\"target\")\ncols.remove(\"id\")\npreprocessed = preprocessing(train[cols])\npreprocessed_submission = preprocessing(submission_test[cols])\nX_train, X_test, y_train, y_test = train_test_split(preprocessed, train[\"target\"], test_size=0.20, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalizer = Normalizer(X_train)\nX_train = normalizer(X_train)\nX_test = normalizer(X_test)\npreprocessed = normalizer(preprocessed)\npreprocessed_submission = normalizer(preprocessed_submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_submission.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def logistic_regression_grid_search(solver: tuple, penalties: tuple) -> GridSearchCV:\n    tol = (0.01, 0.001, 0.0001)\n    C = (0.1, 0.5, 1.0, 1.5, 10.0)\n    params = {\"penalty\": penalties, \"tol\": tol, \"C\": C, \"solver\": solver, \"random_state\": (42,), \"dual\": (False, )}\n    search = GridSearchCV(LogisticRegression(), param_grid=params, verbose=5, return_train_score=True, scoring=make_scorer(roc_auc_score), n_jobs=-1)\n    search.fit(X_train, y_train)\n    return search","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newton = logistic_regression_grid_search((\"newton-cg\",), (\"l2\", \"none\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lbfgs = logistic_regression_grid_search((\"lbfgs\",), (\"l2\", \"none\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"liblinear = logistic_regression_grid_search((\"liblinear\",), (\"l1\",))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sag = logistic_regression_grid_search((\"sag\",), (\"l2\", \"none\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"saga = logistic_regression_grid_search((\"saga\",), (\"l1\", \"l2\", \"elasticnet\", \"none\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(newton.best_score_)\nprint(newton.best_params_)\nprint()\nprint(lbfgs.best_score_)\nprint(lbfgs.best_params_)\nprint()\nprint(liblinear.best_score_)\nprint(liblinear.best_params_)\nprint()\nprint(sag.best_score_)\nprint(sag.best_params_)\nprint()\nprint(saga.best_score_)\nprint(saga.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def linear_svc_grid_search():\n    tol = (0.1, 0.01, 0.001, 0.0001)\n    C = (0.1, 0.5, 1.0, 1.5, 10.0)\n    penalty = (\"l1\", \"l2\")\n    loss = (\"hinge\", \"squared_hinge\")\n    params = {\"penalty\": penalty, \"tol\": tol, \"C\": C, \"loss\": loss, \"dual\": (False, )}\n    search = GridSearchCV(LinearSVC(), param_grid=params, verbose=5, return_train_score=True, scoring=make_scorer(roc_auc_score), n_jobs=-1)\n    search.fit(X_train, y_train)\n    return search\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"svc_search = linear_svc_grid_search()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(svc_search.best_score_)\nprint(svc_search.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_best_threshold(model, X_test, y_test) -> tuple:\n    best = [0, 0]\n    predicted = model.predict_proba(X_test)\n    for i in range(0, 101):\n        threshold = i / 100\n        score = f1_score(y_test, (predicted >= threshold).astype(int)[:, 1])\n        if score >= best[1]:\n            best = [threshold, score]\n    return tuple(best)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_sorted_coefs(model):\n    coefs = model.coef_[0]\n    coefs = tuple((col, coefs[index]) for index, col in enumerate(X_train.columns))\n    coefs = sorted(coefs, key=lambda x: abs(x[1]), reverse=True)\n    for coef in coefs:\n        print(coef)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_lr = sorted((newton, lbfgs, liblinear, sag, saga), key=lambda x: x.best_score_, reverse=True)[0]\nprint(best_lr.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LogisticRegression(**best_lr.best_params_)\nmodel.fit(X_train, y_train)\nprint_sorted_coefs(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_roc_curve(model, X_test, y_test)\nprint(f\"f1 score: {f1_score(y_test, model.predict(X_test))}\")\nprint(f\"roc auc:  {roc_auc_score(y_test, model.predict_proba(X_test)[:, 1])}\")\nbest = find_best_threshold(model, X_test, y_test)\nprint(f\"Best f1 score: {best[1]}; for threshold: {best[0]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_svc = LinearSVC(**svc_search.best_params_)\nmodel_svc.fit(X_train, y_train)\nprint_sorted_coefs(model_svc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_roc_curve(model_svc, X_test, y_test)\nprint(f\"f1 score: {f1_score(y_test, model_svc.predict(X_test))}\")\nprint(f\"roc auc:  {roc_auc_score(y_test, model_svc.predict(X_test))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LogisticRegression(**best_lr.best_params_)\nmodel.fit(preprocessed, train[\"target\"])\nprint_sorted_coefs(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted = model.predict_proba(preprocessed_submission)\npredicted = (predicted >= best[0]).astype(int)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\"id\": np.array(submission_test.id), \"target\": predicted})\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}