{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nimport lightgbm as lgb\nimport gc\n\nimport lightgbm as lgb\nimport xgboost as xgb\n# sklearn\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nimport optuna\n\n\nimport sklearn.metrics\n\nimport numpy as np\nimport pandas as pd\nimport time\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nsns.set()\n\nimport operator as op \nfrom itertools import combinations\n\nfrom sklearn.metrics  import accuracy_score, auc, roc_curve, precision_recall_curve, roc_auc_score, precision_score, recall_score, average_precision_score\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport polars as pl","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-13T11:58:18.965006Z","iopub.execute_input":"2023-09-13T11:58:18.965519Z","iopub.status.idle":"2023-09-13T11:58:18.978640Z","shell.execute_reply.started":"2023-09-13T11:58:18.965461Z","shell.execute_reply":"2023-09-13T11:58:18.976912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntrain = (pl.scan_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\n          .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.hour()\n                 .alias(\"hour\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.weekday()\n                 .alias(\"weekday\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.month()\n                 .alias(\"month\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.day()\n                 .alias(\"day\")\n              )\n          )\n          .drop(\"timestamp\")\n          .collect()\n          .to_pandas()\n)\n\ntest = (pl.scan_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n          .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.hour()\n                 .alias(\"hour\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.weekday()\n                 .alias(\"weekday\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.month()\n                 .alias(\"month\")\n              )\n          )\n                     .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.day()\n                 .alias(\"day\")\n              )\n          )\n          .drop(\"timestamp\")\n          .collect()\n          .to_pandas()\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T11:58:19.421872Z","iopub.execute_input":"2023-09-13T11:58:19.422958Z","iopub.status.idle":"2023-09-13T11:58:58.955807Z","shell.execute_reply.started":"2023-09-13T11:58:19.422919Z","shell.execute_reply":"2023-09-13T11:58:58.954678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all numeric columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n#     start_mem = df.memory_usage().sum() / 1024**2\n#     print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object and col_type != 'datetime64[ns, UTC]':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    \n    df['series_id'] = df['series_id'].astype('category')\n#     end_mem = df.memory_usage().sum() / 1024**2\n#     print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n#     decrease = 100 * (start_mem - end_mem) / start_mem\n#     print(f'Decreased by {decrease:.2f}%')\n    gc.collect()\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:02:28.773151Z","iopub.execute_input":"2023-09-13T12:02:28.773781Z","iopub.status.idle":"2023-09-13T12:02:28.796806Z","shell.execute_reply.started":"2023-09-13T12:02:28.773732Z","shell.execute_reply":"2023-09-13T12:02:28.794169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df,x=True):\n    \n#     print('Start Feat Eng')\n#     print(df.shape)\n    df.sort_values(['series_id'], inplace=True)\n    \n    df['is_weekend'] = np.where(df['weekday']>5, 1, 0).astype(np.int8)\n    df['is_friday'] = np.where(df['weekday']==5, 1, 0).astype(np.int8)\n    df['is_saturday'] = np.where(df['weekday']==6, 1, 0).astype(np.int8)\n    df['step'] = df['step'] % 150\n#     print('Created timestamp derivates')\n#     print(df.shape)\n    gc.collect()\n    \n    # feature cross\n    df['anglez_x_enmo'] = df['anglez'] * df['enmo']\n    df['anglezabs_x_enmo'] = abs(df['anglez']) * df['enmo']\n    df['relu_anglez_x_enmo'] = np.maximum(df['anglez'], 0.) * df['enmo']\n    \n    for col in ['enmo', 'anglez', 'anglez_x_enmo']:\n        for n in [12, 36, 60]:\n            \n            df[f'{col}_diff_{n}'] = df.groupby('series_id')[col].diff(periods=n).astype(np.float32)\n            df[f'{col}_smavg_{n}'] = df[col].rolling(n, center=True).mean().astype(np.float32)\n            df[f'{col}_emavg_{n}'] = df[col].ewm(span=n).mean().astype(np.float32)\n            df[f'{col}_median_{n}'] = df[col].rolling(n, center=True).median().astype(np.float32)\n            df[f'{col}_max_{n}'] = df.groupby('series_id')[col].rolling(n, center=True).max().reset_index(drop=True).astype(np.float32)\n            df[f'{col}_min_{n}'] = df.groupby('series_id')[col].rolling(n, center=True).min().reset_index(drop=True).astype(np.float32)\n            df[f'{col}_skew_{n}'] = df.groupby('series_id')[col].rolling(n).skew().reset_index(drop=True).astype(np.float32)\n            df[f'{col}_kurt_{n}'] = df.groupby('series_id')[col].rolling(n).kurt().reset_index(drop=True).astype(np.float32)\n            gc.collect()\n            \n#         print(f'Created diff and smoothed derivates from {col}')\n#         print(df.shape)\n        \n        for n in range(1, 6):\n            df[f'{col}_lag_{n}'] = df.groupby('series_id')[col].shift(n).astype(np.float32)\n        gc.collect()\n#         print(f'Created lag derivates from {col}')\n#         print(df.shape)\n        \n        for n in range(1, 6):\n            df[f'{col}_lead_{n}'] = df.groupby('series_id')[col].shift(-n).astype(np.float32)\n        gc.collect() \n#         print(f'Created lead derivates from {col}')\n#         print(df.shape)\n\n    df.bfill(inplace=True, limit=5)\n    df.ffill(inplace=True, limit=5)\n    df = df.fillna(0)\n#     if x == True:\n#         df.dropna(inplace=True)\n    \n    df = reduce_mem_usage(df)\n    \n    return df\n\ndef batched_feat_eng(df, batch_size=1000000):\n    \"\"\"\n    Apply feature engineering to the dataframe in batches.\n    \"\"\"\n    total_rows = df.shape[0]\n    print(total_rows)\n    processed_batches = []\n\n    for start in range(0, total_rows, batch_size):\n        end = min(start + batch_size, total_rows)\n        batch = df.iloc[start:end].copy()\n        \n        # Apply the feat_eng function to the batch\n        processed_batch = feat_eng(batch, x=False)\n        processed_batches.append(processed_batch)\n        print(len(processed_batches))\n\n    # Combine processed batches\n    result_df = pd.concat(processed_batches, axis=0)\n    \n    return result_df\n\n# Test the batched feature engineering function\n# batched_df = batched_feat_eng(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:04:52.778430Z","iopub.execute_input":"2023-09-13T12:04:52.778923Z","iopub.status.idle":"2023-09-13T12:04:52.806352Z","shell.execute_reply.started":"2023-09-13T12:04:52.778886Z","shell.execute_reply":"2023-09-13T12:04:52.804522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-13T11:58:59.001871Z","iopub.execute_input":"2023-09-13T11:58:59.002217Z","iopub.status.idle":"2023-09-13T11:58:59.190779Z","shell.execute_reply.started":"2023-09-13T11:58:59.002181Z","shell.execute_reply":"2023-09-13T11:58:59.189650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# parse the timestamp and create an 'hour' feature\n# train['timestamp'] = pd.to_datetime(train['timestamp'],utc=True)\ntrain['awake'] = train['awake'].astype(np.int8)\ntrain = batched_feat_eng(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:04:55.927145Z","iopub.execute_input":"2023-09-13T12:04:55.927670Z","iopub.status.idle":"2023-09-13T12:11:30.416051Z","shell.execute_reply.started":"2023-09-13T12:04:55.927625Z","shell.execute_reply":"2023-09-13T12:11:30.414827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# parse the timestamp and create an 'hour' feature\n# test['timestamp'] = pd.to_datetime(test['timestamp'],utc=True)\ntest = feat_eng(test)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:14:26.975726Z","iopub.execute_input":"2023-09-13T12:14:26.976153Z","iopub.status.idle":"2023-09-13T12:14:30.450960Z","shell.execute_reply.started":"2023-09-13T12:14:26.976118Z","shell.execute_reply":"2023-09-13T12:14:30.449786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id', 'step']\n\nc = train.drop(columns=drop_cols).sample(50000).corr()\n\nX = train.drop(columns=drop_cols+['awake'])\ny = train['awake']","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:14:36.169550Z","iopub.execute_input":"2023-09-13T12:14:36.170078Z","iopub.status.idle":"2023-09-13T12:15:03.091753Z","shell.execute_reply.started":"2023-09-13T12:14:36.170034Z","shell.execute_reply":"2023-09-13T12:15:03.090396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:15:03.093740Z","iopub.execute_input":"2023-09-13T12:15:03.094109Z","iopub.status.idle":"2023-09-13T12:15:03.130258Z","shell.execute_reply.started":"2023-09-13T12:15:03.094065Z","shell.execute_reply":"2023-09-13T12:15:03.129072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Increase the size of the heatmap.\nplt.figure(figsize=(40, 40))\n# Store heatmap object in a variable to easily access it when you want to include more features (such as title).\n# Set the range of values to be displayed on the colormap from -1 to 1, and set the annotation to True to display the correlation values on the heatmap.\nheatmap = sns.heatmap(c.corr(), vmin=-1, vmax=1,cmap=\"coolwarm\", annot=False)\n# Give a title to the heatmap. Pad defines the distance of the title from the top of the heatmap.\nheatmap.set_title('Correlation Heatmap', fontdict={'fontsize':1}, pad=1);","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:15:03.131923Z","iopub.execute_input":"2023-09-13T12:15:03.132668Z","iopub.status.idle":"2023-09-13T12:15:08.434186Z","shell.execute_reply.started":"2023-09-13T12:15:03.132628Z","shell.execute_reply":"2023-09-13T12:15:08.432961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# upper = c.where(np.triu(np.ones(c.shape), k=1).astype(np.bool))\n# to_drop = [column for column in upper.columns if any(upper[column] > 0.9)]\n# X[to_drop]\n# Define the threshold for high correlation\nthreshold = 0.8\ncorrelations_with_target = c['awake'].drop('awake')\n\n# List to store features to keep\nfeatures_to_keep = list(correlations_with_target.index)\n\n# For each feature, if it has a high correlation with another feature (excluding the target), we might consider dropping it\nfor feature in features_to_keep:\n    # Drop if there's any other feature with which it has high correlation, excluding the target\n    high_corr_features = c[feature][(c[feature].abs() > threshold) & (c.index != 'awake')].index\n    for correlated_feature in high_corr_features:\n        if correlated_feature in features_to_keep:  # If the correlated feature is still in our list\n            features_to_keep.remove(correlated_feature)  # Remove the correlated feature\n\nfeatures_to_keep","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:15:08.436749Z","iopub.execute_input":"2023-09-13T12:15:08.437215Z","iopub.status.idle":"2023-09-13T12:15:08.475441Z","shell.execute_reply.started":"2023-09-13T12:15:08.437182Z","shell.execute_reply":"2023-09-13T12:15:08.474021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom sklearn.cluster import KMeans\nfrom sklearn.neighbors import NearestNeighbors\nimport numpy as np\n\n# print(combined_data_no_timestamp.dtypes)\ntrain_df= train[features_to_keep]\n# print(X_train_no_timestamp.dtypes)\ntest_df = test[features_to_keep]\ndef batched_resampling(train, test, batch_size=5000):\n    \"\"\"\n    Resample the training data in batches to find rows most similar to the test dataset.\n    \"\"\"\n    total_rows = train.shape[0]\n    resampled_indices = []\n\n    # Initialize Nearest Neighbors model\n    nbrs = NearestNeighbors(n_neighbors=1)\n\n    for start in range(0, total_rows, batch_size):\n        end = min(start + batch_size, total_rows)\n        train_batch = train.iloc[start:end]\n\n        # Fit the Nearest Neighbors model on the batch\n        nbrs.fit(train_batch)\n\n        # Find the nearest neighbors in the batch for the test data\n        try:\n            _, indices = nbrs.kneighbors(test.sample(100000))\n        except:\n            _, indices = nbrs.kneighbors(test.sample(450))\n\n        # Adjust indices to match the original training data\n        adjusted_indices = [idx + start for idx in indices.flatten()]\n        resampled_indices.extend(adjusted_indices)\n\n    # Extract unique indices and corresponding rows from the training data\n    unique_indices = np.unique(resampled_indices)\n    resampled_train = train.iloc[unique_indices]\n\n    return resampled_train\n\n# Apply batched resampling\nresampled_train_df = batched_resampling(train_df, test_df)\n\nresampled_train_df.shape\n","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:17:21.180253Z","iopub.execute_input":"2023-09-13T12:17:21.180700Z","iopub.status.idle":"2023-09-13T12:19:51.640183Z","shell.execute_reply.started":"2023-09-13T12:17:21.180666Z","shell.execute_reply":"2023-09-13T12:19:51.639114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.loc[resampled_train_df.index,features_to_keep]\ny = train.loc[resampled_train_df.index,'awake']","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:41:28.715407Z","iopub.execute_input":"2023-09-13T12:41:28.715932Z","iopub.status.idle":"2023-09-13T12:41:28.969444Z","shell.execute_reply.started":"2023-09-13T12:41:28.715887Z","shell.execute_reply":"2023-09-13T12:41:28.967588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, valid_x, train_y, valid_y = train_test_split(X, y,test_size=0.2, random_state = 123)\n#         train_x, valid_x, train_y, valid_y = train_test_split(X, y,test_size=0.95, random_state = 123)\n\n#             train_x, valid_x, train_y, valid_y = train_test_split(X, y,test_size=0.25, random_state = 123)\ndtrain_xbg = xgb.DMatrix(train_x, label=train_y)\ndvalid_xbg = xgb.DMatrix(valid_x, label=valid_y)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:29:44.164351Z","iopub.execute_input":"2023-09-13T12:29:44.164668Z","iopub.status.idle":"2023-09-13T12:29:44.737722Z","shell.execute_reply.started":"2023-09-13T12:29:44.164641Z","shell.execute_reply":"2023-09-13T12:29:44.736761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_x, valid_x, train_y, X, c, train","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:29:44.739441Z","iopub.execute_input":"2023-09-13T12:29:44.740181Z","iopub.status.idle":"2023-09-13T12:29:44.800707Z","shell.execute_reply.started":"2023-09-13T12:29:44.740138Z","shell.execute_reply":"2023-09-13T12:29:44.799495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/zulqarnainali/sleep-state-prediction-ipynb\nfrom sklearn.metrics import roc_auc_score, accuracy_score, log_loss  # 📏 Metrics for evaluation\nfrom sklearn.model_selection import cross_validate  # 📚 Cross-validation utility\nfrom sklearn.metrics import RocCurveDisplay  # 📈 ROC curve display\nfrom sklearn.metrics import confusion_matrix  # ⚖️ Confusion matrix for classification\nfrom sklearn.metrics import ConfusionMatrixDisplay  # 📊 Confusion matrix display\nfrom sklearn.metrics import precision_score  # 👆 Precision score for classification\nfrom sklearn.metrics import average_precision_score  # 📉 Average precision score\n# 🌈 Creating a custom color palette using Seaborn\nmy_palette = sns.cubehelix_palette(n_colors=7, start=.46, rot=-.45, dark=.2, hue=0.95, as_cmap=True)\n\n# 📊 Function to display the confusion matrix and ROC curve\ndef show_confusion_roc(oof, title='Model Evaluation Results'):\n    # Creating a 1x2 subplot grid with specified figure size\n    f, ax = plt.subplots(1, 2, figsize=(16, 6))\n    \n    # Creating a DataFrame for predictions and true labels\n    df = pd.DataFrame({'preds': oof[0], 'target': oof[1]})\n    \n    # Calculating and plotting the confusion matrix\n    cm = confusion_matrix(df.target, df.preds.ge(0.5).astype(int))\n    cm_display = ConfusionMatrixDisplay(cm).plot(cmap=my_palette, ax=ax[0])\n    ax[0].grid(False)\n    \n    # Plotting the ROC curve\n    RocCurveDisplay.from_predictions(df.target, df.preds, ax=ax[1])\n    ax[1].grid(True)\n    \n    # Adding a title to the overall plot\n    plt.suptitle(f'{title}', fontsize=12, fontweight='bold')\n    \n    # Ensuring tight layout for better visualization\n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:29:45.679462Z","iopub.execute_input":"2023-09-13T12:29:45.679909Z","iopub.status.idle":"2023-09-13T12:29:45.693525Z","shell.execute_reply.started":"2023-09-13T12:29:45.679875Z","shell.execute_reply":"2023-09-13T12:29:45.692105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Objective:\n\n    def __init__(self):\n        self._xgb = None\n        self.predictions = None\n        self.fpredictions = None\n\n    def __call__(self, trial):\n\n        xgb_preds = np.zeros((math.ceil(len(y)*0.25),1), dtype=np.int)\n        xgb_param = {\n          \"objective\": \"binary:logistic\", # change for multiclass\n          \"eval_metric\": \"auc\",\n          # \"num_class\": 3, # change this up depending on multiclass | dont use with binary\n          \"booster\": trial.suggest_categorical(\"booster2\", [\"gbtree\", \"gblinear\", \"dart\"]),\n          \"lambda\": trial.suggest_loguniform(\"lambda2\", 1e-8, 1.0),\n          \"alpha\": trial.suggest_loguniform(\"alpha2\", 1e-8, 1.0),\n        }\n        if xgb_param[\"booster\"] == \"gbtree\" or xgb_param[\"booster\"] == \"dart\":\n            xgb_param[\"max_depth\"] = trial.suggest_int(\"max_depth2\", 1, 250)\n            xgb_param[\"eta\"] = trial.suggest_loguniform(\"eta2\", 1e-8, 1.0)\n            xgb_param[\"gamma\"] = trial.suggest_loguniform(\"gamma2\", 1e-8, 1.0)\n            xgb_param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy2\", [\"depthwise\", \"lossguide\"])\n        if xgb_param[\"booster\"] == \"dart\":\n            xgb_param[\"sample_type\"] = trial.suggest_categorical(\"sample_type2\", [\"uniform\", \"weighted\"])\n            xgb_param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type2\", [\"tree\", \"forest\"])\n            xgb_param[\"rate_drop\"] = trial.suggest_loguniform(\"rate_drop2\", 1e-8, 1.0)\n            xgb_param[\"skip_drop\"] = trial.suggest_loguniform(\"skip_drop2\", 1e-8, 1.0)\n        # Add a callback for pruning.\n        xgb_pruning_callback = optuna.integration.XGBoostPruningCallback(trial, \"validation-auc\" )\n        xgb_ = xgb.train(xgb_param, dtrain_xbg, evals=[(dvalid_xbg, \"validation\")], verbose_eval=False, callbacks=[xgb_pruning_callback])\n        xgb_preds = xgb_.predict(dvalid_xbg)\n        self._xgb = xgb_\n\n        preds = xgb_preds\n#         print(preds)\n#         preds = preds[:1][0]\n\n\n        self.predictions = preds\n        auc = average_precision_score(valid_y, preds)\n        return auc\n\n    def callback(self, study, trial):\n        if study.best_trial == trial:\n            self.best_xgb = self._xgb\n            self.fpredictions = self.predictions\n#             show_confusion_roc(oof=[self.predictions, valid_y], title='EC1 OOF Evaluation Results')\n#             print(self.predictions)\n            \n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning) # for log error\n\nimport math\nimport optuna\nobjective = Objective()\n\n# Setting SEED \nfrom optuna.samplers import TPESampler\nsampler = TPESampler(seed=10)\n\nstudy = optuna.create_study(\n    pruner=optuna.pruners.MedianPruner(n_warmup_steps=10), direction=\"maximize\",\n    sampler=sampler\n)\nstudy.optimize(objective, n_trials=13, callbacks=[objective.callback]) # change this to 500 + \n\nprint(\"Best trial:\")\ntrial = study.best_trial\n\nprint(\"  Params: \")\nfor key, value in trial.params.items():\n    print(\"    {}: {}\".format(key, value))","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:29:52.400318Z","iopub.execute_input":"2023-09-13T12:29:52.400807Z","iopub.status.idle":"2023-09-13T12:40:31.266723Z","shell.execute_reply.started":"2023-09-13T12:29:52.400772Z","shell.execute_reply":"2023-09-13T12:40:31.265474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = objective.best_xgb\n\nshow_confusion_roc(oof=[objective.fpredictions, valid_y], title='Evaluation Results')","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:40:31.268863Z","iopub.execute_input":"2023-09-13T12:40:31.269209Z","iopub.status.idle":"2023-09-13T12:40:32.190235Z","shell.execute_reply.started":"2023-09-13T12:40:31.269179Z","shell.execute_reply":"2023-09-13T12:40:32.189045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# series_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id', 'step'])\n# test  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n# test['timestamp'] = pd.to_datetime(test['timestamp'],utc=True)\n# print(test.shape)\n# test = feat_eng(test, x=False)\nprint(test.shape)\nX_test = test[features_to_keep]\nprint(test.shape)\ndtest_xbg = xgb.DMatrix(X_test)\nprint(test.shape)\ntest['score'] = m.predict(dtest_xbg)\ntest['not_awake'] = 1-test['score']\n# exponential smoothing of the predictions\ntest['smooth'] = test['not_awake'].ewm(span = 100).mean()\n# re-binarize\ntest['smooth'] = test['smooth'].round()\n\n# https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: \n                            (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False: \n            lstPOI.extend([0]*llg)\n        else: \n            lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest['event'] = get_event(test)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:42:02.923911Z","iopub.execute_input":"2023-09-13T12:42:02.926145Z","iopub.status.idle":"2023-09-13T12:42:02.973416Z","shell.execute_reply.started":"2023-09-13T12:42:02.926038Z","shell.execute_reply":"2023-09-13T12:42:02.972008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test[['series_id','step','event','score']].loc[test[\"event\"] != 0]","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:42:05.810122Z","iopub.execute_input":"2023-09-13T12:42:05.811706Z","iopub.status.idle":"2023-09-13T12:42:05.821054Z","shell.execute_reply.started":"2023-09-13T12:42:05.811653Z","shell.execute_reply":"2023-09-13T12:42:05.819491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:42:08.576162Z","iopub.execute_input":"2023-09-13T12:42:08.576593Z","iopub.status.idle":"2023-09-13T12:42:08.591923Z","shell.execute_reply.started":"2023-09-13T12:42:08.576562Z","shell.execute_reply":"2023-09-13T12:42:08.590415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(test['score'])","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:42:26.264773Z","iopub.execute_input":"2023-09-13T12:42:26.265186Z","iopub.status.idle":"2023-09-13T12:42:26.679028Z","shell.execute_reply.started":"2023-09-13T12:42:26.265150Z","shell.execute_reply":"2023-09-13T12:42:26.677788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.copy().reset_index(drop=True).reset_index(names='row_id')\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-13T12:42:38.119992Z","iopub.execute_input":"2023-09-13T12:42:38.120517Z","iopub.status.idle":"2023-09-13T12:42:38.139318Z","shell.execute_reply.started":"2023-09-13T12:42:38.120477Z","shell.execute_reply":"2023-09-13T12:42:38.137742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}