# %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:42:34.965160Z","iopub.execute_input":"2023-12-05T14:42:34.966686Z","iopub.status.idle":"2023-12-05T14:42:36.528804Z","shell.execute_reply.started":"2023-12-05T14:42:34.966644Z","shell.execute_reply":"2023-12-05T14:42:36.527343Z"}} from sklearn.metrics import confusion_matrix, classification_report, roc_curve, roc_auc_score import matplotlib.pyplot as plt import numpy as np def evaluate_model(set_name, y_set, predictions, probabilities): # Plot the confusion matrix cm = confusion_matrix(y_set, predictions) plt.figure(figsize=(6, 4)) plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues) plt.title(f"Confusion Matrix for {set_name} set") plt.colorbar() classes = [0, 1] tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes) plt.yticks(tick_marks, classes) plt.xlabel("Predicted") plt.ylabel("True") for i in range(len(classes)): for j in range(len(classes)): plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment="center", color="white" if cm[i, j] > cm.max() / 2 else "black") plt.show() # Print the classification report print(f"Classification Report for {set_name} set:\n", classification_report(y_set, predictions)) # Plot the ROC curve fpr, tpr, thresholds = roc_curve(y_set, probabilities) roc_auc = roc_auc_score(y_set, probabilities) plt.figure(figsize=(8, 6)) plt.plot(fpr, tpr, label=f'{set_name} set ROC curve (AUC = {roc_auc:.2f})') plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title(f'Receiver Operating Characteristic (ROC) Curve for {set_name} set') plt.legend(loc="lower right") plt.show() # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:42:36.531390Z","iopub.execute_input":"2023-12-05T14:42:36.531834Z","iopub.status.idle":"2023-12-05T14:42:36.962251Z","shell.execute_reply.started":"2023-12-05T14:42:36.531794Z","shell.execute_reply":"2023-12-05T14:42:36.960716Z"}} import pandas as pd def Categorize_hours(data, num_categories): category_size = 24 // num_categories categories = [] for hour in data.hour: category_label = hour // category_size categories.append(f"{category_label}") int8_list = [int(value) for value in categories] data_quartile = pd.DataFrame(int8_list,columns=["day_quartile"]) return pd.concat([data,data_quartile],axis =1) def add_features(data): data["day of year"] = data.timestamp.dt.dayofyear data["day of week"] = data.timestamp.dt.day_of_week #can also do day_name() for the name of the day data["month"] = data.timestamp.dt.month # can do month_name() for data["days in month"] = data.timestamp.dt.days_in_month data["minute"] = data.timestamp.dt.minute return data def Create_lag_features(data, window_size,data_name1,data_name2): data_frame1 = pd.DataFrame(data[data_name1]) for i in range(1, window_size+1): data_frame1[f'{data_name1} lag_{i}'] = data_frame1[data_name1].shift(i) data_frame1 = data_frame1.drop(data_name1,axis= 1) data_frame2 = pd.DataFrame(data[data_name2]) for i in range(1, window_size+1): data_frame2[f'{data_name2} lag_{i}'] = data_frame2[data_name2].shift(i) data_frame2 = data_frame2.drop(data_name2,axis= 1) return pd.concat([data_frame1,data_frame2],axis =1 ) def Create_rolling_and_expanding_statistics(data, window_size, data_name): rolling_stats = data[f"{data_name}"].rolling(window=window_size) expanding_stats = data[f"{data_name}"].expanding(min_periods=1) rolling_Data = pd.DataFrame({ f"{data_name} rolling mean": rolling_stats.mean().astype("float16"), # can also add diff of this windows mean to the previous windows mean f"{data_name} rolling std": rolling_stats.std().astype("float16"), f"{data_name} rolling median": rolling_stats.median().astype("float16"), f"{data_name} rolling max": rolling_stats.max().astype("float16"), f"{data_name} rolling min": rolling_stats.min().astype("float16") }) expanding_Data = pd.DataFrame({ f"{data_name} expanding mean": expanding_stats.mean().astype("float16"), f"{data_name} expanding std": expanding_stats.std().astype("float16"), f"{data_name} expanding median": expanding_stats.median().astype("float16"), f"{data_name} expanding max": expanding_stats.max().astype("float16"), f"{data_name} expanding min": expanding_stats.min().astype("float16") }) rolling_Data = rolling_Data.drop(f"{data_name}") expanding_Data = expanding_Data.drop(f"{data_name}") return pd.concat([data,rolling_Data,expanding_Data],axis = 1) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:42:36.971177Z","iopub.execute_input":"2023-12-05T14:42:36.976581Z","iopub.status.idle":"2023-12-05T14:42:36.993671Z","shell.execute_reply.started":"2023-12-05T14:42:36.976507Z","shell.execute_reply":"2023-12-05T14:42:36.992271Z"}} import pandas as pd import matplotlib.pyplot as plt from colorama import Fore, Style, init from pprint import pprint def plot(data,graph_type:str,Title:str): plt.figure(figsize=(12, 8)) data.plot(kind=f'{graph_type}', color='skyblue') plt.title(f'{Title}') plt.xlabel('Values') plt.ylabel('Count') plt.xticks(rotation=45) plt.show() def clean_df(df: pd.DataFrame) -> pd.DataFrame: df['date'] = df['timestamp'].str.split('T', expand=True)[0] df['time'] = df['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0] df['timestamp'] = pd.to_datetime(df['date']+' '+df['time']) df['hour'] = df['timestamp'].dt.hour return df def transform_and_rename_dataframe( dataframe: pd.DataFrame, id_map_dataframe: pd.DataFrame, merge_on: str, drop_columns: list[str], rename_columns: dict[str] ) -> pd.DataFrame: """ Transform and rename a DataFrame by merging it with another DataFrame, dropping specified columns, and renaming columns. Args: - dataframe (pd.DataFrame): The input DataFrame. - id_map_dataframe (pd.DataFrame): The DataFrame to merge with. - merge_on (str): The column on which to perform the merge. - drop_columns (list): List of column names to drop. - rename_columns (dict): Dictionary to rename columns. Returns: - pd.DataFrame: The transformed and renamed DataFrame. """ transformed_dataframe = dataframe.merge(id_map_dataframe, on=merge_on, how='left') transformed_dataframe = transformed_dataframe.drop(columns=drop_columns) transformed_dataframe = transformed_dataframe.rename(columns=rename_columns) return transformed_dataframe def CPrint(text: str, colour=Fore.BLUE, style=Style.BRIGHT): print(style+colour+text+Style.RESET_ALL) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:42:36.998520Z","iopub.execute_input":"2023-12-05T14:42:36.999334Z","iopub.status.idle":"2023-12-05T14:42:37.012464Z","shell.execute_reply.started":"2023-12-05T14:42:36.999291Z","shell.execute_reply":"2023-12-05T14:42:37.011343Z"}} # This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the read-only "../input/" directory # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory import os for dirname, _, filenames in os.walk('/kaggle/input'): for filename in filenames: print(os.path.join(dirname, filename)) # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:42:37.014162Z","iopub.execute_input":"2023-12-05T14:42:37.014908Z","iopub.status.idle":"2023-12-05T14:43:32.005299Z","shell.execute_reply.started":"2023-12-05T14:42:37.014852Z","shell.execute_reply":"2023-12-05T14:43:32.004132Z"}} import pandas as pd import numpy as np import missingno as msno import matplotlib.pyplot as plt import matplotlib.cm as cm import seaborn as sns import datetime as dt import plotly.graph_objects as go from plotly.subplots import make_subplots import plotly.express as px import plotly.offline series = "/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet" events = "/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv" train_series = pd.read_parquet(series) train_events = pd.read_csv(events) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:43:32.009738Z","iopub.execute_input":"2023-12-05T14:43:32.010126Z","iopub.status.idle":"2023-12-05T14:43:32.031630Z","shell.execute_reply.started":"2023-12-05T14:43:32.010095Z","shell.execute_reply":"2023-12-05T14:43:32.030355Z"}} train_series.head() # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:43:32.033063Z","iopub.execute_input":"2023-12-05T14:43:32.033411Z","iopub.status.idle":"2023-12-05T14:47:17.426964Z","shell.execute_reply.started":"2023-12-05T14:43:32.033382Z","shell.execute_reply":"2023-12-05T14:47:17.425852Z"}} unique_series_ids = np.unique(train_series.series_id) train_id_map = pd.DataFrame({ "series_id": unique_series_ids, "id_map": np.arange(len(unique_series_ids), dtype=np.uint16) }) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:47:17.428367Z","iopub.execute_input":"2023-12-05T14:47:17.428796Z","iopub.status.idle":"2023-12-05T14:47:17.456490Z","shell.execute_reply.started":"2023-12-05T14:47:17.428764Z","shell.execute_reply":"2023-12-05T14:47:17.454848Z"}} train_id_map.to_parquet("train_id_map.parquet", index=False) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:47:17.458004Z","iopub.execute_input":"2023-12-05T14:47:17.458385Z","iopub.status.idle":"2023-12-05T14:48:14.061296Z","shell.execute_reply.started":"2023-12-05T14:47:17.458355Z","shell.execute_reply":"2023-12-05T14:48:14.059485Z"}} train_series = transform_and_rename_dataframe(train_series, train_id_map, 'series_id', ['series_id'], {'id_map': 'series_id'}) train_events = transform_and_rename_dataframe(train_events, train_id_map, 'series_id', ['series_id'], {'id_map': 'series_id'}) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:14.066702Z","iopub.execute_input":"2023-12-05T14:48:14.067154Z","iopub.status.idle":"2023-12-05T14:48:15.356050Z","shell.execute_reply.started":"2023-12-05T14:48:14.067118Z","shell.execute_reply":"2023-12-05T14:48:15.354964Z"}} for col in train_events.columns: pct_miss = 100*(train_events[col].isnull().sum()/train_events[col].shape[0]) txt = f"Column: {col}\t Percent of NaN value: {pct_miss}" CPrint(f"\n {txt}") msno.bar(train_events, color=(0.5,0.0,0.75)) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:15.357882Z","iopub.execute_input":"2023-12-05T14:48:15.358256Z","iopub.status.idle":"2023-12-05T14:48:15.409422Z","shell.execute_reply.started":"2023-12-05T14:48:15.358225Z","shell.execute_reply":"2023-12-05T14:48:15.408264Z"}} series_by_id = train_events.groupby(['series_id'])['step'].apply(lambda x: x.isnull().any()) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:15.411103Z","iopub.execute_input":"2023-12-05T14:48:15.411810Z","iopub.status.idle":"2023-12-05T14:48:15.663390Z","shell.execute_reply.started":"2023-12-05T14:48:15.411777Z","shell.execute_reply":"2023-12-05T14:48:15.662184Z"}} train_events = clean_df(train_events) complete_series = series_by_id[~series_by_id].index.tolist() train_events_filtered = train_events[train_events['series_id'].isin(complete_series)] train_events_filtered.reset_index(drop=True) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:15.664860Z","iopub.execute_input":"2023-12-05T14:48:15.665322Z","iopub.status.idle":"2023-12-05T14:48:28.154549Z","shell.execute_reply.started":"2023-12-05T14:48:15.665291Z","shell.execute_reply":"2023-12-05T14:48:28.153530Z"}} new_train_events = train_events_filtered.copy() new_train_events.night = new_train_events.night.astype(np.uint16) new_train_events.event = new_train_events.event.replace({'onset':'0', 'wakeup':'1'}).astype(np.uint8) series_data = train_series[train_series['series_id'].isin(complete_series)] # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:28.155829Z","iopub.execute_input":"2023-12-05T14:48:28.156914Z","iopub.status.idle":"2023-12-05T14:48:33.163287Z","shell.execute_reply.started":"2023-12-05T14:48:28.156850Z","shell.execute_reply":"2023-12-05T14:48:33.161926Z"}} train = pd.merge(series_data, new_train_events[["step", "event"]], on="step", how="left") train.event = train.event.bfill(axis = "rows") train.event = train.event.fillna(1) train.event = train.event.astype(np.uint8) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:33.164936Z","iopub.execute_input":"2023-12-05T14:48:33.165500Z","iopub.status.idle":"2023-12-05T14:48:37.558324Z","shell.execute_reply.started":"2023-12-05T14:48:33.165455Z","shell.execute_reply":"2023-12-05T14:48:37.557346Z"}} train.to_parquet('data_ready_featuresengineering.parquet', index=True) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:48:37.559676Z","iopub.execute_input":"2023-12-05T14:48:37.560215Z","iopub.status.idle":"2023-12-05T14:51:22.282891Z","shell.execute_reply.started":"2023-12-05T14:48:37.560184Z","shell.execute_reply":"2023-12-05T14:51:22.281412Z"}} train = clean_df(train) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:22.284514Z","iopub.execute_input":"2023-12-05T14:51:22.285505Z","iopub.status.idle":"2023-12-05T14:51:22.299310Z","shell.execute_reply.started":"2023-12-05T14:51:22.285468Z","shell.execute_reply":"2023-12-05T14:51:22.297761Z"}} total_length = len(train) nu_sections = 4 section_length = total_length // nu_sections section_length test_data = [] train_data = [] val_data = [] test_length = total_length // nu_sections start_vals = [] end_vals = [] for section in range(nu_sections): start_sec = section * test_length end_sec = (section + 1) * test_length start_vals.append(start_sec) end_vals.append(end_sec) for start, end in zip(start_vals, end_vals): train_df_len = int(0.6 * (end - start)) test_df_len = int(0.2 * (end - start)) start = int(start) # Ensure start is an integer train_df = train.loc[start:start + train_df_len] test_df = train.loc[start + train_df_len:start + train_df_len + test_df_len] val_df = train.loc[start + train_df_len + test_df_len:end] test_data.append(test_df) train_data.append(train_df) val_data.append(val_df) print(start, start + train_df_len, start + train_df_len + test_df_len, end) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:22.300321Z","iopub.execute_input":"2023-12-05T14:51:22.300650Z","iopub.status.idle":"2023-12-05T14:51:29.273035Z","shell.execute_reply.started":"2023-12-05T14:51:22.300623Z","shell.execute_reply":"2023-12-05T14:51:29.271721Z"}} for idx,(test, train, val) in enumerate(zip(test_data, train_data, val_data)): train_data[idx] = add_features(train) test_data[idx] = add_features(test) val_data[idx] = add_features(val) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:29.274673Z","iopub.execute_input":"2023-12-05T14:51:29.275024Z","iopub.status.idle":"2023-12-05T14:51:31.623748Z","shell.execute_reply.started":"2023-12-05T14:51:29.274996Z","shell.execute_reply":"2023-12-05T14:51:31.622166Z"}} window_size = 5 for idx,(test, train, val) in enumerate(zip(test_data, train_data, val_data)): train_data[idx] = pd.concat([train,Create_lag_features(train,window_size,"enmo","anglez")],axis = 1) test_data[idx] = pd.concat([test,Create_lag_features(test,window_size,"enmo","anglez")],axis = 1) val_data[idx] = pd.concat([val,Create_lag_features(val,window_size,"enmo","anglez")],axis = 1) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:31.625624Z","iopub.execute_input":"2023-12-05T14:51:31.626136Z","iopub.status.idle":"2023-12-05T14:51:35.729596Z","shell.execute_reply.started":"2023-12-05T14:51:31.626095Z","shell.execute_reply":"2023-12-05T14:51:35.728289Z"}} train_all = pd.DataFrame() test_all = pd.DataFrame() val_all = pd.DataFrame() for test, train, val in zip(test_data, train_data, val_data): train_all = pd.concat([train_all,train],axis = 0) test_all = pd.concat([test_all,test],axis = 0) val_all = pd.concat([val_all,val],axis = 0) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:35.730944Z","iopub.execute_input":"2023-12-05T14:51:35.731318Z","iopub.status.idle":"2023-12-05T14:51:38.505373Z","shell.execute_reply.started":"2023-12-05T14:51:35.731287Z","shell.execute_reply":"2023-12-05T14:51:38.504478Z"}} train_label = train_all.event val_label = val_all.event test_label = test_all.event test_all = test_all.drop("event",axis = 1) train_all = train_all.drop("event",axis = 1) val_all = val_all.drop("event",axis = 1) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:38.506792Z","iopub.execute_input":"2023-12-05T14:51:38.507502Z","iopub.status.idle":"2023-12-05T14:51:38.790450Z","shell.execute_reply.started":"2023-12-05T14:51:38.507460Z","shell.execute_reply":"2023-12-05T14:51:38.789293Z"}} from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.metrics import accuracy_score # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:38.792204Z","iopub.execute_input":"2023-12-05T14:51:38.792650Z","iopub.status.idle":"2023-12-05T14:51:40.252236Z","shell.execute_reply.started":"2023-12-05T14:51:38.792609Z","shell.execute_reply":"2023-12-05T14:51:40.251192Z"}} test_all = test_all.drop(["series_id","timestamp","date","time"],axis =1) train_all = train_all.drop(["series_id","timestamp","date","time"],axis =1) val_all = val_all.drop(["series_id","timestamp","date","time"],axis =1) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:51:40.253860Z","iopub.execute_input":"2023-12-05T14:51:40.254507Z","iopub.status.idle":"2023-12-05T14:54:29.081556Z","shell.execute_reply.started":"2023-12-05T14:51:40.254463Z","shell.execute_reply":"2023-12-05T14:54:29.080291Z"}} X_train, y_train = train_all, train_label X_val, y_val = val_all, val_label X_test, y_test = test_all, test_label model = HistGradientBoostingClassifier(random_state=42) model.fit(X_train, y_train) val_predictions = model.predict(X_val) test_predictions = model.predict(X_test) val_accuracy = accuracy_score(y_val, val_predictions) test_accuracy = accuracy_score(y_test, test_predictions) print("Validation Accuracy:", val_accuracy) print("Test Accuracy:", test_accuracy) test_probabilities = model.predict_proba(X_test)[:, 1] val_probabilities = model.predict_proba(X_val)[:, 1] # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:59:48.766161Z","iopub.execute_input":"2023-12-05T14:59:48.766601Z","iopub.status.idle":"2023-12-05T14:59:48.803540Z","shell.execute_reply.started":"2023-12-05T14:59:48.766571Z","shell.execute_reply":"2023-12-05T14:59:48.802396Z"}} X_train # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:54:29.083270Z","iopub.execute_input":"2023-12-05T14:54:29.083625Z","iopub.status.idle":"2023-12-05T14:54:29.089470Z","shell.execute_reply.started":"2023-12-05T14:54:29.083597Z","shell.execute_reply":"2023-12-05T14:54:29.088342Z"}} sample_submission = "/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv" kaggle_test = "/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet" # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:54:29.090857Z","iopub.execute_input":"2023-12-05T14:54:29.091788Z","iopub.status.idle":"2023-12-05T14:54:29.194940Z","shell.execute_reply.started":"2023-12-05T14:54:29.091742Z","shell.execute_reply":"2023-12-05T14:54:29.193780Z"}} kaggle = pd.read_parquet(kaggle_test) sample_sub = pd.read_csv(sample_submission) kaggle = transform_and_rename_dataframe(kaggle, train_id_map, 'series_id', ['series_id'], {'id_map': 'series_id'}) kaggle = clean_df(kaggle) kaggle = add_features(kaggle) kaggle = pd.concat([kaggle,Create_lag_features(kaggle,5,"enmo","anglez")],axis = 1) kaggle = kaggle.drop(["series_id","timestamp","date","time"],axis =1) kaggle_predictions = model.predict(kaggle) probabilities = model.predict_proba(kaggle)[:, 1] # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T15:17:19.472836Z","iopub.execute_input":"2023-12-05T15:17:19.473281Z","iopub.status.idle":"2023-12-05T15:17:19.480783Z","shell.execute_reply.started":"2023-12-05T15:17:19.473248Z","shell.execute_reply":"2023-12-05T15:17:19.479533Z"}} len(set(probabilities)) # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:54:29.199884Z","iopub.execute_input":"2023-12-05T14:54:29.200490Z","iopub.status.idle":"2023-12-05T14:54:29.236517Z","shell.execute_reply.started":"2023-12-05T14:54:29.200456Z","shell.execute_reply":"2023-12-05T14:54:29.235228Z"}} start = pd.read_parquet(kaggle_test) start["event"] = kaggle_predictions start["event"] = pd.Series(start.event).replace({0:"wakeup",1:"onset"}) start["row_id"] = start.index start.drop(["timestamp","anglez","enmo"],inplace = True,axis = 1) start = start[['row_id', 'series_id','step',"event"]] start["score"] = probabilities start # %% [code] {"execution":{"iopub.status.busy":"2023-12-05T14:54:29.243545Z","iopub.execute_input":"2023-12-05T14:54:29.244008Z","iopub.status.idle":"2023-12-05T14:54:29.257725Z","shell.execute_reply.started":"2023-12-05T14:54:29.243972Z","shell.execute_reply":"2023-12-05T14:54:29.256393Z"}} start.to_csv("submission.csv",index = False)