{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport seaborn as sns\nimport pyarrow.parquet as pq\nfrom re import match\n\n\nPER_ACEPTACION_DE_NAN = 0.88\nY_TARGET = 'awake'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-24T15:34:25.987017Z","iopub.execute_input":"2023-10-24T15:34:25.987388Z","iopub.status.idle":"2023-10-24T15:34:27.393314Z","shell.execute_reply.started":"2023-10-24T15:34:25.987358Z","shell.execute_reply":"2023-10-24T15:34:27.392194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Before reviewing the pre-treatment and training of the data, it is suggested to understand it in detail through \nhttps://www.kaggle.com/code/gvyshnya/to-sleep-or-not-to-sleep-deep-eda-dive**","metadata":{}},{"cell_type":"code","source":"DIR_DATA = \"\"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n    DIR_DATA = dirname\n        \ndf_wake = pd.read_csv(f'{DIR_DATA}/train_events.csv')\nsample_submission = pd.read_csv(f'{DIR_DATA}/sample_submission.csv')\ndf_series = pq.read_table(f'{DIR_DATA}/train_series.parquet').to_pandas()\ndf_wake_test = pq.read_table(f'{DIR_DATA}/test_series.parquet').to_pandas()\n\nprint(\"\\n[1] Read data ohlcv data  from  \", f'{DIR_DATA}' )\nprint(\"train_events \", df_wake.shape)\nprint(\"sample_submission \",sample_submission.shape )\nprint(\"train_file \", df_series.shape)\nprint(\"test_file \", df_wake_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:34:27.395301Z","iopub.execute_input":"2023-10-24T15:34:27.395993Z","iopub.status.idle":"2023-10-24T15:35:40.577788Z","shell.execute_reply.started":"2023-10-24T15:34:27.395959Z","shell.execute_reply":"2023-10-24T15:35:40.576632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DATA FORMAT:\n* **df_wake:** df of TRAINING . each person series_id, when waking and when sleeping, event log in series ['series_id', 'night', 'event', 'step', 'timestamp'], must be related to df_series\n* **df_series:** each person, has a long series of sleep record every minute ['row_id', 'series_id', 'step', 'event', 'score'], is a COMMON relation table for df_wake and df_wake_test\n* **sample_submission:** sample result to send to kaggle\n* **df_wake_test:** df of TEST. each person series_id, when awake and when asleep, event log in the series ['series_id', 'night', 'event', 'step', 'timestamp'], should be related to df_series","metadata":{}},{"cell_type":"code","source":"print(\"Each person who sleeps (series_id) how much data \")\n\nprint(\"df_wake:\\t\\t\", df_wake.columns)\nprint(\"sample_submission:\\t\", sample_submission.columns)\nprint(\"df_series:\\t\\t\", df_series.columns)\nprint(\"df_wake_test:\\t\\t\", df_wake_test.columns)\nprint(df_wake[\"series_id\"].value_counts().head().to_markdown())\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:40.579368Z","iopub.execute_input":"2023-10-24T15:35:40.580114Z","iopub.status.idle":"2023-10-24T15:35:40.619465Z","shell.execute_reply.started":"2023-10-24T15:35:40.580074Z","shell.execute_reply":"2023-10-24T15:35:40.618269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"all df have corresponding data in the other df ¿?\")\ndf_count = df_wake.groupby('series_id').count()\nprint(df_count.head().to_markdown())\n\n\nprint(\"\\ndf_count.night == df_count.event  :\\t\", all(df_count.night == df_count.event))\nprint(\"df_count.night == df_count.step  :\\t\", all(df_count.night == df_count.step))\nprint(\"df_count.night == df_count.timestamp   :\\t\", all(df_count.night == df_count.timestamp))\nprint(\"df_count.step == df_count.timestamp   :\\t\", all(df_count.step == df_count.timestamp))","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:40.621962Z","iopub.execute_input":"2023-10-24T15:35:40.622330Z","iopub.status.idle":"2023-10-24T15:35:40.643294Z","shell.execute_reply.started":"2023-10-24T15:35:40.622302Z","shell.execute_reply":"2023-10-24T15:35:40.642045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"none counts \")\nprint(\"\\ndf_wake None before: \", df_wake.isna().sum())\nprint(\"\\ndf_series None before: \", df_series.isna().sum())\nprint(\"\\ndf_wake_test None before: \", df_wake_test.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:40.644700Z","iopub.execute_input":"2023-10-24T15:35:40.645210Z","iopub.status.idle":"2023-10-24T15:35:52.971693Z","shell.execute_reply.started":"2023-10-24T15:35:40.645180Z","shell.execute_reply":"2023-10-24T15:35:52.970624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**All 000 Fastastico (except df_wake) ¿? or be careful** because the data are related tables, if a record in df_wake does not exist in df_series it is None.","metadata":{}},{"cell_type":"markdown","source":"## PRE PROCES","metadata":{}},{"cell_type":"code","source":"print(\" We consider that if a person has more than 12%\")\nprint(\"of lost nights of sleep, he/she should not train with it. \")\nprint(\"Remove those with more than % of nights missed. Percentage: \",PER_ACEPTACION_DE_NAN )\ndf_count[\"is_retired\"] =0\ndf_count.loc[ ( (df_count['night'] * PER_ACEPTACION_DE_NAN >  df_count['step'])  ), \"is_retired\"] = 1\nprint(\"Examine if the proportion is correct.\")\nprint(\"how many of the records will be deleted due to excess of NONE or lack of records \")\nprint(df_count.groupby('is_retired').count()['step'].to_markdown())\n# is_retired 88%\n# 0    101\n# 1    176\n# is_retired   99%\n# 0     37\n# 1    240","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:52.973023Z","iopub.execute_input":"2023-10-24T15:35:52.973316Z","iopub.status.idle":"2023-10-24T15:35:52.988142Z","shell.execute_reply.started":"2023-10-24T15:35:52.973290Z","shell.execute_reply":"2023-10-24T15:35:52.986713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_keys_clean = df_count.loc[  df_count['is_retired'] == 0 ]\nprint(\" Series with many NANs are withdrawn : shape_anterior : \",df_wake.shape )\ndf_wake = df_wake.loc[  df_wake[\"series_id\"].isin(list(df_keys_clean.index)) ]\nprint(\"Items (series_id) removed due to excess of nulls: \" ,list(df_keys_clean.index))\n\n#si es igual que la anteriror\ndf_wake = df_wake.reset_index(drop=True)\n#order the data to work with shitf()\ndf_wake = df_wake.sort_values([\"series_id\", \"night\",\"event\" ], ascending=True)\nprint(\"\\nIf same night , same series , have wake or onset a nan , both must be removed\")\ndf_wake['match'] = ( df_wake[\"night\"] == df_wake[\"night\"].shift(-1) ) & (df_wake[\"series_id\"] == df_wake[\"series_id\"].shift(-1))\ndf_wake['remove_nan'] =  (df_wake['match'] == True) & ( df_wake[\"step\"].isna() | df_wake[\"step\"].shift(-1).isna())\ndf_wake['remove_nan'] = df_wake['remove_nan'] | ( (df_wake['match'] == False) & ( df_wake[\"step\"].isna() | df_wake[\"step\"].shift(1).isna()) )\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:52.989764Z","iopub.execute_input":"2023-10-24T15:35:52.990274Z","iopub.status.idle":"2023-10-24T15:35:53.021372Z","shell.execute_reply.started":"2023-10-24T15:35:52.990231Z","shell.execute_reply":"2023-10-24T15:35:53.020081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\" before shape : \",df_wake.shape )\nprint(\" Null before:\", df_wake.isna().sum().to_markdown())\ndf_wake = df_wake.loc[ ~ df_wake[\"remove_nan\"] == True ]\ndf_wake = df_wake.drop(columns=['match' , \"remove_nan\"] )\nprint(\"\\nNull after:\", df_wake.isna().sum().to_markdown())#remove aux","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:53.022883Z","iopub.execute_input":"2023-10-24T15:35:53.023297Z","iopub.status.idle":"2023-10-24T15:35:53.041130Z","shell.execute_reply.started":"2023-10-24T15:35:53.023258Z","shell.execute_reply":"2023-10-24T15:35:53.039900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert all(df_wake.groupby([\"series_id\", \"night\" ]).count()[\"step\"] ==2) , \"  after removing the none , not all data are in pairs, there are mismatches.\"\nprint(\" posterior shape : \",df_wake.shape )\n\n#  AUXILIARY CODE TO ONLY TEST WITH THE 3 BEST RECORDS \n# list_series_id_tallar = set(df_wake.groupby([\"series_id\"  ]).count().loc[ (df_wake.groupby([\"series_id\"  ]).count()[\"step\" ]  < 19) ].index)\n# df_series_clean = df_series.loc[  df_series[\"series_id\"].isin(list_series_id_tallar) ].copy()\n# print(\"d_clean/test_series.parquet_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(3) +\".csv   SHAPE: \",df_series_clean.shape )\n# df_series_clean.to_csv(\"d_clean/test_series.parquet_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(3) +\".csv\")\n#EXTRA TALLAR\n# train_filel = df_series[:900].copy()\n# train_filel.describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:53.042456Z","iopub.execute_input":"2023-10-24T15:35:53.042796Z","iopub.status.idle":"2023-10-24T15:35:53.053717Z","shell.execute_reply.started":"2023-10-24T15:35:53.042737Z","shell.execute_reply":"2023-10-24T15:35:53.052600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_series_id_clean = set(df_wake[\"series_id\"])\nprint(\"The CLEAN sleep series to be worked on are:\\n\", len(list_series_id_clean) ,  \" Names(series_id): \", list_series_id_clean)\ndf_series_clean = df_series.loc[  df_series[\"series_id\"].isin(list_series_id_clean) ].copy()\nprint(df_series_clean.describe().to_markdown())\n\nnum_total_series = df_wake.groupby('series_id').count().shape[0]\nprint(\"\\nTotal number of series : \",num_total_series , \"  Margin: \",PER_ACEPTACION_DE_NAN )\n\n#files to debug if dont have good computer\nprint(f\"{DIR_DATA}/train_events.csv_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(num_total_series) +\".csv  SHAPE: \",df_wake.shape )\n# df_wake.to_csv(f\"{DIR_DATA}/train_events.csv_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(num_total_series) +\".csv\")\nprint(f\"{DIR_DATA}/test_series.parquet_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(num_total_series) +\".csv   SHAPE: \",df_series_clean.shape )\n# df_series_clean.to_csv(f\"{DIR_DATA}/test_series.parquet_\"+str(PER_ACEPTACION_DE_NAN)+ \"_Se\"+ str(num_total_series) +\".csv\")\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:35:53.057685Z","iopub.execute_input":"2023-10-24T15:35:53.058022Z","iopub.status.idle":"2023-10-24T15:36:07.230992Z","shell.execute_reply.started":"2023-10-24T15:35:53.057996Z","shell.execute_reply":"2023-10-24T15:36:07.229835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_series = df_series_clean\nprint(\"df_wake : \" ,df_wake.shape) \nprint(\"df_series : \" ,df_series.shape)\nlist_series_id_in_deseries = set(df_series['series_id'])\ndf_wake = df_wake.loc[  df_wake[\"series_id\"].isin(list_series_id_in_deseries) ]\nprint(\"df_wake CLEAN : \" ,df_wake.shape) \nprint(df_wake.dtypes)\nprint(df_series.dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:36:07.232673Z","iopub.execute_input":"2023-10-24T15:36:07.233380Z","iopub.status.idle":"2023-10-24T15:36:12.464510Z","shell.execute_reply.started":"2023-10-24T15:36:07.233338Z","shell.execute_reply":"2023-10-24T15:36:12.463407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_wake[\"awake\"] = df_wake[\"event\"].replace({\"onset\": 1, \"wakeup\": 0})\nprint(\"see how many awakenings and sleeps you have \")\nprint(df_wake.groupby('event').count().to_markdown())\n\ndf = pd.merge(df_series,df_wake, on=['series_id', 'step', 'timestamp'], how='left')\ndf[\"awake\"] = df[\"awake\"].bfill(axis='rows')\ndf['awake'] = df['awake'].fillna(1).astype(\"int\")  # Result: the last event is always a \"wakeup\"\n#fill in the gaps before the \"awake\" column\ndf_check =  df.dropna(subset=['night', 'event']).copy()\ndf_check['timestamp'] = pd.to_datetime(df_check['timestamp'], format=\"%Y-%m-%dT%H:%M:%S%z\")\nfiltered_values_by_seconds  = list(filter(lambda v: match(r'[0-6]\\d:[0-6]\\d:00', v.strftime('%H:%M:%S') ), df_check['timestamp'].values))\nassert df_check.shape[0] == len(filtered_values_by_seconds) , \" Error there is a result row ending other than :00 in seconds  \"\nassert df_series.shape[0] == df.shape[0] , \" Error the row sizes of the df_series after the union are not correct. \"\nassert df_wake.shape[0] == df_check.shape[0] , \" Error the row sizes of the df_wake after the union are not correct. \"\nassert df_wake.shape[1] + len(['night', 'event']) == df_check.shape[1] , \" Error the COLUMN sizes of the df_wake after the union are not correct. \"\ndf\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:36:12.467511Z","iopub.execute_input":"2023-10-24T15:36:12.467861Z","iopub.status.idle":"2023-10-24T15:37:30.546570Z","shell.execute_reply.started":"2023-10-24T15:36:12.467809Z","shell.execute_reply":"2023-10-24T15:37:30.545553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW_PREVIOUS_EVENT = 60 # 5s CADA UNO  60s *5min / (each 5s) = 60 windows 5MINUTOS ATRAS \ndf.loc[ df['night'].notnull() | df['event'].notnull(), 'valid' ] = 1","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:37:30.547826Z","iopub.execute_input":"2023-10-24T15:37:30.548138Z","iopub.status.idle":"2023-10-24T15:37:32.843725Z","shell.execute_reply.started":"2023-10-24T15:37:30.548112Z","shell.execute_reply":"2023-10-24T15:37:32.842762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VIEW GRAPHICS https://www.kaggle.com/code/carlmcbrideellis/zzzs-plots-of-all-277-anglez-and-enmo-series \nDIVE https://www.kaggle.com/code/gvyshnya/to-sleep-or-not-to-sleep-deep-eda-dive \n* anglez - As calculated and described by the GGIR package, angle z is a metric derived from individual accelerometer components that is commonly used in sleep detection. \n     commonly used in sleep detection, and refers to the angle of the arm relative to the vertical axis of the body. \n     of the arm relative to the vertical axis of the body.\n* enmo - As calculated and described by the GGIR package, ENMO is the Euclidean norm minus one of all accelerometer signals, with rounded negative values \n     accelerometer signals, with negative values rounded to zero. Although there is no standard measure of acceleration in this space,\n    this is one of several commonly calculated characteristics.","metadata":{}},{"cell_type":"code","source":"print(\"\\n[2] Auxiliar columns indicators. \")\nprint(\"[2.1] the sleep data is every 1second, it is changed to every minute through averages, the events are always when the second hand is at :00.\")\nprint(\"new columns enter format \\\"_m1_\\\", indicating that seconds were changed to minutes \")\nprint(\"auxiliary variables are also used, the angle is given sine, cosine and tangent.\")\nTRASITO_RESAMPLE  = '1T' # tambien puede ser '5T' 5 minutos\ndef get_resample_means_for_each_serie_id(df_ser):\n    print('resample(', TRASITO_RESAMPLE, ') START series_id: ', ser_id, \" df_shape: \", df_ser.shape)\n    df_ser['timestamp'] = pd.to_datetime(df_ser['timestamp'], format=\"%Y-%m-%dT%H:%M:%S%z\", errors='coerce') #TODO so slow\n    df_ser = df_ser.dropna(subset=['timestamp'])#los que no se trasnforman se deja a nan y se borrar\n    df_ser['timestamp'] = pd.to_datetime(df_ser['timestamp'] , errors='coerce', utc=True) #hay que repetirlo en caso de que falle la primera y recurra al dropna\n    df_ser['date_day_week'] = df_ser['timestamp'].dt.dayofweek\n    df_ser['date_hour'] = df_ser['timestamp'].dt.hour\n    df_ser['date_day_minute'] = (df_ser['timestamp'].dt.hour * 60 + df_ser['timestamp'].dt.minute)\n\n    df_ser = df_ser.set_index('timestamp') \n    df_ser[['anglez_m1_max', 'enmo_m1_max']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).max()\n    df_ser[['anglez_m1_min', 'enmo_m1_min']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).min()\n    df_ser['anglez_m1_dif'] = df_ser['anglez_m1_max'] - df_ser['anglez_m1_min']\n    df_ser['enmo_m1_dif'] = df_ser['enmo_m1_max'] - df_ser['enmo_m1_min']\n    df_ser[['anglez_m1_mean', 'enmo_m1_mean']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).mean()\n    df_ser[['anglez_m1_std', 'enmo_m1_std']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).std() #Desviación típica\n    df_ser[['anglez_m1_med', 'enmo_m1_med']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).median()\n    df_ser[['anglez_m1_var', 'enmo_m1_var']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).var()\n    #VARIANZA rango   desviancion tipica\n    #TODO rango se calcula con el describe \n    #TODOañadir las medias en el pasado \n    #WINDOW_ROLL = 15 #15 minutos anteriores \n    #aa = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).shift(-WINDOW_ROLL).rolling(WINDOW_ROLL +1, min_periods=1)#.mean()\n    #df_ser[['anglez_m1_std', 'enmo_m1_std']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).std() #Desviación típica\n    #df_ser[['anglez_m1_med', 'enmo_m1_med']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).median()\n    #df_ser[['anglez_m1_var', 'enmo_m1_var']] = df_ser[['anglez', 'enmo']].resample(TRASITO_RESAMPLE).var()\n    #ppp = df_ser['anglez', 'enmo'].resample(TRASITO_RESAMPLE).describe(include='int64')\n    \n    df_ser[['step_m1']] = df_ser[['step']].resample(TRASITO_RESAMPLE).mean()\n    # añadir las columnas medias de despierto vs dormido 'anglez_mean, anglez_std, anglez_median, enmo_mean, enmo_std, enmo_median'\n    #typical_price.rolling(period).apply(lambda x: (x - x.mean()).abs().mean())\n    #TRAMPAS no se puede usar la medi , tiene datos de futuro \n    #df_awake_mean = df_ser[['anglez', 'enmo', \"awake\"]].groupby([\"awake\"]).agg([np.mean, np.std, np.median])\n    #.columns = df_awake_mean.columns.map('_'.join)\n    #[[df_awake_mean.columns]] = 0\n    #df_ser.loc[df_ser['awake'] == 0, df_awake_mean.columns] = pd.DataFrame(df_awake_mean.iloc[0]).transpose()[df_awake_mean.columns].values\n    #df_ser.loc[df_ser['awake'] == 1, df_awake_mean.columns] = pd.DataFrame(df_awake_mean.iloc[1]).transpose()[df_awake_mean.columns].values\n    #TODO solo se casa las np.mean, np.std, np.median de despierto vs dormido , pero debe añadirse el dia_despierto vs dia_dormido\n    df_ser = df_ser.dropna(subset=['anglez_m1_max', 'enmo_m1_med'])  # cualquier coluymna con los huecos vale\n    #df_ser['anglez_dif_mean'] = df_ser['anglez_mean'] - df_ser['anglez_m1_mean'] #, anglez_std, anglez_median, enmo_mean, enmo_std, enmo_median'\n    #df_ser['anglez_dif_std'] = df_ser['anglez_std'] - df_ser['anglez_m1_std']\n    #df_ser['anglez_dif_med'] = df_ser['anglez_median'] - df_ser['anglez_m1_med']\n    #df_ser['enmo_dif_mean'] = df_ser['enmo_mean'] - df_ser['enmo_m1_mean']\n    #df_ser['enmo_dif_std'] = df_ser['enmo_std'] - df_ser['enmo_m1_std']\n    #df_ser['enmo_dif_med'] = df_ser['enmo_median'] - df_ser['enmo_m1_med']\n    #df_ser = df_ser.drop(['anglez_mean', 'anglez_std', 'anglez_median', 'enmo_mean', 'enmo_std', 'enmo_median'], axis='columns')\n    #anglez es un angulo\n    df_ser['anglez_m1_sin'] = np.sin(df_ser['anglez_m1_med'] )\n    df_ser['anglez_m1_cos'] = np.cos(df_ser['anglez_m1_med'])\n    df_ser['anglez_m1_tan'] = np.tan(df_ser['anglez_m1_med'])\n    \n    #Generate the Ground True wake_event  do\"onset\": -1, do \"wakeup\": 1   \"awake\" = {\"onset\": 1, \"wakeup\": 0}\n    df_ser['wake_event'] = 1\n    df_ser.loc[((df_ser[\"awake\"] == 1) & (df_ser[\"awake\"].shift() == 0) ), 'wake_event'] = 2 # It pass from 0 to 1\n    df_ser.loc[((df_ser[\"awake\"] == 0) & (df_ser[\"awake\"].shift() == 1) ), 'wake_event'] = 0  # It pass from 1 to 0\n\n    \n    print('resample(', TRASITO_RESAMPLE, ') END   series_id: ', ser_id, \" df_shape: \", df_ser.shape)\n    # df_ser.drop(['step', 'step_m1'], axis=1).describe(include='all').tail(-1)\n    return df_ser\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:38:11.116863Z","iopub.execute_input":"2023-10-24T15:38:11.117247Z","iopub.status.idle":"2023-10-24T15:38:11.136274Z","shell.execute_reply.started":"2023-10-24T15:38:11.117218Z","shell.execute_reply":"2023-10-24T15:38:11.134989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG_LIGHT_NUM_SERIES = 5\nlist_df_serie = []\ndict_df_serie = {}\nnum_series = str(len(set(df['series_id'].values)))\nprint(\"num of series sleep: \",num_series )\nprint('columns : awake\" = {\"onset\": 1, \"wakeup\": 0}')\nfor ser_id in list(set(df['series_id'].values))[DEBUG_LIGHT_NUM_SERIES:DEBUG_LIGHT_NUM_SERIES*2]:\n    df_ser = pd.DataFrame()\n    df_ser = None\n    df_ser = df.loc[df['series_id'] == ser_id].copy()\n    \n    if 'timestamp' not in df_ser:\n        df_ser.insert(loc=1, column='timestamp', value=df_ser.index)\n    df_ser = get_resample_means_for_each_serie_id(df_ser)\n    df_ser.insert(loc=1, column='timestamp', value=df_ser.index) #insertar en la segunda posicion de columna\n    df_ser.reset_index(drop=True, inplace=True)\n    list_df_serie.append(df_ser)\n    dict_df_serie[ser_id] = df_ser\n    print(\"\\tlist_df_serie.append series_id: \", ser_id, \" shape: \",df_ser.shape )\n    \nnum_series = str(len(set(df['series_id'].values)))\ndf_s = pd.concat(list_df_serie).sort_values([\"series_id\", \"timestamp\",\"step\" ], ascending=True).reset_index(drop=True)\nprint(\"\\nDateFrame clean of all series. Num_series: \",num_series ,\" Shape: \", df_s.shape, \"\\n DESCRIBE:\\n\")\n# from tabulate import tabulate\n# print(tabulate(df_s.drop(['step', 'step_m1'], axis=1).describe(include='all').tail(-1), headers='keys', tablefmt='psql'))\n# print(df_s.drop(['step', 'step_m1'], axis=1).describe(include='all').tail(-1))\nprint(f\"{DIR_DATA}/train_events_MED_m1_full_Se\"+ str(num_series) +\".csv  SHAPE: \",df_s.shape )\n# df_s.to_csv(f\"{DIR_DATA}/train_events_MED_m1_full_Se\"+ str(num_series) +\".csv\")\n\n#Crear VENTANAS de 30 minutos anteriores\n#desde la columna valida crea la ventana\n#MINUTES_BEFORE_FOR_WINDOW_TIME = 30\n#rows_valid  = df_s[df_s['valid'] ==1 ].index\n#df_check_time1 = df_s.iloc[np.concatenate([ [ i - MINUTES_BEFORE_FOR_WINDOW_TIME  ] for i  in rows_valid ]) ][\"timestamp\"]\n#df_check_time2 = df_s.iloc[np.concatenate([ [ i  ] for i  in rows_valid ]) ][\"timestamp\"]\n#assert df_check_time1.shape == df_check_time2.shape , \" ERROR los tamaños de las fechas preprocesadas no son iguales \"\n#bool_is_window_time_valid = any(df_check_time1.reset_index(drop=True) < ( df_check_time2.reset_index(drop=True) - pd.DateOffset(hours=24) ))\n#assert not bool_is_window_time_valid , \" ERROR las ventanas de tiempo , no son de 30min, son superiores a 24h , dato null en medio de la ventana  \"\n#rows_valid_bef = [ np.r_[ i - MINUTES_BEFORE_FOR_WINDOW_TIME: i +1 ] for i  in rows_valid ]\n#df_s = df_s.iloc[np.concatenate(rows_valid_bef) ]#one dimension\n#print(\"d_clean/train_events_MED_w\"+str(MINUTES_BEFORE_FOR_WINDOW_TIME)+\"_full_Se\"+ str(num_series) +\".csv  SHAPE: \",df_s.shape )\n#df_s.to_csv(\"d_clean/train_events_MED_w\"+str(MINUTES_BEFORE_FOR_WINDOW_TIME)+\"_full_Se\"+ str(num_series) +\".csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:38:19.815286Z","iopub.execute_input":"2023-10-24T15:38:19.815692Z","iopub.status.idle":"2023-10-24T15:39:19.382415Z","shell.execute_reply.started":"2023-10-24T15:38:19.815660Z","shell.execute_reply":"2023-10-24T15:39:19.381313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TRAIN","metadata":{}},{"cell_type":"code","source":"# ditc_keys_to_debug = dict_df_serie.keys()[6:9]\ndf_raw = df_s\nprint(df_raw.info())\nprint(\"df_raw shape: \",df_raw.shape )\nprint(\"GT awake variable: \")\nprint(df_raw.groupby('awake').count().to_markdown())\nprint(df_raw.groupby('wake_event').count().to_markdown())","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:42:08.904688Z","iopub.execute_input":"2023-10-24T15:42:08.905088Z","iopub.status.idle":"2023-10-24T15:42:09.067065Z","shell.execute_reply.started":"2023-10-24T15:42:08.905058Z","shell.execute_reply":"2023-10-24T15:42:09.065676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [3] Get the target_Y (ground true)\nprint(\"\\n[3] Calculate the target_Y (ground true)\")\nprint( \" remove development columns Have None\")\ndf_raw = df_raw.drop(columns=['awake'], errors='ignore') \ndf_raw = df_raw.drop(columns=[ 'series_id' , \"timestamp\"], errors='ignore') \ndf_raw = df_raw.drop(columns=['night' , \"event\", \"valid\"], errors='ignore') \ndf_raw = df_raw.drop(columns=[ 'series_id' , \"timestamp\"], errors='ignore') \ndf_raw.reset_index(drop=True, inplace=True)\n\ntarget_Y = df_raw.pop('wake_event')\ntarget_Y = pd.get_dummies(target_Y)\n\n#assert not all(df_raw.isna()),  \"TODO Hay algun None en el df \"\nprint(\"[3.1]target_Y.shape: \" ,target_Y.shape)\nfeatures_X_ALL = df_raw\n\n# Create correlation matrix https://stackoverflow.com/questions/29294983/how-to-calculate-correlation-between-all-columns-and-remove-highly-correlated-on\nprint(\"[3.2] DEBUG All technical indicators Extracted Count: \",len(features_X_ALL.columns))\nprint(target_Y.value_counts().to_markdown())\nprint(target_Y.head(12).to_markdown())\n","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:42:12.133380Z","iopub.execute_input":"2023-10-24T15:42:12.133844Z","iopub.status.idle":"2023-10-24T15:42:12.223869Z","shell.execute_reply.started":"2023-10-24T15:42:12.133811Z","shell.execute_reply":"2023-10-24T15:42:12.222503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n[4] Calculation of correlation strength. What are the +-100 best technical indicators and which are noise. Remove strong correlacted columns\")\ndef remove_strong_correlations_columns(df_cor , factor:float):\n    # Create correlation matrix https://stackoverflow.com/questions/29294983/how-to-calculate-correlation-between-all-columns-and-remove-highly-correlated-on\n    corr_matrix = df_cor.corr().abs()  # Select upper triangle of correlation matrix\n    upper = corr_matrix.where( np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))  # Find features with correlation greater than 0.95\n    to_drop = [column for column in upper.columns if any(upper[column] > factor)]\n    print(\"\\tColumns more correlated than factor, will be Removed. Factor: \",factor, \" Columns number: \", len(to_drop) )\n    print(\"\\tColumns repeted (more correlation that factor) : \", to_drop)\n    df_cor.drop(to_drop, axis=1, inplace=True)  # Drop features\n    return df_cor\n\ndef get_best_Y_correlations_columns(df_cor, target_Y, num_columns:int):\n    # Create correlation matrix https://stackoverflow.com/questions/29294983/how-to-calculate-correlation-between-all-columns-and-remove-highly-correlated-on\n    df_cor['aux_target_Y_1'] = target_Y[0] # do sleep \n    df_cor['aux_target_Y_2'] = target_Y[2] # do wakeup \n    df_corr = df_cor.corr()[['aux_target_Y_1','aux_target_Y_2']].abs().head(-2)\n\n    df_corr['sum_strong_corr'] = df_corr['aux_target_Y_1'] + df_corr['aux_target_Y_2']\n    df_corr = df_corr.sort_values(['sum_strong_corr'], ascending=False).head(num_columns)\n    df_cor.drop(['aux_target_Y_1','aux_target_Y_2'], axis=1, inplace=True)\n    return df_corr.index.values\n\nNUMS_FEATURES = int( len(features_X_ALL.columns) *0.95)\nprint(\"[4.1] Columns with a correlation of more than 93% are removed. NumColumns: \",len(features_X_ALL.columns) )\nfeatures_X_ALL = remove_strong_correlations_columns(features_X_ALL,factor=0.93)\nprint(\"[4.2] The 5% columns with the lowest correlation to Ground True are removed. Num columns: \",NUMS_FEATURES )\nfeatures_X_col_corr = get_best_Y_correlations_columns(features_X_ALL.copy(), target_Y, num_columns=NUMS_FEATURES)\nprint(\"[4.3] List good columns Num: \",len(features_X_col_corr) , \" Names: \", features_X_col_corr)\n\nfeatures_X = features_X_ALL[features_X_col_corr]\nprint(\"[4.2] Select the best technical patterns to train with features_W3 Extracted Count: \", len(features_X.columns), \" Names : \",\", \".join(features_X.columns))\nprint(\"[4.3] features_W3 index Dates: from \", features_X.index[0],\" to \", features_X.index[-1], \" Shape: \", features_X.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:42:25.639552Z","iopub.execute_input":"2023-10-24T15:42:25.640769Z","iopub.status.idle":"2023-10-24T15:42:26.319626Z","shell.execute_reply.started":"2023-10-24T15:42:25.640715Z","shell.execute_reply":"2023-10-24T15:42:26.317782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nimport pickle\n\ndef normalise_data(X, split_ratio, scaler_path, todict=False):\n    # Normalise the Data\n    X.replace([np.inf, -np.inf], np.nan, inplace=True)\n    X.dropna(inplace=True)\n    # X = np.float32(X_int)\n    if isinstance(split_ratio, bool) and split_ratio == True:\n        scaler: MinMaxScaler = pickle.load(open(scaler_path, 'rb'))\n    else:\n        scaler = MinMaxScaler(feature_range=(0, 1))\n        train_size = int(len(X) * split_ratio)\n        scaler = scaler.fit(X[:train_size])\n        pickle.dump(scaler, open(scaler_path, 'wb'))\n\n    if isinstance(X, pd.DataFrame):\n        columns = X.columns\n        index = X.index\n        try:\n            output = scaler.transform(X)\n        except ValueError:\n            print(\"DEBUG {ValueError}The feature names should match those that were passed during fit.\")\n            output = scaler.transform(X[scaler.feature_names_in_])\n        output = pd.DataFrame(output, columns=columns, index=index)\n    else:\n        output = scaler.transform(X.reshape((-1, X.shape[-1] if len(X.shape) > 1 else 1))).reshape(X.shape)\n    \n    if todict: return { 'output': output, 'scaler': scaler }\n    else: return output, scaler\n\ndef get_window_data(df: pd.DataFrame, target: pd.Series, window_size: int, todict=False):\n    x = []\n    y = []\n    index = []\n\n    if target is None:\n        df.insert(loc=0, column=\"target\", value=0)\n        df[\"target\"].iloc[1] = 1;\n        df[\"target\"].iloc[2] = 2\n        target = pd.get_dummies(df['target'])\n        df = df.drop(['target'], axis=1)\n        #Por que lo hace en lo anteiror\n        print(\"INFO To create the window_data need to create an artificial Useless Y_target  Count: \", len(df.columns) , \" Names : \", \",\".join(df.columns))\n        print(\"DEBUG features_W3 index Dates:: \", df.index[0], df.index[-1], \" Shape: \", df.shape)\n        target = target[target.index.isin(df.index)]\n        df = df[df.index.isin(target.index)]\n\n    assert len(df) == len(target)\n\n    for i in range(window_size, len(df)+1):\n        # Target is at the same row to the feature\n        assert df[i - window_size: i].index[-1] == target.index[i-1]\n\n        x.append(df[i - window_size: i].values)\n        if isinstance(target, pd.DataFrame): y.append(target.iloc[i-1].values)\n        else: y.append(target.iloc[i-1])\n        index.append(target.index[i-1])\n\n    x = np.array(x, np.float32)\n    try:\n        index = np.array(index, np.datetime64)\n    except Exception as ex:\n        print(\"Exception: \", ex)\n        index = np.array(index)\n    y = np.array(y)\n\n    if todict: return { 'X': x, 'y': y, 'index': index }\n    else: return x, y, index","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:42:30.585451Z","iopub.execute_input":"2023-10-24T15:42:30.586825Z","iopub.status.idle":"2023-10-24T15:42:30.685255Z","shell.execute_reply.started":"2023-10-24T15:42:30.586774Z","shell.execute_reply":"2023-10-24T15:42:30.683912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizated\nprint(\"\\n[5] For correct training, for correct training the values must be normalised to between 0 and 1  \")\n#TIP   how to normalise this data is stored in file 'data/scalers/{symbol}_{REF_MODEL}_scalex.pkl' for use in the future realtime  .\nprint(\"[5.1] Normalise_data path: \", '/kaggle/working/MinMaxScaler.pkl')\nWINDOWS_SIZE = 60 # one hour previous \nfeatures_X, _ = normalise_data(features_X, split_ratio=1, scaler_path='/kaggle/working/MinMaxScaler.pkl')\n\n# Get Window Data\nprint(\"\\n[6] Currently you have for each target_Y value a row of technical indicators, you add a 'window'\"\n      \" to make the decision to predict whether the +-60 rows above will be taken (about 1 hour of previous indicators, splited in 5min).\")\nX, y, index = get_window_data(features_X, target_Y, window_size=WINDOWS_SIZE)\n# TODO the and target_Y does not distinguish between different persons (series_id) \n# TIP currently only 5min windows are used, but there is also a multi-window model that collects the 5min, 15min and 60min windows, ask the author\n# Example 'INFO created array x_array Created x_array.shape : (221641, 60, 17) ,EX: 221641 rows,, windows size 60, 17 tech parents' Maybe add a one minute candle as well. method get_window_data_3Window()\nprint(\"[6.1] get_window_data Shapes Y:\" , y.shape, \" X: \", X.shape)\nprint(\" Index:\" , index)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:43:21.684393Z","iopub.execute_input":"2023-10-24T15:43:21.684951Z","iopub.status.idle":"2023-10-24T15:43:50.082606Z","shell.execute_reply.started":"2023-10-24T15:43:21.684920Z","shell.execute_reply":"2023-10-24T15:43:50.081234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n[7] data split between training and validation \")\nfrom sklearn.model_selection import train_test_split\nX_train, X_eval, y_train, y_eval = train_test_split(X,y,train_size = 0.78,random_state = 123,shuffle = True)\n\nprint(\"[7.1] X_train: \",X_train.shape, \"X_eval: \",X_eval.shape,\"y_train: \",y_train.shape,\"y_eval: \",y_eval.shape)\n# TIP  shuffling does not break the data, as the time series is stored in the time get_window_data(....window_size=WINDOWS_SIZE). ","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:44:28.346885Z","iopub.execute_input":"2023-10-24T15:44:28.347376Z","iopub.status.idle":"2023-10-24T15:44:29.190954Z","shell.execute_reply.started":"2023-10-24T15:44:28.347346Z","shell.execute_reply":"2023-10-24T15:44:29.189634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_save_class_weight(dict_class_weight , path_class_weight):\n    with open(path_class_weight, 'wb') as f:\n        pickle.dump(dict_class_weight, f)\n\ndef get_load_class_weight( path_class_weight):\n    with open(path_class_weight, 'rb') as f:\n        loaded_dict = pickle.load(f)\n    return loaded_dict\n\nfrom sklearn.utils.class_weight import compute_class_weight\n# [8] Get  class_weight\nprint(\"\\n[8] Ground True data are unbalanced  Given that there is a lot of 'do nothing' 1, and very little 'do sleep' 0 or 'do wake' 2,\"\n      \" weight balancing is required, to give more importance to the minorities.  \")\nclass_weight = compute_class_weight('balanced', classes=np.unique(np.argmax(y_train, axis=1)), y=np.argmax(y_train, axis=1))\ndict_class_weight = dict(enumerate(class_weight))\nprint(\"[8.1] Class weight  path: \", f'/kaggle/working/class_weight.pkl' , \"\\n Dict: \" , dict_class_weight)\nset_save_class_weight(dict_class_weight, f'/kaggle/working/class_weight.pkl')\n# TIP tip the balancing to check the unbalance is very necessary, I would also like to add that it penalizes more in the training the false positives,\n# there are other methods for balancing as SMOTE, they have been tested without improvement, in my opinion, I don't like them, they destroy the data, if it is applied it must be very delicate.\n# TODO do much unbalance","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:44:40.126876Z","iopub.execute_input":"2023-10-24T15:44:40.127330Z","iopub.status.idle":"2023-10-24T15:44:40.162726Z","shell.execute_reply.started":"2023-10-24T15:44:40.127300Z","shell.execute_reply":"2023-10-24T15:44:40.161502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Disable Tensorflow useless warnings\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow_addons.metrics import F1Score\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\n\nREF_MODEL = \"A1\"\n# [9] Create a model TF\nprint(\"\\n[9] Creation of the TF model architecture. must respect the input_shape and output_shape and the 'softmax' ,\"\n      \" from there EXPRIMENT combinations \")\ninput_shape = X_train.shape[1:]\noutput_shape = 1 if len(y_train.shape) == 1 else y_train.shape[1]\nprint(\"[9.1] arrays to use the TF model  , input_shape: \", input_shape, \" output_shape: \",output_shape)\n#se deben entrenar con los mismos datos\n# MODEL\nhparams = {'HP_NUM_UNITS1': 182, 'HP_NUM_UNITS2': 64, 'HP_DROPOUT': 0.2, 'HP_L2': 0.0005, 'HP_OPTIMIZER': 'adam'}\nmodel = tf.keras.Sequential([\n    # keras.layers.Dense(WINDOWS_SIZE*3, activation=ACTIVATION_2_ReLU, input_shape = input_shape ),\n    tf.keras.layers.BatchNormalization(input_shape=input_shape),\n    tf.keras.layers.Dense(hparams['HP_NUM_UNITS1'], kernel_regularizer=tf.keras.regularizers.l2(hparams['HP_L2']), activation=\"relu\"),\n    tf.keras.layers.Dropout(hparams['HP_DROPOUT']),\n    tf.keras.layers.Dense(hparams['HP_NUM_UNITS1'], kernel_regularizer=tf.keras.regularizers.l2(hparams['HP_L2']), activation=\"relu\"),  # or elu\n    tf.keras.layers.Dropout(hparams['HP_DROPOUT']),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(hparams['HP_NUM_UNITS1'] // 1.5, kernel_regularizer=tf.keras.regularizers.l2(hparams['HP_L2']), activation=\"relu\"),\n    tf.keras.layers.Dropout(hparams['HP_DROPOUT']),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(hparams['HP_NUM_UNITS2'], kernel_regularizer=tf.keras.regularizers.l2(hparams['HP_L2']), activation=\"relu\"),\n    tf.keras.layers.Dropout(hparams['HP_DROPOUT'] - 0.1),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(output_shape, activation='softmax')  # , bias_initializer = output_bias_k),\n])\noptimizer = tf.keras.optimizers.Adam(learning_rate=hparams['HP_L2'])\nloss = tf.keras.losses.MeanSquaredError(reduction=\"auto\", name=\"mean_squared_error\")\n\nreduce_lr_loss = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=5, verbose=1,min_delta=0.005, min_lr=0.000001, mode='min')\nearly_stopping = keras.callbacks.EarlyStopping(patience=10, monitor=\"val_loss\", mode='min', verbose=1,restore_best_weights=True)\ncallbacks_save = ModelCheckpoint(filepath=f'/kaggle/working/model_awake_{REF_MODEL}.h5',monitor='val_loss', verbose=1, save_best_only=True)\ntf.keras.utils.plot_model(model, to_file=f'/kaggle/working/model_awake_{REF_MODEL}.png')\nprint(\"[9.3] print diagram of the TF model Path: \", f'/kaggle/working/model_awake_{REF_MODEL}.h5')","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:45:25.903457Z","iopub.execute_input":"2023-10-24T15:45:25.904746Z","iopub.status.idle":"2023-10-24T15:45:26.245496Z","shell.execute_reply.started":"2023-10-24T15:45:25.904661Z","shell.execute_reply":"2023-10-24T15:45:26.243868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_metrics(num_classes=3):\n    return [\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall'),\n        F1Score(name='f1_macro', num_classes=num_classes, average='macro'),\n        keras.metrics.AUC(name='auc'),\n        keras.metrics.AUC(name='prc', curve='PR'),  # precision-recall curve\n    ]\n\n# [10] Trian the model\nprint(\"\\n[10] Start training TF model \"  , f'/kaggle/working/model_awake_{REF_MODEL}.h5' )\nprint(\"[10.1] To adjust and measure the model, callbacks is required, the callbacks will stop before the overfit. \")\nmodel.compile(optimizer=optimizer, loss=loss, metrics=get_metrics(num_classes=len(target_Y.columns)))\n# TIP f1 is the BEST metric to unbalance clases\nprint(\"[10.2] X_train: \",X_train.shape, \"y_train: \",y_train.shape)\nhistory = model.fit(x=X_train, y=y_train, batch_size=512, epochs=5, class_weight=dict_class_weight, verbose=1 ,validation_split=0.7,  callbacks=[reduce_lr_loss, early_stopping, callbacks_save])\n# TIP on a monitor= you can stop training when you get the best precision    recall  f1-score , but the one that works best is val_loss.\nprint(\"[10.3] Model initial_weights saved : \",  f'/kaggle/working/model_awake_{REF_MODEL}._initial_weights.tf')\nmodel.save_weights( f'/kaggle/working/model_awake_{REF_MODEL}._initial_weights.tf')","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:45:47.628897Z","iopub.execute_input":"2023-10-24T15:45:47.632512Z","iopub.status.idle":"2023-10-24T15:50:03.703556Z","shell.execute_reply.started":"2023-10-24T15:45:47.632437Z","shell.execute_reply":"2023-10-24T15:50:03.702005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot as plt\n#matplotlib.use('TKAgg')  # avoid error could not find or load the Qt platform plugin windows\n\ndef plot_history_data_acc_loss_f1(history, path_png):\n    # print(history.history.keys())\n    plt.figure(1, figsize=(8, 16)).tight_layout()\n    # summarize history for accuracy\n    plt.subplot(4, 1, 1)\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    # plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['accuracy', 'val_accuracy'], loc='lower right') #The strings 'upper left', 'upper right', 'lower left', 'lower right'\n    # summarize history for loss\n    plt.subplot(4, 1, 2)\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    # plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['loss', 'val_loss'], loc='upper right')\n    # summarize history for f1_score\n    if \"f1_macro\" in history.history:\n        plt.subplot(4, 1, 3)\n        plt.plot(history.history['f1_macro'])\n        plt.plot(history.history['val_f1_macro'])\n        plt.title('model f1')\n        plt.ylabel('f1_score')\n        plt.xlabel('epoch')\n        plt.legend(['f1_macro', 'val_f1_macro'], loc='lower right')\n    # summarize history for lr\n    plt.subplot(4, 1, 4)\n    plt.plot(history.history['lr'])\n    # plt.title('learning rate')\n    plt.ylabel('learning rate')\n    plt.xlabel('epoch')\n    plt.legend(['lr'], loc='upper right')\n    plt.subplots_adjust(left=0.2, top=0.9, right=0.9, bottom=0.1, hspace=0.5, wspace=0.8)\n    print(\"PLOT data: \", path_png)\n    plt.savefig(path_png)\n    plt.close()\n\ndef plot_metrics_loss_prc_precision_recall(history, path_png):\n    metrics = ['loss', 'prc', 'precision', 'recall']\n    for n, metric in enumerate(metrics):\n        name = metric.replace(\"_\",\" \").capitalize()\n        plt.subplot(2,2,n+1)\n        plt.plot(history.epoch, history.history[metric],  label='Train')\n        plt.plot(history.epoch, history.history['val_'+metric], label='Val')\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        if metric == 'loss':\n          plt.ylim([0, plt.ylim()[1]])\n        elif metric == 'auc':\n          plt.ylim([0.8,1])\n        else:\n          plt.ylim([0,1])\n\n    plt.legend()\n    print(\"PLOT data: \", path_png)\n    plt.savefig(path_png)\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:57:20.365510Z","iopub.execute_input":"2023-10-24T15:57:20.366430Z","iopub.status.idle":"2023-10-24T15:57:20.380460Z","shell.execute_reply.started":"2023-10-24T15:57:20.366382Z","shell.execute_reply":"2023-10-24T15:57:20.379346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\ndef get_info_model_evaluation(y_pred,y_test ):\n    print(y_pred.shape, y_test.shape)\n    info_predict = classification_report(y_true=y_test, y_pred=y_pred, digits=3)\n    print(info_predict)\n    df_info_predict = pd.DataFrame({\"y_pred\": y_pred, \"y_test\": y_test})\n    # print(f'outputs/model_info/{symbol}_{indicator_timeframe}.csv')\n    df_info_predict.to_csv(f'/kaggle/working/model_awake_{REF_MODEL}_result.csv', sep='\\t')\n    df_details = pd.DataFrame()\n    df_details['count'] = df_info_predict.value_counts().to_frame()\n    df_details['per%'] = df_info_predict.value_counts(normalize=True).mul(100).round(2)\n    return df_details, df_info_predict\n\nprint(\"File history : \", f'/kaggle/working/model_awake_{REF_MODEL}_history_train.csv')\npd.DataFrame(history.history).round(5).to_csv(f'/kaggle/working/model_awake_{REF_MODEL}_history_train.csv')\nplot_metrics_loss_prc_precision_recall(history, path_png=f'/kaggle/working/model_awake_{REF_MODEL}_history_PRC.png')\nplot_history_data_acc_loss_f1(history, path_png=f'/kaggle/working/model_awake_{REF_MODEL}_history_Loss.png')\n\n# [11] predict and eval\nprint('\\n[11] Do a predict and eval. Load path: ', f'/kaggle/working/model_awake_{REF_MODEL}.h5')\nprint(\"[11.1] array to predit x_test.shape: \", X_eval.shape, \" y_eval.shape: \", y_eval.shape)\n# load_class_weight = get_load_class_weight(f'data/class_weight/{symbol}_{REF_MODEL}_class_weight.pkl')\nresampled_model_2 = keras.models.load_model(f'/kaggle/working/model_awake_{REF_MODEL}.h5')\ny_pred_1 = resampled_model_2.predict(X_eval, batch_size=64)\ny_pred_1 = np.argmax(np.squeeze(y_pred_1), axis=1)\ny_test_a = np.argmax(y_eval, axis=1)\n# TIP argmax is used to detect which of the 3 preconditions ('do nothing', 'do buy' or 'do sell') is the best,\n# but it can be adjusted to be more accurate and discard half results. ex: only validating the ones above 0.65\nprint(\"[11.2] read the evaluation: \", X_eval.shape)\nget_info_model_evaluation(y_pred_1, y_test_a)\n# TIP The ideal would be to increase the precision (in 1 buy and 2 sell) above 50%, but without sacrificing the recall f1-score values.\n# TIP check the y_pred y_test table , to know what is the proportion of failures and successes in accurate prediction.","metadata":{"execution":{"iopub.status.busy":"2023-10-24T15:57:26.176297Z","iopub.execute_input":"2023-10-24T15:57:26.176734Z","iopub.status.idle":"2023-10-24T15:57:36.366641Z","shell.execute_reply.started":"2023-10-24T15:57:26.176699Z","shell.execute_reply":"2023-10-24T15:57:36.365354Z"},"trusted":true},"execution_count":null,"outputs":[]}]}