{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"1. Biblioteki\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas_profiling\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport xgboost as xgb\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.metrics import confusion_matrix\nfrom lightgbm.sklearn import LGBMClassifier\nfrom lightgbm.sklearn import LGBMRegressor\n\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n\nfrom sklearn.model_selection import KFold\nfrom functools import partial\n\nimport traceback\n\n#import optuna","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:30.382209Z","iopub.execute_input":"2023-04-26T14:02:30.383448Z","iopub.status.idle":"2023-04-26T14:02:30.391623Z","shell.execute_reply.started":"2023-04-26T14:02:30.383388Z","shell.execute_reply":"2023-04-26T14:02:30.390515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1.1 Ustawienia flag","metadata":{}},{"cell_type":"code","source":"\ndata_reducing = False ## -------------------------------------------memory reducing technique\nfeature_reducing = 'fi' ## -----------------------------------------'pca' or 'fi'\nonline = False ## --------------------------------------------------use online training\nsampling = 'hybrid' ## ---------------------------------------------'under' or 'over' or 'hybrid'","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:30.394264Z","iopub.execute_input":"2023-04-26T14:02:30.395016Z","iopub.status.idle":"2023-04-26T14:02:30.408318Z","shell.execute_reply.started":"2023-04-26T14:02:30.394965Z","shell.execute_reply":"2023-04-26T14:02:30.406905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. Ładowanie danych","metadata":{}},{"cell_type":"code","source":"path = \"../input/bosch-production-line-performance\" #ścieżka do folderu z plikami\n\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\nSEED = 0\nCHUNKSIZE = 5000\nNROWS = 50000\n\nTRAIN_NUMERIC = \"{0}/train_numeric.csv.zip\".format(path) # tu jest response \nTRAIN_DATE = \"{0}/train_date.csv.zip\".format(path) #data kazdego testu z train_numeric\nTEST_NUMERIC = \"{0}/test_numeric.csv.zip\".format(path)# tu mamy przewidzieć response (czyli czy jest failure czy nie)\nTEST_DATE = \"{0}/test_date.csv.zip\".format(path)#data kazdego testu z test_numeric\ntrain_num_part = pd.read_csv(path+'/train_numeric.csv.zip', nrows=1000)\ntrain_date_part = pd.read_csv(path+'/train_date.csv.zip', nrows=1000)\nTRAIN_NUMERIC_100 = pd.read_csv(path+'/train_date.csv.zip', nrows=100)#100 linijek z train_numeric\n\n# Read a fraction of the train_numeric file\ntrain_numeric = pd.read_csv(TRAIN_NUMERIC, nrows=1183748//100)\n\nTRAIN_NUMERIC_PARQUET = \"../input/parquet2/train_numeric.parquet\"\n\n# Convert the Pandas DataFrame to a PyArrow Table\ntable = pa.Table.from_pandas(train_numeric)\n\n# Write the Table to a Parquet file using PyArrow\n#pq.write_table(\n#    table=table,\n#    where=TRAIN_NUMERIC_PARQUET\n#)\n\n# Verify the shape of the DataFrame\nprint(train_numeric.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:30.410418Z","iopub.execute_input":"2023-04-26T14:02:30.410832Z","iopub.status.idle":"2023-04-26T14:02:31.699576Z","shell.execute_reply.started":"2023-04-26T14:02:30.410792Z","shell.execute_reply":"2023-04-26T14:02:31.698335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pomijam dane kategoryczne z powodu nieprzydatności do EDA","metadata":{}},{"cell_type":"markdown","source":"W pliku train_date.csv tworzona jest zmienna pochodna.\nCzas pomiaru jest inny dla każdej stacji. \nTO DO:\n1.obliczyć min max dla każdej\nstart_station - nazwa stacji, w której produkt zaczyna swoją drogę\nend_station - nazwa stacji końcowej\ntime_diff - czas trwania procesu\n\n","metadata":{}},{"cell_type":"code","source":"\n# 1. tworzenie serii z kolumnami\ntrain_date_part.count()\n# 2. wybór indeksu serii i s0 w środku\ntrain_date_part.count().reset_index()['index'].str.split('_', expand=True)[1]\n# 3. zostawienie jednego do każdeh stacji i usiniecie kolumny id\ntrain_date_part.count().reset_index()['index'].str.split('_', expand=True)[1].drop_duplicates()[1:]\n# 4. wybór kolumny odpowiadającej indeksowi każdej stacji\ntemp = train_date_part.count().reset_index()['index'].str.split('_', expand=True)[1].drop_duplicates()[1:].index\ndatecols = train_date_part.columns[temp]\ndatecols","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:31.702011Z","iopub.execute_input":"2023-04-26T14:02:31.702931Z","iopub.status.idle":"2023-04-26T14:02:31.760686Z","shell.execute_reply.started":"2023-04-26T14:02:31.702885Z","shell.execute_reply":"2023-04-26T14:02:31.759729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date_part[\"start_station\"] = -1  # inicjalizacja\ntrain_date_part[\"end_station\"] = -1\n\nfor col in train_date_part[datecols]:\n    station_name = col.split(\"_\")[1] # S0\n    station_num = int(col.split(\"_\")[1][1:]) # 0\n    notnulls = (~train_date_part[col].isnull()) # (maska logiczna) wyszukiwanie wartości innich niż NaN\n    \n    # Jeżeli jest wartość NaN to minął on stację. Jeżeli jest jakaś wartość to jest to zmierzony czas na tej stacji.\n    # start_station musi mieć jakąś wartość (nie może być NaN) dlatego zamiast NaN musi być -1\n    # end_station musi być not null \n    train_date_part.loc[(notnulls & (train_date_part.start_station == -1)), \"start_station\"] = station_num\n    train_date_part.loc[notnulls, \"end_station\"] = station_num\n\nstart_time = train_date_part.drop(columns=[\"Id\", \"start_station\", \"end_station\"]).min(axis=1) # start_time\nend_time = train_date_part.drop(columns=[\"Id\", \"start_station\", \"end_station\"]).max(axis=1) # end_time\ntime_diff = end_time - start_time  # Obliczanie czasu trwania procesu\ntrain_date_part['time_diff'] = time_diff\ntrain_date = train_date_part[[\"Id\", \"start_station\", \"end_station\", \"time_diff\"]]\ntrain_date","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:31.763366Z","iopub.execute_input":"2023-04-26T14:02:31.764493Z","iopub.status.idle":"2023-04-26T14:02:31.890648Z","shell.execute_reply.started":"2023-04-26T14:02:31.764449Z","shell.execute_reply":"2023-04-26T14:02:31.889639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tworzenie tabeli ze wszystkich rekordów","metadata":{}},{"cell_type":"code","source":"extract_cols = [\"Id\", \"start_station\", \"end_station\", \"time_diff\"]\ntrain_date = pd.DataFrame(columns=extract_cols)\n\nfor chunk in pd.read_csv(path+'/train_date.csv.zip', usecols=[\"Id\"] + datecols.tolist(), chunksize=300000):\n\n    chunk[\"start_station\"] = -1 \n    chunk[\"end_station\"] = -1\n    for col in chunk[datecols]: \n        station_num = int(col.split(\"_\")[1][1:])\n        notnulls = (~chunk[col].isnull()) \n        chunk.loc[(notnulls & (chunk.start_station == -1)), \"start_station\"] = station_num\n        chunk.loc[notnulls, \"end_station\"] = station_num\n        \n    start_time = chunk[datecols].min(axis=1)\n    end_time = chunk[datecols].max(axis=1)\n    time_diff = end_time - start_time\n    chunk['time_diff'] = time_diff\n    chunk = chunk[extract_cols]\n    train_date = pd.concat([train_date, chunk])\n    \ntrain_date","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:02:31.891886Z","iopub.execute_input":"2023-04-26T14:02:31.892806Z","iopub.status.idle":"2023-04-26T14:03:13.271452Z","shell.execute_reply.started":"2023-04-26T14:02:31.892764Z","shell.execute_reply":"2023-04-26T14:03:13.270174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnullcols = train_num_part.columns[train_num_part.isnull().mean() < 1]\nnullcols","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:13.273227Z","iopub.execute_input":"2023-04-26T14:03:13.273581Z","iopub.status.idle":"2023-04-26T14:03:13.288536Z","shell.execute_reply.started":"2023-04-26T14:03:13.273547Z","shell.execute_reply":"2023-04-26T14:03:13.287257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if feature_reducing == 'pca':\n    from sklearn.decomposition import TruncatedSVD ## dla rzadkich danych wejściowych\n\n    train_num = pd.read_csv(path+'/train_numeric.csv.zip', usecols=nullcols)\n\n    X = train_num.drop([\"Id\", \"Response\"], axis=1).fillna(0)  # 1183747 x 954 features\n    svd = TruncatedSVD(n_components=15, n_iter=30, random_state=42) \n    X_svd = svd.fit_transform(X)\n    print(X_svd.shape) # zmniejszenie liczby z 954 features -> mniejszej liczby features.","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:13.290292Z","iopub.execute_input":"2023-04-26T14:03:13.290750Z","iopub.status.idle":"2023-04-26T14:03:13.300641Z","shell.execute_reply.started":"2023-04-26T14:03:13.290701Z","shell.execute_reply":"2023-04-26T14:03:13.299445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if feature_reducing == 'pca':\n    sv_cols = [f\"SV{i}\" for i in range(1, X_svd.shape[1]+1)]\n    df_svd = pd.DataFrame(data=X_svd, columns=sv_cols)\n    df_svd[\"Id\"] = train_num[\"Id\"] # Dodanie kolumny identyfikatora\n    df_svd[\"Response\"] = train_num[\"Response\"] # Dodanie wartości docelowej\n    display(df_svd)\n    \n    # zwalnianie pamięci\n    del train_num\n    del X\n    del X_svd\n\n    gc.collect()\n    \n    # łącznie informacji z train_date i train_numeric\n    X_train_1 = pd.merge(df_svd, train_date, on=\"Id\")\n    display(X_train_1)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:13.302048Z","iopub.execute_input":"2023-04-26T14:03:13.302447Z","iopub.status.idle":"2023-04-26T14:03:13.313970Z","shell.execute_reply.started":"2023-04-26T14:03:13.302411Z","shell.execute_reply":"2023-04-26T14:03:13.312528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if feature_reducing == 'fi':\n    usecols = ['Id',\n               'L3_S30_F3514', 'L0_S9_F200', 'L3_S29_F3430', 'L0_S11_F314',\n               'L0_S0_F18', 'L3_S35_F3896', 'L0_S12_F350', 'L3_S36_F3918',\n               'L0_S0_F20', 'L3_S30_F3684', 'L1_S24_F1632', 'L0_S2_F48',\n               'L3_S29_F3345', 'L0_S18_F449', 'L0_S21_F497', 'L3_S29_F3433',\n               'L3_S30_F3764', 'L0_S1_F24', 'L3_S30_F3554', 'L0_S11_F322',\n               'L3_S30_F3564', 'L3_S29_F3327', 'L0_S2_F36', 'L0_S9_F180',\n               'L3_S33_F3855', 'L0_S0_F4', 'L0_S21_F477', 'L0_S5_F114',\n               'L0_S6_F122', 'L1_S24_F1122', 'L0_S9_F165', 'L0_S18_F439',\n               'L1_S24_F1490', 'L0_S6_F132', 'L3_S29_F3379', 'L3_S29_F3336',\n               'L0_S3_F80', 'L3_S30_F3749', 'L1_S24_F1763', 'L0_S10_F219',\n               'Response'][:15] + [\"Response\"]\n\n    train_num = pd.read_csv(path+\"/train_numeric.csv.zip\", usecols=usecols)\n    display(train_num) # 1183747 x 42","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:13.315814Z","iopub.execute_input":"2023-04-26T14:03:13.316206Z","iopub.status.idle":"2023-04-26T14:03:45.732923Z","shell.execute_reply.started":"2023-04-26T14:03:13.316168Z","shell.execute_reply":"2023-04-26T14:03:45.731484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if feature_reducing == 'fi':\n    # łącznie informacji z train_date i train_numeric\n    X_train_1 = pd.merge(train_num.fillna(0), train_date, on=\"Id\")\n    display(X_train_1)\n    \n    del train_num\n    del train_date\n    import gc\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:45.738618Z","iopub.execute_input":"2023-04-26T14:03:45.739368Z","iopub.status.idle":"2023-04-26T14:03:46.794285Z","shell.execute_reply.started":"2023-04-26T14:03:45.739321Z","shell.execute_reply":"2023-04-26T14:03:46.792914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X_train_1.Response.value_counts()) # 0.995 : 0.005 = 995 : 5 = 199 : 1\nsns.countplot(data=X_train_1, x=\"Response\")","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:46.796219Z","iopub.execute_input":"2023-04-26T14:03:46.796602Z","iopub.status.idle":"2023-04-26T14:03:47.077866Z","shell.execute_reply.started":"2023-04-26T14:03:46.796566Z","shell.execute_reply":"2023-04-26T14:03:47.076453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obróbka danych\n\n\nKlasy nie są równe","metadata":{}},{"cell_type":"markdown","source":"Badanie korelacji","metadata":{}},{"cell_type":"code","source":"%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:47.079357Z","iopub.execute_input":"2023-04-26T14:03:47.079731Z","iopub.status.idle":"2023-04-26T14:03:47.085554Z","shell.execute_reply.started":"2023-04-26T14:03:47.079694Z","shell.execute_reply":"2023-04-26T14:03:47.084355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['L3_S38_F3960', 'L3_S33_F3865', 'L3_S38_F3956', 'L3_S33_F3857',\n       'L3_S29_F3321', 'L1_S24_F1846', 'L3_S32_F3850', 'L3_S29_F3354',\n       'L3_S29_F3324', 'L3_S35_F3889', 'L0_S1_F28', 'L1_S24_F1844',\n       'L3_S29_F3376', 'L0_S0_F22', 'L3_S33_F3859', 'L3_S38_F3952', \n       'L3_S30_F3754', 'L2_S26_F3113', 'L3_S30_F3759', 'L0_S5_F114']","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:47.087003Z","iopub.execute_input":"2023-04-26T14:03:47.087381Z","iopub.status.idle":"2023-04-26T14:03:47.098041Z","shell.execute_reply.started":"2023-04-26T14:03:47.087344Z","shell.execute_reply":"2023-04-26T14:03:47.096685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = pd.read_csv(path+\"/train_numeric.csv.zip\", nrows = 1).columns.values\nimp_idxs = [np.argwhere(feature_name == numeric_cols)[0][0] for feature_name in feature_names]\ntrain = pd.read_csv(path+\"/train_numeric.csv.zip\", index_col = 0, header = 0, usecols = [0, len(numeric_cols) - 1] + imp_idxs)\ntrain = train[feature_names + ['Response']]","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:03:47.099833Z","iopub.execute_input":"2023-04-26T14:03:47.100386Z","iopub.status.idle":"2023-04-26T14:04:20.938553Z","shell.execute_reply.started":"2023-04-26T14:03:47.100334Z","shell.execute_reply":"2023-04-26T14:04:20.937135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" X_neg, X_pos = train[train['Response'] == 0].iloc[:, :-1], train[train['Response']==1].iloc[:, :-1]","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:20.940341Z","iopub.execute_input":"2023-04-26T14:04:20.940743Z","iopub.status.idle":"2023-04-26T14:04:21.051063Z","shell.execute_reply.started":"2023-04-26T14:04:20.940702Z","shell.execute_reply":"2023-04-26T14:04:21.049770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sprawdzenie ilości danych dla poszczególnych features","metadata":{}},{"cell_type":"code","source":"FIGSIZE = (13,4)\nnon_missing = pd.DataFrame(pd.concat([(X_neg.count()/X_neg.shape[0]).to_frame('negatywne próbki'),\n                                      (X_pos.count()/X_pos.shape[0]).to_frame('pozytywne próbki'),], axis = 1))\nnon_missing_sort = non_missing.sort_values(['negatywne próbki'])\nnon_missing_sort.plot.barh(title = 'Sprawdzenie nierownosci danych', figsize = FIGSIZE)\nplt.gca().invert_yaxis()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:21.053039Z","iopub.execute_input":"2023-04-26T14:04:21.053773Z","iopub.status.idle":"2023-04-26T14:04:21.613696Z","shell.execute_reply.started":"2023-04-26T14:04:21.053710Z","shell.execute_reply":"2023-04-26T14:04:21.612417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Kowariancje dla próbek ujemnych i dodatnich osobno","metadata":{}},{"cell_type":"code","source":"FIGSIZE = (13,4)\n_, (ax1, ax2) = plt.subplots(1,2, figsize = FIGSIZE)\nMIN_PERIODS = 100\n\ntriang_mask = np.zeros((X_pos.shape[1], X_pos.shape[1]))\ntriang_mask[np.triu_indices_from(triang_mask)] = True\n\nax1.set_title('Ujemne')\nsns.heatmap(X_neg.corr(min_periods = MIN_PERIODS), mask = triang_mask, square=True,  ax = ax1)\n\nax2.set_title('Dodatnie')\nsns.heatmap(X_pos.corr(min_periods = MIN_PERIODS), mask = triang_mask, square=True,  ax = ax2)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:21.615682Z","iopub.execute_input":"2023-04-26T14:04:21.616502Z","iopub.status.idle":"2023-04-26T14:04:23.725833Z","shell.execute_reply.started":"2023-04-26T14:04:21.616447Z","shell.execute_reply":"2023-04-26T14:04:23.724428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Różnica miedzy nimi :","metadata":{}},{"cell_type":"code","source":"sns.heatmap(X_pos.corr(min_periods = MIN_PERIODS) -X_neg.corr(min_periods = MIN_PERIODS), mask = triang_mask, square=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:23.727556Z","iopub.execute_input":"2023-04-26T14:04:23.728785Z","iopub.status.idle":"2023-04-26T14:04:25.594490Z","shell.execute_reply.started":"2023-04-26T14:04:23.728724Z","shell.execute_reply":"2023-04-26T14:04:25.592664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Korelacja między brakami danych w różnych features ","metadata":{}},{"cell_type":"code","source":"nan_pos, nan_neg = np.isnan(X_pos), np.isnan(X_neg)\n\ntriang_mask = np.zeros((X_pos.shape[1], X_pos.shape[1]))\ntriang_mask[np.triu_indices_from(triang_mask)] = True\n\nFIGSIZE = (13,4)\n_, (ax1, ax2) = plt.subplots(1,2, figsize = FIGSIZE)\nMIN_PERIODS = 100\n\nax1.set_title('Ujemne')\nsns.heatmap(nan_neg.corr(),   square=True, mask = triang_mask, ax = ax1)\n\nax2.set_title('Dodatnie')\nsns.heatmap(nan_pos.corr(), square=True, mask = triang_mask,  ax = ax2)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:25.596442Z","iopub.execute_input":"2023-04-26T14:04:25.597310Z","iopub.status.idle":"2023-04-26T14:04:27.862910Z","shell.execute_reply.started":"2023-04-26T14:04:25.597261Z","shell.execute_reply":"2023-04-26T14:04:27.861577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Różnica między macierzami korelacji brakujących wartości potwierdza hipoteze zastosowania próbkowania do dalszej zabawy z danymi","metadata":{}},{"cell_type":"code","source":"sns.heatmap(nan_neg.corr() - nan_pos.corr(), mask = triang_mask, square=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:27.864584Z","iopub.execute_input":"2023-04-26T14:04:27.864967Z","iopub.status.idle":"2023-04-26T14:04:29.719187Z","shell.execute_reply.started":"2023-04-26T14:04:27.864920Z","shell.execute_reply":"2023-04-26T14:04:29.718195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sprawdzenie czasu wykonywania pomiarów w train numeric i test numeric","metadata":{}},{"cell_type":"code","source":"def get_station_times(dates, withId=False):\n    times = []\n    cols = list(dates.columns)\n    if 'Id' in cols:\n        cols.remove('Id')\n    for feature_name in cols:\n        if withId:\n            df = dates[['Id', feature_name]].copy()\n            df.columns = ['Id', 'time']\n        else:\n            df = dates[[feature_name]].copy()\n            df.columns = ['time']\n        df['station'] = feature_name.split('_')[1][1:]\n        df = df.dropna()\n        times.append(df)\n    return pd.concat(times)\n\nstation_times = get_station_times(TRAIN_NUMERIC_100, withId=True).sort_values(by=['Id', 'station'])\nprint(station_times[:5])\nprint(station_times.shape)\nmin_station_times = station_times.groupby(['Id', 'station']).min()['time']\nmax_station_times = station_times.groupby(['Id', 'station']).max()['time']\nprint(np.mean(1. * (min_station_times == max_station_times)))","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:29.720377Z","iopub.execute_input":"2023-04-26T14:04:29.721183Z","iopub.status.idle":"2023-04-26T14:04:32.900318Z","shell.execute_reply.started":"2023-04-26T14:04:29.721143Z","shell.execute_reply":"2023-04-26T14:04:32.898897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_cols = TRAIN_NUMERIC_100.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\ndate_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\nprint(date_cols) \n\ntrain_date = pd.read_csv(path+'/train_date.csv.zip', usecols=date_cols)\nprint(train_date.shape)\ntrain_station_times = get_station_times(train_date, withId=False)\nprint(train_station_times.shape)\ntrain_time_cnt = train_station_times.groupby('time').count()[['station']].reset_index()\ntrain_time_cnt.columns = ['time', 'cnt']\nprint(train_time_cnt.shape)\n\ntest_date = pd.read_csv(path+'/test_date.csv.zip', usecols=date_cols)\nprint(test_date.shape)\ntest_station_times = get_station_times(test_date, withId=False)\nprint(test_station_times.shape)\ntest_time_cnt = test_station_times.groupby('time').count()[['station']].reset_index()\ntest_time_cnt.columns = ['time', 'cnt']\nprint(test_time_cnt.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:04:32.901659Z","iopub.execute_input":"2023-04-26T14:04:32.905488Z","iopub.status.idle":"2023-04-26T14:06:03.023848Z","shell.execute_reply.started":"2023-04-26T14:04:32.905428Z","shell.execute_reply":"2023-04-26T14:06:03.022474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\nplt.plot(train_time_cnt['time'].values, train_time_cnt['cnt'].values, 'b.', alpha=0.1, label='train')\nplt.plot(test_time_cnt['time'].values, test_time_cnt['cnt'].values, 'r.', alpha=0.1, label='test')\nplt.title('Original date values')\nplt.ylabel('Number of records')\nplt.xlabel('Time')\nfig.savefig('original_date_values.png', dpi=300)\nplt.show()\n\nprint((train_time_cnt['time'].min(), train_time_cnt['time'].max()))\nprint((test_time_cnt['time'].min(), test_time_cnt['time'].max()))","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:03.025295Z","iopub.execute_input":"2023-04-26T14:06:03.027540Z","iopub.status.idle":"2023-04-26T14:06:05.202552Z","shell.execute_reply.started":"2023-04-26T14:06:03.027494Z","shell.execute_reply":"2023-04-26T14:06:05.201323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TWOPLOT:","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\ntest = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\ntrain[\"StartTime\"] = -1\ntest[\"StartTime\"] = -1","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:05.204500Z","iopub.execute_input":"2023-04-26T14:06:05.205420Z","iopub.status.idle":"2023-04-26T14:06:07.846935Z","shell.execute_reply.started":"2023-04-26T14:06:05.205276Z","shell.execute_reply":"2023-04-26T14:06:07.845963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nrows = 0\n\nfor tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n    feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n\n    stime_tr = tr[feats].min(axis=1).values\n    stime_te = te[feats].min(axis=1).values\n\n    train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n    test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n\n    nrows += CHUNKSIZE\n    if nrows >= NROWS:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:07.848421Z","iopub.execute_input":"2023-04-26T14:06:07.848823Z","iopub.status.idle":"2023-04-26T14:06:19.498720Z","shell.execute_reply.started":"2023-04-26T14:06:07.848785Z","shell.execute_reply":"2023-04-26T14:06:19.497437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ntrain = train.shape[0]\ntrain_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n\ntrain_test['feat1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n\ntrain_test['feat2'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\ntrain = train_test.iloc[:ntrain, :]","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:19.500426Z","iopub.execute_input":"2023-04-26T14:06:19.501667Z","iopub.status.idle":"2023-04-26T14:06:19.596287Z","shell.execute_reply.started":"2023-04-26T14:06:19.501619Z","shell.execute_reply":"2023-04-26T14:06:19.595010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def twoplot(df, col, xaxis=None):\n   #twoplot do pokazywania zaleznosci jak inne dane wpływają na response i tu mamy pokazane osobno dla response 0 i 1\n    if col not in df.columns.values:\n        print('ERROR: %s not a column' % col)\n    ndf = pd.DataFrame(index = df.index)\n    ndf[col] = df[col]\n    ndf[xaxis] = df[xaxis] if xaxis else df.index\n    ndf['Response'] = df['Response']\n    sns.set(font_scale=2)\n    g = sns.FacetGrid(ndf, col=\"Response\", hue=\"Response\",height=5,aspect=5)\n    sns.set(font_scale=2)\n    g.map(plt.scatter, xaxis, col, alpha=.7, s=1)\n    g.add_legend();\n    \n    del ndf","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:19.597949Z","iopub.execute_input":"2023-04-26T14:06:19.598386Z","iopub.status.idle":"2023-04-26T14:06:19.606985Z","shell.execute_reply.started":"2023-04-26T14:06:19.598339Z","shell.execute_reply":"2023-04-26T14:06:19.605494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"twoplot(train, 'feat1')","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:19.615995Z","iopub.execute_input":"2023-04-26T14:06:19.616481Z","iopub.status.idle":"2023-04-26T14:06:20.754744Z","shell.execute_reply.started":"2023-04-26T14:06:19.616441Z","shell.execute_reply":"2023-04-26T14:06:20.753452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"twoplot(train, 'feat2')","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:20.756341Z","iopub.execute_input":"2023-04-26T14:06:20.756766Z","iopub.status.idle":"2023-04-26T14:06:21.967340Z","shell.execute_reply.started":"2023-04-26T14:06:20.756695Z","shell.execute_reply":"2023-04-26T14:06:21.965909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def df_load(file, batch_size, skip = 1):\n    #sprawdzanie anomaliiiiiii\n\n    reader = pd.read_csv(file, chunksize=batch_size, dtype=np.float16)\n\n    df = pd.concat((chunk for i, chunk in enumerate(reader) if i % skip == 0))\n\n    return df\n\n    \ndf_num = df_load(TRAIN_NUMERIC, batch_size=10000)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:06:21.968962Z","iopub.execute_input":"2023-04-26T14:06:21.969484Z","iopub.status.idle":"2023-04-26T14:07:42.941869Z","shell.execute_reply.started":"2023-04-26T14:06:21.969432Z","shell.execute_reply":"2023-04-26T14:07:42.940581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask.dataframe as dd\ndef df_load(file, batch_size, skip=1):\n    # Load data in batches and skip rows as needed\n    df = dd.read_parquet(file, engine='pyarrow')\n    df = df[df.columns[::skip]]\n    df = df.compute()\n\n    return df\n\npath = \"../bosh\"\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\nSEED = 0\nCHUNKSIZE = 5000\nNROWS = 50000\n\nTRAIN_NUMERIC_PARQUET = \"../input/parquet2/train_numeric.parquet\"\n\n# Read the Parquet file using Dask\ndf_num = df_load(TRAIN_NUMERIC_PARQUET, batch_size=10000)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:42.943451Z","iopub.execute_input":"2023-04-26T14:07:42.944618Z","iopub.status.idle":"2023-04-26T14:07:43.871254Z","shell.execute_reply.started":"2023-04-26T14:07:42.944535Z","shell.execute_reply":"2023-04-26T14:07:43.869967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_num.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:43.873087Z","iopub.execute_input":"2023-04-26T14:07:43.873611Z","iopub.status.idle":"2023-04-26T14:07:43.904375Z","shell.execute_reply.started":"2023-04-26T14:07:43.873555Z","shell.execute_reply":"2023-04-26T14:07:43.902945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(420)\nnum_samples = 20\nrandom_picks = (np.array(np.random.random(num_samples)) * df_num.shape[1]).astype(int)\nrandom_picks = sorted(random_picks)\nprint(random_picks)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:43.908181Z","iopub.execute_input":"2023-04-26T14:07:43.909171Z","iopub.status.idle":"2023-04-26T14:07:43.917043Z","shell.execute_reply.started":"2023-04-26T14:07:43.908950Z","shell.execute_reply":"2023-04-26T14:07:43.915645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feature in random_picks[:1]:\n    fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n    # Plot linear scale feature density\n    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, ax=axes[0], label='%d response 0 samples' % len(df_num.iloc[:, feature][(df_num.Response == 0)].dropna()))\n    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', ax=axes[0], label='%d response 1 samples' % len(df_num.iloc[:, feature][(df_num.Response == 1)].dropna()))\n    # Plot log scale feature density\n    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, ax=axes[1], log=True)\n    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', ax=axes[1], log=True)\n\n    axes[0].legend(loc='upper left')\n    plt.suptitle('Feature column \"%s\" histogram' % df_num.columns[feature], fontsize=15)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:43.918905Z","iopub.execute_input":"2023-04-26T14:07:43.920089Z","iopub.status.idle":"2023-04-26T14:07:45.042089Z","shell.execute_reply.started":"2023-04-26T14:07:43.920034Z","shell.execute_reply":"2023-04-26T14:07:45.041125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBOOST","metadata":{}},{"cell_type":"code","source":"df_names = [\"S\"+str(i) for i in range(52)]\n[df_names.append(\"L\"+str(i)) for i in range(4)] \ndf_names.append(\"all\")\ndatabase_file = \"bosch_data.h5\"\n#będziemy wykorzystywać hdf sotre jako baze danych","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:45.043609Z","iopub.execute_input":"2023-04-26T14:07:45.044224Z","iopub.status.idle":"2023-04-26T14:07:45.050276Z","shell.execute_reply.started":"2023-04-26T14:07:45.044182Z","shell.execute_reply":"2023-04-26T14:07:45.048853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_numeric_chunk(chunk_numeric, name):\n    # select feature columns \n    if name == \"all\":\n        sub_columns = chunk_numeric.columns[chunk_numeric.columns.str.contains(r'L\\d+_S\\d+_F\\d+.*')] \n    else:\n        sub_columns = chunk_numeric.columns[chunk_numeric.columns.str.contains(r'.*'+name+'_.*')]\n\n\n    # select correct subchunk and drop empty rows from subchunk\n    sub_chunk = chunk_numeric[sub_columns]\n    sub_chunk.dropna(how=\"all\", inplace=True)\n\n    # engineer features\n    sub_chunk[name+\"_numerical_min\"] = sub_chunk.min(axis=1)\n    sub_chunk[name+\"_numerical_max\"] = sub_chunk.max(axis=1)\n    sub_chunk[name+\"_numerical_mean\"] = sub_chunk.mean(axis=1)\n\n    # only keep engineered features for lines and all\n    if \"L\" ==name[0] or name==\"all\":\n        \n        sub_chunk.drop(sub_chunk.columns[sub_chunk.columns.str.contains(r'L\\d+_S\\d+_F\\d+.*')], inplace = True, axis=1)\n\n    # add Response Column to all\n    if name == \"all\":\n        sub_chunk[\"Response\"] = chunk_numeric[\"Response\"]\n\n    return sub_chunk\n\n\ndef process_row(row):\n    first = row[row.first_valid_index()]\n    last = row[row.last_valid_index()]\n    NAs = row.isnull().sum()\n    return first, last, NAs\n\n\ndef process_date_chunk(chunk_date,name):\n    if name == \"all\":\n        sub_columns = chunk_date.columns[chunk_date.columns.str.contains(r'L\\d+_S\\d+_D\\d+.*')] \n    else:\n        sub_columns = chunk_date.columns[chunk_date.columns.str.contains(r'.*'+name+'_.*')]\n\n    # select correct subchunk and drop empty rows from subchunk \n    sub_chunk = chunk_date[sub_columns]\n    sub_chunk.dropna(how=\"all\", inplace=True)\n    if sub_chunk.empty:\n        return sub_chunk\n    if name[0] == \"S\":\n        return sub_chunk\n\n    sub_chunk[name+\"_date_min\"] = sub_chunk.min(axis=1)\n    sub_chunk[name+\"_date_max\"] = sub_chunk.max(axis=1)    \n    sub_chunk[name+\"_first\"], sub_chunk[name+\"_last\"], sub_chunk[name+\"_NAs\"] = zip(*sub_chunk[sub_columns].apply(process_row, axis=1))\n    sub_chunk[name+\"_time\"]=sub_chunk[name+\"_date_max\"]-sub_chunk[name+\"_date_min\"]\n    sub_chunk.drop(sub_chunk.columns[sub_chunk.columns.str.contains(r'L\\d+_S\\d+_D\\d+.*')], inplace = True, axis=1)\n    return sub_chunk\n\n\ndef process_categorical_chunk(chunk_categorical, name):\n    if name[0] == \"S\":\n        sub_columns = chunk_categorical.columns[chunk_categorical.columns.str.contains(r'.*'+name+'_.*')]\n        sub_chunk = chunk_categorical[sub_columns]\n        sub_chunk.dropna(how=\"all\", inplace=True)\n        return sub_chunk\n    return None","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:07:45.052352Z","iopub.execute_input":"2023-04-26T14:07:45.053289Z","iopub.status.idle":"2023-04-26T14:07:45.071187Z","shell.execute_reply.started":"2023-04-26T14:07:45.053236Z","shell.execute_reply":"2023-04-26T14:07:45.070076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Łączenie danych z train_numeric, train_date i train_categorical\nstore =  pd.HDFStore(database_file)\n\nchunksize = 10 ** 4\n\nnumber_of_lines_to_load=12*10**4\n\nheader_numeric = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", nrows= 1 ,engine=\"c\",compression=\"zip\").columns\nheader_date = pd.read_csv(\"../input/bosch-production-line-performance/train_date.csv.zip\", nrows= 1 ,engine=\"c\",compression=\"zip\").columns\nheader_categorical = pd.read_csv(\"../input/bosch-production-line-performance/train_categorical.csv.zip\", nrows= 1 ,engine=\"c\",compression=\"zip\").columns\n\nconvert = lambda error: float(error[1:]) if error != \"\"  else np.nan\nconverter = {cat:convert for cat in header_categorical[1:]}\ntry: \n    for loaded in range(1,number_of_lines_to_load,chunksize):\n        \n\n        # wczytanie danych\n        chunk_numeric = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", skiprows = loaded, nrows= chunksize ,engine=\"c\", names = header_numeric, index_col=\"Id\",compression=\"zip\")\n        chunk_date = pd.read_csv(\"../input/bosch-production-line-performance/train_date.csv.zip\", skiprows = loaded, nrows= chunksize, engine=\"c\", names = header_date, index_col=\"Id\",compression=\"zip\")\n        chunk_categorical = pd.read_csv(\"../input/bosch-production-line-performance/train_categorical.csv.zip\", skiprows = loaded, nrows= chunksize , \n            names = header_categorical,index_col=\"Id\",dtype=\"object\",engine=\"c\", converters=converter,compression=\"zip\")# dtype=\"float\",  sep = r',T*')\n\n        for name in df_names:            \n            \n            # przetworzenie danych\n            numeric_processed = process_numeric_chunk(chunk_numeric, name)\n            date_processed = process_date_chunk(chunk_date, name)\n            categorical_processed = process_categorical_chunk(chunk_categorical, name)\n            \n            # Łączenie danych\n            processed_joined = numeric_processed.join(date_processed, on=\"Id\")\n            if categorical_processed is not None:\n                processed_joined = processed_joined.join(categorical_processed, on=\"Id\")\n\n            # Dodanie do reszty danych\n            if processed_joined is not None:\n                store.append(name,processed_joined.iloc[:-1,:])\n        # Zatrzymanie\n        print(loaded)\n        if loaded>=number_of_lines_to_load:\n            break\n\n    store.close()\nexcept Exception as e:\n    traceback.print_exc()\n\n    store.close()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:18:23.622990Z","iopub.execute_input":"2023-04-26T14:18:23.624325Z","iopub.status.idle":"2023-04-26T14:23:18.280042Z","shell.execute_reply.started":"2023-04-26T14:18:23.624257Z","shell.execute_reply":"2023-04-26T14:23:18.278580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#feature engineering na utworzonym wcześniej zbiorze danych\nstore =  pd.HDFStore(database_file)\ntry:\n    for name in df_names[-5:]:\n        chunk = store.get(name)\n        chunk[name+\"_time_dt\"] = chunk[name+\"_time\"]- chunk[name+\"_time\"].shift(1)\n        chunk[name+\"_time_idt\"]= chunk[name+\"_time\"]- chunk[name+\"_time\"].shift(-1)\n        chunk[name+\"_NAs_dt\"] = chunk[name+\"_NAs\"]- chunk[name+\"_NAs\"].shift(1)\n        chunk[name+\"_NAs_idt\"]= chunk[name+\"_NAs\"]- chunk[name+\"_NAs\"].shift(-1)\n        \n        store.put(name,value=chunk, format=\"table\")\n\n    table_date = store.get(\"all\")\n\n    table_date[\"P1\"] = np.logical_or(table_date[\"all_first\"] == table_date[\"all_first\"].shift(1), table_date[\"all_first\"] == table_date[\"all_first\"].shift(-1))\n\n    i = 1\n    p1 = table_date[table_date[\"P1\"]==True]\n    p1[\"ord\"] = np.logical_and(table_date[\"P1\"],table_date[\"P1\"].shift(1)==0)*1\n\n    while(len(p1[(p1[\"ord\"]==0)])>0):\n        left = p1[(p1[\"ord\"]==0)&(p1[\"ord\"].shift(1) ==i)]\n        left[\"ord\"]=i+1\n        p1.update(left)\n        i+=1\n    p1[\"group_len\"] = np.nan\n    p1[\"group_len\"] = p1[p1[\"ord\"].shift(-1)==1][\"ord\"]\n    p1.fillna(method=\"bfill\", inplace=True)\n\n    table_date[\"ord\"] = 0\n    table_date[\"group_len\"] = 0\n    table_date.update(p1)\n    table_date[\"P1\"]=table_date[\"P1\"].astype(bool)\n    store.put(\"all\",value=table_date, format=\"table\")\nexcept Exception as e:\n    traceback.print_exc()\n    store.close()\n    # w powyższych pętlach pobierane są wartości z bazy danych i dodawana jest kolumna P1\n    #która ma wartość TRUE jeśli wartości w kollumnie \"all_first\" są takie same jak wartości \n    #w poprzednim lub następnym wierszu\n    #następnie tworzona jest kolumna ord, która ma wartość 1 jeśli wiersz spełnia warunek P1\n    #a poprzedni wiersz nie spełniał tego warunku","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:18.283859Z","iopub.execute_input":"2023-04-26T14:23:18.284659Z","iopub.status.idle":"2023-04-26T14:23:19.400619Z","shell.execute_reply.started":"2023-04-26T14:23:18.284602Z","shell.execute_reply":"2023-04-26T14:23:19.399128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#wczytanie określonych kolumn z bazy danych w celu przeprowadzenia na nich analizy\ncolumns = ['Response','P1', 'ord', 'group_len', 'all_first','L3_time', 'L3_first', 'L3_last', 'L3_time_dt', 'L3_time_idt', 'L3_NAs_dt', 'L3_NAs_idt',\n           'L3_S38_F3960', 'L3_S33_F3865', 'L3_S38_F3956', 'L3_S33_F3857',\n       'L3_S29_F3321', 'L1_S24_F1846', 'L3_S32_F3850', 'L3_S29_F3354',\n       'L3_S29_F3324', 'L3_S35_F3889', 'L0_S1_F28', 'L1_S24_F1844',\n       'L3_S29_F3376', 'L0_S0_F22', 'L3_S33_F3859', 'L3_S38_F3952', \n       'L3_S30_F3754', 'L2_S26_F3113', 'L3_S30_F3759', 'L0_S5_F114']\n# Wybór tabeli do załadowania z HDF\nload = {\"all\":[]}\nfor col in columns:\n    if col in ['Response','P1', 'ord', 'group_len']:\n        load[\"all\"].append(col)\n    for name in df_names:\n        # if feature is from station get the station \n        if name +\"_\" in col:\n            if name not in load.keys():\n                load[name]=[]\n            load[name].append(col)\n\nstore =  pd.HDFStore(database_file)\ntable = pd.DataFrame()\ntry:\n    for name in load.keys():\n\n        if table.empty:\n            table = store.select(name, columns = load[name])\n        else:\n            table = pd.concat([table,store.select(name, columns=load[name])], axis=1)\n    store.close()\nexcept Exception as e:\n    traceback.print_exc()\n    store.close()\ntable","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:19.402536Z","iopub.execute_input":"2023-04-26T14:23:19.403006Z","iopub.status.idle":"2023-04-26T14:23:20.890872Z","shell.execute_reply.started":"2023-04-26T14:23:19.402967Z","shell.execute_reply":"2023-04-26T14:23:20.889599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(table.drop([\"Response\"], axis=1),\n                                                    table[\"Response\"],test_size=0.2, \n                                                    random_state=4, shuffle=True)\nX_train, X_val, y_train, y_val = train_test_split(X_train,y_train,test_size=0.2, \n                                                  random_state=4, shuffle=True)\n\ndtrain  = xgb.DMatrix(X_train,y_train)\ndval = xgb.DMatrix(X_val, y_val)\ndtest = xgb.DMatrix(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:20.892544Z","iopub.execute_input":"2023-04-26T14:23:20.892902Z","iopub.status.idle":"2023-04-26T14:23:20.995142Z","shell.execute_reply.started":"2023-04-26T14:23:20.892866Z","shell.execute_reply":"2023-04-26T14:23:20.994039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_round = 65\nevallist = [ (dtrain, 'train'),(dval, 'eval')]\nparam = {\"nthread\": 8, \"max_depth\" : 10, \"eta\":0.1, \"subsample\" : 0.9, \n         \"colsample_bytree\" : 0.5, \"objective\": \"binary:hinge\", \"booster\":\"gbtree\"}\n#max_depth- głebokość drzewa decyzyjnego, zbyt duża - overfitting, za mała - underfitting\n#eta- szybkość uczenia (za duża - skoki, za mała - wolno działa)\n#subsample- 0-1 frakcja cech wykorzystywana w każdym węźle drzewa (wysoka overfitting niska underfitting)\n#colsample_bytree- 0-1 to co powyzej\n#objective alternatywne opcje to np binary:logistic albo binary:softmax\n#num_round ilość rund uczenia\n#early_stopping za wysokie to za szybko zatrzyma a za niskie to przeuczy\nbst = xgb.train(param, dtrain, num_round,evallist,early_stopping_rounds = 5)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:20.998667Z","iopub.execute_input":"2023-04-26T14:23:20.999528Z","iopub.status.idle":"2023-04-26T14:23:21.642326Z","shell.execute_reply.started":"2023-04-26T14:23:20.999473Z","shell.execute_reply":"2023-04-26T14:23:21.641153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = bst.predict(dtest)\n#prob = np.sort(y_pred)[-int(len(y_pred)*0.006)]\n#y_pred = y_pred>prob\nprint(\"Matthews Correlation Coeffinciant:\",matthews_corrcoef(y_test,y_pred))\nc = confusion_matrix(y_test,y_pred)\nc","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.643957Z","iopub.execute_input":"2023-04-26T14:23:21.646826Z","iopub.status.idle":"2023-04-26T14:23:21.688685Z","shell.execute_reply.started":"2023-04-26T14:23:21.646781Z","shell.execute_reply":"2023-04-26T14:23:21.687401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb.plot_importance(bst, max_num_features=20)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.690616Z","iopub.execute_input":"2023-04-26T14:23:21.691001Z","iopub.status.idle":"2023-04-26T14:23:21.905315Z","shell.execute_reply.started":"2023-04-26T14:23:21.690963Z","shell.execute_reply":"2023-04-26T14:23:21.904063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store = pd.HDFStore(database_file)\n\n# print all dataset names\nprint(store.keys())\n# close the HDFStore\nstore.close()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.907141Z","iopub.execute_input":"2023-04-26T14:23:21.907576Z","iopub.status.idle":"2023-04-26T14:23:21.959496Z","shell.execute_reply.started":"2023-04-26T14:23:21.907536Z","shell.execute_reply":"2023-04-26T14:23:21.958164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store =  pd.HDFStore(database_file)\nwith pd.HDFStore(database_file) as store:\n    column_names = store.select('S47', start=0, stop=1).columns\n    print(column_names)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.961257Z","iopub.execute_input":"2023-04-26T14:23:21.961956Z","iopub.status.idle":"2023-04-26T14:23:21.982315Z","shell.execute_reply.started":"2023-04-26T14:23:21.961902Z","shell.execute_reply":"2023-04-26T14:23:21.980878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PANDAS PROFILING DLA DANYCH UCZĄCYCH Z XGBOOSTA","metadata":{}},{"cell_type":"code","source":"datapandasp = pd.DataFrame(table)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.984042Z","iopub.execute_input":"2023-04-26T14:23:21.984915Z","iopub.status.idle":"2023-04-26T14:23:21.990252Z","shell.execute_reply.started":"2023-04-26T14:23:21.984862Z","shell.execute_reply":"2023-04-26T14:23:21.989224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas_profiling import ProfileReport\nprof = ProfileReport(datapandasp)\nprof.to_file(output_file='output.html')","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:21.991649Z","iopub.execute_input":"2023-04-26T14:23:21.991999Z","iopub.status.idle":"2023-04-26T14:23:28.824178Z","shell.execute_reply.started":"2023-04-26T14:23:21.991966Z","shell.execute_reply":"2023-04-26T14:23:28.822899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prof.to_notebook_iframe()","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:28.825557Z","iopub.execute_input":"2023-04-26T14:23:28.825897Z","iopub.status.idle":"2023-04-26T14:23:28.875182Z","shell.execute_reply.started":"2023-04-26T14:23:28.825863Z","shell.execute_reply":"2023-04-26T14:23:28.874016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Próbkowanie","metadata":{}},{"cell_type":"markdown","source":"Są trzy metody próbkowania (sytuacje):\n\n\n\n        zbiór pierwszy      zbiór drugi\n\n            150000              1000                ---->       1000:1000\n\n            150000              1000                ---->       150000:150000\n\n            150000              1000                ---->       2000:2000","metadata":{}},{"cell_type":"markdown","source":"Usuniecie danych gdzie time_diff jest NaN a train_data nie ma informacji.","metadata":{}},{"cell_type":"code","source":"X_train_1 = X_train_1.dropna() # wszystkich 1183767  usunięto 582\ndata_pyca = X_train_1.dropna()\nX_train_1","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:28.876822Z","iopub.execute_input":"2023-04-26T14:23:28.877976Z","iopub.status.idle":"2023-04-26T14:23:29.746809Z","shell.execute_reply.started":"2023-04-26T14:23:28.877922Z","shell.execute_reply":"2023-04-26T14:23:29.745528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. rodzaj próbkowania","metadata":{}},{"cell_type":"code","source":"if sampling == 'under':\n    N = 20000\n    normal = X_train_1[X_train_1.Response == 0]\n    abnormal = X_train_1[X_train_1.Response == 1]\n    # --------------------------------------------------------------------------------------------------undersampling\n    normal = normal.sample(n=N, random_state=42)\n    print(normal.shape, abnormal.shape)\n    \n    temp = pd.concat([normal, abnormal])\n    X_s = temp.drop(columns=[\"Id\", \"Response\"])\n    y_s = temp.Response","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:29.751069Z","iopub.execute_input":"2023-04-26T14:23:29.751827Z","iopub.status.idle":"2023-04-26T14:23:29.758576Z","shell.execute_reply.started":"2023-04-26T14:23:29.751782Z","shell.execute_reply":"2023-04-26T14:23:29.757413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. rodzaj próbkowania","metadata":{}},{"cell_type":"code","source":"if sampling == 'over':\n    # 2) ------------------------------------------------------------------------------------------------oversampling\n    from imblearn.over_sampling import SMOTE\n\n    X = X_train_1.drop(columns=[\"Id\", \"Response\"])\n    y = X_train_1.Response\n\n    X_s, y_s = SMOTE(random_state=42).fit_resample(X, y)\n    print(X_s.shape, y_s.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:29.759910Z","iopub.execute_input":"2023-04-26T14:23:29.760333Z","iopub.status.idle":"2023-04-26T14:23:29.778957Z","shell.execute_reply.started":"2023-04-26T14:23:29.760272Z","shell.execute_reply":"2023-04-26T14:23:29.777557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. rodzaj próbkowania","metadata":{}},{"cell_type":"code","source":"if sampling == 'hybrid':\n    # 3) ------------------------------------------------------------------------------------------------Hybrid approach\n    N_sample = 20000\n    normal = X_train_1[X_train_1.Response == 0].sample(n=N_sample, random_state=42)\n    abnormal = X_train_1[X_train_1.Response == 1]\n    temp = pd.concat([normal, abnormal])\n\n    from imblearn.over_sampling import SMOTE\n\n    X = temp.drop(columns=[\"Id\", \"Response\"])\n    y = temp.Response\n\n    X_s, y_s = SMOTE(random_state=42).fit_resample(X, y)\n    print(X_s.shape, y_s.shape) ","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:29.780850Z","iopub.execute_input":"2023-04-26T14:23:29.781541Z","iopub.status.idle":"2023-04-26T14:23:31.418899Z","shell.execute_reply.started":"2023-04-26T14:23:29.781491Z","shell.execute_reply":"2023-04-26T14:23:31.417259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"zmiana float na int","metadata":{}},{"cell_type":"code","source":"X_s['start_station'] = X_s.start_station.astype(np.int8) \nX_s['end_station'] = X_s.end_station.astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:31.420419Z","iopub.execute_input":"2023-04-26T14:23:31.420830Z","iopub.status.idle":"2023-04-26T14:23:31.436203Z","shell.execute_reply.started":"2023-04-26T14:23:31.420791Z","shell.execute_reply":"2023-04-26T14:23:31.434429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Podział danych treningowych        8:2 train:test","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX = X_s\ny = y_s\n\nX_train_1, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\nprint(X_train_1.shape, X_val.shape, y_train.shape, y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:31.438242Z","iopub.execute_input":"2023-04-26T14:23:31.438787Z","iopub.status.idle":"2023-04-26T14:23:31.472293Z","shell.execute_reply.started":"2023-04-26T14:23:31.438733Z","shell.execute_reply":"2023-04-26T14:23:31.470686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Uczenie sieci i wyniki","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nmodel = RandomForestClassifier(n_estimators=20, max_depth=7, random_state=42)\nmodel.fit(X_train_1, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:31.474150Z","iopub.execute_input":"2023-04-26T14:23:31.474963Z","iopub.status.idle":"2023-04-26T14:23:32.217420Z","shell.execute_reply.started":"2023-04-26T14:23:31.474906Z","shell.execute_reply":"2023-04-26T14:23:32.216292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"zmiana metryki za każdym razem","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import matthews_corrcoef # MCC\nevaluation_metric = matthews_corrcoef","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:32.219345Z","iopub.execute_input":"2023-04-26T14:23:32.219731Z","iopub.status.idle":"2023-04-26T14:23:32.226943Z","shell.execute_reply.started":"2023-04-26T14:23:32.219694Z","shell.execute_reply":"2023-04-26T14:23:32.225410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Przewidywanie na poziomie\")\npred_train = model.predict(X_train_1)\npred_test = model.predict(X_val)\n\ntrain_score = evaluation_metric(y_train, pred_train)\ntest_score = evaluation_metric(y_val, pred_test)\n\nprint(\"Train Score : %.4f\" % train_score)\nprint(\"Test Score : %.4f\" % test_score)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:23:32.228584Z","iopub.execute_input":"2023-04-26T14:23:32.229136Z","iopub.status.idle":"2023-04-26T14:23:32.332551Z","shell.execute_reply.started":"2023-04-26T14:23:32.229066Z","shell.execute_reply":"2023-04-26T14:23:32.330962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Problem z instalacją pycaret w kaggle","metadata":{}},{"cell_type":"code","source":"!pip uninstall scikit-learn -y\n!pip install -U pycaret scikit-learn","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:40:28.686701Z","iopub.execute_input":"2023-04-26T14:40:28.687245Z","iopub.status.idle":"2023-04-26T14:41:24.397162Z","shell.execute_reply.started":"2023-04-26T14:40:28.687197Z","shell.execute_reply":"2023-04-26T14:41:24.395286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pycaret.classification import *\nfrom pycaret.datasets import get_data","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:42:01.352468Z","iopub.execute_input":"2023-04-26T14:42:01.353006Z","iopub.status.idle":"2023-04-26T14:42:06.495770Z","shell.execute_reply.started":"2023-04-26T14:42:01.352957Z","shell.execute_reply":"2023-04-26T14:42:06.493926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_pycaret = TRAIN_NUMERIC_100","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:42:19.367803Z","iopub.execute_input":"2023-04-26T14:42:19.369479Z","iopub.status.idle":"2023-04-26T14:42:19.376479Z","shell.execute_reply.started":"2023-04-26T14:42:19.369392Z","shell.execute_reply":"2023-04-26T14:42:19.373943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#exp_clf = setup(data_pycaret, target=\"Response\", fold_shuffle=True, session_id=2,silent = True, imputation_type=\"iterative\", categorical_iterative_imputer=\"lightgbm\", numeric_iterative_imputer=\"lightgbm\", iterative_imputation_iters = 5)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T14:42:26.908026Z","iopub.execute_input":"2023-04-26T14:42:26.909074Z","iopub.status.idle":"2023-04-26T14:42:26.915456Z","shell.execute_reply.started":"2023-04-26T14:42:26.909019Z","shell.execute_reply":"2023-04-26T14:42:26.914057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}