{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-25T04:05:44.930187Z","iopub.execute_input":"2024-10-25T04:05:44.930707Z","iopub.status.idle":"2024-10-25T04:05:46.189469Z","shell.execute_reply.started":"2024-10-25T04:05:44.930647Z","shell.execute_reply":"2024-10-25T04:05:46.188126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This code is completely for biginner\n\n###  I would explain following preprocessing steps\n\n1. feature selection(by reducing raws)\n2. make a preprocess pipeline\n3. make a training pipeline","metadata":{}},{"cell_type":"markdown","source":"## before explaining\n\n### Why I use pipeline for training?\n\n- The answer is it is just a useful. no more reasons\n\n### How useful??\n\n- read below and you'll notice how useful it is at the end","metadata":{}},{"cell_type":"markdown","source":"## Import libralies\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport yaml\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer, KNNImputer\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing   import OneHotEncoder, StandardScaler\nfrom sklearn.pipeline        import make_pipeline, Pipeline\nfrom sklearn.compose         import ColumnTransformer\nfrom sklearn.metrics         import accuracy_score\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.linear_model      import Lasso, LassoCV\nfrom sklearn.feature_selection import RFE, RFECV, SequentialFeatureSelector\nfrom sklearn.ensemble          import RandomForestClassifier\nfrom sklearn.feature_selection import SelectKBest, chi2\nfrom sklearn.feature_selection import SelectPercentile, f_classif\nfrom sklearn.feature_selection import f_regression,  mutual_info_regression, chi2\nfrom sklearn.feature_selection import RFE, RFECV, SequentialFeatureSelector\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.svm import SVR\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.decomposition         import PCA, KernelPCA","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:05:46.191700Z","iopub.execute_input":"2024-10-25T04:05:46.192253Z","iopub.status.idle":"2024-10-25T04:05:49.408171Z","shell.execute_reply.started":"2024-10-25T04:05:46.192200Z","shell.execute_reply":"2024-10-25T04:05:49.406914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## set a config class\n\n\n### reason\n\n- if you make and make variables in it , you can recall any variables from any cells in this notebook","metadata":{}},{"cell_type":"code","source":"class Conifg:\n    train_path = '/kaggle/input/brist1d/train.csv'\n    test_path = '/kaggle/input/brist1d/test.csv'\n    sample_path= '/kaggle/input/brist1d/sample_submission.csv'\n\nconfig=Conifg()\n\nconfig.train_path","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:05:49.409740Z","iopub.execute_input":"2024-10-25T04:05:49.410475Z","iopub.status.idle":"2024-10-25T04:05:49.420075Z","shell.execute_reply.started":"2024-10-25T04:05:49.410421Z","shell.execute_reply":"2024-10-25T04:05:49.418299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## impiot csv file","metadata":{}},{"cell_type":"code","source":"df_train=pd.read_csv(config.train_path,low_memory=False)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:05:49.423578Z","iopub.execute_input":"2024-10-25T04:05:49.424083Z","iopub.status.idle":"2024-10-25T04:06:10.639824Z","shell.execute_reply.started":"2024-10-25T04:05:49.424030Z","shell.execute_reply":"2024-10-25T04:06:10.638731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=pd.read_csv(config.test_path,low_memory=False)\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:10.641117Z","iopub.execute_input":"2024-10-25T04:06:10.641501Z","iopub.status.idle":"2024-10-25T04:06:10.929409Z","shell.execute_reply.started":"2024-10-25T04:06:10.641460Z","shell.execute_reply":"2024-10-25T04:06:10.928220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## make a definition\n\n\nto apply same preprocessing for both train and test data.</br>\n\n\nThis data has so many missing values ,so **<span style = 'color : red;'>imputing and cutting features would be critical for improving the accuracy</span>**\n\nI decided to cut columns with more than 17,000 missing values were eliminated. I felt that too many missing places could not be used for learning.\n\n\nI Do not delete lines for test data because if I cut lines in the test data, the submission lines will not match and I will not be able to submit the data.","metadata":{}},{"cell_type":"code","source":"# check the number of missing values for raws\nprint(df_train.isnull().sum(axis=1))\nprint(len(df_train.columns))","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:10.930691Z","iopub.execute_input":"2024-10-25T04:06:10.931018Z","iopub.status.idle":"2024-10-25T04:06:11.471407Z","shell.execute_reply.started":"2024-10-25T04:06:10.930985Z","shell.execute_reply":"2024-10-25T04:06:11.470275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the number of missing value for columns\nprint(df_train.isnull().sum(axis=0))\nprint(len(df_train.index))","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:11.472771Z","iopub.execute_input":"2024-10-25T04:06:11.473131Z","iopub.status.idle":"2024-10-25T04:06:11.968222Z","shell.execute_reply.started":"2024-10-25T04:06:11.473090Z","shell.execute_reply":"2024-10-25T04:06:11.967165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices=np.where(df_train.isnull().sum(axis=1)<100)[0]\nprint(len(indices))\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:11.969654Z","iopub.execute_input":"2024-10-25T04:06:11.970096Z","iopub.status.idle":"2024-10-25T04:06:12.520122Z","shell.execute_reply.started":"2024-10-25T04:06:11.970051Z","shell.execute_reply":"2024-10-25T04:06:12.518803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_time(df,thresholds,mode):\n    '''\n    Convert time data to datetime type and divide into hours and minutes\n    Cut variables themselves if the number of missing values exceeds 170000\n    '''\n    df['time']=pd.to_datetime(df['time'],format='%H:%M:%S')\n    df['hour']=df['time'].dt.hour\n    df['minute']=df['time'].dt.minute\n    df.drop('time',axis=1,inplace=True)\n\n    for column in df.columns:\n        if df[column].isnull().sum() >170000:\n            df=df.drop(column,axis=1)\n\n    if mode=='train':\n        df=df[df.isnull().sum(axis=1)<thresholds]\n    else:\n        pass\n    \n    df=df.drop(['id','p_num'],axis=1)\n    return df\n\n\n\ndf_train=expand_time(df_train,100,mode='train')\ndf_test=expand_time(df_test,100,mode='test')","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:12.521424Z","iopub.execute_input":"2024-10-25T04:06:12.521797Z","iopub.status.idle":"2024-10-25T04:06:58.681160Z","shell.execute_reply.started":"2024-10-25T04:06:12.521755Z","shell.execute_reply":"2024-10-25T04:06:58.679889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_cols=df_train.select_dtypes(include=['float64', 'int64']).columns.tolist()\ncategorical_cols=[column for column in df_train.columns if column not in numerical_cols]\n\nnumerical_cols.remove('bg+1:00')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:58.684898Z","iopub.execute_input":"2024-10-25T04:06:58.685342Z","iopub.status.idle":"2024-10-25T04:06:58.823851Z","shell.execute_reply.started":"2024-10-25T04:06:58.685294Z","shell.execute_reply":"2024-10-25T04:06:58.822313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set a 'bg+1:00' as a target and rests as a \nX_train=df_train.drop('bg+1:00',axis=1)\ny_train=df_train['bg+1:00']","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:58.825386Z","iopub.execute_input":"2024-10-25T04:06:58.825888Z","iopub.status.idle":"2024-10-25T04:06:58.932560Z","shell.execute_reply.started":"2024-10-25T04:06:58.825833Z","shell.execute_reply":"2024-10-25T04:06:58.931431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Okay, this definition seems to work correctly , Let's check a dataframe","metadata":{}},{"cell_type":"code","source":"display(X_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:58.933974Z","iopub.execute_input":"2024-10-25T04:06:58.934442Z","iopub.status.idle":"2024-10-25T04:06:58.986137Z","shell.execute_reply.started":"2024-10-25T04:06:58.934387Z","shell.execute_reply":"2024-10-25T04:06:58.985031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Huh? (currently being implemented）\n According to a data explanation,cals is **'total calories burnt in the last 5 minutes'**</br>\n so , It's weird that they have the same value regardless of time, now thay're getting all messy and putting in the same numbers?","metadata":{}},{"cell_type":"code","source":"# let's take a look at the last index as a example\nprint(X_train.iloc[-1,:].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:58.987591Z","iopub.execute_input":"2024-10-25T04:06:58.987964Z","iopub.status.idle":"2024-10-25T04:06:59.008582Z","shell.execute_reply.started":"2024-10-25T04:06:58.987915Z","shell.execute_reply":"2024-10-25T04:06:59.007267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df=X_train[X_train['cals-0:25'] == 6.75 ]\n\ndisplay(sample_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:59.009885Z","iopub.execute_input":"2024-10-25T04:06:59.010324Z","iopub.status.idle":"2024-10-25T04:06:59.065176Z","shell.execute_reply.started":"2024-10-25T04:06:59.010280Z","shell.execute_reply":"2024-10-25T04:06:59.063934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### missing value completion\nThere is room for improvement here, as we use a simple imputer.","metadata":{}},{"cell_type":"code","source":"imputer = SimpleImputer(strategy='mean')\n#imputer = KNNImputer(n_neighbors=3, weights='distance')\n\nX_train[numerical_cols] = imputer.fit_transform(X_train[numerical_cols])\ndf_test[numerical_cols] = imputer.transform(df_test[numerical_cols])\n\n\nX_train","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:06:59.066682Z","iopub.execute_input":"2024-10-25T04:06:59.067066Z","iopub.status.idle":"2024-10-25T04:07:00.581431Z","shell.execute_reply.started":"2024-10-25T04:06:59.067023Z","shell.execute_reply":"2024-10-25T04:07:00.580187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:07:00.582973Z","iopub.execute_input":"2024-10-25T04:07:00.583429Z","iopub.status.idle":"2024-10-25T04:07:00.658245Z","shell.execute_reply.started":"2024-10-25T04:07:00.583383Z","shell.execute_reply":"2024-10-25T04:07:00.656996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## make a pipeline\n\n- Create a separate pre-processing pipeline and training pipeline and merge the flows into one with make_pipeline.","metadata":{}},{"cell_type":"code","source":"preprocessor = ColumnTransformer([\n    #('ohe', OneHotEncoder(), categorical_cols),\n    ('std', StandardScaler(), numerical_cols),\n], remainder='passthrough')\n\nfeature_selection_pipeline = Pipeline([\n    ('preprocessor',preprocessor),\n    #('selector',  SelectFromModel(LassoCV(random_state=42,alphas=[0.001, 0.01, 0.1]))),\n    ('selector', SelectKBest(f_regression, k=100))\n])\n\n    #('selector',SelectFromModel(LassoCV(random_state=42,alphas=[0.001, 0.01, 0.1]))) ,# By default, 5-fold cross validation\n\n\nfeature_selection_pipeline","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:07:00.659675Z","iopub.execute_input":"2024-10-25T04:07:00.660078Z","iopub.status.idle":"2024-10-25T04:07:00.694757Z","shell.execute_reply.started":"2024-10-25T04:07:00.660029Z","shell.execute_reply":"2024-10-25T04:07:00.693524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the feature selection pipeline and transform the data\nX_train_selected = feature_selection_pipeline.fit_transform(X_train, y_train)\n\n# (Optional) Print selected features\nall_features = feature_selection_pipeline.named_steps['preprocessor'].get_feature_names_out()\nselected_features = feature_selection_pipeline.named_steps['selector'].get_support()\nprint(\"Selected Features:\\n\", all_features[selected_features])","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:07:00.696327Z","iopub.execute_input":"2024-10-25T04:07:00.696799Z","iopub.status.idle":"2024-10-25T04:07:02.142683Z","shell.execute_reply.started":"2024-10-25T04:07:00.696752Z","shell.execute_reply":"2024-10-25T04:07:02.139215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Specify the model to be ensemble by estimater.","metadata":{}},{"cell_type":"code","source":"estimators = [\n              ('svr',SVR()),\n              #('lgbm', lgb.LGBMRegressor()),\n            ('knn', KNeighborsRegressor(n_neighbors=4)),\n                ]\n\nvoting_reg = VotingRegressor(estimators)\n\ntraining_pipeline = make_pipeline(\n      feature_selection_pipeline,\n      voting_reg,\n\n  )\n\ntraining_pipeline","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:07:02.148758Z","iopub.execute_input":"2024-10-25T04:07:02.152681Z","iopub.status.idle":"2024-10-25T04:07:02.245757Z","shell.execute_reply.started":"2024-10-25T04:07:02.152602Z","shell.execute_reply":"2024-10-25T04:07:02.244633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## validate with cross validation","metadata":{}},{"cell_type":"code","source":"cv = KFold(n_splits=3, shuffle=False)\nscores = cross_val_score(training_pipeline, X_train, y_train, cv=cv, scoring='neg_mean_squared_error')\nprint(\"Cross-Validation RMSE:\", scores)\nprint(f\"RMSE: {np.mean(scores).round(3)} ± {2*np.std(scores).round(3)}\" )","metadata":{"execution":{"iopub.status.busy":"2024-10-25T04:07:02.247167Z","iopub.execute_input":"2024-10-25T04:07:02.247640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_pipeline.fit(X_train,y_train)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred=training_pipeline.predict(df_test)\npred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## I will fix this error tomorrow","metadata":{}},{"cell_type":"code","source":"sub=pd.read_csv(config.sample_path)\nsub['bg+1:00']=pred\n\nsub\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}