{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84253,"databundleVersionId":9421888,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-25T15:57:17.071321Z","iopub.execute_input":"2024-08-25T15:57:17.071737Z","iopub.status.idle":"2024-08-25T15:57:17.504284Z","shell.execute_reply.started":"2024-08-25T15:57:17.071696Z","shell.execute_reply":"2024-08-25T15:57:17.502967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.feature_selection import RFE\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:18.378982Z","iopub.execute_input":"2024-08-25T15:57:18.379539Z","iopub.status.idle":"2024-08-25T15:57:19.243036Z","shell.execute_reply.started":"2024-08-25T15:57:18.379498Z","shell.execute_reply":"2024-08-25T15:57:19.241886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/first-steps-in-data-science-new/train (7).csv\")\ntest = pd.read_csv(\"/kaggle/input/first-steps-in-data-science-new/test (4).csv\")\nsubmission = pd.read_csv('/kaggle/input/first-steps-in-data-science-new/sample.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:20.368966Z","iopub.execute_input":"2024-08-25T15:57:20.369526Z","iopub.status.idle":"2024-08-25T15:57:23.152959Z","shell.execute_reply.started":"2024-08-25T15:57:20.369486Z","shell.execute_reply":"2024-08-25T15:57:23.151847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:24.537865Z","iopub.execute_input":"2024-08-25T15:57:24.538292Z","iopub.status.idle":"2024-08-25T15:57:24.583472Z","shell.execute_reply.started":"2024-08-25T15:57:24.538252Z","shell.execute_reply":"2024-08-25T15:57:24.582424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:25.468398Z","iopub.execute_input":"2024-08-25T15:57:25.468821Z","iopub.status.idle":"2024-08-25T15:57:25.482807Z","shell.execute_reply.started":"2024-08-25T15:57:25.468782Z","shell.execute_reply":"2024-08-25T15:57:25.481451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z_scores = np.abs((train - train.mean()) / train.std())\n\nthreshold = 3.5\n\ntrain_no_outliers = train[(z_scores < threshold).all(axis=1)]","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:26.408399Z","iopub.execute_input":"2024-08-25T15:57:26.408820Z","iopub.status.idle":"2024-08-25T15:57:26.465304Z","shell.execute_reply.started":"2024-08-25T15:57:26.408783Z","shell.execute_reply":"2024-08-25T15:57:26.464227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_no_outliers = train_no_outliers.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:28.332818Z","iopub.execute_input":"2024-08-25T15:57:28.333617Z","iopub.status.idle":"2024-08-25T15:57:28.344783Z","shell.execute_reply.started":"2024-08-25T15:57:28.333573Z","shell.execute_reply":"2024-08-25T15:57:28.343685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split,GridSearchCV\nX = train_no_outliers.drop(columns=['Unnamed: 0','target'])\ny = train_no_outliers['target']","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:29.414131Z","iopub.execute_input":"2024-08-25T15:57:29.414587Z","iopub.status.idle":"2024-08-25T15:57:29.422665Z","shell.execute_reply.started":"2024-08-25T15:57:29.414543Z","shell.execute_reply":"2024-08-25T15:57:29.421621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import PolynomialFeatures\npoly = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)\nX_interaction = poly.fit_transform(X)\nX_interaction_df = pd.DataFrame(X_interaction, columns=poly.get_feature_names_out(X.columns))\nX_combined = pd.concat([X, X_interaction_df], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:30.951985Z","iopub.execute_input":"2024-08-25T15:57:30.952448Z","iopub.status.idle":"2024-08-25T15:57:33.644053Z","shell.execute_reply.started":"2024-08-25T15:57:30.952409Z","shell.execute_reply":"2024-08-25T15:57:33.642914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX = scaler.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:37.496857Z","iopub.execute_input":"2024-08-25T15:57:37.497357Z","iopub.status.idle":"2024-08-25T15:57:37.528521Z","shell.execute_reply.started":"2024-08-25T15:57:37.497278Z","shell.execute_reply":"2024-08-25T15:57:37.527455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:38.467860Z","iopub.execute_input":"2024-08-25T15:57:38.468281Z","iopub.status.idle":"2024-08-25T15:57:38.489143Z","shell.execute_reply.started":"2024-08-25T15:57:38.468242Z","shell.execute_reply":"2024-08-25T15:57:38.488183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeRegressor\nmodel_1 = DecisionTreeRegressor()\nmodel_1.fit(X_train,y_train)\ny_pred = model_1.predict(X_test)\nprint(f\"Stacking RMSE: {mean_squared_error(y_test, y_pred, squared=False):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:39.298926Z","iopub.execute_input":"2024-08-25T15:57:39.299315Z","iopub.status.idle":"2024-08-25T15:57:40.528431Z","shell.execute_reply.started":"2024-08-25T15:57:39.299280Z","shell.execute_reply":"2024-08-25T15:57:40.527402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nmodel_2 = LinearRegression()\nmodel_2.fit(X_train,y_train)\ny_pred = model_2.predict(X_test)\nprint(f\"Stacking RMSE: {mean_squared_error(y_test, y_pred, squared=False):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:42.179980Z","iopub.execute_input":"2024-08-25T15:57:42.180425Z","iopub.status.idle":"2024-08-25T15:57:42.325424Z","shell.execute_reply.started":"2024-08-25T15:57:42.180383Z","shell.execute_reply":"2024-08-25T15:57:42.324466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_grid = {\n    'fit_intercept': [True, False],\n    'positive': [True, False]\n}\n\ngrid_search = GridSearchCV(estimator=model_2, param_grid=param_grid, cv=5, scoring='neg_mean_squared_error')\ngrid_search.fit(X_train, y_train)\nprint(f\"Best parameters: {grid_search.best_params_}\")\nprint(f\"Best RMSE: {(-grid_search.best_score_)**0.5:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:43.463943Z","iopub.execute_input":"2024-08-25T15:57:43.464361Z","iopub.status.idle":"2024-08-25T15:57:46.175385Z","shell.execute_reply.started":"2024-08-25T15:57:43.464296Z","shell.execute_reply":"2024-08-25T15:57:46.174253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVR\nmodel_3 = SVR()\nmodel_3.fit(X_train,y_train)\ny_pred = model_3.predict(X_test)\nprint(f\"Stacking RMSE: {mean_squared_error(y_test, y_pred, squared=False):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:47.775496Z","iopub.execute_input":"2024-08-25T15:57:47.775937Z","iopub.status.idle":"2024-08-25T15:57:52.828843Z","shell.execute_reply.started":"2024-08-25T15:57:47.775899Z","shell.execute_reply":"2024-08-25T15:57:52.827769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import RFE\nfrom sklearn.metrics import mean_squared_error\n\n# Use the best estimator from the grid search\nbest_model = grid_search.best_estimator_\n\n# Feature selection with RFE using the best model\nselector = RFE(estimator=best_model, n_features_to_select=200)\nselector.fit(X_train, y_train)\n\n# Transform the training and testing data\nX_train_selected = selector.transform(X_train)\nX_test_selected = selector.transform(X_test)\n\n# Predict and evaluate\ny_pred = best_model.predict(X_test_selected)\nprint(f\"Stacking RMSE: {mean_squared_error(y_test, y_pred, squared=False):.2f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:57:54.072227Z","iopub.execute_input":"2024-08-25T15:57:54.072661Z","iopub.status.idle":"2024-08-25T15:57:54.181483Z","shell.execute_reply.started":"2024-08-25T15:57:54.072624Z","shell.execute_reply":"2024-08-25T15:57:54.180393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import StackingRegressor, BaggingRegressor\n\nbase_regressors = [\n     ('rf', BaggingRegressor(estimator=DecisionTreeRegressor(), n_estimators=10, random_state=42)),\n     ('svm', SVR())\n]\n\nmeta_regressor = LinearRegression()\n    \nstacking_reg = StackingRegressor(estimators=base_regressors, final_estimator=meta_regressor, cv=5)\n\nstacking_reg.fit(X_train, y_train)\n\ny_pred = stacking_reg.predict(X_test)\nprint(f\"Stacking RMSE: {mean_squared_error(y_test, y_pred, squared=False):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:59:06.087954Z","iopub.execute_input":"2024-08-25T15:59:06.089068Z","iopub.status.idle":"2024-08-25T16:00:06.846889Z","shell.execute_reply.started":"2024-08-25T15:59:06.089012Z","shell.execute_reply":"2024-08-25T16:00:06.845567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import Lasso, Ridge\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.datasets import make_regression\nfrom sklearn.model_selection import train_test_split\n\n# Define Lasso and Ridge models\nlasso = Lasso()\nridge = Ridge()\n\n# Define parameter grids for GridSearchCV\nlasso_param_grid = {\n    'alpha': [0.01, 0.1, 1, 10, 100]  # Regularization strength\n}\n\nridge_param_grid = {\n    'alpha': [0.01, 0.1, 1, 10, 100]  # Regularization strength\n}\n\n# Setup GridSearchCV for Lasso\nlasso_grid_search = GridSearchCV(estimator=lasso, param_grid=lasso_param_grid, cv=5, scoring='neg_mean_squared_error')\nlasso_grid_search.fit(X_train, y_train)\n\n# Setup GridSearchCV for Ridge\nridge_grid_search = GridSearchCV(estimator=ridge, param_grid=ridge_param_grid, cv=5, scoring='neg_mean_squared_error')\nridge_grid_search.fit(X_train, y_train)\n\n# Get the best model and make predictions for Lasso\nbest_lasso = lasso_grid_search.best_estimator_\ny_pred_lasso = best_lasso.predict(X_test)\nlasso_rmse = mean_squared_error(y_test, y_pred_lasso, squared=False)\nprint(f\"Best Lasso Alpha: {lasso_grid_search.best_params_['alpha']}\")\nprint(f\"Lasso RMSE: {lasso_rmse:.2f}\")\n\n# Get the best model and make predictions for Ridge\nbest_ridge = ridge_grid_search.best_estimator_\ny_pred_ridge = best_ridge.predict(X_test)\nridge_rmse = mean_squared_error(y_test, y_pred_ridge, squared=False)\nprint(f\"Best Ridge Alpha: {ridge_grid_search.best_params_['alpha']}\")\nprint(f\"Ridge RMSE: {ridge_rmse:.2f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T16:00:06.849272Z","iopub.execute_input":"2024-08-25T16:00:06.849657Z","iopub.status.idle":"2024-08-25T16:00:10.381757Z","shell.execute_reply.started":"2024-08-25T16:00:06.849622Z","shell.execute_reply":"2024-08-25T16:00:10.380148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import StackingRegressor, BaggingRegressor\nfrom sklearn.linear_model import Lasso, Ridge, LinearRegression\nfrom sklearn.svm import SVR\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split, GridSearchCV\n\n# Define base regressors\nbase_regressors = [\n    ('rf', BaggingRegressor(estimator=DecisionTreeRegressor(), n_estimators=10, random_state=42)),\n    ('svm', SVR())\n]\n\n# Define parameter grids for GridSearchCV for different meta-regressors\nparam_grids = {\n    'linear': {},\n    'lasso': {'final_estimator__alpha': [0.01, 0.1, 1, 10]},\n    'ridge': {'final_estimator__alpha': [0.01, 0.1, 1, 10]}\n}\n\n# Function to perform GridSearchCV for a given meta-regressor\ndef perform_grid_search(meta_regressor, param_grid):\n    stacking_reg = StackingRegressor(estimators=base_regressors, final_estimator=meta_regressor, cv=5)\n    grid_search = GridSearchCV(estimator=stacking_reg, param_grid=param_grid, cv=5, scoring='neg_mean_squared_error')\n    grid_search.fit(X_train, y_train)\n    return grid_search\n\n# Perform GridSearchCV for each meta-regressor\nresults = {}\nfor name, meta_regressor in zip(['linear', 'lasso', 'ridge'], [LinearRegression(), Lasso(), Ridge()]):\n    print(f\"Evaluating {name}...\")\n    grid_search = perform_grid_search(meta_regressor, param_grids[name])\n    results[name] = grid_search\n\n# Get the best model and make predictions\nbest_name = min(results, key=lambda k: results[k].best_score_)\nbest_grid_search = results[best_name]\nbest_model = best_grid_search.best_estimator_\ny_pred = best_model.predict(X_test)\n\n# Calculate and print RMSE\nstacking_rmse = mean_squared_error(y_test, y_pred, squared=False)\nprint(f\"Best Meta-Regressor: {best_name}\")\nprint(f\"Best Meta-Regressor Alpha: {best_grid_search.best_params_.get('final_estimator__alpha', 'N/A')}\")\nprint(f\"Stacking RMSE: {stacking_rmse:.2f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T16:00:10.383531Z","iopub.execute_input":"2024-08-25T16:00:10.383923Z","iopub.status.idle":"2024-08-25T16:35:36.482667Z","shell.execute_reply.started":"2024-08-25T16:00:10.383882Z","shell.execute_reply":"2024-08-25T16:35:36.481535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=best_model.predict(test.drop(columns=['Unnamed: 0']))\nsubmission['target'] = preds\nsubmission.to_csv(\"submission.csv\", index = False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-25T16:36:47.131021Z","iopub.execute_input":"2024-08-25T16:36:47.131464Z","iopub.status.idle":"2024-08-25T16:37:15.250057Z","shell.execute_reply.started":"2024-08-25T16:36:47.131425Z","shell.execute_reply":"2024-08-25T16:37:15.248828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}