{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84253,"databundleVersionId":9421888,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.linear_model import LinearRegression, LarsCV, BayesianRidge\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.model_selection import KFold\nfrom sklearn import feature_selection\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import MinMaxScaler\nimport os\n\n# Load data\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ntrain = pd.read_csv('/kaggle/input/first-steps-in-data-science-new/train (7).csv')\ntest = pd.read_csv('/kaggle/input/first-steps-in-data-science-new/test (4).csv')\n\n# Initialize parameters\nn_folds = 500\nbest_rmse = float('inf')\nbest_test_preds = None\n\n# Split data into features and target\ntrain_data = train.iloc[:, :-1]\ntrain_target = train.iloc[:, -1]\n\n# Normalize the data\nscaler = MinMaxScaler()\ntrain_data = scaler.fit_transform(train_data)\ntest_data = scaler.transform(test.iloc[:, :])\n\n# Convert back to DataFrame for feature selection\ntrain_data = pd.DataFrame(train_data, columns=train.columns[:-1])\ntest_data = pd.DataFrame(test_data, columns=train.columns[:-1])\n\n# Feature selection\nestimator = LinearRegression()\nselector = feature_selection.RFECV(estimator)\nselector.fit(train_data, train_target)\nrfe_train_data = train_data.iloc[:, selector.support_]\ntest_data = test_data.iloc[:, selector.support_]\n\n# Define base models\nbase_models = [\n    ('lars', LarsCV()),\n    ('linear', LinearRegression()),\n    ('bayesian_ridge', BayesianRidge())\n]\n\n# Define the stacking model\nstacking_model = StackingRegressor(\n    estimators=base_models,\n    final_estimator=LinearRegression()  # You can change this to another model if needed\n)\n\n# K-Fold cross-validation\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\nweights = []\ntest_preds = np.zeros((test_data.shape[0],))\n\n# Cross-validation and prediction\nfor fold, (train_idx, val_idx) in enumerate(kf.split(train)):\n    X_train, y_train = rfe_train_data.iloc[train_idx], train_target[train_idx]\n    X_val, y_val = rfe_train_data.iloc[val_idx], train_target[val_idx]\n    stacking_model.fit(X_train, y_train)\n    val_preds = stacking_model.predict(X_val)\n    fold_rmse = mean_squared_error(y_val, val_preds, squared=False)\n    weights.append(1 / fold_rmse)\n    print(f'Fold {fold+1} RMSE: {fold_rmse}')\n    test_preds += stacking_model.predict(test_data) * weights[fold]\n\n# Normalize test predictions\ntest_preds /= sum(weights)\n\noverall_rmse = np.mean([mean_squared_error(train_target[val_idx], stacking_model.predict(rfe_train_data.iloc[val_idx]), squared=False) for _, val_idx in kf.split(train)])\nprint(f'Overall RMSE: {overall_rmse}')\n\n# Check if this is the best performing model so far\nif overall_rmse < best_rmse:\n    best_rmse = overall_rmse\n    best_test_preds = test_preds\n\nprint(f'Final RMSE: {best_rmse}')\nprint(best_test_preds)\n\n# Prepare submission\nsubmission = pd.read_csv('/kaggle/input/first-steps-in-data-science-new/sample.csv')\nsubmission['target'] = best_test_preds\nsubmission.to_csv('submission.csv', index=False)\n","metadata":{},"execution_count":null,"outputs":[]}]}