{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **1. Import Libraries**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split \nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import GridSearchCV\n\nimport gc\n\n#importing for using regex to find columns with specific names\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **2. Import Data**","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None) #Stop view restriction of maximum number of columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:31:24.136262Z","iopub.execute_input":"2024-11-29T17:31:24.136739Z","iopub.status.idle":"2024-11-29T17:31:24.142144Z","shell.execute_reply.started":"2024-11-29T17:31:24.136704Z","shell.execute_reply":"2024-11-29T17:31:24.140854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load datasets\ndef load_datasets():\n    try:\n        # Load CSV files\n        train_df=pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\n        test_df=pd.read_csv(\"/kaggle/input/brist1d/test.csv\")\n        sample_submission=pd.read_csv(\"/kaggle/input/brist1d/sample_submission.csv\")\n        return train_df, test_df, sample_submission\n    \n    except Exception as e:\n        print(f\"Error loading datasets: {e}\")\n        return None\n    \ntrain_df, test_df, sample, = load_datasets()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to reduce memory usage\ndef reduce_memory(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f\"Memory usage before optimization: {start_mem:.2f} MB\")\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            if str(col_type).startswith(\"int\"):\n                df[col] = pd.to_numeric(df[col], downcast=\"integer\")\n            elif str(col_type).startswith(\"float\"):\n                df[col] = pd.to_numeric(df[col], downcast=\"float\")\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f\"Memory usage after optimization: {end_mem:.2f} MB\")\n    print(f\"Reduced by {(100 * (start_mem - end_mem) / start_mem):.1f}%\")\n\n    return df\n\n# Apply memory reduction\ntrain_df = reduce_memory(train_df)\ntest_df = reduce_memory(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:31:34.254280Z","iopub.execute_input":"2024-11-29T17:31:34.254647Z","iopub.status.idle":"2024-11-29T17:31:37.350094Z","shell.execute_reply.started":"2024-11-29T17:31:34.254612Z","shell.execute_reply":"2024-11-29T17:31:37.348855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **3. Data Exploration**","metadata":{}},{"cell_type":"code","source":"print(\"Names of the Columns:\",\"\\n\")\nprint(train_df.columns.tolist(), \"\\n\")\n\nprint(\"Size:\", train_df.size, \"\\n\")\nprint(\"Shape of the dataframe:\", train_df.shape,\"\\n\")\n\nprint(\"dtypes:\", \"\\n\")\ndtypes_dict = train_df.dtypes.apply(str).to_dict()\nimport json\nprint(json.dumps(dtypes_dict, indent=4), \"\\n\")\n\n\nmissing_counts = train_df.isnull().sum()\nprint(\"Missing Counts:\", \"\\n\")\nprint(missing_counts, \"\\n\")\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The following cells are intended to do a more in-depth analysis of missing values in the dataframe:**","metadata":{}},{"cell_type":"code","source":"# Find columns with only null values\ncolumns_null_values = train_df.isnull().sum() == len(train_df)\n\n# Grab all the columns with True values\nnull_columns = columns_null_values[columns_null_values]\n\n# Print results\nprint(f\"Number of columns with all null values: {len(null_columns)}\")\nprint(\"Columns with all null values:\")\nprint(null_columns.index.tolist())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The code below is intended to perform an exploratory analysis to see if we can use the activity column for feature engineering in the future:** \n\nIf commented out, then it means that activity columns had too many missing values and they were dropped for now.\n","metadata":{}},{"cell_type":"code","source":"# Identify activity columns\nactivity_columns = [col for col in train_df.columns if re.match(r'^activity.*', col.lower())]\n\nprint(\"Columns that start with 'activity':\", activity_columns)\n\n# Dictionary to store the most common activity value for each column\nmost_common_activities = {}\n\n# Identify unique activity values and count their occurrences across columns\nunique_values_counts = {}\n\n# Loop through each activity column\nfor column in activity_columns:\n    # Get most common value for each column\n    most_common_value = train_df[column].mode()[0]  # mode() returns a Series, so we take the first value\n    most_common_activities[column] = most_common_value\n    \n    # Count value occurrences in the column\n    column_value_counts = train_df[column].value_counts(dropna=True)\n    \n    # Update the global unique values counts\n    for value, count in column_value_counts.items(): \n        if value in unique_values_counts:\n            unique_values_counts[value] += count \n        else:\n            unique_values_counts[value] = count\n\n# Output the results\nprint(\"Most common activity for each column:\",\"\\n\") \nprint( most_common_activities, \"\\n\")\nprint(\"Unique activity counts across all columns:\", unique_values_counts, \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T20:51:22.962976Z","iopub.execute_input":"2024-11-29T20:51:22.963374Z","iopub.status.idle":"2024-11-29T20:51:23.576468Z","shell.execute_reply.started":"2024-11-29T20:51:22.963335Z","shell.execute_reply":"2024-11-29T20:51:23.575492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **4.Data Cleaning**","metadata":{}},{"cell_type":"code","source":"# Encode categorical variables  \ncategorical_columns = train_df.select_dtypes(include=\"category\").columns  \n\nfor col in categorical_columns:  \n    # Initialize the LabelEncoder  \n    le = LabelEncoder()  \n    \n    # Fit and transform the categorical column  \n    train_df[col] = le.fit_transform(train_df[col])  \n    test_df[col] = le.fit_transform(test_df[col])\n    # Save the encoder for inverse transformation later  ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Remove columns with more than 60% missing values\nmin_non_missing = int(0.6 * len(train_df))  # Minimum required non-missing values\ntrain_df = train_df.loc[:, train_df.notnull().sum() >= min_non_missing]\n\n# Step 2: Filter numeric columns for variance calculation\nnumeric_columns = train_df.select_dtypes(include=[\"number\"])\n\n# Step 3: Compute variance for numeric columns\nvariances = numeric_columns.var()\n\n# Step 4: Identify and drop low-variance columns\nlow_variance_threshold = 1\nlow_variance_cols = variances[variances < low_variance_threshold].index.tolist()\ntrain_df = train_df.drop(columns=low_variance_cols)\n\n# Output for reference\nprint(f\"Columns dropped due to low variance (threshold < {low_variance_threshold}): {low_variance_cols}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:31:40.599332Z","iopub.execute_input":"2024-11-29T17:31:40.599901Z","iopub.status.idle":"2024-11-29T17:31:41.973436Z","shell.execute_reply.started":"2024-11-29T17:31:40.599859Z","shell.execute_reply":"2024-11-29T17:31:41.972068Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **5.Feauture Selection and Feature Engineering**","metadata":{}},{"cell_type":"code","source":"# Selected columns\nselected_columns = [\n     \"bg-2:45\", \"bg-2:30\", \"bg-2:15\", \"bg-2:00\", \"bg-1:45\", \n    \"bg-1:30\", \"bg-1:15\",\"bg-1:00\", \"bg-0:45\", \"bg-0:30\", \n    \"bg-0:15\", \"bg-0:00\", \"bg+1:00\", \n    \"activity-3:35\", \"activity-4:05\", \"activity-3:50\", \n    \"activity-4:20\", \"activity-4:15\", \"activity-4:25\", \n    \"activity-3:05\", \"activity-3:20\", \"activity-0:05\", \n    \"activity-2:50\", \"activity-2:35\", \"activity-2:05\", \n    \"activity-0:20\", \"activity-0:15\", \"activity-0:10\", \n    \"activity-0:00\", \"activity-4:35\", \"activity-4:30\", \n    \"id\", \"activity-4:40\", \"activity-5:30\", \"time\", \n    \"p_num\", \"activity-5:55\", \"activity-5:50\", \n    \"activity-5:45\", \"activity-5:40\", \"activity-5:35\", \n    \"activity-5:25\", \"activity-4:45\", \"activity-5:20\", \n    \"activity-5:15\", \"activity-5:10\", \"activity-5:05\", \n    \"activity-5:00\", \"activity-4:55\", \"activity-4:50\", \n]\n\n# Select columns for train and test\ntrain_df = train_df[selected_columns]\ntest_df = test_df[[col for col in selected_columns if col in test_df.columns]]\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T20:51:29.067653Z","iopub.execute_input":"2024-11-29T20:51:29.068082Z","iopub.status.idle":"2024-11-29T20:51:29.189655Z","shell.execute_reply.started":"2024-11-29T20:51:29.068045Z","shell.execute_reply":"2024-11-29T20:51:29.188677Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Code below is used only for running imputing only in the sample of our dataframe for debugging purposes.**","metadata":{}},{"cell_type":"code","source":"# # Select 10% of the training data to see if code below runs \n# train_sample = train_df.sample(frac=0.002, random_state=42)\n# test_sample = test_df.sample(frac=0.002, random_state=42)\n\n\n#if you decide to run this code, modify the code below to run it on this sample instead of train_df and test_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:31:42.020906Z","iopub.execute_input":"2024-11-29T17:31:42.021233Z","iopub.status.idle":"2024-11-29T17:31:42.025716Z","shell.execute_reply.started":"2024-11-29T17:31:42.021203Z","shell.execute_reply":"2024-11-29T17:31:42.024621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_trainColl = train_df.select_dtypes(\"number\")  \nnumeric_testColl = test_df.select_dtypes(\"number\")  \n\nimputer = KNNImputer(n_neighbors=5)\ntrain_imputed = imputer.fit_transform(numeric_trainColl)\ntrain_df = pd.DataFrame(train_imputed, columns=numeric_trainColl.columns)\n\nimputer = KNNImputer(n_neighbors=5)\ntest_imputed = imputer.fit_transform(numeric_testColl)\ntest_df = pd.DataFrame(test_imputed, columns=numeric_testColl.columns)\ngc.collect()\n\ndel numeric_trainColl, numeric_testColl, train_imputed, test_imputed\n\nprint(\"Remaining missing values in train:\", train_df.isnull().sum().sum())\nprint(\"Remaining missing values in test:\", test_df.isnull().sum().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T20:51:35.539732Z","iopub.execute_input":"2024-11-29T20:51:35.540126Z","iopub.status.idle":"2024-11-29T20:57:23.975819Z","shell.execute_reply.started":"2024-11-29T20:51:35.540089Z","shell.execute_reply":"2024-11-29T20:57:23.974816Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **6.Linear Regression**","metadata":{}},{"cell_type":"markdown","source":"Linear Regression is a simple model used to predict a target variable by fitting a linear equation to the data, assuming a linear relationship between the features and the target. It works well for problems where this assumption holds and provides interpretable coefficients for each feature. In some cases, extensions like Ridge (L2 regularization) or Lasso (L1 regularization) can improve performance by controlling overfitting, using a regularization strength parameter (\nα\nα).","metadata":{}},{"cell_type":"code","source":"# Identify features and target\nX = train_df.drop(columns=['bg+1:00'])  # Drop the target column\ny = train_df['bg+1:00']  # The target column\n\n# Normalize/Scale Features\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\n\n# Split the data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n\n# Initialize and train the Linear Regression model\nmodel_linear = LinearRegression()\nmodel_linear.fit(X_train, y_train)\n\n# Predict values on the test set\ny_pred = model_linear.predict(X_test)\n\n# Evaluate the model\nmse = mean_squared_error(y_test, y_pred)\nprint(f\"Mean Squared Error: {mse}\")\n\nr2 = r2_score(y_test, y_pred)\nprint(f\"R-squared: {r2}\")\n\n# Visualizing the results\nplt.scatter(y_test, y_pred)\nplt.xlabel(\"True Values\")\nplt.ylabel(\"Predicted Values\")\nplt.title(\"True vs Predicted Values\")\nplt.show()\n\n# Visualizing residuals\nresiduals = y_test - y_pred\nplt.scatter(y_pred, residuals)\nplt.axhline(y=0, color='r', linestyle='--')\nplt.xlabel(\"Predicted Values\")\nplt.ylabel(\"Residuals\")\nplt.title(\"Residuals vs Predicted Values\")\nplt.show()\n\n# Model coefficients (Feature importance)\ncoefficients = model_linear.coef_\nfeature_names = X.columns\ncoeff_df = pd.DataFrame({\"Feature\": feature_names, \"Coefficient\": coefficients})\nprint(coeff_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We are adding polynomial features to the data and checking if the model improves. If adding polynomial features significantly improves the model’s performance (through R² or MSE), it suggests that the relationship is polynomial.","metadata":{}},{"cell_type":"code","source":"\nfrom sklearn.preprocessing import PolynomialFeatures\n\n# Create polynomial features\npoly = PolynomialFeatures(degree=2)\nX_poly = poly.fit_transform(X)\n\n# Fit linear regression with polynomial features\npoly_model = LinearRegression()\npoly_model.fit(X_poly, y)\n\n# Evaluate performance\ny_pred_poly = poly_model.predict(X_poly)\nmse_poly = mean_squared_error(y, y_pred_poly)\n\n# Compare with linear model\nlinear_model = LinearRegression()\nlinear_model.fit(X, y)\ny_pred_linear = linear_model.predict(X)\nmse_linear = mean_squared_error(y, y_pred_linear)\n\nprint(f\"Linear MSE: {mse_linear}\")\nprint(f\"Polynomial MSE: {mse_poly}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:15.606804Z","iopub.execute_input":"2024-11-29T17:41:15.607129Z","iopub.status.idle":"2024-11-29T17:41:53.575793Z","shell.execute_reply.started":"2024-11-29T17:41:15.607096Z","shell.execute_reply":"2024-11-29T17:41:53.573509Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In regression analysis, metrics like R2 and RMSE (Root Mean Squared Error) are crucial for evaluating model performance. The R2 score, also called the coefficient of determination, measures how well the model explains the variance in the target variable. Its value ranges from 0 to 1, where a higher value indicates that the model captures a larger portion of the variability in the data. An R2 score close to 1 suggests a strong fit, while a score near 0 implies that the model does not explain the target well. RMSE, on the other hand, quantifies the average magnitude of prediction errors by taking the square root of the mean of squared differences between actual and predicted values. A lower RMSE indicates better model performance, as it suggests that the predictions are closer to the true values. These metrics, used together, provide a comprehensive understanding of the model's accuracy and predictive capability. While R2 focuses on variance explanation, RMSE highlights the error magnitude, making them complementary for regression evaluation.","metadata":{}},{"cell_type":"markdown","source":"# 7. KNN model","metadata":{}},{"cell_type":"markdown","source":"K-Nearest Neighbors (KNN) is a non-parametric model that predicts the target by averaging the values (for regression) or selecting the majority class (for classification) among the \nk closest data points in the feature space. It is better suited for non-linear relationships but is sensitive to the choice of k, the distance metric (e.g., Euclidean or Manhattan), and whether neighbors are weighted by distance. Feature scaling is crucial for KNN to work effectively.","metadata":{}},{"cell_type":"code","source":"# Scale features for KNN\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\n# KNN Regression\nknn = KNeighborsRegressor(n_neighbors=5)  # Adjust n_neighbors as needed\nknn.fit(X_train_scaled, y_train)\ny_pred_knn = knn.predict(X_test_scaled)\n\n# Evaluate\nmse_knn = mean_squared_error(y_test, y_pred_knn)\nr2_knn = r2_score(y_test, y_pred)\nrmse_knn = np.sqrt(mean_squared_error(y_test, y_pred))\nprint(f\"KNN Regression MSE: {mse_knn}\")\nprint(f\"KNN Regression RMSE: {rmse_knn}\")\nprint(f\"KNN Regression R2: {r2_knn}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:41:53.577633Z","iopub.execute_input":"2024-11-29T17:41:53.578051Z","iopub.status.idle":"2024-11-29T17:42:10.008730Z","shell.execute_reply.started":"2024-11-29T17:41:53.578010Z","shell.execute_reply":"2024-11-29T17:42:10.006232Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Gradient Boosting Machines (GBM)","metadata":{}},{"cell_type":"markdown","source":"Gradient boosting methods (e.g., XGBoost, LightGBM, and CatBoost) are powerful machine learning algorithms for regression problems, particularly when dealing with high-dimensional and non-linear data. These methods work by iteratively building weak learners (decision trees) and focusing on the mistakes of previous trees. They are often more accurate than random forests when tuned correctly. Key hyperparameters include the learning rate, number of estimators, maximum depth, and subsampling rate.","metadata":{}},{"cell_type":"code","source":"# Initialize the GradientBoostingRegressor\ngbm = GradientBoostingRegressor(n_estimators=100, random_state=42)\n\n# Train the model\ngbm.fit(X_train, y_train)\n\n# Make predictions\ny_pred = gbm.predict(X_test)\n\n# Evaluate the model with RMSE, MSE, MAE, and R2\nrmse = np.sqrt(mean_squared_error(y_test, y_pred))\nmse = mean_squared_error(y_test, y_pred)\nr2 = r2_score(y_test, y_pred)\n\n# Print the results\nprint(\"Root Mean Squared Error (RMSE):\", rmse)\nprint(\"Mean Squared Error (MSE):\", mse)\nprint(\"R-squared (R2):\", r2)\n\n# Hyperparameter tuning with GridSearchCV\nparam_grid = {\n    'n_estimators': [50, 100, 200],\n    'learning_rate': [0.01, 0.1, 0.2],\n    'max_depth': [3, 5, 7],\n}\n\n# # Initialize GridSearchCV\n# grid_search = GridSearchCV(estimator=gbm, param_grid=param_grid, cv=3, scoring='neg_mean_squared_error')\n\n# # Fit the grid search\n# grid_search.fit(X_train, y_train)\n\n# # Best parameters from grid search\n# print(\"Best parameters from GridSearchCV:\", grid_search.best_params_)\n\n# # Retrain with the best parameters\n# best_gbm = grid_search.best_estimator_\n\n# # Retrain on the entire training set\n# best_gbm.fit(X_train, y_train)\n\n# # Make predictions with the best model\n# y_pred_final = best_gbm.predict(X_test)\n\n# # Evaluate again with the final model\n# final_rmse = np.sqrt(mean_squared_error(y_test, y_pred_final))\n# final_mse = mean_squared_error(y_test, y_pred_final)\n# final_r2 = r2_score(y_test, y_pred_final)\n\n# # Print the final evaluation metrics\n# print(\"\\nFinal Evaluation with Best Model:\")\n# print(\"Final RMSE:\", final_rmse)\n# print(\"Final MSE:\", final_mse)\n# print(\"Final R2:\", final_r2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-29T17:42:10.009907Z","iopub.execute_input":"2024-11-29T17:42:10.010220Z","iopub.status.idle":"2024-11-29T17:43:15.578837Z","shell.execute_reply.started":"2024-11-29T17:42:10.010188Z","shell.execute_reply":"2024-11-29T17:43:15.577830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Base models: Predictions on train and test sets\ny_pred_linear_train = model_linear.predict(X_train)\ny_pred_linear_test = model_linear.predict(X_test)\n\ny_pred_knn_train = knn.predict(X_train_scaled)\ny_pred_knn_test = knn.predict(X_test_scaled)\n\n# Stack predictions as new features\nstacked_train = np.column_stack((y_pred_linear_train, y_pred_knn_train))  # Combine base models' train predictions\nstacked_test = np.column_stack((y_pred_linear_test, y_pred_knn_test))    # Combine base models' test predictions\n\n# Meta-model: Train a linear regression on the stacked features\nmeta_model = LinearRegression()\nmeta_model.fit(stacked_train, y_train)\n\n# Predict using the stacked test data\ny_pred_stacked = meta_model.predict(stacked_test)\n\n# Evaluate the stacked model\nmse_stacked = mean_squared_error(y_test, y_pred_stacked)\nr2_stacked = r2_score(y_test, y_pred_stacked)\nrmse_stacked = np.sqrt(mean_squared_error(y_test, y_pred_stacked))\n\nprint(f\"Stacked Model MSE: {mse_stacked}\")\nprint(f\"Stacked Model R-squared: {r2_stacked}\")\nprint(f\"Stacked Model RMSE: {rmse_stacked}\")\n\n# Visualizing predictions\nplt.scatter(y_test, y_pred_stacked)\nplt.xlabel(\"True Values\")\nplt.ylabel(\"Stacked Model Predictions\")\nplt.title(\"True vs Stacked Model Predictions\")\nplt.show()\n\n# Visualizing residuals for the stacked model\nresiduals_stacked = y_test - y_pred_stacked\nplt.scatter(y_pred_stacked, residuals_stacked)\nplt.axhline(y=0, color='r', linestyle='--')\nplt.xlabel(\"Stacked Predictions\")\nplt.ylabel(\"Residuals\")\nplt.title(\"Residuals vs Stacked Predictions\")\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scale the test data\ntest_scaled = scaler.transform(test_df) \n\n# Prepare stacked test data\nbase_predictions_test = np.column_stack((\n    model_linear.predict(test_scaled),  # Predictions from Linear Regression\n    knn.predict(test_scaled)           # Predictions from KNN\n))\n\n# Make predictions with the stacked model\nstacked_predictions = meta_model.predict(base_predictions_test).flatten()\n\n# Create a submission DataFrame\nsubmission = pd.DataFrame({\n    'id': test_df['id'],               \n    'bg+1:00': stacked_predictions.flatten()     # Predicted target values\n})\n\n# Save to CSV\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file created:\", submission.shape)\n\n# Clear memory\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_df['id'].head())  # Display first few values\nprint(test_df['id'].isnull().sum())  # Check for missing values in 'id'\nprint(test_df.shape[0], test_df['id'].nunique())  # Compare total rows vs unique IDs","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}