{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84253,"databundleVersionId":9421888,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import LinearRegression\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:15.211327Z","iopub.execute_input":"2024-08-25T15:12:15.211699Z","iopub.status.idle":"2024-08-25T15:12:16.899452Z","shell.execute_reply.started":"2024-08-25T15:12:15.211666Z","shell.execute_reply":"2024-08-25T15:12:16.898308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/first-steps-in-data-science-new/train (7).csv\") \ntest = pd.read_csv(\"/kaggle/input/first-steps-in-data-science-new/test (4).csv\")","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:20.062516Z","iopub.execute_input":"2024-08-25T15:12:20.063067Z","iopub.status.idle":"2024-08-25T15:12:22.744169Z","shell.execute_reply.started":"2024-08-25T15:12:20.063025Z","shell.execute_reply":"2024-08-25T15:12:22.743309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train= train.drop(train.columns[0], axis=1)\ntest= test.drop(test.columns[0], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:25.345936Z","iopub.execute_input":"2024-08-25T15:12:25.346355Z","iopub.status.idle":"2024-08-25T15:12:25.376711Z","shell.execute_reply.started":"2024-08-25T15:12:25.346314Z","shell.execute_reply":"2024-08-25T15:12:25.375599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(train.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:28.016409Z","iopub.execute_input":"2024-08-25T15:12:28.017219Z","iopub.status.idle":"2024-08-25T15:12:28.029555Z","shell.execute_reply.started":"2024-08-25T15:12:28.017165Z","shell.execute_reply":"2024-08-25T15:12:28.028588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z_scores = np.abs((train - train.mean()) / train.std())\n\n# Set a threshold\nthreshold = 3.5\n\n# Filter out rows with Z-score greater than the threshold\ntrain_no_outliers = train[(z_scores < threshold).all(axis=1)]","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:33.042165Z","iopub.execute_input":"2024-08-25T15:12:33.043093Z","iopub.status.idle":"2024-08-25T15:12:33.088356Z","shell.execute_reply.started":"2024-08-25T15:12:33.043050Z","shell.execute_reply":"2024-08-25T15:12:33.087285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_no_outliers.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:36.196159Z","iopub.execute_input":"2024-08-25T15:12:36.196575Z","iopub.status.idle":"2024-08-25T15:12:36.202666Z","shell.execute_reply.started":"2024-08-25T15:12:36.196536Z","shell.execute_reply":"2024-08-25T15:12:36.201730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_no_outliers = train_no_outliers.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:38.306264Z","iopub.execute_input":"2024-08-25T15:12:38.306961Z","iopub.status.idle":"2024-08-25T15:12:38.316628Z","shell.execute_reply.started":"2024-08-25T15:12:38.306919Z","shell.execute_reply":"2024-08-25T15:12:38.315586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_no_outliers.drop(columns=['target'])\ny = train_no_outliers['target']","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:40.601384Z","iopub.execute_input":"2024-08-25T15:12:40.602302Z","iopub.status.idle":"2024-08-25T15:12:40.609265Z","shell.execute_reply.started":"2024-08-25T15:12:40.602259Z","shell.execute_reply":"2024-08-25T15:12:40.608431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize MinMaxScaler with the desired range (-1 to 1)\nscaler = MinMaxScaler(feature_range=(-1, 1))\n\n# Fit and transform the data\nX_final = pd.DataFrame(scaler.fit_transform(X), columns=X.columns)\ntest_final = pd.DataFrame(scaler.transform(test), columns = test.columns)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:12:42.626838Z","iopub.execute_input":"2024-08-25T15:12:42.627259Z","iopub.status.idle":"2024-08-25T15:12:42.683390Z","shell.execute_reply.started":"2024-08-25T15:12:42.627220Z","shell.execute_reply":"2024-08-25T15:12:42.682515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import PolynomialFeatures,StandardScaler\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import mean_squared_error\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:21:13.368024Z","iopub.execute_input":"2024-08-25T15:21:13.368971Z","iopub.status.idle":"2024-08-25T15:21:13.373398Z","shell.execute_reply.started":"2024-08-25T15:21:13.368929Z","shell.execute_reply":"2024-08-25T15:21:13.372420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of folds for cross-validation\nn_folds = 15\n\n# List to store the weights (based on RMSE) for each fold\nweights = []\n\n# Initialize KFold cross-validation\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n\n# Assign your features (train data) and target (labels)\ntrain_data = X_final\ntrain_target = y\n\n# Initialize the base model (Linear Regression) for RFE\nestimator = LinearRegression()\n\n# Initialize RFE (Recursive Feature Elimination) to select the top 150 features\nselector = RFE(estimator, n_features_to_select=150, step=1)\n\n# Fit the RFE model on the training data to select important features\nselector.fit(train_data, train_target)\n\n# Create a new training dataset with only the selected features\nrfe_train_data = train_data.iloc[:, selector.support_]\n\n# Apply the same feature selection to the test data\nrfe_test_data = test_final.iloc[:, selector.support_]\n\n# Initialize an array to store the predictions for the test set\ntest_preds = np.zeros((rfe_test_data.shape[0],))\n\n# Loop over each fold in the KFold cross-validation\nfor fold, (train_idx, val_idx) in enumerate(kf.split(rfe_train_data)):\n    # Split the data into training and validation sets for the current fold\n    X_train, y_train = rfe_train_data.iloc[train_idx], train_target.iloc[train_idx]\n    X_val, y_val = rfe_train_data.iloc[val_idx], train_target.iloc[val_idx]\n\n    # Initialize the pipeline for the current fold\n    pipeline = Pipeline([\n        ('poly_features', PolynomialFeatures(degree=2)),  # Adjust the degree for more complexity\n        ('scaler', StandardScaler()),                     # Feature scaling\n        ('ridge_reg', Ridge(alpha=1.0))                   # Ridge regularization to prevent overfitting\n    ])\n\n    # Fit the pipeline model\n    pipeline.fit(X_train, y_train)\n\n    # Predict the target for the validation set\n    val_preds = pipeline.predict(X_val)\n\n    # Calculate the RMSE for the current fold\n    fold_rmse = mean_squared_error(y_val, val_preds, squared=False)\n\n    # Calculate the weight for the current fold (inverse of RMSE)\n    weights.append(1 / fold_rmse)\n\n    # Print the RMSE for the current fold\n    print(f'Fold {fold+1} RMSE: {fold_rmse}')\n\n    # Predict the target for the test data and update test_preds\n    # Multiply the predictions by the weight for the current fold\n    test_preds += pipeline.predict(rfe_test_data) * weights[fold]\n\n# Normalize the test predictions by dividing by the sum of the weights\ntest_preds /= sum(weights)\n\n# Print the final test predictions\nprint(test_preds)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:25:49.295144Z","iopub.execute_input":"2024-08-25T15:25:49.295587Z","iopub.status.idle":"2024-08-25T15:29:21.390262Z","shell.execute_reply.started":"2024-08-25T15:25:49.295549Z","shell.execute_reply":"2024-08-25T15:29:21.387444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/first-steps-in-data-science-new/sample.csv\")\nsubmission['target'] = test_preds\nsubmission.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T15:29:42.204349Z","iopub.execute_input":"2024-08-25T15:29:42.204773Z","iopub.status.idle":"2024-08-25T15:29:42.271628Z","shell.execute_reply.started":"2024-08-25T15:29:42.204732Z","shell.execute_reply":"2024-08-25T15:29:42.270705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}