{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"},{"sourceId":123522,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":103967,"modelId":128179}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip uninstall scikit-learn -y\n!pip install scikit-learn==1.5.2\n#!pip uninstall pandas -y\n#!pip install pandas==2.1.4\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-29T16:54:22.034825Z","iopub.execute_input":"2024-09-29T16:54:22.035252Z","iopub.status.idle":"2024-09-29T16:54:22.437493Z","shell.execute_reply.started":"2024-09-29T16:54:22.035213Z","shell.execute_reply":"2024-09-29T16:54:22.436008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install optuna\n!pip uninstall lightgbm -y\n!pip install lightgbm --config-settings=cmake.define.USE_CUDA=ON","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:54:23.432400Z","iopub.execute_input":"2024-09-29T16:54:23.432967Z","iopub.status.idle":"2024-09-29T16:54:48.277525Z","shell.execute_reply.started":"2024-09-29T16:54:23.432920Z","shell.execute_reply":"2024-09-29T16:54:48.275948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm\nimport numpy as np\ndef check_gpu_support():\n    try:\n        data = np.random.rand(50, 2)\n        label = np.random.randint(2, size=50)\n        train_data = lightgbm.Dataset(data, label=label)\n        params = {'num_iterations': 1, 'device': 'gpu'}\n        gbm = lightgbm.train(params, train_set=train_data)\n        return True\n    except Exception as e:\n        return False\ncheck_gpu_support()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:54:48.280511Z","iopub.execute_input":"2024-09-29T16:54:48.280931Z","iopub.status.idle":"2024-09-29T16:54:49.644888Z","shell.execute_reply.started":"2024-09-29T16:54:48.280888Z","shell.execute_reply":"2024-09-29T16:54:49.644044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# check xgboost version\nimport xgboost as xg","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:54:49.646110Z","iopub.execute_input":"2024-09-29T16:54:49.646648Z","iopub.status.idle":"2024-09-29T16:54:49.679419Z","shell.execute_reply.started":"2024-09-29T16:54:49.646594Z","shell.execute_reply":"2024-09-29T16:54:49.678307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.metrics import mean_squared_error\nimport gc  # Import garbage collection for manual memory management\n\ndef remove_columns(df, columns_to_remove):\n    \"\"\"\n    Remove specified columns from the DataFrame.\n\n    :param df: Input DataFrame.\n    :param columns_to_remove: List of column names to remove.\n    :return: DataFrame with specified columns removed.\n    \"\"\"\n    df = df.drop(columns=columns_to_remove, errors='ignore')\n    return df\n\n# Function to reduce memory usage by downcasting numerical data types\ndef reduce_memory_usage(df):\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            if pd.api.types.is_integer_dtype(df[col]):\n                df[col] = pd.to_numeric(df[col], downcast='integer')\n            else:\n                df[col] = pd.to_numeric(df[col], downcast='float')\n        else:\n            # Convert object types to category type for memory efficiency\n            df[col] = df[col].astype('category')\n    \n    return df\n\n# Function to preprocess data and train model with Optuna for hyperparameter tuning\ndef preprocess_and_train_with_optuna(dataset_path, target_variable,columns_to_remove=None, n_trials=50, use_knn_imputer=True, chunk_size=5000):\n    # Load the dataset in chunks for memory efficiency\n    chunks = pd.read_csv(dataset_path, chunksize=chunk_size)\n\n    # Concatenate chunks into a single DataFrame while reducing memory usage\n    data = pd.concat([reduce_memory_usage(chunk) for chunk in chunks])\n    \n    if columns_to_remove is not None:\n        data = remove_columns(data, columns_to_remove)\n        \n    # Explicit garbage collection to free up memory\n    gc.collect()\n\n    # Check if target variable exists in dataset\n    if target_variable not in data.columns:\n        raise ValueError(f\"{target_variable} not found in the dataset\")\n\n    # Separate features and target\n    X = data.drop(columns=[target_variable])\n    y = data[target_variable]\n\n    # Optimize target column by converting to float32\n    y = y.astype(np.float32)\n\n    # Identify numerical and categorical columns\n    numerical_features = X.select_dtypes(include=['int32', 'int64', 'float32', 'float64']).columns\n    categorical_features = X.select_dtypes(include=['object', 'category']).columns\n\n    # Intelligent imputation for numerical features\n    if use_knn_imputer:\n        # Use KNN Imputer for numerical columns (more advanced, but can be heavy on memory)\n        numerical_imputer = KNNImputer(n_neighbors=3)\n    else:\n        # Use Median Imputer for numerical columns\n        numerical_imputer = SimpleImputer(strategy='median')\n\n    # Preprocessing for numerical features (KNN Imputation or Median Imputation and Scaling)\n    numerical_transformer = Pipeline(steps=[\n        ('imputer', numerical_imputer),\n        ('scaler', StandardScaler())\n    ])\n\n    # Intelligent imputation for categorical features (add 'Missing' as a new category)\n    categorical_transformer = Pipeline(steps=[\n        ('imputer', SimpleImputer(strategy='constant', fill_value='Missing')),\n        ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))\n    ])\n\n    # Combine numerical and categorical transformers\n    preprocessor = ColumnTransformer(\n        transformers=[\n            ('num', numerical_transformer, numerical_features),\n            ('cat', categorical_transformer, categorical_features)\n        ])\n\n    # Define the objective function for Optuna\n    def objective(trial):\n        # Define hyperparameter search space\n        xgboost_param = {\n            'device':'gpu',\n            'objective': 'reg:squarederror',\n            'eval_metric': 'rmse',\n            'tree_method': 'gpu_hist',  # Enable GPU for XGBoost\n            'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),\n            'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),\n            'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 0.1),\n            'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n            'max_depth': trial.suggest_int('max_depth', 3, 20),\n            'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n            'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n        }\n\n        # Create LightGBM model\n        model = Pipeline(steps=[\n            ('preprocessor', preprocessor),\n            ('regressor', xg.XGBRegressor(**xgboost_param, random_state=42))\n        ])\n        print(\"Training\")\n        # Cross-validation with 5-fold to evaluate the performance using RMSE\n        cv_scores = cross_val_score(model, X, y, cv=5, scoring='neg_root_mean_squared_error', n_jobs=-1)\n        rmse = -np.mean(cv_scores)  # RMSE (scikit-learn uses negative RMSE by default)\n        return rmse\n\n    # Create an Optuna study and optimize it\n    study = optuna.create_study(direction='minimize')\n   \n    study.optimize(objective, n_trials=n_trials)\n\n    # Get the best hyperparameters\n    best_params = study.best_params\n    print(f\"Best parameters: {best_params}\")\n\n    # Train the final model using the best hyperparameters\n    best_model = Pipeline(steps=[\n        ('preprocessor', preprocessor),\n        ('regressor', lgb.LGBMRegressor(**best_params, random_state=42))\n    ])\n\n    # Split the data into training and testing sets\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n    # Train the model\n    best_model.fit(X_train, y_train)\n\n    # Predict on the test set\n    y_pred = best_model.predict(X_test)\n\n    # Calculate and print final performance using RMSE\n    mse = mean_squared_error(y_test, y_pred)\n    rmse = np.sqrt(mse)\n    print(f'Test RMSE: {rmse}')\n\n    # Explicit garbage collection\n    gc.collect()\n\n    return best_model, study\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:54:49.681708Z","iopub.execute_input":"2024-09-29T16:54:49.682096Z","iopub.status.idle":"2024-09-29T16:54:49.965835Z","shell.execute_reply.started":"2024-09-29T16:54:49.682058Z","shell.execute_reply":"2024-09-29T16:54:49.964844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Predict function that accepts a DataFrame\ndef predict(model, input_data,columns_to_remove = None):\n    \"\"\"\n    Predicts target values using the trained model on new input data.\n\n    :param model: Trained regression model pipeline\n    :param input_data: Pandas DataFrame containing input features\n    :return: Predicted values\n    \"\"\"\n    # Ensure input data is in DataFrame format\n    if not isinstance(input_data, pd.DataFrame):\n        raise ValueError(\"Input data must be a pandas DataFrame\")\n    \n    if columns_to_remove is not None:\n        data = remove_columns(input_data, columns_to_remove)\n    \n    # Predict using the trained model\n    predictions = model.predict(data)\n\n    return predictions     ","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:56:00.721179Z","iopub.execute_input":"2024-09-29T16:56:00.721632Z","iopub.status.idle":"2024-09-29T16:56:00.732007Z","shell.execute_reply.started":"2024-09-29T16:56:00.721593Z","shell.execute_reply":"2024-09-29T16:56:00.730857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open('/kaggle/input/trainedlightgbm/scikitlearn/default/1/finalmodel.pkl', 'rb') as file:\n    model = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:54:59.439231Z","iopub.execute_input":"2024-09-29T16:54:59.439660Z","iopub.status.idle":"2024-09-29T16:54:59.472976Z","shell.execute_reply.started":"2024-09-29T16:54:59.439621Z","shell.execute_reply":"2024-09-29T16:54:59.471958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example DataFrame for prediction (replace with actual data)\ntest =  pd.read_csv('/kaggle/input/brist1d/test.csv')\n\n# Get predictions\npredictions = predict(model, test,columns_to_remove=['id'])\n","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:56:04.717020Z","iopub.execute_input":"2024-09-29T16:56:04.717555Z","iopub.status.idle":"2024-09-29T16:56:05.430322Z","shell.execute_reply.started":"2024-09-29T16:56:04.717509Z","shell.execute_reply":"2024-09-29T16:56:05.429099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['bg+1:00'] = predictions","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:57:53.814981Z","iopub.execute_input":"2024-09-29T16:57:53.815414Z","iopub.status.idle":"2024-09-29T16:57:53.822028Z","shell.execute_reply.started":"2024-09-29T16:57:53.815377Z","shell.execute_reply":"2024-09-29T16:57:53.820826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_sub_df = test[['id','bg+1:00']]","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:58:23.151079Z","iopub.execute_input":"2024-09-29T16:58:23.151544Z","iopub.status.idle":"2024-09-29T16:58:23.158039Z","shell.execute_reply.started":"2024-09-29T16:58:23.151501Z","shell.execute_reply":"2024-09-29T16:58:23.156718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_sub_df.to_csv('submission.csv',index = False)","metadata":{},"execution_count":null,"outputs":[]}]}