{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":85540,"databundleVersionId":9725394,"sourceType":"competition"},{"sourceId":9915259,"sourceType":"datasetVersion","datasetId":6092985}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **0. Before You Start**","metadata":{}},{"cell_type":"markdown","source":"1. This is based on the former work, where I get the embeddings from both images and text message.\n   [https://www.kaggle.com/code/kaiyisung/image-caption-and-embeddings-with-openai](http://)\n3. **Improvement in the future:**  I actually ask it too detect how many windows it contained in the photo, but I forgot to extract it out into the tabular data.","metadata":{}},{"cell_type":"code","source":"import torch\nhas_mps = torch.backends.mps.is_built()\ndevice = \"mps\" if has_mps else \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:45.530313Z","iopub.execute_input":"2024-11-23T09:24:45.530679Z","iopub.status.idle":"2024-11-23T09:24:48.473779Z","shell.execute_reply.started":"2024-11-23T09:24:45.530649Z","shell.execute_reply":"2024-11-23T09:24:48.472876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:48.475644Z","iopub.execute_input":"2024-11-23T09:24:48.476597Z","iopub.status.idle":"2024-11-23T09:24:48.479961Z","shell.execute_reply.started":"2024-11-23T09:24:48.476564Z","shell.execute_reply":"2024-11-23T09:24:48.479259Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **1. Data processing**","metadata":{}},{"cell_type":"markdown","source":"Since I saved the whole embeddings as a column in the tabular data, I'll have to flatten it so that it'll turn into a 1500+ columns dataFrame.","metadata":{}},{"cell_type":"code","source":"import ast\nimport re\nimport numpy as np\nimport pandas as pd\n\n# Load training and submission data\ndf_train = pd.read_csv(\"/kaggle/input/new-com-orig/new_com_orig.csv\", na_values=['NA', '?'])\ndf_submit = pd.read_csv(\"/kaggle/input/new-com-orig/new_com_orig.csv\", na_values=['NA', '?'])\n\n# Filter out rows where 'target_homevalue' is missing\ndf_train = df_train.dropna(subset=['target_homevalue'])\ndf_submit = df_submit[df_submit['target_homevalue'].isna()]\n\n# Updated function to parse and flatten embeddings\ndef parse_and_flatten_embeddings(x):\n    try:\n        \"\"\n        # If the value is a string, attempt to parse it as a list\n        if isinstance(x, str):\n            parsed = ast.literal_eval(x)\n\n            # If parsing results in a list, iterate through its elements\n            if isinstance(parsed, list):\n                flattened_list = []\n                for element in parsed:\n                    # If an element is a string representation of a list, parse it\n                    if isinstance(element, str) and element.startswith('[') and element.endswith(']'):\n                        element = ast.literal_eval(element)\n\n                    # If element is now a list, extend the flattened list with its contents\n                    if isinstance(element, list):\n                        flattened_list.extend(element)\n                    else:\n                        flattened_list.append(float(element))\n\n                return flattened_list\n\n        # If x is already a list (parsed correctly), return it\n        elif isinstance(x, list):\n            return x\n        else:\n            return np.nan\n    except (ValueError, SyntaxError):\n        return np.nan\n\n# Apply parsing to the embeddings column\ndf_train['embeddings_final_all_features'] = df_train['embeddings_final_all_features'].apply(parse_and_flatten_embeddings)\ndf_submit['embeddings_final_all_features'] = df_submit['embeddings_final_all_features'].apply(parse_and_flatten_embeddings)\n\ntemp = pd.DataFrame(data=df_train['embeddings_final_all_features'].tolist())\ntemp['target_homevalue'] = df_train['target_homevalue'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:48.480862Z","iopub.execute_input":"2024-11-23T09:24:48.481106Z","iopub.status.idle":"2024-11-23T09:24:57.777160Z","shell.execute_reply.started":"2024-11-23T09:24:48.481081Z","shell.execute_reply":"2024-11-23T09:24:57.776474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n# Split into features and target\nX = temp.drop(columns=[\"target_homevalue\"])\ny = temp[\"target_homevalue\"]\n\n# Train-test split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:57.779623Z","iopub.execute_input":"2024-11-23T09:24:57.780311Z","iopub.status.idle":"2024-11-23T09:24:58.359883Z","shell.execute_reply.started":"2024-11-23T09:24:57.780269Z","shell.execute_reply":"2024-11-23T09:24:58.359106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **2. features Permutations**","metadata":{}},{"cell_type":"code","source":"# from catboost import CatBoostRegressor\n\n# # Initialize the model\n# cat_model = CatBoostRegressor(\n#     iterations=1200,         # Number of boosting iterations\n#     learning_rate=0.03,      # Learning rate\n#     depth=6,                 # Depth of the tree\n#     loss_function=\"MAE\",     # Loss function; \"MAE\" or \"RMSE\" for regression\n#     verbose=100,             # Show training progress\n#     task_type='GPU'\n# )\n\n# # Train the model\n# cat_model.fit(X_train, y_train, eval_set=(X_test, y_test), early_stopping_rounds=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:58.360883Z","iopub.execute_input":"2024-11-23T09:24:58.361274Z","iopub.status.idle":"2024-11-23T09:24:58.365496Z","shell.execute_reply.started":"2024-11-23T09:24:58.361242Z","shell.execute_reply":"2024-11-23T09:24:58.364604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Predict on the test set\n# y_pred = cat_model.predict(X_test)\n\n# #Evaluate the value\n# from sklearn.metrics import mean_absolute_error, mean_squared_error\n\n# mae = mean_absolute_error(y_test, y_pred)\n# mse = mean_squared_error(y_test, y_pred)\n\n# print(f\"Mean Absolute Error: {mae}\")\n# print(f\"Mean Squared Error: {mse}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:24:58.366795Z","iopub.execute_input":"2024-11-23T09:24:58.367202Z","iopub.status.idle":"2024-11-23T09:24:58.382245Z","shell.execute_reply.started":"2024-11-23T09:24:58.367151Z","shell.execute_reply":"2024-11-23T09:24:58.381399Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **2.1 Optnua the cat model**","metadata":{}},{"cell_type":"markdown","source":"I didn't use it this time.","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Suggest hyperparameters\n    learning_rate = trial.suggest_loguniform(\"learning_rate\", 0.001, 0.1)\n    depth = trial.suggest_int(\"depth\", 4, 10)\n    l2_leaf_reg = trial.suggest_loguniform(\"l2_leaf_reg\", 1e-3, 10.0)\n    bagging_temperature = trial.suggest_uniform(\"bagging_temperature\", 0, 1)\n    iterations = trial.suggest_int(\"iterations\", 100, 1500, 10)\n    \n    # Initialize the model with suggested hyperparameters\n    cat_model = CatBoostRegressor(\n        iterations=iterations,                # Fixed number of iterations\n        learning_rate=learning_rate,\n        depth=depth,\n        l2_leaf_reg=l2_leaf_reg,\n        bagging_temperature=bagging_temperature,\n        loss_function=\"RMSE\",            # You are optimizing MAE\n        verbose=0,                      # Suppress training logs for faster runs\n        task_type=\"GPU\"                 # Use GPU for faster training\n    )\n    \n    # Train the model\n    cat_model.fit(X_train, y_train, eval_set=(X_test, y_test), early_stopping_rounds=50, verbose=0)\n    \n    # Predict and calculate MAE\n    y_pred = cat_model.predict(X_test)\n    mae = mean_absolute_error(y_test, y_pred)\n    \n    return mae  # Return the evaluation metric to minimize\n\n# Create an Optuna study\nstudy = optuna.create_study(direction=\"minimize\", study_name=\"CatBoost Optimization\")\n\n# Optimize the study\nstudy.optimize(objective, n_trials=50)\n\n# Print best parameters and score\nprint(\"Best Parameters: \", study.best_params)\nprint(\"Best RMSE: \", study.best_value)\n\n# Train the best model using the best parameters\nbest_params = study.best_params\ncat_model = CatBoostRegressor(\n    iterations=best_params['iterations'],\n    learning_rate=best_params[\"learning_rate\"],\n    depth=best_params[\"depth\"],\n    l2_leaf_reg=best_params[\"l2_leaf_reg\"],\n    bagging_temperature=best_params[\"bagging_temperature\"],\n    loss_function=\"RMSE\",\n    verbose=0,\n    task_type=\"GPU\"\n)\n\n# Fit the model with the best parameters\ncat_model.fit(X_train, y_train, eval_set=(X_test, y_test), early_stopping_rounds=50)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T09:26:22.008959Z","iopub.execute_input":"2024-11-23T09:26:22.009786Z","iopub.status.idle":"2024-11-23T09:56:42.952470Z","shell.execute_reply.started":"2024-11-23T09:26:22.009748Z","shell.execute_reply":"2024-11-23T09:56:42.951510Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **2.2 Features Permutation**","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\ndef permutation_importance(model, X_val, y_val, n_repeats=10):\n    # Convert X_val to DataFrame if it's a numpy array\n    if isinstance(X_val, np.ndarray):\n        X_val = pd.DataFrame(X_val)\n\n    # Calculate the baseline score\n    baseline_score = mean_absolute_error(y_val, model.predict(X_val))\n    feature_importances = {}\n\n    # Iterate over each feature in the validation set with a progress bar\n    for col in tqdm(X_val.columns, desc=\"Calculating feature importances\"):\n        scores = []\n        for _ in range(n_repeats):\n            # Shuffle the feature values\n            X_val_shuffled = X_val.copy()\n            X_val_shuffled[col] = np.random.permutation(X_val_shuffled[col])\n\n            # Calculate the score with the shuffled feature\n            shuffled_score = mean_absolute_error(y_val, model.predict(X_val_shuffled))\n            scores.append(shuffled_score - baseline_score)\n\n        # Average the score increase across all repeats\n        feature_importances[col] = np.mean(scores)\n\n    # Sort features by importance\n    feature_importances = pd.Series(feature_importances).sort_values(ascending=False)\n    return feature_importances","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T10:32:16.119902Z","iopub.execute_input":"2024-11-23T10:32:16.120237Z","iopub.status.idle":"2024-11-23T10:32:16.126763Z","shell.execute_reply.started":"2024-11-23T10:32:16.120207Z","shell.execute_reply":"2024-11-23T10:32:16.125853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate permutation feature importance\nimportances = permutation_importance(cat_model, X_test, y_test)\nprint(importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T10:32:19.418228Z","iopub.execute_input":"2024-11-23T10:32:19.419020Z","iopub.status.idle":"2024-11-23T10:54:29.463123Z","shell.execute_reply.started":"2024-11-23T10:32:19.418987Z","shell.execute_reply":"2024-11-23T10:54:29.462273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **3. Training the model**","metadata":{}},{"cell_type":"markdown","source":"## **3.1 Selecting important features**","metadata":{}},{"cell_type":"code","source":"# Set a threshold to filter important features\nimportance_threshold = 0.01  # This can be adjusted based on your needs\n\n# Get the list of important feature indices above the threshold\nimportant_feature_indices = importances[importances > importance_threshold].index.tolist()\nprint(f\"Number of important features: {len(important_feature_indices)}\")\n\nX = temp.drop(columns = ['target_homevalue'])\nX = X[important_feature_indices]\nX['target_homevalue'] = temp['target_homevalue']\n\nX_submit = pd.DataFrame(data=df_submit['embeddings_final_all_features'].tolist())\nX_submit = X_submit[important_feature_indices]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T10:54:29.528996Z","iopub.execute_input":"2024-11-23T10:54:29.529716Z","iopub.status.idle":"2024-11-23T10:54:29.589841Z","shell.execute_reply.started":"2024-11-23T10:54:29.529673Z","shell.execute_reply":"2024-11-23T10:54:29.588826Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3.2 AutoML**","metadata":{}},{"cell_type":"code","source":"import tempfile\nfrom autogluon.tabular import TabularPredictor\n\ntarget = 'target_homevalue'\n\n# Create a temporary directory to avoid saving models to a permanent folder\ntemp_model_path = tempfile.mkdtemp()\n\n# Train AutoGluon model with the desired adjustments\npredictor = TabularPredictor(label=target, eval_metric='mae',problem_type='regression', path=temp_model_path).fit(\n    X,\n    presets='best_quality',  # Using the best quality preset for higher accuracy\n    time_limit=8000,  # Control the training time (in seconds)\n    num_bag_folds=10,  # Enable ensembling by using bagging with 5 folds\n    num_stack_levels=2,  # Enable stacking to further improve model performance\n    ag_args_fit={'num_gpus': 1},# Set to True if using GPU for model training\n    save_bag_folds=True,\n    verbosity=0  # Suppress logging output (0: most silent, 4: most detailed)\n)\n\nleaderboard = predictor.leaderboard(silent=True)\nleaderboard.to_csv('leaderboard_2.csv', index=False)\nleaderboard.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T10:54:29.591491Z","iopub.execute_input":"2024-11-23T10:54:29.591769Z","iopub.status.idle":"2024-11-23T13:07:51.340273Z","shell.execute_reply.started":"2024-11-23T10:54:29.591743Z","shell.execute_reply":"2024-11-23T13:07:51.339384Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3.3 Submit: Ridge**","metadata":{}},{"cell_type":"code","source":"# Generate OOF predictions\noof_predictions = predictor.get_oof_pred()\n\n# Load test predictions\ntest_predictions = predictor.predict(X_submit)\n\nfrom sklearn.linear_model import Ridge\nimport pandas as pd\n\n# Assuming oof_predictions and true labels are available\noof_data = pd.DataFrame(oof_predictions)\noof_data['target'] = temp['target_homevalue']  # Replace `temp` with your actual training data, ensure that temp has the original target\n\n# Train a custom ensemble model using the OOF predictions\nmeta_model = Ridge()  # Example meta-learner (you could use other regression models or experiment with stacking)\nmeta_model.fit(oof_data.drop(columns=['target']), oof_data['target'])\n\n# Create final predictions using the trained meta-model\ntest_oof_predictions = pd.DataFrame(test_predictions)\nfinal_predictions = meta_model.predict(test_oof_predictions)\n\n# Create the submission file\nsub2 = pd.DataFrame({\n    'homeid': df_submit['homeid'],\n    'target_homevalue': final_predictions\n})\n\n# Save the submission file\nsub2.to_csv('sub02.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T13:17:12.030877Z","iopub.execute_input":"2024-11-23T13:17:12.031283Z","iopub.status.idle":"2024-11-23T13:17:21.017125Z","shell.execute_reply.started":"2024-11-23T13:17:12.031248Z","shell.execute_reply":"2024-11-23T13:17:21.016239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3.4 Submit: original**","metadata":{}},{"cell_type":"code","source":"X_submit\n\n# Use AutoGluon to predict class labels (0 or 1) for 'depression'\ny_test_pred = predictor.predict(X_submit)\n\n#Ensure alinged correctly\ndf_submit = df_submit.reset_index(drop=True)\ny_test_pred = pd.Series(y_test_pred).reset_index(drop=True)\n\n# Create the submission file\nsub1 = pd.DataFrame({\n    'homeid': df_submit['homeid'],\n    'target_homevalue': y_test_pred\n})\n\n# Save the submission file\nsub1.to_csv('sub01.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T13:17:21.018535Z","iopub.execute_input":"2024-11-23T13:17:21.018803Z","iopub.status.idle":"2024-11-23T13:17:29.812234Z","shell.execute_reply.started":"2024-11-23T13:17:21.018776Z","shell.execute_reply":"2024-11-23T13:17:29.811298Z"}},"outputs":[],"execution_count":null}]}