{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"},{"sourceId":9492506,"sourceType":"datasetVersion","datasetId":5758231},{"sourceId":198469277,"sourceType":"kernelVersion"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Environment and imports","metadata":{}},{"cell_type":"code","source":"import warnings\nimport sys\n\n# Ignore DeprecationWarnings and RuntimeWarnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\nsys.path.append('/kaggle/input/gresearch-crypto')\nsys.path.append('/kaggle/input/source')\nsys.path.append('/kaggle/input/g-research-crypto-forecasting/gresearch_crypto')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:13.562821Z","iopub.execute_input":"2024-09-27T08:01:13.563457Z","iopub.status.idle":"2024-09-27T08:01:13.579815Z","shell.execute_reply.started":"2024-09-27T08:01:13.563378Z","shell.execute_reply":"2024-09-27T08:01:13.578480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport pickle\nimport shutil\n\nimport numpy as np\nimport pandas as pd\n\nfrom lightgbm import LGBMRegressor\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n# import gresearch_crypto","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-27T08:01:13.581344Z","iopub.execute_input":"2024-09-27T08:01:13.581791Z","iopub.status.idle":"2024-09-27T08:01:15.800397Z","shell.execute_reply.started":"2024-09-27T08:01:13.581748Z","shell.execute_reply":"2024-09-27T08:01:15.798922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the path to the saved models\nmodel_dir = '/kaggle/input/gresearch-crypto/outputs/trained_models'\ndata_folder = '/kaggle/input/g-research-crypto-forecasting/'\nasset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\n%ls $model_dir","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:15.802314Z","iopub.execute_input":"2024-09-27T08:01:15.803141Z","iopub.status.idle":"2024-09-27T08:01:17.070160Z","shell.execute_reply.started":"2024-09-27T08:01:15.803080Z","shell.execute_reply":"2024-09-27T08:01:17.068444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_dir = 'crypto_project_results'\nif not os.path.exists(results_dir):\n    os.makedirs(results_dir)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:17.075389Z","iopub.execute_input":"2024-09-27T08:01:17.075923Z","iopub.status.idle":"2024-09-27T08:01:17.082820Z","shell.execute_reply.started":"2024-09-27T08:01:17.075873Z","shell.execute_reply":"2024-09-27T08:01:17.081282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance_dir = os.path.join(results_dir, 'feature_importance_per_asset')\nif not os.path.exists(feature_importance_dir):\n    os.makedirs(feature_importance_dir)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:17.084800Z","iopub.execute_input":"2024-09-27T08:01:17.085433Z","iopub.status.idle":"2024-09-27T08:01:17.094534Z","shell.execute_reply.started":"2024-09-27T08:01:17.085382Z","shell.execute_reply":"2024-09-27T08:01:17.093202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Loading data and trained models","metadata":{}},{"cell_type":"code","source":"# Get the list of asset IDs for which models were saved\nmodel_files = [f for f in os.listdir(model_dir) if f.startswith('best_model_asset_')]\n\n# Extract asset IDs from filenames, keeping the decimal part (e.g., '0.0', '1.0', etc.)\nasset_ids = [f.split('_')[-1].split('.')[0] + '.' + f.split('_')[-1].split('.')[1] for f in model_files]\nasset_ids = sorted(asset_ids, key=lambda x: float(x))\n\n# Initialize dictionaries to store models, scalers, and features\nmodels = {}\nscalers = {}\nfeature_lists = {}\n\n# Load models, scalers, and feature lists for each asset\nfor asset_id in asset_ids:\n    print(f\"Loading data for Asset ID {asset_id}\")\n    \n    # Load the model if it exists\n    model_filename = os.path.join(model_dir, f'best_model_asset_{asset_id}.pkl')\n    if os.path.exists(model_filename):\n        with open(model_filename, 'rb') as file:\n            model = pickle.load(file)\n        models[asset_id] = model\n    else:\n        print(f\"Model file not found for Asset ID {asset_id}\")\n    \n    # Load the scaler if it exists\n    scaler_filename = os.path.join(model_dir, f'scaler_asset_{asset_id}.pkl')\n    if os.path.exists(scaler_filename):\n        with open(scaler_filename, 'rb') as file:\n            scaler = pickle.load(file)\n        scalers[asset_id] = scaler\n    else:\n        print(f\"Scaler file not found for Asset ID {asset_id}\")\n    \n    # Load the feature list if it exists\n    features_filename = os.path.join(model_dir, f'features_asset_{asset_id}.pkl')\n    if os.path.exists(features_filename):\n        with open(features_filename, 'rb') as file:\n            features = pickle.load(file)\n        feature_lists[asset_id] = features\n    else:\n        print(f\"Feature list file not found for Asset ID {asset_id}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:17.096832Z","iopub.execute_input":"2024-09-27T08:01:17.097285Z","iopub.status.idle":"2024-09-27T08:01:17.313808Z","shell.execute_reply.started":"2024-09-27T08:01:17.097243Z","shell.execute_reply":"2024-09-27T08:01:17.312402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load test data\ntest_df = pd.read_csv(os.path.join(data_folder, 'supplemental_train.csv')) \n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:17.315405Z","iopub.execute_input":"2024-09-27T08:01:17.315762Z","iopub.status.idle":"2024-09-27T08:01:22.278591Z","shell.execute_reply.started":"2024-09-27T08:01:17.315724Z","shell.execute_reply":"2024-09-27T08:01:22.277112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of test data: \", test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:22.280166Z","iopub.execute_input":"2024-09-27T08:01:22.280525Z","iopub.status.idle":"2024-09-27T08:01:22.310893Z","shell.execute_reply.started":"2024-09-27T08:01:22.280488Z","shell.execute_reply":"2024-09-27T08:01:22.309494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data Preprocessing and Feature Engineering","metadata":{}},{"cell_type":"code","source":"%%time\nfrom utils import fill_timestamps\nfrom feature_engineering import Features\n\npreprocessed_assets = []\nunique_ids = test_df.Asset_ID.unique()\nfor id in unique_ids:\n    asset_df = fill_timestamps(test_df, id)\n    asset_df.loc[:, 'Asset_ID'] = id\n    preprocessed_assets.append(asset_df)\n    \npreprocessed_df = pd.concat(preprocessed_assets)\npreprocessed_df.reset_index(inplace=True)\n\nsorted_indices = np.argsort(preprocessed_df['timestamp'])\npreprocessed_df = preprocessed_df.iloc[sorted_indices]\n\n\n# Initialize an empty DataFrame to store the engineered features\nengineered_test_df = pd.DataFrame()\n\n# Apply feature engineering for each asset\nfor asset_id in unique_ids:\n    print(f\"Processing asset ID {asset_id}\")\n    \n    # Select data for the current asset\n    asset_df = preprocessed_df[preprocessed_df['Asset_ID'] == asset_id].copy()\n    \n    # Apply the feature engineering methods\n    feature_engineer = Features(asset_df)\n    \n    # Apply all the feature engineering methods\n    asset_df = feature_engineer.price_based_features()\n    asset_df = feature_engineer.volume_based_features()\n    asset_df = feature_engineer.volatility_features(window=15)\n    asset_df = feature_engineer.momentum_features(window=15)\n    asset_df = feature_engineer.trend_features(window=14)\n    asset_df = feature_engineer.lag_rolling_features(lags=[1, 2, 3], windows=[5, 10, 20])\n    asset_df = feature_engineer.statistical_features(window=20)\n    asset_df = feature_engineer.time_based_features()\n    asset_df = feature_engineer.fill(True, True)  # ffill and bfill\n    \n    # Append to the final DataFrame\n    engineered_test_df = pd.concat([engineered_test_df, asset_df], ignore_index=True)\n    \n    # Free up memory\n    del asset_df\n    gc.collect()\n\n# Clean up\ndel test_df, preprocessed_assets, preprocessed_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:22.312924Z","iopub.execute_input":"2024-09-27T08:01:22.313441Z","iopub.status.idle":"2024-09-27T08:01:45.491993Z","shell.execute_reply.started":"2024-09-27T08:01:22.313385Z","shell.execute_reply":"2024-09-27T08:01:45.490501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Scaling data and preparing the predictions dataframe","metadata":{}},{"cell_type":"code","source":"predictions_list = []\n\nfor asset_id in asset_ids:\n    print(f\"Making predictions for Asset ID {asset_id}\")\n    \n    # Convert asset_id to float if necessary\n    asset_id_float = float(asset_id)\n    \n    # Get the data for the current asset\n    asset_data = engineered_test_df[engineered_test_df['Asset_ID'] == asset_id_float].copy()\n    \n    # Check if there's data for the asset\n    if asset_data.empty:\n        print(f\"No data available for Asset ID {asset_id}\")\n        continue\n    \n    # Check if we have a model for this asset\n    if asset_id not in models:\n        print(f\"No model available for Asset ID {asset_id}\")\n        continue\n    \n    model = models[asset_id]\n    scaler = scalers[asset_id]\n    FEATURES = feature_lists[asset_id]\n    \n    # Ensure that all required features are present in the test data\n    missing_features = set(FEATURES) - set(asset_data.columns)\n    if missing_features:\n        print(f\"Missing features for Asset ID {asset_id}: {missing_features}\")\n        # Fill missing features with zero\n        for feat in missing_features:\n            asset_data[feat] = 0.0\n    \n    X_test = asset_data[FEATURES]\n    \n    # Replace infinite values and fill NaNs as in training\n    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n    X_test.fillna(0, inplace=True)\n    \n    # Scale the data using the saved scaler\n    X_test_scaled = scaler.transform(X_test)\n    \n    # Make predictions\n    y_pred = model.predict(X_test_scaled)\n    \n    # Store the predictions along with timestamps and asset IDs\n    temp_df = pd.DataFrame({\n        'timestamp': asset_data['timestamp'],\n        'Asset_ID': asset_id_float,\n        'Target': asset_data['Target'],  # Actual target values\n        'Prediction': y_pred\n    }, index=asset_data.index)\n    \n    predictions_list.append(temp_df)\n    \n    # Free up memory\n    del asset_data, X_test, X_test_scaled, y_pred, temp_df\n    gc.collect()\n    \npredictions_df = pd.concat(predictions_list)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:01:45.494151Z","iopub.execute_input":"2024-09-27T08:01:45.495013Z","iopub.status.idle":"2024-09-27T08:02:45.849641Z","shell.execute_reply.started":"2024-09-27T08:01:45.494953Z","shell.execute_reply":"2024-09-27T08:02:45.848451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:02:45.851157Z","iopub.execute_input":"2024-09-27T08:02:45.851528Z","iopub.status.idle":"2024-09-27T08:02:45.867086Z","shell.execute_reply.started":"2024-09-27T08:02:45.851490Z","shell.execute_reply":"2024-09-27T08:02:45.865938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Evaluation","metadata":{}},{"cell_type":"code","source":"# Merge asset weights\nasset_weights = asset_details[['Asset_ID', 'Weight']]\nfinal_predictions_df = predictions_df.merge(asset_weights, on='Asset_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:11.377451Z","iopub.execute_input":"2024-09-27T08:03:11.377972Z","iopub.status.idle":"2024-09-27T08:03:11.584690Z","shell.execute_reply.started":"2024-09-27T08:03:11.377926Z","shell.execute_reply":"2024-09-27T08:03:11.583450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from utils import weighted_correlation, evaluate_predictions\nevaluation_results = []\nfor asset_id in asset_ids:\n    asset_id_float = float(asset_id)\n    asset_df = final_predictions_df[final_predictions_df['Asset_ID'] == asset_id_float]\n    \n    if asset_df.empty:\n        continue\n    \n    # Evaluate\n    rmse, corr = evaluate_predictions(asset_df)\n    if np.isnan(rmse):\n        continue\n    evaluation_results.append({\n        'Asset_ID': asset_id_float,\n        'RMSE': rmse,\n        'Correlation': corr\n    })\n    print(f\"Asset ID {asset_id}: RMSE = {rmse:.6f}, Correlation = {corr:.6f}\")\n\nevaluation_df = pd.DataFrame(evaluation_results)\n\n# Calculate overall weighted correlation\nvalid_df = final_predictions_df.dropna(subset=['Target', 'Prediction', 'Weight'])\nweights = valid_df['Weight']\nactual = valid_df['Target']\npredicted = valid_df['Prediction']\n\n\noverall_corr = weighted_correlation(actual, predicted, weights)\nprint(f\"Weighted Correlation across all assets: {overall_corr:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:12.513666Z","iopub.execute_input":"2024-09-27T08:03:12.514850Z","iopub.status.idle":"2024-09-27T08:03:13.272415Z","shell.execute_reply.started":"2024-09-27T08:03:12.514795Z","shell.execute_reply":"2024-09-27T08:03:13.270956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluation_df.to_csv(os.path.join(results_dir, 'evaluation_df.csv'), index=False)\nevaluation_df\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:13.583253Z","iopub.execute_input":"2024-09-27T08:03:13.583755Z","iopub.status.idle":"2024-09-27T08:03:13.606340Z","shell.execute_reply.started":"2024-09-27T08:03:13.583710Z","shell.execute_reply":"2024-09-27T08:03:13.605199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Feature importance","metadata":{}},{"cell_type":"code","source":"# Initialize a DataFrame to store feature importances\nfeature_importance_df = pd.DataFrame()\n\nfor asset_id in asset_ids:\n    asset_id_float = float(asset_id)\n    if asset_id in models:\n        model = models[asset_id]\n        FEATURES = feature_lists[asset_id]\n        \n        # Get feature importances\n        importance = model.feature_importances_\n        \n        # Create a DataFrame\n        temp_df = pd.DataFrame({\n            'feature': FEATURES,\n            'importance': importance,\n            'Asset_ID': asset_id_float\n        })\n        feature_importance_df = pd.concat([feature_importance_df, temp_df], ignore_index=True)\n    else:\n        print(f\"No model found for Asset ID {asset_id}\")\n        \nfeature_importance_df.to_csv(os.path.join(results_dir, 'feature_importance_df.csv'), index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:14.838339Z","iopub.execute_input":"2024-09-27T08:03:14.838837Z","iopub.status.idle":"2024-09-27T08:03:14.869576Z","shell.execute_reply.started":"2024-09-27T08:03:14.838795Z","shell.execute_reply":"2024-09-27T08:03:14.868315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate mean importance per feature","metadata":{}},{"cell_type":"code","source":"mean_importance = feature_importance_df.groupby('feature')['importance'].mean().reset_index()\nmean_importance.sort_values(by='importance', ascending=False, inplace=True)\n\n# Display the top features\nprint(\"Top Features Across All Assets:\")\nmean_importance.head(20)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:15.848160Z","iopub.execute_input":"2024-09-27T08:03:15.849215Z","iopub.status.idle":"2024-09-27T08:03:15.870264Z","shell.execute_reply.started":"2024-09-27T08:03:15.849161Z","shell.execute_reply":"2024-09-27T08:03:15.868826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.barplot(data=mean_importance.head(20), x='importance', y='feature')\nplt.title('Top 20 Feature Importances Across All Assets')\nplt.tight_layout()\nplt.savefig(os.path.join(results_dir, 'top_20_feature_importances_all_assets.png'))\nplt.show()\nplt.close()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:16.177669Z","iopub.execute_input":"2024-09-27T08:03:16.178564Z","iopub.status.idle":"2024-09-27T08:03:17.116437Z","shell.execute_reply.started":"2024-09-27T08:03:16.178513Z","shell.execute_reply":"2024-09-27T08:03:17.114902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for asset_id in asset_ids:\n    asset_id_float = float(asset_id)\n    \n    # Filter feature importance for the current asset\n    asset_feature_importance = feature_importance_df.loc[feature_importance_df['Asset_ID'] == asset_id_float]\n    asset_feature_importance.sort_values(by='importance', ascending=False, inplace=True)\n    \n    # Plot top 20 features for the asset\n    plt.figure(figsize=(12, 8))\n    sns.barplot(data=asset_feature_importance.head(20), x='importance', y='feature')\n    plt.title(f'Top 20 Feature Importances for Asset ID {asset_id_float}')\n    plt.tight_layout()\n    \n    # Save the plot\n    asset_filename = f'asset_{asset_id_float}_feature_importances.png'\n    plt.savefig(os.path.join(feature_importance_dir, asset_filename))\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:26.712805Z","iopub.execute_input":"2024-09-27T08:03:26.713338Z","iopub.status.idle":"2024-09-27T08:03:35.661074Z","shell.execute_reply.started":"2024-09-27T08:03:26.713269Z","shell.execute_reply":"2024-09-27T08:03:35.659656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive(results_dir, 'zip', results_dir)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T08:03:35.663373Z","iopub.execute_input":"2024-09-27T08:03:35.663778Z","iopub.status.idle":"2024-09-27T08:03:35.711945Z","shell.execute_reply.started":"2024-09-27T08:03:35.663736Z","shell.execute_reply":"2024-09-27T08:03:35.710665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}