{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========================================================================\n# INSTALL REQUIRED PACKAGES\n# ========================================================================\n!pip install lightgbm -q\n!pip install scikit-learn --upgrade -q\n\n# ========================================================================\n# IMPORTS\n# ========================================================================\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.feature_selection import VarianceThreshold, SelectKBest, f_classif\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.metrics import classification_report, confusion_matrix, f1_score, make_scorer\nfrom lightgbm import LGBMClassifier\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set display options\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 100)\nplt.style.use('seaborn-v0_8-darkgrid')\n\nprint(\"=\" * 80)\nprint(\"DETECTING REVERSAL POINTS IN US EQUITIES - COMPLETE PIPELINE\")\nprint(\"=\" * 80)\n\n# ========================================================================\n# 1. LOAD DATA\n# ========================================================================\nprint(\"\\n📁 LOADING DATA...\")\nprint(\"-\" * 40)\n\ntrain = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\ntest = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\nsample_sub = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/sample_submission.csv')\n\nprint(f\"✓ Train shape: {train.shape}\")\nprint(f\"✓ Test shape: {test.shape}\")\nprint(f\"✓ Sample submission shape: {sample_sub.shape}\")\n\n# ========================================================================\n# 2. INITIAL EXPLORATION\n# ========================================================================\nprint(\"\\n🔍 DATA EXPLORATION\")\nprint(\"-\" * 40)\n\n# Identify columns\nmetadata_cols = ['ticker_id', 't', 'class_label']\nfeature_cols = [col for col in train.columns if col not in metadata_cols + ['id']]\nprint(f\"Number of features: {len(feature_cols):,}\")\nprint(f\"Metadata columns: {metadata_cols}\")\n\n# Target distribution\nprint(\"\\n📊 Target Distribution:\")\ntarget_dist = train['class_label'].value_counts()\nfor cls, count in target_dist.items():\n    pct = count / len(train) * 100\n    print(f\"  {cls}: {count:,} ({pct:.1f}%)\")\n\n# Check for ID column in test\nif 'id' not in test.columns:\n    test['id'] = range(len(test))\n    print(\"\\n⚠️ Added ID column to test set\")\n\n# Ticker distribution\nprint(\"\\n📈 Ticker Distribution:\")\nticker_counts = train['ticker_id'].value_counts()\nprint(f\"  Unique tickers: {ticker_counts.nunique()}\")\nprint(f\"  Samples per ticker: {ticker_counts.iloc[0]} (all equal)\")\n\n# Date range\nprint(f\"\\n📅 Date range: {train['t'].min()} to {train['t'].max()}\")\n\n# Missing values\ntrain_missing = train[feature_cols].isnull().sum().sum()\ntest_missing = test[feature_cols].isnull().sum().sum()\nprint(f\"\\n🔢 Missing values:\")\nprint(f\"  Train: {train_missing:,}\")\nprint(f\"  Test: {test_missing:,}\")\n\n# ========================================================================\n# 3. VISUALIZATIONS\n# ========================================================================\nprint(\"\\n📊 CREATING VISUALIZATIONS...\")\nprint(\"-\" * 40)\n\nfig = plt.figure(figsize=(16, 10))\ngs = fig.add_gridspec(3, 3, hspace=0.3, wspace=0.3)\n\n# 1. Target distribution bar chart\nax1 = fig.add_subplot(gs[0, 0])\ncolors = ['#e74c3c' if x == 'H' else '#3498db' if x == 'L' else '#95a5a6' for x in target_dist.index]\nbars = ax1.bar(target_dist.index, target_dist.values, color=colors, edgecolor='black', linewidth=1.5)\nax1.set_title('Target Class Distribution', fontsize=12, fontweight='bold')\nax1.set_xlabel('Class')\nax1.set_ylabel('Count')\nax1.grid(axis='y', alpha=0.3)\nfor bar, val in zip(bars, target_dist.values):\n    ax1.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 50, \n             f'{val:,}', ha='center', fontsize=10)\n\n# 2. Class distribution pie chart\nax2 = fig.add_subplot(gs[0, 1])\ncolors = ['#e74c3c', '#3498db', '#95a5a6']\nwedges, texts, autotexts = ax2.pie(target_dist.values, labels=target_dist.index, \n                                    autopct='%1.1f%%', colors=colors, \n                                    explode=(0.05, 0.05, 0), startangle=90)\nax2.set_title('Class Balance', fontsize=12, fontweight='bold')\nfor autotext in autotexts:\n    autotext.set_color('white')\n    autotext.set_fontweight('bold')\n\n# 3. Target by ticker\nax3 = fig.add_subplot(gs[0, 2])\ntarget_by_ticker = pd.crosstab(train['ticker_id'], train['class_label'])\ntarget_by_ticker_pct = target_by_ticker.div(target_by_ticker.sum(axis=1), axis=0) * 100\ntarget_by_ticker_pct.plot(kind='bar', stacked=True, ax=ax3, \n                          color=['#e74c3c', '#3498db', '#95a5a6'],\n                          edgecolor='black', linewidth=0.5)\nax3.set_title('Target Distribution by Ticker (%)', fontsize=12, fontweight='bold')\nax3.set_xlabel('Ticker ID')\nax3.set_ylabel('Percentage')\nax3.legend(title='Class', loc='upper right')\nax3.set_xticklabels(ax3.get_xticklabels(), rotation=0)\n\n# 4. Temporal distribution\nax4 = fig.add_subplot(gs[1, :])\ntrain_temp = train.copy()\ntrain_temp['t'] = pd.to_datetime(train_temp['t'])\ntrain_temp['month'] = train_temp['t'].dt.to_period('M')\ntemporal = pd.crosstab(train_temp['month'], train_temp['class_label'])\ntemporal.plot(ax=ax4, color=['#e74c3c', '#3498db', '#95a5a6'], \n              linewidth=2, marker='o', markersize=4)\nax4.set_title('Classes Over Time', fontsize=12, fontweight='bold')\nax4.set_xlabel('Month')\nax4.set_ylabel('Count')\nax4.legend(title='Class')\nax4.grid(alpha=0.3)\n\n# 5. Feature statistics - sample analysis\nax5 = fig.add_subplot(gs[2, 0])\nsample_features = feature_cols[:500]  # Analyze first 500 features\nfeature_means = train[sample_features].mean()\nax5.hist(feature_means, bins=50, color='#9b59b6', edgecolor='black', alpha=0.7)\nax5.set_title('Distribution of Feature Means\\n(First 500 features)', fontsize=12, fontweight='bold')\nax5.set_xlabel('Mean Value')\nax5.set_ylabel('Count')\nax5.grid(axis='y', alpha=0.3)\n\n# 6. Feature variance distribution\nax6 = fig.add_subplot(gs[2, 1])\nfeature_stds = train[sample_features].std()\nax6.hist(feature_stds, bins=50, color='#1abc9c', edgecolor='black', alpha=0.7)\nax6.set_title('Distribution of Feature Std Dev\\n(First 500 features)', fontsize=12, fontweight='bold')\nax6.set_xlabel('Standard Deviation')\nax6.set_ylabel('Count')\nax6.grid(axis='y', alpha=0.3)\n\n# 7. Zero variance features\nax7 = fig.add_subplot(gs[2, 2])\nzero_var_count = (train[feature_cols].var() == 0).sum()\nnon_zero_var_count = len(feature_cols) - zero_var_count\nsizes = [non_zero_var_count, zero_var_count]\nlabels = [f'Non-zero variance\\n({non_zero_var_count:,})', \n          f'Zero variance\\n({zero_var_count:,})']\ncolors = ['#27ae60', '#e74c3c']\nax7.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%', startangle=90)\nax7.set_title('Feature Variance Analysis', fontsize=12, fontweight='bold')\n\nplt.suptitle('EXPLORATORY DATA ANALYSIS', fontsize=14, fontweight='bold', y=1.02)\nplt.tight_layout()\nplt.show()\n\n# ========================================================================\n# 4. FEATURE PREPROCESSING\n# ========================================================================\nprint(\"\\n⚙️ FEATURE PREPROCESSING\")\nprint(\"-\" * 40)\n\n# Extract features and target\nX_train_raw = train[feature_cols].values\ny_train = train['class_label'].values\nX_test_raw = test[feature_cols].values\n\nprint(f\"Original dimensions: {X_train_raw.shape}\")\n\n# Handle infinite and NaN values\nprint(\"Handling infinite and NaN values...\")\nX_train = np.nan_to_num(X_train_raw, nan=0.0, posinf=0.0, neginf=0.0)\nX_test = np.nan_to_num(X_test_raw, nan=0.0, posinf=0.0, neginf=0.0)\n\n# ========================================================================\n# 5. AGGRESSIVE FEATURE REDUCTION (due to 68K features)\n# ========================================================================\nprint(\"\\n🎯 AGGRESSIVE FEATURE REDUCTION PIPELINE\")\nprint(\"-\" * 40)\n\n# Step 1: Remove zero/low variance features\nprint(\"Step 1: Removing low variance features...\")\nvar_threshold = VarianceThreshold(threshold=0.01)\nX_train_var = var_threshold.fit_transform(X_train)\nX_test_var = var_threshold.transform(X_test)\nprint(f\"  → Reduced from {X_train.shape[1]:,} to {X_train_var.shape[1]:,} features\")\n\n# Step 2: Remove highly correlated features (if still too many)\nif X_train_var.shape[1] > 5000:\n    print(\"Step 2: Removing highly correlated features...\")\n    # Calculate correlation matrix for a subset\n    subset_size = min(5000, X_train_var.shape[1])\n    np.random.seed(42)\n    subset_indices = np.random.choice(X_train_var.shape[1], subset_size, replace=False)\n    X_subset = X_train_var[:, subset_indices]\n    \n    # Keep these subset features for now\n    X_train_var = X_subset\n    X_test_var = X_test_var[:, subset_indices]\n    print(f\"  → Randomly sampled {subset_size:,} features to manage memory\")\n\n# Step 3: Standardize\nprint(\"Step 3: Standardization...\")\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train_var)\nX_test_scaled = scaler.transform(X_test_var)\n\n# Step 4: Select best features\nprint(\"Step 4: Selecting best features...\")\nk_features = min(1000, X_train_scaled.shape[1])\nselector = SelectKBest(f_classif, k=k_features)\nX_train_selected = selector.fit_transform(X_train_scaled, y_train)\nX_test_selected = selector.transform(X_test_scaled)\nprint(f\"  → Selected top {k_features:,} features\")\n\n# Step 5: PCA\nprint(\"Step 5: PCA dimensionality reduction...\")\nn_components = min(100, X_train_selected.shape[0] - 1, X_train_selected.shape[1])\npca = PCA(n_components=n_components, random_state=42)\nX_train_final = pca.fit_transform(X_train_selected)\nX_test_final = pca.transform(X_test_selected)\nexplained_var = pca.explained_variance_ratio_.sum()\nprint(f\"  → Reduced to {n_components} components\")\nprint(f\"  → Explained variance: {explained_var:.3f}\")\n\n# ========================================================================\n# 6. PREPARE FOR MODELING\n# ========================================================================\nprint(\"\\n🎨 PREPARING FOR MODELING\")\nprint(\"-\" * 40)\n\n# Encode labels\nle = LabelEncoder()\ny_train_encoded = le.fit_transform(y_train)\n\nprint(\"Class encoding:\")\nfor orig, enc in zip(le.classes_, le.transform(le.classes_)):\n    count = np.sum(y_train_encoded == enc)\n    print(f\"  {orig} → {enc} (count: {count:,})\")\n\n# ========================================================================\n# 7. SIMPLE BASELINE MODEL (due to extreme imbalance)\n# ========================================================================\nprint(\"\\n🚀 TRAINING BASELINE MODELS\")\nprint(\"-\" * 40)\n\n# Use a simple, robust model first\nprint(\"Training LightGBM with balanced class weights...\")\nlgb_model = LGBMClassifier(\n    n_estimators=200,\n    max_depth=5,\n    learning_rate=0.05,\n    num_leaves=31,\n    min_child_samples=30,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    class_weight='balanced',\n    random_state=42,\n    verbosity=-1,\n    force_col_wise=True\n)\n\n# Simple train-val split for quick validation\nfrom sklearn.model_selection import train_test_split\nX_tr, X_val, y_tr, y_val = train_test_split(\n    X_train_final, y_train_encoded, \n    test_size=0.2, random_state=42, \n    stratify=y_train_encoded\n)\n\n# Train model\nlgb_model.fit(X_tr, y_tr)\n\n# Validate\nval_preds = lgb_model.predict(X_val)\nval_f1 = f1_score(y_val, val_preds, average='macro')\nprint(f\"Validation F1 Macro: {val_f1:.4f}\")\n\n# Train on full data\nprint(\"\\nRetraining on full dataset...\")\nlgb_model.fit(X_train_final, y_train_encoded)\n\n# Also train a Random Forest as backup\nprint(\"\\nTraining Random Forest as ensemble candidate...\")\nrf_model = RandomForestClassifier(\n    n_estimators=200,\n    max_depth=6,\n    min_samples_split=30,\n    min_samples_leaf=15,\n    class_weight='balanced',\n    random_state=42,\n    n_jobs=-1\n)\nrf_model.fit(X_train_final, y_train_encoded)\n\n# ========================================================================\n# 8. ENSEMBLE PREDICTIONS\n# ========================================================================\nprint(\"\\n🎯 GENERATING ENSEMBLE PREDICTIONS\")\nprint(\"-\" * 40)\n\n# Get predictions from both models\nlgb_preds = lgb_model.predict_proba(X_test_final)\nrf_preds = rf_model.predict_proba(X_test_final)\n\n# Weighted average (giving more weight to LightGBM)\nensemble_probs = 0.7 * lgb_preds + 0.3 * rf_preds\ntest_preds_encoded = np.argmax(ensemble_probs, axis=1)\ntest_preds = le.inverse_transform(test_preds_encoded)\n\n# Analyze predictions\nprint(\"Prediction distribution:\")\npred_counts = pd.Series(test_preds).value_counts()\nfor cls, count in pred_counts.items():\n    pct = count / len(test_preds) * 100\n    print(f\"  {cls}: {count:,} ({pct:.1f}%)\")\n\n# ========================================================================\n# 9. CREATE SUBMISSION\n# ========================================================================\nprint(\"\\n📝 CREATING SUBMISSION\")\nprint(\"-\" * 40)\n\n# Create submission dataframe\nsubmission = pd.DataFrame({\n    'id': test['id'],\n    'class_label': test_preds\n})\n\n# Verify submission\nprint(f\"Submission shape: {submission.shape}\")\nprint(f\"Unique predictions: {submission['class_label'].unique()}\")\nprint(\"\\nFirst 20 predictions:\")\nprint(submission.head(20))\n\n# Check distribution\nprint(\"\\nFinal submission class distribution:\")\nfor cls in ['H', 'L', 'None']:\n    count = (submission['class_label'] == cls).sum()\n    pct = count / len(submission) * 100\n    print(f\"  {cls}: {count:,} ({pct:.1f}%)\")\n\n# Save submission\nsubmission.to_csv('submission.csv', index=False)\nprint(\"\\n✅ Submission saved as 'submission.csv'\")\n\n# ========================================================================\n# 10. MODEL PERFORMANCE SUMMARY\n# ========================================================================\nprint(\"\\n\" + \"=\" * 80)\nprint(\"📊 FINAL SUMMARY\")\nprint(\"=\" * 80)\n\nprint(f\"\"\"\nPipeline Summary:\n----------------\n- Original features: {len(feature_cols):,}\n- After variance filter: {X_train_var.shape[1]:,}\n- After selection: {k_features:,}\n- Final PCA components: {n_components}\n- Explained variance: {explained_var:.1%}\n\nModel Details:\n-------------\n- Primary Model: LightGBM (70% weight)\n- Secondary Model: Random Forest (30% weight)\n- Validation F1 Macro: {val_f1:.4f}\n- Class balancing: Applied via class_weight='balanced'\n\nSubmission Stats:\n----------------\n- Total predictions: {len(submission):,}\n- Classes predicted: {submission['class_label'].nunique()}\n- File saved: submission.csv\n\nKey Strategies Used:\n-------------------\n✓ Aggressive feature reduction (68K → 100 components)\n✓ Class weight balancing for extreme imbalance\n✓ Ensemble of two robust models\n✓ Conservative hyperparameters to avoid overfitting\n\nRecommendations for Improvement:\n--------------------------------\n1. Feature Engineering:\n   - Create rolling statistics per ticker\n   - Add time-based features (day of week, month)\n   - Engineer ticker-specific patterns\n   \n2. Advanced Modeling:\n   - Try XGBoost with custom objective\n   - Implement focal loss for imbalance\n   - Use time-series specific validation\n   \n3. Post-Processing:\n   - Calibrate probabilities\n   - Apply business rules for H/L patterns\n   - Consider minimum distance between signals\n\"\"\")\n\nprint(\"\\n🎉 Pipeline completed successfully!\")\nprint(\"=\" * 80)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}