{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":114135,"databundleVersionId":14604229,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-25T12:45:11.930865Z","iopub.execute_input":"2025-12-25T12:45:11.931274Z","iopub.status.idle":"2025-12-25T12:45:14.441303Z","shell.execute_reply.started":"2025-12-25T12:45:11.931238Z","shell.execute_reply":"2025-12-25T12:45:14.440226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport gc\nfrom tqdm import tqdm\n\nprint(\"Loading competition data...\")\nprint(\"-\" * 50)\n\n# =========================\n# 1. LOAD DATA\n# =========================\nTRAIN_CSV = \"/kaggle/input/detecting-reversal-points-in-us-equities/new_comptetition_data/train.csv\"\nTEST_CSV = \"/kaggle/input/detecting-reversal-points-in-us-equities/new_comptetition_data/test.csv\"\nSUBMISSION_CSV = \"/kaggle/input/detecting-reversal-points-in-us-equities/new_comptetition_data/sample_submission.csv\"\n\n# Load data\ntrain = pd.read_csv(TRAIN_CSV, low_memory=False)\ntest = pd.read_csv(TEST_CSV, low_memory=False) if os.path.exists(TEST_CSV) else pd.DataFrame()\nsubmission = pd.read_csv(SUBMISSION_CSV, low_memory=False)\n\nprint(f\"✓ Train shape: {train.shape}\")\nprint(f\"✓ Test shape: {test.shape}\")\nprint(f\"✓ Submission shape: {submission.shape}\")\n\n# =========================\n# 2. DATA PREPARATION\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"DATA PREPARATION\")\nprint(\"=\" * 50)\n\n# Target processing\ntarget_col = 'class_label'\ny_raw = train[target_col].copy()\n\n# Simple mapping\ndef map_label(x):\n    if pd.isna(x):\n        return \"N\"\n    x = str(x).upper()\n    if 'H' in x:\n        return \"H\"\n    elif 'L' in x:\n        return \"L\"\n    else:\n        return \"N\"\n\ny = y_raw.apply(map_label)\nprint(\"Class distribution:\")\nprint(y.value_counts())\n\nfrom sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\ny_encoded = le.fit_transform(y)\n\n# Prepare features - drop uninformative columns\nprint(\"\\nPreparing features...\")\n\n# First, identify common columns between train and test\ncommon_columns = list(set(train.columns) & set(test.columns))\nprint(f\"Common columns between train and test: {len(common_columns)}\")\n\n# Remove target column from common columns if present\nif target_col in common_columns:\n    common_columns.remove(target_col)\n\n# Remove ID columns and other non-features\nnon_feature_cols = ['id', 'row_id', 'index', 'Unnamed: 0', 'train_id', 'test_id', 'ticker_id', 't']\ncommon_columns = [col for col in common_columns if col not in non_feature_cols]\n\n# Check for constant columns\nprint(\"Checking for constant columns...\")\nconstant_cols = []\nfor col in tqdm(common_columns[:1000], desc=\"Analyzing columns\"):\n    try:\n        if train[col].nunique() <= 1:\n            constant_cols.append(col)\n    except:\n        continue\n\nprint(f\"Found {len(constant_cols)} constant columns\")\ncommon_columns = [col for col in common_columns if col not in constant_cols]\n\n# Create feature matrices\nX = train[common_columns].copy()\nX_test = test[common_columns].copy() if len(test) > 0 else pd.DataFrame()\n\nprint(f\"\\nFeature matrix shapes:\")\nprint(f\"X_train: {X.shape}\")\nprint(f\"X_test: {X_test.shape}\")\n\n# Convert to float32 to save memory\nfor col in X.columns:\n    X[col] = X[col].astype('float32')\n    if len(X_test) > 0 and col in X_test.columns:\n        X_test[col] = X_test[col].astype('float32')\n\n# =========================\n# 3. SMART FEATURE SELECTION\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"SMART FEATURE SELECTION\")\nprint(\"=\" * 50)\n\n# Select top features by variance\nprint(\"Selecting top features by variance...\")\nn_features_to_keep = 100\n\n# Calculate variance for each column\nvariances = []\nfor col in tqdm(X.columns, desc=\"Calculating variances\"):\n    try:\n        var = X[col].var()\n        if pd.notna(var):\n            variances.append((col, var))\n    except:\n        continue\n\n# Sort by variance and take top N\nvariances.sort(key=lambda x: x[1], reverse=True)\ntop_cols = [col for col, var in variances[:n_features_to_keep]]\nprint(f\"Selected top {len(top_cols)} features by variance\")\n\n# Keep only selected columns\nX = X[top_cols]\nif len(X_test) > 0:\n    # Ensure test has the same columns\n    X_test = X_test[[col for col in top_cols if col in X_test.columns]]\n    # Add missing columns with zeros\n    missing_cols = set(top_cols) - set(X_test.columns)\n    for col in missing_cols:\n        X_test[col] = 0\n    X_test = X_test[top_cols]  # Reorder to match train\n\nprint(f\"\\nFinal feature shapes:\")\nprint(f\"Train: {X.shape}\")\nprint(f\"Test: {X_test.shape}\")\n\n# =========================\n# 4. SIMPLE FEATURE ENGINEERING\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"SIMPLE FEATURE ENGINEERING\")\nprint(\"=\" * 50)\n\ndef create_simple_features(df, prefix=\"\"):\n    \"\"\"Create simple but effective features\"\"\"\n    features = pd.DataFrame(index=df.index)\n    \n    for col in tqdm(df.columns, desc=f\"Creating {prefix}features\"):\n        try:\n            series = df[col].fillna(0)\n            \n            # Basic transformations\n            features[f'{prefix}{col}_lag1'] = series.shift(1)\n            features[f'{prefix}{col}_lag3'] = series.shift(3)\n            features[f'{prefix}{col}_diff1'] = series.diff(1)\n            features[f'{prefix}{col}_diff3'] = series.diff(3)\n            \n            # Rolling statistics\n            features[f'{prefix}{col}_roll_mean_5'] = series.rolling(5, min_periods=3).mean()\n            features[f'{prefix}{col}_roll_std_5'] = series.rolling(5, min_periods=3).std()\n            features[f'{prefix}{col}_roll_max_5'] = series.rolling(5, min_periods=3).max()\n            features[f'{prefix}{col}_roll_min_5'] = series.rolling(5, min_periods=3).min()\n            \n            # Price position\n            features[f'{prefix}{col}_zscore'] = (series - series.mean()) / (series.std() + 1e-10)\n            \n        except:\n            continue\n    \n    return features.fillna(0)\n\nprint(\"Creating features for train set...\")\ntrain_features = create_simple_features(X, \"train_\")\n\nprint(\"Creating features for test set...\")\ntest_features = create_simple_features(X_test, \"test_\") if len(X_test) > 0 else pd.DataFrame()\n\n# Combine original and engineered features\nprint(\"\\nCombining features...\")\nX_final = pd.concat([X, train_features], axis=1).fillna(0)\nif len(X_test) > 0:\n    X_test_final = pd.concat([X_test, test_features], axis=1).fillna(0)\nelse:\n    X_test_final = pd.DataFrame()\n\nprint(f\"Final feature dimensions:\")\nprint(f\"Train: {X_final.shape}\")\nprint(f\"Test: {X_test_final.shape}\")\n\n# =========================\n# 5. MODEL TRAINING\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"MODEL TRAINING\")\nprint(\"=\" * 50)\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, accuracy_score, log_loss\n\nnum_classes = len(le.classes_)\nprint(f\"Number of classes: {num_classes}\")\nprint(f\"Classes: {le.classes_}\")\n\n# LightGBM parameters\nparams = {\n    'objective': 'multiclass',\n    'num_class': num_classes,\n    'metric': 'multi_logloss',\n    'boosting_type': 'gbdt',\n    'learning_rate': 0.05,\n    'num_leaves': 31,\n    'max_depth': 7,\n    'min_child_samples': 20,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 0.1,\n    'reg_lambda': 0.1,\n    'n_jobs': -1,\n    'random_state': 42,\n    'verbose': -1,\n}\n\n# Cross-validation\nn_folds = 5\nskf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)\n\n# Initialize arrays for predictions\noof_preds = np.zeros((len(X_final), num_classes))\ntest_preds = np.zeros((len(X_test_final), num_classes)) if len(X_test_final) > 0 else None\n\nprint(f\"\\nStarting {n_folds}-fold cross-validation...\")\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X_final, y_encoded)):\n    print(f\"\\n{'='*30}\")\n    print(f\"FOLD {fold + 1}/{n_folds}\")\n    print(f\"{'='*30}\")\n    \n    # Split data\n    X_train, X_val = X_final.iloc[train_idx], X_final.iloc[val_idx]\n    y_train, y_val = y_encoded[train_idx], y_encoded[val_idx]\n    \n    # Apply class weights\n    from sklearn.utils.class_weight import compute_class_weight\n    class_weights = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)\n    sample_weights = np.array([class_weights[y] for y in y_train])\n    \n    # Create datasets\n    train_data = lgb.Dataset(X_train, y_train, weight=sample_weights)\n    val_data = lgb.Dataset(X_val, y_val, reference=train_data)\n    \n    # Train model\n    model = lgb.train(\n        params,\n        train_data,\n        num_boost_round=1000,\n        valid_sets=[val_data],\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=100, verbose=False),\n            lgb.log_evaluation(period=100, show_stdv=False)\n        ]\n    )\n    \n    # Get predictions\n    val_pred = model.predict(X_val, num_iteration=model.best_iteration)\n    oof_preds[val_idx] = val_pred\n    \n    # Calculate fold metrics\n    val_pred_class = np.argmax(val_pred, axis=1)\n    fold_acc = accuracy_score(y_val, val_pred_class)\n    fold_f1 = f1_score(y_val, val_pred_class, average='macro')\n    \n    print(f\"Fold {fold + 1} - Accuracy: {fold_acc:.4f}, F1: {fold_f1:.4f}\")\n    \n    # Predict on test set\n    if test_preds is not None:\n        test_pred = model.predict(X_test_final, num_iteration=model.best_iteration)\n        test_preds += test_pred / n_folds\n    \n    # Clean up\n    del model, train_data, val_data\n    gc.collect()\n\n# Calculate overall OOF metrics\nprint(f\"\\n{'='*50}\")\nprint(\"OVERALL PERFORMANCE\")\nprint(f\"{'='*50}\")\n\noof_preds_class = np.argmax(oof_preds, axis=1)\noverall_acc = accuracy_score(y_encoded, oof_preds_class)\noverall_f1 = f1_score(y_encoded, oof_preds_class, average='macro')\n\nprint(f\"Overall Accuracy: {overall_acc:.4f}\")\nprint(f\"Overall Macro F1: {overall_f1:.4f}\")\n\n# Display class-wise performance\nprint(\"\\nClass-wise performance:\")\nfor class_idx, class_name in enumerate(le.classes_):\n    mask = y_encoded == class_idx\n    if mask.any():\n        class_acc = accuracy_score(y_encoded[mask], oof_preds_class[mask])\n        print(f\"{class_name}: Accuracy = {class_acc:.4f}, Count = {mask.sum()}\")\n\n# =========================\n# 6. PREDICTION & SUBMISSION\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"MAKING PREDICTIONS\")\nprint(\"=\" * 50)\n\nif test_preds is not None:\n    # Convert to class predictions\n    test_preds_class = np.argmax(test_preds, axis=1)\n    \n    # Apply temporal smoothing\n    print(\"Applying temporal smoothing...\")\n    smoothed = test_preds_class.copy()\n    \n    # Simple majority voting in sliding window\n    window_size = 5\n    for i in range(window_size, len(smoothed)):\n        window = smoothed[i-window_size:i]\n        unique, counts = np.unique(window, return_counts=True)\n        most_common = unique[np.argmax(counts)]\n        if counts.max() >= window_size * 0.6:  # 60% agreement\n            smoothed[i] = most_common\n    \n    # Convert back to original labels\n    final_predictions = le.inverse_transform(smoothed)\n    \n    # Display distribution\n    print(\"\\nPredictions distribution:\")\n    unique, counts = np.unique(final_predictions, return_counts=True)\n    for label, count in zip(unique, counts):\n        print(f\"{label}: {count} ({count/len(final_predictions)*100:.1f}%)\")\n    \n    # Prepare submission\n    submission['class_label'] = final_predictions\n    \n    # Validate predictions\n    valid_labels = set(le.classes_)\n    invalid_mask = ~submission['class_label'].isin(valid_labels)\n    \n    if invalid_mask.any():\n        print(f\"\\n⚠️ Warning: {invalid_mask.sum()} invalid predictions found\")\n        submission.loc[invalid_mask, 'class_label'] = 'N'\n    \n    # Show sample predictions\n    print(\"\\nFirst 10 predictions:\")\n    print(submission[['id', 'class_label']].head(10))\n    \n    # Save submission\n    submission.to_csv(\"submission.csv\", index=False)\n    print(f\"\\n✓ Submission saved to 'submission.csv'\")\n    \n    # Performance estimation\n    print(f\"\\nExpected performance based on validation:\")\n    print(f\"Accuracy: {overall_acc:.4f}\")\n    print(f\"F1 Score: {overall_f1:.4f}\")\n    \n    if overall_f1 > 0.95:\n        print(\"🎯 Excellent! Target score 0.99520 is achievable!\")\n    elif overall_f1 > 0.9:\n        print(\"✅ Very good performance!\")\n    elif overall_f1 > 0.8:\n        print(\"⚠️ Decent performance, consider feature engineering improvements\")\n    else:\n        print(\"❌ Needs significant improvement\")\n    \n    # Additional diagnostics\n    print(\"\\n\" + \"=\" * 50)\n    print(\"DIAGNOSTICS\")\n    print(\"=\" * 50)\n    \n    # Check for class imbalance in predictions\n    pred_counts = submission['class_label'].value_counts()\n    print(\"\\nSubmission class distribution:\")\n    for label in le.classes_:\n        count = pred_counts.get(label, 0)\n        percentage = count / len(submission) * 100\n        print(f\"{label}: {count} ({percentage:.1f}%)\")\n    \n    # Compare with train distribution\n    print(\"\\nTrain vs Submission distribution:\")\n    train_counts = y.value_counts()\n    for label in le.classes_:\n        train_count = train_counts.get(label, 0)\n        train_pct = train_count / len(train) * 100\n        sub_count = pred_counts.get(label, 0)\n        sub_pct = sub_count / len(submission) * 100\n        print(f\"{label}: Train={train_pct:.1f}%, Submission={sub_pct:.1f}%\")\n    \nelse:\n    print(\"No test data available for submission\")\n\n# =========================\n# 7. FINAL MODEL (TRAIN ON ALL DATA)\n# =========================\nprint(\"\\n\" + \"=\" * 50)\nprint(\"TRAINING FINAL MODEL\")\nprint(\"=\" * 50)\n\nprint(\"Training final model on all training data...\")\nfinal_model = lgb.train(\n    params,\n    lgb.Dataset(X_final, y_encoded),\n    num_boost_round=500\n)\n\n# Save feature importance\nfeature_importance = pd.DataFrame({\n    'feature': X_final.columns,\n    'importance': final_model.feature_importance(importance_type='gain')\n}).sort_values('importance', ascending=False)\n\nprint(\"\\nTop 10 most important features:\")\nprint(feature_importance.head(10))\n\n# Save feature importance\nfeature_importance.to_csv(\"feature_importance.csv\", index=False)\nprint(\"✓ Feature importance saved to 'feature_importance.csv'\")\n\nprint(\"\\n\" + \"=\" * 50)\nprint(\"NOTEBOOK COMPLETED!\")\nprint(\"=\" * 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T12:46:17.594574Z","iopub.execute_input":"2025-12-25T12:46:17.594844Z","iopub.status.idle":"2025-12-25T12:50:58.734164Z","shell.execute_reply.started":"2025-12-25T12:46:17.594818Z","shell.execute_reply":"2025-12-25T12:50:58.733489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# 📊 DATA VISUALIZATION\n# =========================\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\n\nprint(\"=== DATA VISUALIZATION ===\")\n\n# 1. TARGET DISTRIBUTION\nplt.figure(figsize=(6,4))\nsns.countplot(x=y, palette=\"viridis\")\nplt.title(\"Target Distribution (H / L / N)\")\nplt.xlabel(\"Class Label\")\nplt.ylabel(\"Count\")\nplt.show()\n\n# 2. FEATURE SUMMARY (first 20 numeric features)\nnumeric_cols = X.select_dtypes(include=[np.number]).columns[:20]\n\nplt.figure(figsize=(14,10))\nX[numeric_cols].hist(bins=30, figsize=(14,10), layout=(5,4))\nplt.suptitle(\"Distribution of First 20 Numeric Features\", y=1.02)\nplt.show()\n\n# 3. CORRELATION HEATMAP (small subset)\nsubset_cols = X.columns[:50]  # reduce for speed\ncorr = X[subset_cols].corr()\n\nplt.figure(figsize=(12,8))\nsns.heatmap(corr, cmap=\"coolwarm\", center=0)\nplt.title(\"Correlation Heatmap (First 50 Features)\")\nplt.show()\n\n# 4. PCA VISUALIZATION (2D)\nprint(\"Running PCA on 500-feature set...\")\n\npca = PCA(n_components=2, random_state=42)\nX_pca = pca.fit_transform(X)\n\nplt.figure(figsize=(8,6))\nscatter = plt.scatter(\n    X_pca[:,0], X_pca[:,1],\n    c=y_encoded, cmap=\"viridis\", alpha=0.6\n)\nplt.colorbar(scatter, ticks=[0,1,2], label=\"Encoded Class\")\nplt.title(\"PCA (2 Components) – Class Separation\")\nplt.xlabel(\"PC1\")\nplt.ylabel(\"PC2\")\nplt.show()\n\n# 5. FEATURE VARIANCE (Top 30 most varying)\nfeature_var = X.var().sort_values(ascending=False)[:30]\n\nplt.figure(figsize=(10,6))\nsns.barplot(x=feature_var.values, y=feature_var.index, palette=\"plasma\")\nplt.title(\"Top 30 Features by Variance\")\nplt.xlabel(\"Variance\")\nplt.ylabel(\"Feature Name\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T12:51:14.923207Z","iopub.execute_input":"2025-12-25T12:51:14.923809Z","iopub.status.idle":"2025-12-25T12:51:18.900431Z","shell.execute_reply.started":"2025-12-25T12:51:14.923779Z","shell.execute_reply":"2025-12-25T12:51:18.899884Z"}},"outputs":[],"execution_count":null}]}