{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":162826,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":138462,"modelId":161088}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n# Ultra-Advanced EDA with Enhanced 7B LLM Intelligence\n# ==============================================================================\n\n# Package Installation\n!pip install pandas numpy matplotlib seaborn plotly -q\n!pip install scikit-learn scipy statsmodels -q\n!pip install transformers torch accelerate bitsandbytes -q\n!pip install missingno psutil gputil -q\n\n# ==============================================================================\n# Imports\n# ==============================================================================\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.io as pio\nimport missingno as msno\n\nimport json\nimport os\nimport warnings\nfrom datetime import datetime\nfrom collections import defaultdict, Counter\nfrom IPython.display import display, HTML, Markdown\nimport time\nimport gc # For garbage collection\nimport psutil # For system monitoring\n\n# Import torch and transformers with advanced features\ntry:\n import torch\n from transformers import (\n AutoTokenizer, \n AutoModelForCausalLM,\n BitsAndBytesConfig,\n TextStreamer\n )\n TORCH_AVAILABLE = True\n \n # Check GPU availability and memory\n if torch.cuda.is_available():\n gpu_name = torch.cuda.get_device_name(0)\n gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3\n print(f\"๐ŸŽฎ GPU detected: {gpu_name} ({gpu_memory:.1f} GB)\")\n \n # Clear GPU cache\n torch.cuda.empty_cache()\n gc.collect()\n else:\n print(\"โš ๏ธ No GPU detected - will use CPU (slower)\")\n \nexcept ImportError:\n print(\"Warning: torch or transformers not available. Continuing without LLM features.\")\n TORCH_AVAILABLE = False\n\nwarnings.filterwarnings('ignore')\n\n# Statistical analysis\nfrom scipy import stats\nfrom scipy.stats import normaltest, skew, kurtosis\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier\n\n# Display settings\npd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', 50)\nplt.style.use('seaborn-v0_8-whitegrid')\nsns.set_palette(\"husl\")\n\n# Create output directories\nfor folder in ['eda_visualizations', 'eda_reports', 'eda_insights']:\n os.makedirs(folder, exist_ok=True)\n\n# ==============================================================================\n# Display Functions with Clear Insight Separation\n# ==============================================================================\n\ndef display_statistical_insight(title, metrics, source=\"Statistical Analysis\"):\n \"\"\"Display hard-coded statistical insights with clear labeling\"\"\"\n html_content = f\"\"\"\n
\n
\n

๐Ÿ“Š {title}

\n \n {source}\n \n
\n
\n
\n \"\"\"\n \n for key, value in metrics.items():\n # Format the value appropriately\n if isinstance(value, float):\n if value == float('inf'):\n formatted_value = \"โˆž\"\n elif abs(value) > 0.01:\n formatted_value = f\"{value:,.2f}\"\n else:\n formatted_value = f\"{value:.2e}\"\n elif isinstance(value, dict):\n formatted_value = f\"{len(value)} items\"\n elif isinstance(value, int):\n formatted_value = f\"{value:,}\"\n else:\n formatted_value = str(value)\n \n html_content += f\"\"\"\n
\n
\n {key.replace('_', ' ').title()}\n
\n
\n {formatted_value}\n
\n
\n \"\"\"\n \n html_content += \"\"\"\n
\n
\n
\n \"\"\"\n display(HTML(html_content))\n\ndef display_llm_insight(title, insight, recommendation, confidence, source=\"AI Analysis (7B Model)\"):\n \"\"\"Display LLM-generated insights with clear AI labeling\"\"\"\n # Determine confidence level and colors\n if confidence > 0.8:\n conf_level = \"high\"\n c1, c2 = \"48bb78\", \"38a169\" # Green\n elif confidence > 0.5:\n conf_level = \"medium\"\n c1, c2 = \"ed8936\", \"dd6b20\" # Orange\n else:\n conf_level = \"low\"\n c1, c2 = \"f56565\", \"e53e3e\" # Red\n \n html_content = f\"\"\"\n
\n
\n

๐Ÿค– {title}

\n
\n \n {source}\n \n \n Confidence: {confidence:.1%}\n \n
\n
\n \n
\n

\n ๐Ÿ” AI-Generated Insight\n

\n

\n {insight}\n

\n
\n \n
\n

\n ๐Ÿ’ก AI Recommendation\n

\n

\n {recommendation}\n

\n
\n
\n \"\"\"\n display(HTML(html_content))\n\ndef display_comparison_insights(statistical_summary, ai_insight):\n \"\"\"Display side-by-side comparison of statistical and AI insights\"\"\"\n html_content = f\"\"\"\n
\n

\n ๐Ÿ”„ Insights Comparison\n

\n
\n
\n

\n ๐Ÿ“Š Statistical Analysis\n

\n

\n {statistical_summary}\n

\n
\n
\n

\n ๐Ÿค– AI Analysis (7B Model)\n

\n

\n {ai_insight}\n

\n
\n
\n
\n \"\"\"\n display(HTML(html_content))\n\n# ==============================================================================\n# Enhanced LLM Class with 7B Model Support\n# ==============================================================================\n\nclass AdvancedLLM7B:\n \"\"\"Enhanced LLM with 7B model support and optimizations\"\"\"\n \n def __init__(self, model_path=\"/kaggle/input/qwen2.5-coder/transformers/7b/1\"):\n self.model = None\n self.tokenizer = None\n self.model_path = model_path\n \n if TORCH_AVAILABLE:\n self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n print(f\"๐Ÿš€ Initializing 7B LLM on {self.device}\")\n \n # Initialize with optimizations for 7B model\n self._initialize_model()\n else:\n print(\"โš ๏ธ Torch not available. Using fallback methods.\")\n \n def _initialize_model(self):\n \"\"\"Initialize the 7B model with memory optimizations\"\"\"\n try:\n # Clear memory before loading\n if torch.cuda.is_available():\n torch.cuda.empty_cache()\n gc.collect()\n \n # Initialize tokenizer\n print(\"๐Ÿ“š Loading tokenizer...\")\n self.tokenizer = AutoTokenizer.from_pretrained(\n self.model_path, \n trust_remote_code=True,\n use_fast=True # Use fast tokenizer for better performance\n )\n \n # Configure quantization for memory efficiency\n bnb_config = None\n if torch.cuda.is_available():\n try:\n bnb_config = BitsAndBytesConfig(\n load_in_8bit=True, # Use 8-bit quantization for 7B model\n bnb_8bit_compute_dtype=torch.float16,\n bnb_8bit_use_double_quant=True,\n bnb_8bit_quant_type=\"nf4\"\n )\n print(\"โœ… 8-bit quantization configured\")\n except Exception as e:\n print(f\"โš ๏ธ Quantization not available: {str(e)[:100]}\")\n bnb_config = None\n \n # Load model with optimizations\n print(\"๐Ÿง  Loading 7B model (this may take a moment)...\")\n \n if bnb_config:\n # Load with quantization\n self.model = AutoModelForCausalLM.from_pretrained(\n self.model_path,\n quantization_config=bnb_config,\n device_map=\"auto\",\n trust_remote_code=True,\n torch_dtype=torch.float16,\n low_cpu_mem_usage=True\n )\n else:\n # Load without quantization (fallback)\n self.model = AutoModelForCausalLM.from_pretrained(\n self.model_path,\n torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,\n device_map=\"auto\",\n trust_remote_code=True,\n low_cpu_mem_usage=True\n )\n \n self.model.eval()\n \n # Set pad token\n if self.tokenizer.pad_token is None:\n self.tokenizer.pad_token = self.tokenizer.eos_token\n \n # Get model info\n param_count = sum(p.numel() for p in self.model.parameters()) / 1e9\n print(f\"โœ… 7B LLM initialized successfully ({param_count:.1f}B parameters)\")\n \n if torch.cuda.is_available():\n memory_used = torch.cuda.memory_allocated() / 1024**3\n print(f\"๐Ÿ“Š GPU memory used: {memory_used:.2f} GB\")\n \n except Exception as e:\n print(f\"โš ๏ธ LLM initialization failed: {str(e)[:200]}\")\n print(\"Falling back to simpler model or rule-based analysis\")\n self.model = None\n \n def analyze(self, analysis_type, data_context, max_length=150):\n \"\"\"Generate analysis with the 7B model\"\"\"\n if self.model is None or not TORCH_AVAILABLE:\n return self._fallback_analysis(analysis_type, data_context)\n \n try:\n # Create more sophisticated prompt for 7B model\n prompt = self._create_advanced_prompt(analysis_type, data_context)\n \n # Tokenize with better truncation\n inputs = self.tokenizer(\n prompt, \n return_tensors=\"pt\", \n truncation=True, \n max_length=512, # Increased for 7B model\n padding=True\n )\n \n # Move to device\n if torch.cuda.is_available():\n inputs = {k: v.cuda() for k, v in inputs.items()}\n \n # Generate with better parameters for 7B model\n with torch.no_grad():\n outputs = self.model.generate(\n **inputs,\n max_new_tokens=max_length,\n temperature=0.7,\n do_sample=True,\n top_p=0.9,\n top_k=50,\n repetition_penalty=1.1,\n pad_token_id=self.tokenizer.pad_token_id,\n eos_token_id=self.tokenizer.eos_token_id\n )\n \n # Decode response\n response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)\n \n # Remove prompt from response\n if prompt in response:\n response = response[len(prompt):].strip()\n \n # Parse and structure response\n return self._parse_llm_response(response, analysis_type)\n \n except torch.cuda.OutOfMemoryError:\n print(\"โš ๏ธ GPU out of memory. Clearing cache and using fallback...\")\n torch.cuda.empty_cache()\n gc.collect()\n return self._fallback_analysis(analysis_type, data_context)\n \n except Exception as e:\n print(f\"โš ๏ธ LLM error: {str(e)[:100]}\")\n return self._fallback_analysis(analysis_type, data_context)\n \n def _create_advanced_prompt(self, analysis_type, data_context):\n \"\"\"Create sophisticated prompts for the 7B model\"\"\"\n \n # Format context data\n context_str = json.dumps(data_context, default=str, indent=2)[:400]\n \n if analysis_type == \"class_imbalance\":\n prompt = f\"\"\"You are a data science expert analyzing class imbalance in a machine learning dataset.\n\nDataset Statistics:\n{context_str}\n\nTask: Provide a concise analysis with:\n1. One key insight about the class imbalance\n2. One specific recommendation for handling it\n\nFormat your response as:\nINSIGHT: [Your insight here]\nRECOMMENDATION: [Your recommendation here]\n\nResponse:\"\"\"\n\n elif analysis_type == \"feature_analysis\":\n prompt = f\"\"\"You are a data science expert analyzing feature characteristics.\n\nFeature Statistics:\n{context_str}\n\nTask: Provide a concise analysis with:\n1. One key insight about the features\n2. One recommendation for feature engineering\n\nFormat your response as:\nINSIGHT: [Your insight here]\nRECOMMENDATION: [Your recommendation here]\n\nResponse:\"\"\"\n\n elif analysis_type == \"temporal_patterns\":\n prompt = f\"\"\"You are analyzing temporal patterns in financial time series data.\n\nTemporal Statistics:\n{context_str}\n\nTask: Provide:\n1. One key temporal pattern observation\n2. One recommendation for time series modeling\n\nFormat your response as:\nINSIGHT: [Your insight here]\nRECOMMENDATION: [Your recommendation here]\n\nResponse:\"\"\"\n\n else:\n prompt = f\"\"\"Analyze this data and provide insights.\n\nData:\n{context_str}\n\nTask: Provide one key insight and one recommendation.\n\nFormat your response as:\nINSIGHT: [Your insight here]\nRECOMMENDATION: [Your recommendation here]\n\nResponse:\"\"\"\n \n return prompt\n \n def _parse_llm_response(self, response, analysis_type):\n \"\"\"Parse and structure the LLM response\"\"\"\n \n # Initialize defaults\n insight = \"Analysis completed\"\n recommendation = \"Consider further investigation\"\n \n # Try to parse structured response\n lines = response.split('\\n')\n for line in lines:\n line = line.strip()\n if line.startswith('INSIGHT:'):\n insight = line.replace('INSIGHT:', '').strip()\n elif line.startswith('RECOMMENDATION:'):\n recommendation = line.replace('RECOMMENDATION:', '').strip()\n \n # If parsing fails, use heuristics\n if insight == \"Analysis completed\" and len(response) > 20:\n sentences = [s.strip() for s in response.split('.') if s.strip()]\n if sentences:\n insight = sentences[0][:200]\n if len(sentences) > 1:\n recommendation = sentences[1][:200]\n \n # Calculate confidence based on response quality\n confidence = self._calculate_confidence(response, analysis_type)\n \n return {\n \"insight\": insight,\n \"recommendation\": recommendation,\n \"confidence\": confidence\n }\n \n def _calculate_confidence(self, response, analysis_type):\n \"\"\"Calculate confidence score based on response quality\"\"\"\n \n # Base confidence for 7B model\n confidence = 0.75\n \n # Adjust based on response length\n if len(response) > 100:\n confidence += 0.1\n if len(response) > 200:\n confidence += 0.05\n \n # Check for structured response\n if 'INSIGHT:' in response and 'RECOMMENDATION:' in response:\n confidence += 0.1\n \n # Specific analysis type adjustments\n if analysis_type == \"class_imbalance\" and any(\n word in response.lower() for word in ['imbalance', 'smote', 'weight', 'resample']\n ):\n confidence += 0.05\n \n return min(confidence, 0.95)\n \n def _fallback_analysis(self, analysis_type, data_context):\n \"\"\"Enhanced fallback analysis when LLM is unavailable\"\"\"\n \n insights_map = {\n \"class_imbalance\": {\n \"severe\": (\"Severe class imbalance detected requiring immediate attention\", \n \"Implement SMOTE with careful cross-validation or use ensemble methods with class weights\"),\n \"moderate\": (\"Moderate class imbalance present in the dataset\",\n \"Consider cost-sensitive learning or balanced random forest\"),\n \"balanced\": (\"Classes are relatively well-balanced\",\n \"Standard algorithms should work well, focus on feature engineering\")\n },\n \"feature_analysis\": {\n \"high_cardinality\": (\"High cardinality features detected that may cause overfitting\",\n \"Apply dimensionality reduction or feature selection techniques\"),\n \"missing_data\": (\"Significant missing data patterns identified\",\n \"Implement sophisticated imputation strategies or use tree-based models\"),\n \"normal\": (\"Feature distribution appears suitable for modeling\",\n \"Proceed with standard preprocessing and feature scaling\")\n },\n \"temporal_patterns\": {\n \"trending\": (\"Clear temporal trends detected in the data\",\n \"Use time-aware validation and consider trend decomposition\"),\n \"seasonal\": (\"Seasonal patterns identified in time series\",\n \"Apply seasonal decomposition or use models like SARIMA\"),\n \"stationary\": (\"Data appears relatively stationary over time\",\n \"Standard time series models should be effective\")\n }\n }\n \n # Determine subcategory based on data context\n if analysis_type == \"class_imbalance\":\n ratio = data_context.get(\"class_balance_ratio\", 1)\n if ratio > 10 or ratio == float('inf'):\n subcategory = \"severe\"\n elif ratio > 3:\n subcategory = \"moderate\"\n else:\n subcategory = \"balanced\"\n else:\n subcategory = \"normal\"\n \n # Get appropriate insight and recommendation\n insights = insights_map.get(analysis_type, {}).get(\n subcategory, \n (\"Analysis completed successfully\", \"Review metrics for optimization opportunities\")\n )\n \n return {\n \"insight\": insights[0],\n \"recommendation\": insights[1],\n \"confidence\": 0.6\n }\n \n def cleanup(self):\n \"\"\"Clean up model and free memory\"\"\"\n if self.model is not None:\n del self.model\n self.model = None\n \n if self.tokenizer is not None:\n del self.tokenizer\n self.tokenizer = None\n \n if torch.cuda.is_available():\n torch.cuda.empty_cache()\n \n gc.collect()\n print(\"โœ… Model cleanup completed\")\n\n# ==============================================================================\n# Enhanced EDA Engine\n# ==============================================================================\n\nclass EnhancedEDAEngine:\n \"\"\"Enhanced EDA engine with 7B model insights\"\"\"\n \n def __init__(self, llm_model):\n self.llm = llm_model\n self.statistical_insights = []\n self.ai_insights = []\n self.analysis_cache = {}\n \n def analyze_with_dual_insights(self, df, analysis_name, compute_stats_func, \n analysis_type=\"general\"):\n \"\"\"Perform analysis with cached insights\"\"\"\n \n print(f\"\\n๐Ÿ”ฌ Analyzing: {analysis_name}\")\n print(\"=\" * 80)\n \n # Check cache\n cache_key = f\"{analysis_name}_{len(df)}_{df.shape[1]}\"\n if cache_key in self.analysis_cache:\n print(\"๐Ÿ“ฆ Using cached analysis\")\n stats, ai_analysis = self.analysis_cache[cache_key]\n else:\n # Compute statistics\n stats = compute_stats_func(df)\n \n # Generate AI insights with 7B model\n ai_analysis = self.llm.analyze(analysis_type, stats)\n \n # Cache results\n self.analysis_cache[cache_key] = (stats, ai_analysis)\n \n # Display statistical insights\n display_statistical_insight(\n f\"{analysis_name} - Statistical Metrics\",\n stats,\n source=\"Statistical Analysis\"\n )\n \n # Display AI insights\n display_llm_insight(\n f\"{analysis_name} - AI Analysis\",\n ai_analysis[\"insight\"],\n ai_analysis[\"recommendation\"],\n ai_analysis[\"confidence\"],\n source=\"7B Model Analysis\"\n )\n \n # Store insights\n self.statistical_insights.append({\n \"name\": analysis_name,\n \"stats\": stats\n })\n \n self.ai_insights.append({\n \"name\": analysis_name,\n \"analysis\": ai_analysis\n })\n \n # Display comparison\n stats_summary = self._summarize_stats(stats)\n display_comparison_insights(stats_summary, ai_analysis[\"insight\"])\n \n return stats, ai_analysis\n \n def _summarize_stats(self, stats):\n \"\"\"Create text summary of statistics\"\"\"\n summary_parts = []\n \n for key, value in list(stats.items())[:5]: # Top 5 metrics\n if isinstance(value, (int, float)):\n if value == float('inf'):\n summary_parts.append(f\"{key.replace('_', ' ').title()}: Infinity\")\n elif isinstance(value, float):\n summary_parts.append(f\"{key.replace('_', ' ').title()}: {value:.2f}\")\n else:\n summary_parts.append(f\"{key.replace('_', ' ').title()}: {value}\")\n \n return \". \".join(summary_parts)\n \n def create_advanced_dashboard(self, df, target_col):\n \"\"\"Create advanced visualization dashboard\"\"\"\n print(\"\\n๐ŸŽจ Creating Advanced Dashboard...\")\n \n fig = make_subplots(\n rows=3, cols=3,\n subplot_titles=(\n 'Target Distribution', 'Feature Types', 'Missing Values Pattern',\n 'Temporal Distribution', 'Feature Correlations', 'Data Quality Score',\n 'Class Balance', 'Feature Importance', 'Outlier Detection'\n ),\n specs=[\n [{'type': 'bar'}, {'type': 'domain'}, {'type': 'heatmap'}],\n [{'type': 'scatter'}, {'type': 'heatmap'}, {'type': 'indicator'}],\n [{'type': 'pie'}, {'type': 'bar'}, {'type': 'box'}]\n ],\n vertical_spacing=0.12,\n horizontal_spacing=0.12\n )\n \n # 1. Target distribution\n if target_col in df.columns:\n target_dist = df[target_col].value_counts()\n fig.add_trace(\n go.Bar(\n x=target_dist.index.astype(str), \n y=target_dist.values,\n marker_color=['#FF6B6B', '#4ECDC4', '#95A5A6'],\n text=target_dist.values,\n textposition='outside'\n ),\n row=1, col=1\n )\n \n # 2. Feature types pie\n numeric_count = len(df.select_dtypes(include=[np.number]).columns)\n categorical_count = len(df.select_dtypes(include=['object']).columns)\n \n fig.add_trace(\n go.Pie(\n labels=['Numeric', 'Categorical'], \n values=[numeric_count, categorical_count],\n hole=0.3,\n marker_colors=['#3498db', '#e74c3c']\n ),\n row=1, col=2\n )\n \n # 3. Missing values heatmap (top 20 features)\n missing_df = df.isnull().sum().sort_values(ascending=False)[:20]\n if len(missing_df) > 0:\n missing_matrix = df[missing_df.index].isnull().astype(int).values[:100] # First 100 rows\n fig.add_trace(\n go.Heatmap(\n z=missing_matrix.T,\n colorscale='RdYlBu',\n showscale=False\n ),\n row=1, col=3\n )\n \n # 4. Temporal distribution (if date column exists)\n if 't' in df.columns:\n df['t_parsed'] = pd.to_datetime(df['t'])\n temporal_counts = df.groupby('t_parsed').size()\n fig.add_trace(\n go.Scatter(\n x=temporal_counts.index,\n y=temporal_counts.values,\n mode='lines+markers',\n line=dict(color='#9b59b6', width=2),\n marker=dict(size=4)\n ),\n row=2, col=1\n )\n \n # 5. Feature correlations (numeric features only, top 10x10)\n numeric_cols = df.select_dtypes(include=[np.number]).columns[:10]\n if len(numeric_cols) > 1:\n corr_matrix = df[numeric_cols].corr()\n fig.add_trace(\n go.Heatmap(\n z=corr_matrix.values,\n x=numeric_cols,\n y=numeric_cols,\n colorscale='RdBu',\n zmid=0,\n showscale=False\n ),\n row=2, col=2\n )\n \n # 6. Data quality score\n quality_score = (1 - df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100\n fig.add_trace(\n go.Indicator(\n mode=\"gauge+number\",\n value=quality_score,\n title={'text': \"Data Quality %\"},\n gauge={\n 'axis': {'range': [0, 100]},\n 'bar': {'color': \"darkblue\"},\n 'steps': [\n {'range': [0, 50], 'color': \"lightgray\"},\n {'range': [50, 80], 'color': \"gray\"}\n ],\n 'threshold': {\n 'line': {'color': \"red\", 'width': 4},\n 'thickness': 0.75,\n 'value': 90\n }\n }\n ),\n row=2, col=3\n )\n \n # 7. Class balance pie\n if target_col in df.columns:\n fig.add_trace(\n go.Pie(\n labels=target_dist.index.astype(str),\n values=target_dist.values,\n hole=0.4,\n marker_colors=['#1abc9c', '#f39c12', '#8e44ad']\n ),\n row=3, col=1\n )\n \n # 8. Feature importance placeholder (would need model)\n fig.add_trace(\n go.Bar(\n x=['Feature' + str(i) for i in range(5)],\n y=np.random.random(5),\n marker_color='#27ae60'\n ),\n row=3, col=2\n )\n \n # 9. Outlier detection (box plot for first numeric column)\n if len(numeric_cols) > 0:\n fig.add_trace(\n go.Box(\n y=df[numeric_cols[0]].dropna().values[:1000], # First 1000 values\n marker_color='#e67e22',\n name=numeric_cols[0]\n ),\n row=3, col=3\n )\n \n # Update layout\n fig.update_layout(\n height=900,\n showlegend=False,\n title_text=\"Enhanced EDA Dashboard with 7B Model Insights\",\n title_font_size=20\n )\n \n fig.show()\n \n def generate_comprehensive_report(self):\n \"\"\"Generate comprehensive final report\"\"\"\n print(\"\\n๐Ÿ“‘ Generating Comprehensive Report...\")\n print(\"=\" * 80)\n \n html_report = \"\"\"\n
\n
\n

\n ๐Ÿ“Š Comprehensive EDA Report\n

\n

\n Powered by 7B Language Model Analysis\n

\n
\n \"\"\"\n \n # Executive Summary\n html_report += \"\"\"\n
\n

\n ๐Ÿ“‹ Executive Summary\n

\n
\n \"\"\"\n \n # Add summary cards\n total_analyses = len(self.statistical_insights)\n avg_confidence = np.mean([ai['analysis']['confidence'] for ai in self.ai_insights]) if self.ai_insights else 0\n \n summary_cards = [\n (\"Total Analyses\", str(total_analyses), \"#3498db\"),\n (\"Average AI Confidence\", f\"{avg_confidence:.1%}\", \"#2ecc71\"),\n (\"Model Used\", \"Qwen 7B\", \"#9b59b6\"),\n (\"Insights Generated\", str(len(self.ai_insights)), \"#e74c3c\")\n ]\n \n for title, value, color in summary_cards:\n html_report += f\"\"\"\n
\n
\n {title}\n
\n
\n {value}\n
\n
\n \"\"\"\n \n html_report += \"\"\"\n
\n
\n \"\"\"\n \n # Statistical Insights Section\n html_report += \"\"\"\n
\n

\n ๐Ÿ“ˆ Statistical Insights\n

\n \"\"\"\n \n for i, insight in enumerate(self.statistical_insights, 1):\n html_report += f\"\"\"\n
\n

\n {i}. {insight['name']}\n

\n
\n \"\"\"\n \n # Display key metrics\n for key, value in list(insight['stats'].items())[:6]:\n if isinstance(value, (int, float)) and value != float('inf'):\n if isinstance(value, float):\n formatted_value = f\"{value:.3f}\"\n else:\n formatted_value = f\"{value:,}\"\n \n html_report += f\"\"\"\n
\n
\n {key.replace('_', ' ').title()}\n
\n
\n {formatted_value}\n
\n
\n \"\"\"\n \n html_report += \"\"\"\n
\n
\n \"\"\"\n \n html_report += \"
\"\n \n # AI Insights Section\n html_report += \"\"\"\n
\n

\n ๐Ÿค– AI-Generated Insights (7B Model)\n

\n \"\"\"\n \n for i, insight in enumerate(self.ai_insights, 1):\n confidence = insight['analysis']['confidence']\n conf_color = \"#48bb78\" if confidence > 0.8 else \"#ed8936\" if confidence > 0.5 else \"#f56565\"\n \n html_report += f\"\"\"\n
\n
\n

\n {i}. {insight['name']}\n

\n \n Confidence: {confidence:.1%}\n \n
\n
\n

\n ๐Ÿ” Insight: {insight['analysis']['insight']}\n

\n

\n ๐Ÿ’ก Recommendation: {insight['analysis']['recommendation']}\n

\n
\n
\n \"\"\"\n \n html_report += \"
\"\n \n # Footer\n html_report += \"\"\"\n
\n

\n โœจ Analysis Complete\n

\n

\n Report generated with advanced 7B language model insights\n

\n
\n
\n \"\"\"\n \n display(HTML(html_report))\n \n # Save report to file\n with open('eda_reports/comprehensive_report.html', 'w') as f:\n f.write(html_report)\n print(\"๐Ÿ“ Report saved to: eda_reports/comprehensive_report.html\")\n\n# ==============================================================================\n# Advanced Analysis Functions\n# ==============================================================================\n\ndef perform_advanced_analyses(eda_engine, train_df, test_df):\n \"\"\"Perform comprehensive advanced analyses\"\"\"\n \n # Analysis 1: Basic Statistics\n def compute_basic_stats(df):\n return {\n \"total_samples\": int(len(df)),\n \"total_features\": int(len(df.columns)),\n \"numeric_features\": int(len(df.select_dtypes(include=[np.number]).columns)),\n \"categorical_features\": int(len(df.select_dtypes(include=['object']).columns)),\n \"missing_values\": int(df.isnull().sum().sum()),\n \"missing_percentage\": float((df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100),\n \"memory_usage_mb\": float(df.memory_usage(deep=True).sum() / 1024**2),\n \"duplicate_rows\": int(df.duplicated().sum())\n }\n \n eda_engine.analyze_with_dual_insights(\n train_df, \"Basic Dataset Statistics\", \n compute_basic_stats, \"general\"\n )\n \n # Analysis 2: Class Imbalance\n def compute_class_imbalance(df):\n if 'class_label_mapped' in df.columns:\n target_dist = df['class_label_mapped'].value_counts()\n min_class = target_dist.min() if len(target_dist) > 0 else 0\n max_class = target_dist.max() if len(target_dist) > 0 else 0\n \n return {\n \"target_classes\": int(len(target_dist)),\n \"minority_class\": str(target_dist.idxmin()) if len(target_dist) > 0 else \"N/A\",\n \"minority_class_size\": int(min_class),\n \"majority_class\": str(target_dist.idxmax()) if len(target_dist) > 0 else \"N/A\",\n \"majority_class_size\": int(max_class),\n \"class_balance_ratio\": float(max_class / min_class) if min_class > 0 else float('inf'),\n \"minority_percentage\": float((min_class / len(df)) * 100) if len(df) > 0 else 0\n }\n return {\"error\": \"No target column found\"}\n \n eda_engine.analyze_with_dual_insights(\n train_df, \"Class Imbalance Analysis\", \n compute_class_imbalance, \"class_imbalance\"\n )\n \n # Analysis 3: Feature Statistics\n def compute_feature_stats(df):\n numeric_cols = df.select_dtypes(include=[np.number]).columns\n \n stats = {\n \"total_numeric_features\": int(len(numeric_cols)),\n \"features_with_nulls\": int((df.isnull().sum() > 0).sum()),\n \"features_all_null\": int((df.isnull().sum() == len(df)).sum()),\n \"constant_features\": int((df.nunique() == 1).sum())\n }\n \n if len(numeric_cols) > 0:\n stats.update({\n \"avg_feature_variance\": float(df[numeric_cols].var().mean()),\n \"avg_feature_skewness\": float(df[numeric_cols].skew().mean()),\n \"high_correlation_pairs\": int((df[numeric_cols].corr().abs() > 0.9).sum().sum() // 2 - len(numeric_cols))\n })\n \n return stats\n \n eda_engine.analyze_with_dual_insights(\n train_df, \"Feature Characteristics\", \n compute_feature_stats, \"feature_analysis\"\n )\n \n # Analysis 4: Temporal Patterns (if applicable)\n def compute_temporal_stats(df):\n if 't' in df.columns:\n df['t_parsed'] = pd.to_datetime(df['t'])\n \n return {\n \"date_range_days\": int((df['t_parsed'].max() - df['t_parsed'].min()).days),\n \"unique_dates\": int(df['t_parsed'].nunique()),\n \"samples_per_date_avg\": float(len(df) / df['t_parsed'].nunique()),\n \"weekend_samples\": int(df[df['t_parsed'].dt.dayofweek.isin([5, 6])].shape[0]),\n \"weekday_samples\": int(df[~df['t_parsed'].dt.dayofweek.isin([5, 6])].shape[0]),\n \"first_date\": str(df['t_parsed'].min().date()),\n \"last_date\": str(df['t_parsed'].max().date())\n }\n return {\"error\": \"No temporal column found\"}\n \n eda_engine.analyze_with_dual_insights(\n train_df, \"Temporal Patterns\", \n compute_temporal_stats, \"temporal_patterns\"\n )\n \n # Analysis 5: Train vs Test Comparison\n def compute_train_test_comparison(train, test):\n return {\n \"train_samples\": int(len(train)),\n \"test_samples\": int(len(test)),\n \"train_test_ratio\": float(len(train) / len(test)) if len(test) > 0 else float('inf'),\n \"common_columns\": int(len(set(train.columns) & set(test.columns))),\n \"train_only_columns\": int(len(set(train.columns) - set(test.columns))),\n \"test_only_columns\": int(len(set(test.columns) - set(train.columns)))\n }\n \n # Create a combined df for this analysis\n comparison_stats = compute_train_test_comparison(train_df, test_df)\n display_statistical_insight(\n \"Train vs Test Comparison\",\n comparison_stats,\n source=\"Statistical Analysis\"\n )\n\n# ==============================================================================\n# Main Execution Function\n# ==============================================================================\n\ndef main():\n \"\"\"Main execution function with 7B model\"\"\"\n \n # Display header\n display(HTML(\"\"\"\n
\n

\n ๐Ÿš€ Advanced EDA with 7B Model Intelligence\n

\n

\n Powered by Qwen 2.5 7B Language Model\n

\n

\n Statistical Analysis + Advanced AI Insights\n

\n
\n \"\"\"))\n \n # Load data\n print(\"\\n๐Ÿ“Š Loading competition data...\")\n train_path = '/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv'\n test_path = '/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv'\n \n try:\n train = pd.read_csv(train_path)\n test = pd.read_csv(test_path)\n print(f\"โœ… Training data loaded: {train.shape}\")\n print(f\"โœ… Test data loaded: {test.shape}\")\n except FileNotFoundError:\n print(\"โŒ Data files not found. Please check the paths.\")\n return None\n \n # Map target variable\n print(\"\\n๐Ÿ”„ Mapping target variables...\")\n target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n print(\"โœ… Target mapping completed\")\n \n # Initialize components\n print(\"\\n๐Ÿค– Initializing 7B Language Model...\")\n llm_model = AdvancedLLM7B()\n eda_engine = EnhancedEDAEngine(llm_model)\n \n # Perform comprehensive analyses\n print(\"\\n๐Ÿ”ฌ Starting comprehensive analysis...\")\n perform_advanced_analyses(eda_engine, train, test)\n \n # Generate advanced dashboard\n eda_engine.create_advanced_dashboard(train, 'class_label_mapped')\n \n # Generate comprehensive report\n eda_engine.generate_comprehensive_report()\n \n # Cleanup\n print(\"\\n๐Ÿงน Cleaning up resources...\")\n llm_model.cleanup()\n \n # Display completion message\n display(HTML(\"\"\"\n
\n

\n โœจ Analysis Complete with 7B Model Insights\n

\n

\n All reports and visualizations have been generated successfully\n

\n
\n \"\"\"))\n \n return {\n \"status\": \"complete\",\n \"train_shape\": train.shape,\n \"test_shape\": test.shape,\n \"model_used\": \"Qwen 2.5 7B\"\n }\n\n# ==============================================================================\n# Execute Main Function\n# ==============================================================================\n\nif __name__ == \"__main__\":\n results = main()\n if results:\n print(\"\\n๐Ÿ“ˆ Final Results:\")\n for key, value in results.items():\n print(f\" โ€ข {key}: {value}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}