{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"},{"sourceId":13794373,"sourceType":"datasetVersion","datasetId":8782279}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import the required library","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set up plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:50:57.713301Z","iopub.execute_input":"2025-11-21T16:50:57.714374Z","iopub.status.idle":"2025-11-21T16:51:01.284670Z","shell.execute_reply.started":"2025-11-21T16:50:57.714340Z","shell.execute_reply":"2025-11-21T16:51:01.283702Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load the data","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.286284Z","iopub.execute_input":"2025-11-21T16:51:01.286915Z","iopub.status.idle":"2025-11-21T16:51:01.305818Z","shell.execute_reply.started":"2025-11-21T16:51:01.286882Z","shell.execute_reply":"2025-11-21T16:51:01.304827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the dataset\n# Note: You'll need to download the dataset from the provided Kaggle link first\n# and adjust the file path accordingly\n\ntry:\n    # Try loading the main data file\n    df = pd.read_tsv('/kaggle/input/my-cafa6-submission/submission.tsv')\n    # df = pd.read_csv('/kaggle/input/my-cafa6-submission/data.csv')\nexcept:\n    # If the file structure is different, you might need to adjust this\n    print(\"Please ensure the dataset is properly loaded\")\n    # For demonstration, I'll create a sample structure\n    # In practice, replace this with your actual data loading\n    \n    # Sample data structure (remove this when using real data)\n    np.random.seed(42)\n    n_samples = 1000\n    df = pd.DataFrame({\n        'protein_id': [f'PROT_{i}' for i in range(n_samples)],\n        'sequence_length': np.random.randint(50, 500, n_samples),\n        'molecular_weight': np.random.uniform(10000, 50000, n_samples),\n        'isoelectric_point': np.random.uniform(4.0, 10.0, n_samples),\n        'hydrophobicity': np.random.uniform(-1.0, 1.0, n_samples),\n        'stability': np.random.uniform(0, 1, n_samples),\n        'function_category': np.random.choice(['Enzyme', 'Transport', 'Structural', 'Regulatory', 'Signaling'], n_samples),\n        'expression_level': np.random.exponential(2, n_samples),\n        'conservation_score': np.random.beta(2, 5, n_samples),\n        'target_value': np.random.normal(0, 1, n_samples)\n    })\n\nprint(\"Dataset loaded successfully!\")\nprint(f\"Dataset shape: {df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.306850Z","iopub.execute_input":"2025-11-21T16:51:01.307187Z","iopub.status.idle":"2025-11-21T16:51:01.327618Z","shell.execute_reply.started":"2025-11-21T16:51:01.307164Z","shell.execute_reply":"2025-11-21T16:51:01.326594Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Explore the data","metadata":{}},{"cell_type":"code","source":"# Basic dataset information\nprint(\"=\"*50)\nprint(\"DATASET BASIC INFORMATION\")\nprint(\"=\"*50)\nprint(f\"Dataset Shape: {df.shape}\")\nprint(f\"Number of Features: {df.shape[1]}\")\nprint(f\"Number of Samples: {df.shape[0]}\")\nprint(\"\\n\" + \"=\"*50)\nprint(\"COLUMN NAMES AND DATA TYPES\")\nprint(\"=\"*50)\nprint(df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.329407Z","iopub.execute_input":"2025-11-21T16:51:01.329785Z","iopub.status.idle":"2025-11-21T16:51:01.353735Z","shell.execute_reply.started":"2025-11-21T16:51:01.329763Z","shell.execute_reply":"2025-11-21T16:51:01.352784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# First few rows of the dataset\nprint(\"First 10 rows of the dataset:\")\ndf.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.354651Z","iopub.execute_input":"2025-11-21T16:51:01.354921Z","iopub.status.idle":"2025-11-21T16:51:01.403653Z","shell.execute_reply.started":"2025-11-21T16:51:01.354900Z","shell.execute_reply":"2025-11-21T16:51:01.402332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Statistical summary\nprint(\"Statistical Summary of Numerical Features:\")\ndf.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.404653Z","iopub.execute_input":"2025-11-21T16:51:01.405459Z","iopub.status.idle":"2025-11-21T16:51:01.444649Z","shell.execute_reply.started":"2025-11-21T16:51:01.405391Z","shell.execute_reply":"2025-11-21T16:51:01.443885Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data analysis OR handling missing value","metadata":{}},{"cell_type":"code","source":"# Check for missing values\nprint(\"Missing Values Analysis:\")\nmissing_data = df.isnull().sum()\nmissing_percent = (missing_data / len(df)) * 100\n\nmissing_df = pd.DataFrame({\n    'Missing Count': missing_data,\n    'Missing Percentage': missing_percent\n})\nmissing_df = missing_df[missing_df['Missing Count'] > 0]\n\nif missing_df.empty:\n    print(\"No missing values found!\")\nelse:\n    display(missing_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.445406Z","iopub.execute_input":"2025-11-21T16:51:01.445725Z","iopub.status.idle":"2025-11-21T16:51:01.457952Z","shell.execute_reply.started":"2025-11-21T16:51:01.445693Z","shell.execute_reply":"2025-11-21T16:51:01.457122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## EDA (Exploratory data analysis)","metadata":{}},{"cell_type":"code","source":"# Create a comprehensive EDA visualization dashboard\nfig = plt.figure(figsize=(20, 15))\n\n# 1. Distribution of numerical features\nnumerical_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n\n# Plot distributions for first 6 numerical features\nfor i, col in enumerate(numerical_cols[:6]):\n    plt.subplot(3, 3, i+1)\n    sns.histplot(data=df, x=col, kde=True, bins=30)\n    plt.title(f'Distribution of {col}')\n    plt.xticks(rotation=45)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:01.458840Z","iopub.execute_input":"2025-11-21T16:51:01.459134Z","iopub.status.idle":"2025-11-21T16:51:03.465142Z","shell.execute_reply.started":"2025-11-21T16:51:01.459104Z","shell.execute_reply":"2025-11-21T16:51:03.463946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Correlation Heatmap\nplt.figure(figsize=(12, 10))\ncorrelation_matrix = df[numerical_cols].corr()\nmask = np.triu(np.ones_like(correlation_matrix, dtype=bool))\n\nsns.heatmap(correlation_matrix, mask=mask, annot=True, cmap='coolwarm', \n            center=0, square=True, linewidths=0.5, fmt='.2f')\nplt.title('Correlation Heatmap of Numerical Features', fontsize=16, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:38.299093Z","iopub.execute_input":"2025-11-21T16:51:38.299418Z","iopub.status.idle":"2025-11-21T16:51:38.709735Z","shell.execute_reply.started":"2025-11-21T16:51:38.299391Z","shell.execute_reply":"2025-11-21T16:51:38.708916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pairplot for numerical features (first 5 for clarity)\nif len(numerical_cols) >= 3:\n    pairplot_cols = numerical_cols[:5]\n    sns.pairplot(df[pairplot_cols], diag_kind='kde', corner=True, plot_kws={'alpha': 0.6})\n    plt.suptitle('Pairplot of Numerical Features', y=1.02, fontsize=16, fontweight='bold')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:42.046147Z","iopub.execute_input":"2025-11-21T16:51:42.046586Z","iopub.status.idle":"2025-11-21T16:51:45.620597Z","shell.execute_reply.started":"2025-11-21T16:51:42.046506Z","shell.execute_reply":"2025-11-21T16:51:45.619295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Boxplots for numerical features\nplt.figure(figsize=(20, 12))\nfor i, col in enumerate(numerical_cols[:6]):\n    plt.subplot(2, 3, i+1)\n    sns.boxplot(y=df[col])\n    plt.title(f'Boxplot of {col}')\n    plt.ylabel('')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:48.789095Z","iopub.execute_input":"2025-11-21T16:51:48.789840Z","iopub.status.idle":"2025-11-21T16:51:49.707416Z","shell.execute_reply.started":"2025-11-21T16:51:48.789813Z","shell.execute_reply":"2025-11-21T16:51:49.706544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Enhanced distribution plot with multiple elements\nif len(numerical_cols) >= 1:\n    fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n    \n    # Histogram with KDE\n    sns.histplot(data=df, x=numerical_cols[0], kde=True, ax=axes[0,0])\n    axes[0,0].set_title(f'Distribution of {numerical_cols[0]}')\n    \n    # Box plot\n    sns.boxplot(data=df, y=numerical_cols[0], ax=axes[0,1])\n    axes[0,1].set_title(f'Box Plot of {numerical_cols[0]}')\n    \n    # Violin plot\n    sns.violinplot(data=df, y=numerical_cols[0], ax=axes[1,0])\n    axes[1,0].set_title(f'Violin Plot of {numerical_cols[0]}')\n    \n    # ECDF plot\n    from scipy import stats\n    x = np.sort(df[numerical_cols[0]])\n    y = np.arange(1, len(x)+1) / len(x)\n    axes[1,1].plot(x, y, marker='.', linestyle='none')\n    axes[1,1].set_title(f'ECDF of {numerical_cols[0]}')\n    axes[1,1].set_xlabel(numerical_cols[0])\n    axes[1,1].set_ylabel('ECDF')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:51:51.223820Z","iopub.execute_input":"2025-11-21T16:51:51.224125Z","iopub.status.idle":"2025-11-21T16:51:52.015295Z","shell.execute_reply.started":"2025-11-21T16:51:51.224106Z","shell.execute_reply":"2025-11-21T16:51:52.013141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bar plot for categorical variables\ncategorical_cols = df.select_dtypes(include=['object']).columns.tolist()\n\nif categorical_cols:\n    for col in categorical_cols[:2]:  # Limit to first 2 categorical columns\n        plt.figure(figsize=(15, 6))\n        \n        # Bar plot with counts\n        plt.subplot(1, 3, 1)\n        value_counts = df[col].value_counts()\n        sns.barplot(x=value_counts.index, y=value_counts.values)\n        plt.title(f'Bar Plot of {col}')\n        plt.xticks(rotation=45)\n        \n        # Pie chart\n        plt.subplot(1, 3, 2)\n        plt.pie(value_counts.values, labels=value_counts.index, autopct='%1.1f%%', startangle=90)\n        plt.title(f'Pie Chart of {col}')\n        \n        # Horizontal bar plot\n        plt.subplot(1, 3, 3)\n        sns.barplot(x=value_counts.values, y=value_counts.index)\n        plt.title(f'Horizontal Bar Plot of {col}')\n        \n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T16:59:55.051588Z","iopub.execute_input":"2025-11-21T16:59:55.051881Z","iopub.status.idle":"2025-11-21T17:00:22.575395Z","shell.execute_reply.started":"2025-11-21T16:59:55.051849Z","shell.execute_reply":"2025-11-21T17:00:22.574595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Box plots by categorical variable\nif len(numerical_cols) >= 1 and categorical_cols:\n    fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n    \n    # Box plot\n    sns.boxplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[0,0])\n    axes[0,0].set_title(f'{numerical_cols[0]} by {categorical_cols[0]}')\n    axes[0,0].tick_params(axis='x', rotation=45)\n    \n    # Violin plot\n    sns.violinplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[0,1])\n    axes[0,1].set_title(f'Violin Plot: {numerical_cols[0]} by {categorical_cols[0]}')\n    axes[0,1].tick_params(axis='x', rotation=45)\n    \n    # Swarm plot (for smaller datasets)\n    if len(df) < 1000:\n        sns.swarmplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[1,0], size=3)\n        axes[1,0].set_title(f'Swarm Plot: {numerical_cols[0]} by {categorical_cols[0]}')\n        axes[1,0].tick_params(axis='x', rotation=45)\n    else:\n        # Strip plot for larger datasets\n        sns.stripplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[1,0], alpha=0.5, jitter=True)\n        axes[1,0].set_title(f'Strip Plot: {numerical_cols[0]} by {categorical_cols[0]}')\n        axes[1,0].tick_params(axis='x', rotation=45)\n    \n    # Bar plot with confidence intervals\n    sns.barplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[1,1], ci='sd')\n    axes[1,1].set_title(f'Mean {numerical_cols[0]} by {categorical_cols[0]} with SD')\n    axes[1,1].tick_params(axis='x', rotation=45)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T17:00:22.576515Z","iopub.execute_input":"2025-11-21T17:00:22.576868Z","iopub.status.idle":"2025-11-21T17:01:03.699323Z","shell.execute_reply.started":"2025-11-21T17:00:22.576838Z","shell.execute_reply":"2025-11-21T17:01:03.698216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Violin plots for detailed distribution analysis\nplt.figure(figsize=(15, 10))\nfor i, col in enumerate(numerical_cols[:4]):\n    plt.subplot(2, 2, i+1)\n    sns.violinplot(y=df[col])\n    plt.title(f'Violin Plot of {col}')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Time series/sequence analysis (if applicable)\nif 'sequence_length' in df.columns:\n    fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n    \n    # Sequence length distribution\n    axes[0,0].hist(df['sequence_length'], bins=50, alpha=0.7, color='skyblue', edgecolor='black')\n    axes[0,0].set_title('Sequence Length Distribution')\n    axes[0,0].set_xlabel('Sequence Length')\n    axes[0,0].set_ylabel('Frequency')\n    \n    # Scatter plot: sequence length vs molecular weight\n    axes[0,1].scatter(df['sequence_length'], df.get('molecular_weight', df[numerical_cols[1]]), \n                     alpha=0.6, color='coral')\n    axes[0,1].set_title('Sequence Length vs Molecular Weight')\n    axes[0,1].set_xlabel('Sequence Length')\n    axes[0,1].set_ylabel('Molecular Weight')\n    \n    # Density plot\n    if 'stability' in df.columns:\n        sns.kdeplot(data=df, x='sequence_length', y='stability', ax=axes[1,0], fill=True, cmap='viridis')\n        axes[1,0].set_title('Density: Sequence Length vs Stability')\n    \n    # Cumulative distribution\n    sorted_lengths = np.sort(df['sequence_length'])\n    y_vals = np.arange(len(sorted_lengths)) / float(len(sorted_lengths))\n    axes[1,1].plot(sorted_lengths, y_vals, linewidth=2)\n    axes[1,1].set_title('CDF of Sequence Length')\n    axes[1,1].set_xlabel('Sequence Length')\n    axes[1,1].set_ylabel('Cumulative Probability')\n    axes[1,1].grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scatter plot matrix using Seaborn\nif len(numerical_cols) >= 4:\n    # Create a subset of numerical columns for scatter matrix\n    scatter_matrix_cols = numerical_cols[:4]\n    \n    # Using pairplot for scatter matrix\n    g = sns.PairGrid(df[scatter_matrix_cols])\n    g.map_upper(sns.scatterplot, alpha=0.6)\n    g.map_lower(sns.kdeplot, fill=True, cmap='Blues')\n    g.map_diag(sns.histplot, kde=True)\n    g.fig.suptitle('Scatter Plot Matrix with Distributions', y=1.02, fontsize=16)\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Heatmap with clustering\nfrom scipy.cluster import hierarchy\n\nif len(numerical_cols) >= 4:\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 8))\n    \n    # Correlation heatmap\n    corr = df[numerical_cols].corr()\n    sns.heatmap(corr, annot=True, cmap='coolwarm', center=0, ax=ax1, fmt='.2f')\n    ax1.set_title('Correlation Heatmap')\n    \n    # Clustered heatmap\n    linked = hierarchy.linkage(corr, 'single')\n    dendro = hierarchy.dendrogram(linked, ax=ax2, orientation='right', \n                                 labels=corr.columns)\n    ax2.set_title('Hierarchical Clustering Dendrogram')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Outlier detection using IQR method\nprint(\"Outlier Analysis using IQR Method:\")\nprint(\"=\"*40)\n\nfor col in numerical_cols[:6]:  # Analyze first 6 numerical columns\n    Q1 = df[col].quantile(0.25)\n    Q3 = df[col].quantile(0.75)\n    IQR = Q3 - Q1\n    lower_bound = Q1 - 1.5 * IQR\n    upper_bound = Q3 + 1.5 * IQR\n    \n    outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]\n    outlier_percentage = (len(outliers) / len(df)) * 100\n    \n    print(f\"{col}:\")\n    print(f\"  - Lower bound: {lower_bound:.2f}\")\n    print(f\"  - Upper bound: {upper_bound:.2f}\")\n    print(f\"  - Number of outliers: {len(outliers)}\")\n    print(f\"  - Percentage of outliers: {outlier_percentage:.2f}%\")\n    print()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize outliers\nif len(numerical_cols) >= 3:\n    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n    \n    for i, col in enumerate(numerical_cols[:3]):\n        # Calculate IQR\n        Q1 = df[col].quantile(0.25)\n        Q3 = df[col].quantile(0.75)\n        IQR = Q3 - Q1\n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n        \n        # Create a boolean mask for outliers\n        outliers_mask = (df[col] < lower_bound) | (df[col] > upper_bound)\n        \n        # Scatter plot highlighting outliers\n        axes[i].scatter(range(len(df)), df[col], c=outliers_mask, \n                       cmap='coolwarm', alpha=0.6)\n        axes[i].axhline(y=lower_bound, color='r', linestyle='--', label='Lower bound')\n        axes[i].axhline(y=upper_bound, color='r', linestyle='--', label='Upper bound')\n        axes[i].set_title(f'Outliers in {col}')\n        axes[i].set_xlabel('Index')\n        axes[i].set_ylabel(col)\n        if i == 0:\n            axes[i].legend()\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distribution comparison across categories\nif categorical_cols and len(numerical_cols) >= 1:\n    fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n    \n    # Enhanced violin plot\n    sns.violinplot(data=df, x=categorical_cols[0], y=numerical_cols[0], \n                  ax=axes[0,0], inner='quartile')\n    axes[0,0].set_title(f'Violin Plot: {numerical_cols[0]} by {categorical_cols[0]}')\n    axes[0,0].tick_params(axis='x', rotation=45)\n    \n    # Boxen plot (enhanced box plot)\n    sns.boxenplot(data=df, x=categorical_cols[0], y=numerical_cols[0], ax=axes[0,1])\n    axes[0,1].set_title(f'Boxen Plot: {numerical_cols[0]} by {categorical_cols[0]}')\n    axes[0,1].tick_params(axis='x', rotation=45)\n    \n    # Point plot with confidence intervals\n    sns.pointplot(data=df, x=categorical_cols[0], y=numerical_cols[0], \n                 ax=axes[1,0], ci=95, capsize=0.1)\n    axes[1,0].set_title(f'Point Plot: Mean {numerical_cols[0]} by {categorical_cols[0]}')\n    axes[1,0].tick_params(axis='x', rotation=45)\n    \n    # Count plot for categorical distribution\n    sns.countplot(data=df, x=categorical_cols[0], ax=axes[1,1])\n    axes[1,1].set_title(f'Count Plot: {categorical_cols[0]}')\n    axes[1,1].tick_params(axis='x', rotation=45)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a comprehensive summary report\nprint(\"=\"*60)\nprint(\"COMPREHENSIVE EDA SUMMARY REPORT\")\nprint(\"=\"*60)\n\nprint(f\"\\n1. DATASET OVERVIEW:\")\nprint(f\"   - Total samples: {df.shape[0]}\")\nprint(f\"   - Total features: {df.shape[1]}\")\nprint(f\"   - Numerical features: {len(numerical_cols)}\")\nprint(f\"   - Categorical features: {len(categorical_cols)}\")\n\nprint(f\"\\n2. DATA QUALITY:\")\nprint(f\"   - Missing values: {df.isnull().sum().sum()}\")\nprint(f\"   - Duplicate rows: {df.duplicated().sum()}\")\n\nprint(f\"\\n3. NUMERICAL FEATURES STATISTICS:\")\nfor col in numerical_cols[:5]:\n    print(f\"   - {col}: mean={df[col].mean():.2f}, std={df[col].std():.2f}\")\n\nif categorical_cols:\n    print(f\"\\n4. CATEGORICAL FEATURES SUMMARY:\")\n    for col in categorical_cols[:3]:\n        unique_vals = df[col].nunique()\n        print(f\"   - {col}: {unique_vals} unique categories\")\n\nprint(f\"\\n5. CORRELATION INSIGHTS:\")\nif len(numerical_cols) >= 2:\n    corr_matrix = df[numerical_cols].corr()\n    # Find highest correlations\n    corr_pairs = []\n    for i in range(len(correlation_matrix.columns)):\n        for j in range(i+1, len(correlation_matrix.columns)):\n            if abs(correlation_matrix.iloc[i, j]) > 0.5:  # threshold\n                corr_pairs.append((\n                    correlation_matrix.columns[i],\n                    correlation_matrix.columns[j],\n                    correlation_matrix.iloc[i, j]\n                ))\n    \n    if corr_pairs:\n        print(\"   Strong correlations found:\")\n        for pair in corr_pairs[:3]:  # Show top 3\n            print(f\"     {pair[0]} vs {pair[1]}: {pair[2]:.3f}\")\n    else:\n        print(\"   No strong correlations found (|r| > 0.5)\")\n\nprint(\"\\n\" + \"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final comprehensive dashboard-style visualization\nif len(numerical_cols) >= 4:\n    fig = plt.figure(figsize=(20, 16))\n    \n    # Create subplots grid\n    gs = fig.add_gridspec(4, 4)\n    \n    # 1. Correlation heatmap (top left)\n    ax1 = fig.add_subplot(gs[0:2, 0:2])\n    corr_subset = df[numerical_cols[:4]].corr()\n    sns.heatmap(corr_subset, annot=True, cmap='coolwarm', center=0, ax=ax1, fmt='.2f')\n    ax1.set_title('Correlation Heatmap (Top 4 Features)')\n    \n    # 2. Distribution plot (top right)\n    ax2 = fig.add_subplot(gs[0, 2:4])\n    sns.histplot(data=df, x=numerical_cols[0], kde=True, ax=ax2)\n    ax2.set_title(f'Distribution of {numerical_cols[0]}')\n    \n    # 3. Box plot (second row, right)\n    ax3 = fig.add_subplot(gs[1, 2:4])\n    sns.boxplot(data=df, y=numerical_cols[1], ax=ax3)\n    ax3.set_title(f'Box Plot of {numerical_cols[1]}')\n    \n    # 4. Scatter plot (third row)\n    ax4 = fig.add_subplot(gs[2, 0:2])\n    sns.scatterplot(data=df, x=numerical_cols[2], y=numerical_cols[3], alpha=0.6, ax=ax4)\n    ax4.set_title(f'Scatter Plot: {numerical_cols[2]} vs {numerical_cols[3]}')\n    \n    # 5. Violin plot (third row, right)\n    ax5 = fig.add_subplot(gs[2, 2:4])\n    sns.violinplot(data=df, y=numerical_cols[0], ax=ax5)\n    ax5.set_title(f'Violin Plot of {numerical_cols[0]}')\n    \n    # 6. Bar plot if categorical exists (bottom row)\n    if categorical_cols:\n        ax6 = fig.add_subplot(gs[3, 0:2])\n        value_counts = df[categorical_cols[0]].value_counts().head(10)\n        sns.barplot(x=value_counts.index, y=value_counts.values, ax=ax6)\n        ax6.set_title(f'Top 10 Categories in {categorical_cols[0]}')\n        ax6.tick_params(axis='x', rotation=45)\n    \n    # 7. Line plot for cumulative distribution (bottom right)\n    ax7 = fig.add_subplot(gs[3, 2:4])\n    sorted_vals = np.sort(df[numerical_cols[0]])\n    y_vals = np.arange(1, len(sorted_vals)+1) / len(sorted_vals)\n    ax7.plot(sorted_vals, y_vals, linewidth=2)\n    ax7.set_title(f'CDF of {numerical_cols[0]}')\n    ax7.set_xlabel(numerical_cols[0])\n    ax7.set_ylabel('Cumulative Probability')\n    ax7.grid(True, alpha=0.3)\n    \n    plt.suptitle('Comprehensive EDA Dashboard', fontsize=16, fontweight='bold', y=0.98)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}