{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":105375,"databundleVersionId":12708774,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:16:23.692656Z","iopub.execute_input":"2025-09-11T17:16:23.693240Z","iopub.status.idle":"2025-09-11T17:16:24.229001Z","shell.execute_reply.started":"2025-09-11T17:16:23.693219Z","shell.execute_reply":"2025-09-11T17:16:24.228361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/kaggle_test_data.csv\"\nTRAIN_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/train_data.csv\"\nMEMBERS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/members.csv\"\nUSER_LOGS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/user_logs.csv\"\nTRANSACTIONS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/transactions.csv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:33:49.925078Z","iopub.execute_input":"2025-09-11T17:33:49.925686Z","iopub.status.idle":"2025-09-11T17:33:49.929464Z","shell.execute_reply.started":"2025-09-11T17:33:49.925665Z","shell.execute_reply":"2025-09-11T17:33:49.928692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/kaggle_test_data.csv')\ntest","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:17:14.201281Z","iopub.execute_input":"2025-09-11T17:17:14.201564Z","iopub.status.idle":"2025-09-11T17:17:14.269770Z","shell.execute_reply.started":"2025-09-11T17:17:14.201542Z","shell.execute_reply":"2025-09-11T17:17:14.269159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"members = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/members.csv')\nmembers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:17:43.398886Z","iopub.execute_input":"2025-09-11T17:17:43.399471Z","iopub.status.idle":"2025-09-11T17:17:54.746280Z","shell.execute_reply.started":"2025-09-11T17:17:43.399448Z","shell.execute_reply":"2025-09-11T17:17:54.745496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/members.csv')\ntrain_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:18:17.781422Z","iopub.execute_input":"2025-09-11T17:18:17.781698Z","iopub.status.idle":"2025-09-11T17:18:25.666957Z","shell.execute_reply.started":"2025-09-11T17:18:17.781675Z","shell.execute_reply":"2025-09-11T17:18:25.666368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transactions = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/transactions.csv')\ntransactions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:18:54.028091Z","iopub.execute_input":"2025-09-11T17:18:54.028375Z","iopub.status.idle":"2025-09-11T17:18:56.559206Z","shell.execute_reply.started":"2025-09-11T17:18:54.028355Z","shell.execute_reply":"2025-09-11T17:18:56.558594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_logs = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/user_logs.csv')\nuser_logs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:19:37.020479Z","iopub.execute_input":"2025-09-11T17:19:37.021167Z","iopub.status.idle":"2025-09-11T17:20:08.704833Z","shell.execute_reply.started":"2025-09-11T17:19:37.021142Z","shell.execute_reply":"2025-09-11T17:20:08.704152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add these to your existing imports cell\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom bokeh.plotting import figure, show, output_notebook\nfrom bokeh.models import HoverTool\n\n# Set a nice style for Seaborn plots\nsns.set_theme(style=\"whitegrid\")\n\n# Initialize Bokeh for notebook output\noutput_notebook()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:32:48.678247Z","iopub.execute_input":"2025-09-11T17:32:48.678943Z","iopub.status.idle":"2025-09-11T17:32:50.708348Z","shell.execute_reply.started":"2025-09-11T17:32:48.678916Z","shell.execute_reply":"2025-09-11T17:32:50.707748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This cell is from your original notebook. Run it before the ID guessing cell.\n\ndef guess_id_column(df):\n    candidates = [\"msno\", \"member_id\", \"customer_id\", \"user_id\", \"userid\", \"id\"]\n    for c in candidates:\n        for col in df.columns:\n            if col.lower() == c:\n                return col\n    # Fallback: look for columns ending in _id\n    for col in df.columns:\n        if col.lower().endswith(\"_id\"):\n            return col\n    return None\n\ndef guess_date_columns(df):\n    # Common names\n    date_like = []\n    for col in df.columns:\n        cl = col.lower()\n        if (\"date\" in cl) or (\"_dt\" in cl) or (\"time\" in cl) or (\"_at\" in cl):\n            date_like.append(col)\n    # Also test columns that parse cleanly as dates for a small sample\n    for col in df.columns:\n        if col in date_like:\n            continue\n        try:\n            pd.to_datetime(df[col].dropna().sample(min(20, df[col].dropna().shape[0])), errors=\"raise\", infer_datetime_format=True)\n            date_like.append(col)\n        except Exception:\n            pass\n    return list(dict.fromkeys(date_like))\n\ndef parse_dates_inplace(df, date_cols):\n    for c in date_cols:\n        try:\n            df[c] = pd.to_datetime(df[c], errors=\"coerce\", infer_datetime_format=True)\n        except Exception:\n            df[c] = pd.to_datetime(df[c], errors=\"coerce\")\n\ndef num_cat_cols(df, max_cardinality=30):\n    numeric = df.select_dtypes(include=[np.number]).columns.tolist()\n    categorical = [c for c in df.columns if c not in numeric]\n    # Split low-cardinality categoricals (for summaries) vs free text\n    low_card = [c for c in categorical if df[c].nunique(dropna=False) <= max_cardinality]\n    return numeric, categorical, low_card\n\ndef describe_categoricals(df, cols, topn=10):\n    out = {}\n    for c in cols:\n        vc = df[c].value_counts(dropna=False).head(topn)\n        out[c] = vc\n    return out\n\ndef missing_report(df):\n    m = df.isna().sum().sort_values(ascending=False).to_frame(\"missing\")\n    m[\"pct_missing\"] = (m[\"missing\"] / len(df)).round(4)\n    return m\n\ndef duplicate_report(df):\n    return df.duplicated().sum()\n\ndef basic_outlier_bounds(s, k=1.5):\n    q1, q3 = s.quantile(0.25), s.quantile(0.75)\n    iqr = q3 - q1\n    lo, hi = q1 - k*iqr, q3 + k*iqr\n    return lo, hi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:35:14.765330Z","iopub.execute_input":"2025-09-11T17:35:14.765869Z","iopub.status.idle":"2025-09-11T17:35:14.775704Z","shell.execute_reply.started":"2025-09-11T17:35:14.765846Z","shell.execute_reply":"2025-09-11T17:35:14.775003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This cell is from your original notebook. Run it before the new plotting code.\n\nMEMBER_ID = guess_id_column(members) or \"msno\"\nTRANS_ID = guess_id_column(transactions) or MEMBER_ID\n\nmember_date_cols = guess_date_columns(members)\ntrans_date_cols = guess_date_columns(transactions)\ntest_date_cols = guess_date_columns(test) if test is not None else []\n\nprint(\"Guessed MEMBER_ID:\", MEMBER_ID)\nprint(\"Guessed TRANS_ID:\", TRANS_ID)\nprint(\"Member date-like cols:\", member_date_cols)\nprint(\"Trans date-like cols:\", trans_date_cols)\nprint(\"Test date-like cols:\", test_date_cols)\n\nparse_dates_inplace(members, member_date_cols)\nparse_dates_inplace(transactions, trans_date_cols)\nif test is not None:\n    parse_dates_inplace(test, test_date_cols)\n\n# Ensure ID columns are strings for joins\nmembers[MEMBER_ID] = members[MEMBER_ID].astype(str)\ntransactions[TRANS_ID] = transactions[TRANS_ID].astype(str)\nif test is not None and MEMBER_ID in test.columns:\n    test[MEMBER_ID] = test[MEMBER_ID].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:35:28.778710Z","iopub.execute_input":"2025-09-11T17:35:28.779483Z","iopub.status.idle":"2025-09-11T17:35:33.575408Z","shell.execute_reply.started":"2025-09-11T17:35:28.779459Z","shell.execute_reply":"2025-09-11T17:35:33.574593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## This code enhances the '# Members Feature Audit' section\n\n# Use the variables already defined in your notebook\nnum_cols = members.select_dtypes(include=[np.number]).columns.tolist()\ncat_cols = [c for c in members.columns if c not in num_cols and c != MEMBER_ID]\n\nprint(\"--- Enhanced Numeric Histograms (Seaborn) ---\")\nfor col in num_cols:\n    plt.figure(figsize=(10, 6))\n    sns.histplot(members[col], kde=True, bins=40)\n    plt.title(f\"Distribution for {col}\", fontsize=16)\n    plt.show()\n\nprint(\"\\\\n--- Enhanced Categorical Bar Charts (Plotly) ---\")\nfor col in cat_cols[:10]: # Limit to top 10 categoricals\n    # Get value counts\n    counts = members[col].value_counts(dropna=False).head(20).reset_index()\n    counts.columns = [col, 'count'] # Rename columns for Plotly\n\n    # Create interactive bar chart\n    fig = px.bar(\n        counts,\n        x=col,\n        y='count',\n        title=f\"Top 20 Categories for {col}\",\n        labels={col: col, 'count': 'Number of Members'},\n        text='count' # Display count on bars\n    )\n    fig.update_traces(textposition='outside')\n    fig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:35:40.600366Z","iopub.execute_input":"2025-09-11T17:35:40.601164Z","iopub.status.idle":"2025-09-11T17:35:42.369205Z","shell.execute_reply.started":"2025-09-11T17:35:40.601133Z","shell.execute_reply":"2025-09-11T17:35:42.368553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## This code enhances the '# Members Feature Audit' section\n\n# Use the variables already defined in your notebook\nnum_cols = members.select_dtypes(include=[np.number]).columns.tolist()\ncat_cols = [c for c in members.columns if c not in num_cols and c != MEMBER_ID]\n\nprint(\"--- Enhanced Numeric Histograms (Seaborn) ---\")\nfor col in num_cols:\n    plt.figure(figsize=(10, 6))\n    sns.histplot(members[col], kde=True, bins=40)\n    plt.title(f\"Distribution for {col}\", fontsize=16)\n    plt.show()\n\nprint(\"\\\\n--- Enhanced Categorical Bar Charts (Plotly) ---\")\nfor col in cat_cols[:10]: # Limit to top 10 categoricals\n    # Get value counts\n    counts = members[col].value_counts(dropna=False).head(20).reset_index()\n    counts.columns = [col, 'count'] # Rename columns for Plotly\n\n    # Create interactive bar chart\n    fig = px.bar(\n        counts,\n        x=col,\n        y='count',\n        title=f\"Top 20 Categories for {col}\",\n        labels={col: col, 'count': 'Number of Members'},\n        text='count' # Display count on bars\n    )\n    fig.update_traces(textposition='outside')\n    fig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:36:15.326781Z","iopub.execute_input":"2025-09-11T17:36:15.327376Z","iopub.status.idle":"2025-09-11T17:36:15.805270Z","shell.execute_reply.started":"2025-09-11T17:36:15.327350Z","shell.execute_reply":"2025-09-11T17:36:15.804488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## This function is a better replacement for plot_corr_heatmap\n\ndef plot_corr_heatmap_seaborn(df, title):\n    num = df.select_dtypes(include=[np.number])\n    if num.shape[1] < 2:\n        print(f\"{title}: Not enough numeric columns for correlation heatmap.\")\n        return\n\n    corr = num.corr()\n\n    # Create the heatmap\n    plt.figure(figsize=(12, 8))\n    sns.heatmap(\n        corr,\n        annot=True,      # Show the correlation values\n        fmt=\".2f\",       # Format to two decimal places\n        cmap=\"coolwarm\", # Use a diverging colormap\n        linewidths=.5\n    )\n    plt.title(title, fontsize=16)\n    plt.show()\n\n\n# How to use it (in the '# Correlation Heatmap' section)\nplot_corr_heatmap_seaborn(members, \"Members: Numeric Correlations (Seaborn)\")\nplot_corr_heatmap_seaborn(transactions, \"Transactions: Numeric Correlations (Seaborn)\")\n# Also great for RFM features!\nif 'rfm_agg' in locals():\n    plot_corr_heatmap_seaborn(rfm_agg, \"RFM Features Correlation (Seaborn)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:36:51.954581Z","iopub.execute_input":"2025-09-11T17:36:51.955346Z","iopub.status.idle":"2025-09-11T17:36:51.961290Z","shell.execute_reply.started":"2025-09-11T17:36:51.955319Z","shell.execute_reply":"2025-09-11T17:36:51.960489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## This code enhances the '# Cohort Analysis' section visualizations\n\n# Assuming 'retention' DataFrame is calculated as in your notebook\nif 'retention' in locals():\n    retention_percent = retention * 100\n\n    print(\"--- Cohort Retention Heatmap (Seaborn) ---\")\n    plt.figure(figsize=(14, 10))\n    sns.heatmap(\n        retention_percent,\n        annot=True,\n        fmt=\".1f\", # Format to one decimal place\n        cmap=\"viridis\",\n        cbar_kws={'label': 'Retention Rate (%)'}\n    )\n    plt.title(\"Monthly Cohort Retention (%)\", fontsize=16)\n    plt.xlabel(\"Months Since First Purchase\")\n    plt.ylabel(\"Cohort (YYYY-MM)\")\n    plt.show()\n\n\n    print(\"\\\\n--- Interactive Cohort Retention Heatmap (Plotly) ---\")\n    fig = go.Figure(data=go.Heatmap(\n        z=retention_percent.values,\n        x=retention_percent.columns,\n        y=retention_percent.index.astype(str),\n        hoverongaps=False,\n        colorscale='Viridis',\n        colorbar={'title': 'Retention Rate (%)'}\n    ))\n\n    fig.update_layout(\n        title='Interactive Monthly Cohort Retention',\n        xaxis_title=\"Months Since First Purchase\",\n        yaxis_title=\"Cohort (YYYY-MM)\"\n    )\n    fig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:37:05.170543Z","iopub.execute_input":"2025-09-11T17:37:05.170800Z","iopub.status.idle":"2025-09-11T17:37:05.176331Z","shell.execute_reply.started":"2025-09-11T17:37:05.170782Z","shell.execute_reply":"2025-09-11T17:37:05.175410Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## This is a new section demonstrating Bokeh for RFM analysis\n\nif 'rfm_agg' in locals():\n    print(\"--- Interactive RFM Scatter Plot (Bokeh) ---\")\n\n    # Define the tooltips to show on hover\n    # The @ symbol refers to columns in the data source\n    tooltips = [\n        (\"Member ID\", f\"@{TRANS_ID}\"),\n        (\"Frequency\", \"@frequency\"),\n        (\"Monetary\", \"@monetary{$0,0.00}\"), # Format as currency\n        (\"Recency (days)\", \"@recency_days\")\n    ]\n\n    # Create a figure\n    p = figure(\n        height=400,\n        width=700,\n        title=\"RFM: Frequency vs. Monetary Value\",\n        x_axis_label=\"Frequency (Number of Transactions)\",\n        y_axis_label=\"Monetary Value (Total Spent)\",\n        tools=\"pan,wheel_zoom,box_zoom,reset,save\" # Interactive tools\n    )\n\n    # Add a scatter plot glyph\n    p.scatter(\n        x='frequency',\n        y='monetary',\n        source=rfm_agg,\n        size=8,\n        alpha=0.6,\n        legend_label=\"Customer\"\n    )\n\n    # Add the hover tool\n    p.add_tools(HoverTool(tooltips=tooltips))\n\n    # Show the plot\n    show(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:39:47.571046Z","iopub.execute_input":"2025-09-11T17:39:47.571694Z","iopub.status.idle":"2025-09-11T17:39:47.578479Z","shell.execute_reply.started":"2025-09-11T17:39:47.571660Z","shell.execute_reply":"2025-09-11T17:39:47.577578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\n\n# Set plotting styles for better aesthetics\nsns.set_theme(style=\"whitegrid\")\nplt.rcParams['figure.figsize'] = (12, 7)\n\n# --- 1. Load Data (Corrected) ---\n# Define file paths\nTEST_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/kaggle_test_data.csv\"\nTRAIN_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/train_data.csv\"\nMEMBERS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/members.csv\"\nUSER_LOGS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/user_logs.csv\"\nTRANSACTIONS_PATH = \"/kaggle/input/customer-retention-datathon-brisbane-edition/transactions.csv\"\n\n# Read data into pandas DataFrames\nprint(\"Loading data...\")\ntrain = pd.read_csv(TRAIN_PATH)\ntest = pd.read_csv(TEST_PATH)\n# Added low_memory=False to handle the DtypeWarning\nmembers = pd.read_csv(MEMBERS_PATH, low_memory=False)\ntransactions = pd.read_csv(TRANSACTIONS_PATH)\nuser_logs = pd.read_csv(USER_LOGS_PATH)\n\n# --- Convert Date Columns AFTER loading ---\n# This is the correct place to specify the date format\ntransactions['transaction_date'] = pd.to_datetime(transactions['transaction_date'], format='%Y%m%d')\ntransactions['membership_expire_date'] = pd.to_datetime(transactions['membership_expire_date'], format='%Y%m%d')\n\nprint(\"Data loading and date conversion complete.\")\n\n\n# --- 2. Initial Data Inspection ---\nprint(\"\\n--- Data Overview ---\")\ndef inspect_df(df, name):\n    print(f\"\\n--- {name} DataFrame ---\")\n    print(f\"Shape: {df.shape}\")\n    print(\"\\nInfo:\")\n    df.info()\n    print(\"\\nMissing Values:\")\n    print(df.isna().sum() / len(df) * 100) # Show missing values as a percentage\n    print(f\"\\nDuplicates: {df.duplicated().sum()}\")\n\ninspect_df(train, \"Train\")\ninspect_df(members, \"Members\")\ninspect_df(transactions, \"Transactions\")\ninspect_df(user_logs, \"User Logs\")\n\n# --- 3. Members Analysis ---\nprint(\"\\n--- Visualizing Members Data ---\")\n\n## Age ('bd') Distribution\n# We'll filter for a reasonable age range (1 to 100) to get a clean plot\ncleaned_bd = members[(members['bd'] > 0) & (members['bd'] < 100)]['bd']\nsns.histplot(cleaned_bd, bins=50, kde=True)\nplt.title('Distribution of Member Age (1-100)', fontsize=16)\nplt.xlabel('Age (bd)')\nplt.ylabel('Count')\nplt.show()\n\n## Gender Distribution (Interactive)\ngender_counts = members['gender'].value_counts().reset_index()\nfig = px.pie(gender_counts, values='count', names='gender',\n             title='Member Gender Distribution',\n             color_discrete_sequence=px.colors.sequential.RdBu)\nfig.show()\n\n# --- 4. Transactions Analysis ---\nprint(\"\\n--- Visualizing Transactions Data ---\")\n\n## Daily Transaction Count\ndaily_transactions = transactions.groupby('transaction_date').size().reset_index(name='count')\nfig = px.line(daily_transactions, x='transaction_date', y='count',\n              title='Daily Transaction Volume')\nfig.update_xaxes(rangeslider_visible=True)\nfig.show()\n\n## Payment Plan Days Distribution\nsns.countplot(data=transactions, x='payment_plan_days', order=transactions['payment_plan_days'].value_counts().index[:10])\nplt.title('Top 10 Most Common Payment Plan Durations (in Days)', fontsize=16)\nplt.xlabel('Payment Plan (Days)')\nplt.ylabel('Count')\nplt.show()\n\n## Price vs. Amount Paid\nsns.scatterplot(data=transactions.sample(10000), x='plan_list_price', y='actual_amount_paid', alpha=0.5)\nplt.title('Plan List Price vs. Actual Amount Paid (Sample of 10k)', fontsize=16)\nplt.xlabel('Plan List Price')\nplt.ylabel('Actual Amount Paid')\nplt.plot([0, 2000], [0, 2000], 'r--', label='Price = Paid') # Add a line for reference\nplt.legend()\nplt.show()\n\n# --- 5. User Logs Analysis ---\nprint(\"\\n--- Visualizing User Logs Data ---\")\n\n## Listening Habits (log scale)\nfig, axes = plt.subplots(1, 2, figsize=(18, 6))\nsns.histplot(user_logs['total_secs'], ax=axes[0], bins=100, log_scale=True, color='skyblue')\naxes[0].set_title('Distribution of Total Seconds Listened Per Day (Log Scale)')\nsns.histplot(user_logs['num_unq'], ax=axes[1], bins=100, log_scale=True, color='salmon')\naxes[1].set_title('Distribution of Unique Songs Played Per Day (Log Scale)')\nplt.tight_layout()\nplt.show()\n\n## Correlation Heatmap of Listening Patterns\nlog_features = ['num_25', 'num_50', 'num_75', 'num_985', 'num_100', 'num_unq', 'total_secs']\ncorr = user_logs[log_features].corr()\nsns.heatmap(corr, annot=True, fmt='.2f', cmap='viridis')\nplt.title('Correlation Matrix of User Listening Habits', fontsize=16)\nplt.show()\n\nprint(\"\\nEDA script finished.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-11T17:42:39.431149Z","iopub.execute_input":"2025-09-11T17:42:39.431430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport gc # Garbage Collector interface\n\nprint(\"Starting the script...\")\n\n# --- 1. Memory Optimization (Helper Function) ---\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n# --- 2. Load Data ---\nprint(\"Loading data...\")\ntrain = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/train_data.csv')\ntest = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/kaggle_test_data.csv')\nmembers = reduce_mem_usage(pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/members.csv'))\ntransactions = reduce_mem_usage(pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/transactions.csv'))\nuser_logs = reduce_mem_usage(pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/user_logs.csv'))\n\n# --- 3. Feature Engineering ---\nprint(\"Performing feature engineering...\")\n\n# Transactions Features\ntransactions['transaction_date'] = pd.to_datetime(transactions['transaction_date'], format='%Y%m%d')\ntransactions['membership_expire_date'] = pd.to_datetime(transactions['membership_expire_date'], format='%Y%m%d')\ntransactions['discount'] = transactions['plan_list_price'] - transactions['actual_amount_paid']\ntransactions['days_to_expire'] = (transactions['membership_expire_date'] - transactions['transaction_date']).dt.days\ntransactions['is_discount'] = (transactions['plan_list_price'] > transactions['actual_amount_paid']).astype(int)\n\ntransactions_agg = transactions.groupby('msno').agg({\n    'payment_plan_days': ['mean', 'last', 'std'],\n    'plan_list_price': ['mean', 'last'],\n    'actual_amount_paid': ['mean', 'last'],\n    'is_auto_renew': ['last'],\n    'is_cancel': ['sum', 'last'],\n    'discount': ['mean', 'last'],\n    'days_to_expire': ['mean', 'std'],\n    'is_discount': ['mean', 'sum']\n})\ntransactions_agg.columns = ['_'.join(col).strip() for col in transactions_agg.columns.values]\ntransactions_agg.rename(columns={'msno_': 'msno'}, inplace=True)\n\n# User Logs Features\nuser_logs['total_songs'] = user_logs['num_25'] + user_logs['num_50'] + user_logs['num_75'] + user_logs['num_985'] + user_logs['num_100']\nuser_logs['completion_rate'] = user_logs['num_100'] / user_logs['total_songs']\nuser_logs['uniqueness_rate'] = user_logs['num_unq'] / user_logs['total_songs']\n\nuser_logs_agg = user_logs.groupby('msno').agg({\n    'num_25': ['mean', 'sum'], 'num_50': ['mean', 'sum'], 'num_75': ['mean', 'sum'],\n    'num_985': ['mean', 'sum'], 'num_100': ['mean', 'sum'], 'num_unq': ['mean', 'sum'],\n    'total_secs': ['mean', 'sum', 'std'], 'completion_rate': ['mean', 'std'], 'uniqueness_rate': ['mean']\n})\nuser_logs_agg.columns = ['_'.join(col).strip() for col in user_logs_agg.columns.values]\nuser_logs_agg.rename(columns={'msno_': 'msno'}, inplace=True)\n\n# --- 4. Merge Data ---\nprint(\"Merging all data sources...\")\ntrain = pd.merge(train, members, on='msno', how='left')\ntrain = pd.merge(train, transactions_agg, on='msno', how='left')\ntrain = pd.merge(train, user_logs_agg, on='msno', how='left')\n\ntest = pd.merge(test, members, on='msno', how='left')\ntest = pd.merge(test, transactions_agg, on='msno', how='left')\ntest = pd.merge(test, user_logs_agg, on='msno', how='left')\n\ndel members, transactions, user_logs, transactions_agg, user_logs_agg; gc.collect()\n\n# --- 5. Data Cleaning & Preparation ---\nprint(\"Cleaning and preparing final dataframes...\")\ndef clean_and_prepare(df):\n    df['bd'] = df['bd'].apply(lambda x: np.nan if x <= 0 or x >= 100 else x)\n    df['bd'] = df['bd'].fillna(df['bd'].median())\n    df['gender'].fillna('unknown', inplace=True)\n    df['registration_init_time'] = pd.to_datetime(df['registration_init_time'], format='%Y%m%d', errors='coerce')\n    df['account_tenure'] = (pd.to_datetime('2017-04-30') - df['registration_init_time']).dt.days\n    df.fillna(0, inplace=True)\n    return df\n\ntrain = clean_and_prepare(train)\ntest = clean_and_prepare(test)\n\n# --- 6. Label Encoding ---\nprint(\"Encoding categorical features...\")\ny = train['is_churn']\ntrain = train.drop(['is_churn', 'msno', 'registration_init_time'], axis=1)\ntest = test.drop(['msno', 'registration_init_time'], axis=1)\n\ncategorical_cols = [col for col in train.columns if train[col].dtype == 'object']\nfor col in categorical_cols:\n    le = LabelEncoder()\n    train[col] = le.fit_transform(train[col])\n    # Handle new categories in test set\n    test[col] = test[col].map(lambda s: s if s in le.classes_ else 'unknown')\n    le_classes = le.classes_.tolist()\n    if 'unknown' not in le_classes: le_classes.append('unknown')\n    le.classes_ = np.array(le_classes)\n    test[col] = le.transform(test[col])\n\n# --- 7. Model Training (Cross-Validation) ---\nprint(\"Starting final model training with cross-validation...\")\n\n# These parameters are a strong starting point. Replace them with your own Optuna results.\nparams = {\n    'objective': 'binary', 'metric': 'binary_logloss', 'boosting_type': 'gbdt',\n    'n_estimators': 2000, 'learning_rate': 0.02, 'num_leaves': 80,\n    'max_depth': 7, 'min_child_samples': 100, 'subsample': 0.9,\n    'colsample_bytree': 0.7, 'random_state': 42, 'n_jobs': -1\n}\n\nN_SPLITS = 3\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=42)\ntest_preds = np.zeros(len(test))\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train, y)):\n    print(f\"===== FOLD {fold+1}/{N_SPLITS} =====\")\n    X_train, y_train = train.iloc[train_idx], y.iloc[train_idx]\n    X_val, y_val = train.iloc[val_idx], y.iloc[val_idx]\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(X_train, y_train,\n              eval_set=[(X_val, y_val)],\n              eval_metric='binary_logloss',\n              callbacks=[lgb.early_stopping(100, verbose=True)])\n\n    test_preds += model.predict_proba(test)[:, 1] / N_SPLITS\n    gc.collect()\n\n# --- 8. Create Submission File ---\nprint(\"Creating submission file...\")\nsubmission = pd.read_csv('/kaggle/input/customer-retention-datathon-brisbane-edition/kaggle_test_data.csv')\nsubmission['is_churn'] = test_preds\nsubmission.to_csv('submission_final.csv', index=False)\n\nprint(\"Script finished successfully! `submission_final(l).csv` is ready.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T14:00:32.581265Z","iopub.execute_input":"2025-06-17T14:00:32.581580Z","iopub.status.idle":"2025-06-17T14:04:58.107320Z","shell.execute_reply.started":"2025-06-17T14:00:32.581562Z","shell.execute_reply":"2025-06-17T14:04:58.106543Z"}},"outputs":[],"execution_count":null}]}