{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":113366,"databundleVersionId":13605326,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport re\nimport string\nfrom wordcloud import WordCloud","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:49.007671Z","iopub.execute_input":"2025-09-16T09:24:49.007882Z","iopub.status.idle":"2025-09-16T09:24:51.264533Z","shell.execute_reply.started":"2025-09-16T09:24:49.007862Z","shell.execute_reply":"2025-09-16T09:24:51.263799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install wordcloud","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:51.266104Z","iopub.execute_input":"2025-09-16T09:24:51.266500Z","iopub.status.idle":"2025-09-16T09:24:55.869718Z","shell.execute_reply.started":"2025-09-16T09:24:51.266478Z","shell.execute_reply":"2025-09-16T09:24:55.868882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Base_path = \"/kaggle/input/comments-classification/Dataset\"\ntrain_path = \"/kaggle/input/comments-classification/Dataset/train.csv\"\ntest_path = \"/kaggle/input/comments-classification/Dataset/test.csv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:55.870752Z","iopub.execute_input":"2025-09-16T09:24:55.871011Z","iopub.status.idle":"2025-09-16T09:24:55.875640Z","shell.execute_reply.started":"2025-09-16T09:24:55.870985Z","shell.execute_reply":"2025-09-16T09:24:55.874823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(f\"{train_path}\")\ntest = pd.read_csv(f\"{test_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:55.876565Z","iopub.execute_input":"2025-09-16T09:24:55.876876Z","iopub.status.idle":"2025-09-16T09:24:59.350194Z","shell.execute_reply.started":"2025-09-16T09:24:55.876848Z","shell.execute_reply":"2025-09-16T09:24:59.349316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_samples = len(train)\nprint(\"Total samples:\", total_samples)\n\n# Percentage of class 0\ncount_0 = (train['psychotic_depression'] == 0).sum()\npercent_0 = (count_0 / total_samples) * 100\nprint(f\"Percentage of 0: {percent_0:.2f}%\")\n\n# Percentage of class 1\ncount_1 = (train['psychotic_depression'] == 1).sum()\npercent_1 = (count_1 / total_samples) * 100\nprint(f\"Percentage of 1: {percent_1:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:59.351197Z","iopub.execute_input":"2025-09-16T09:24:59.351430Z","iopub.status.idle":"2025-09-16T09:24:59.364639Z","shell.execute_reply.started":"2025-09-16T09:24:59.351411Z","shell.execute_reply":"2025-09-16T09:24:59.363860Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**We Can See Here that a Large Part of Dataset is just biased towards non-psychotic depression**","metadata":{}},{"cell_type":"markdown","source":"**Wordclouds - Frequent words**:\n\nNow, let's take a look at words that are associated with these classes.\n\nChart Description: The visuals here are word clouds (ie) more frequent words appear bigger.\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ntext_class0 = \" \".join(train[train['psychotic_depression']==0]['comment_text'])\ntext_class1 = \" \".join(train[train['psychotic_depression']==1]['comment_text'])\n\n# Create word clouds\nwordcloud0 = WordCloud(width=1600, height=800, background_color='white').generate(text_class0)\nwordcloud1 = WordCloud(width=1600, height=800, background_color='black', colormap='autumn').generate(text_class1)\n\n# Plot\nplt.figure(figsize=(16, 8))\nplt.imshow(wordcloud0, interpolation='bilinear')\nplt.axis('off')\nplt.title(\"Class 0 - Comments\")\n\nplt.figure(figsize=(16, 8))\nplt.imshow(wordcloud1, interpolation='bilinear')\nplt.axis('off')\nplt.title(\"Class 1 - Comments\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:24:59.365466Z","iopub.execute_input":"2025-09-16T09:24:59.365682Z","iopub.status.idle":"2025-09-16T09:25:34.768479Z","shell.execute_reply.started":"2025-09-16T09:24:59.365662Z","shell.execute_reply":"2025-09-16T09:25:34.767705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cleaned_bad_words = [\n    'mothjer', 'fuck', 'fucker', 'fucking', 'shit', 'asshole', 'bitch', 'bastard', 'cunt', 'damn', 'crap', 'dick', 'cock', 'balls', 'boobs', 'buttsecks',\n    'anal', 'rape', 'sex', 'sexsex', 'fucksex', 'suck', 'go fuck', 'fuck u', 'faggot', 'fag', 'super gay',\n    'idiot', 'moron', 'loser', 'dickhead', 'retarded', 'stupid', 'jerk', 'scum', 'trash',\n    'nigger', 'nigga', 'jew', 'racist', 'nazi', 'hitler', 'heil', 'mexicans', 'assad',\n    'die', 'kill', 'banned', 'poop', 'wanker', 'fggt', 'cocksucker', 'offfuck'\n]\n\n# Convert to set for faster lookup\nbad_words_set = set(cleaned_bad_words)\n\n# Function to count bad words in text\ndef count_bad_words(text):\n    text_lower = str(text).lower()\n    count = 0\n    for bad_word in bad_words_set:\n        count += text_lower.count(bad_word.lower())\n    return count\n\n# Add 'bad_word_count' feature to train dataset\ntrain['bad_word_count'] = train['comment_text'].apply(count_bad_words)\n\n# Add 'bad_word_count' feature to text dataset\ntest['bad_word_count'] = test['comment_text'].apply(count_bad_words)\n\n# Preview the results\nprint(train[['comment_text', 'bad_word_count']].head())\nprint(test[['comment_text', 'bad_word_count']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:25:34.771064Z","iopub.execute_input":"2025-09-16T09:25:34.771339Z","iopub.status.idle":"2025-09-16T09:25:44.433724Z","shell.execute_reply.started":"2025-09-16T09:25:34.771317Z","shell.execute_reply":"2025-09-16T09:25:44.432909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation = train['bad_word_count'].corr(train['psychotic_depression'])\nprint(f\"Correlation between bad_word_count and psychotic_depression: {correlation:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:25:44.434595Z","iopub.execute_input":"2025-09-16T09:25:44.434906Z","iopub.status.idle":"2025-09-16T09:25:44.467526Z","shell.execute_reply.started":"2025-09-16T09:25:44.434877Z","shell.execute_reply":"2025-09-16T09:25:44.466771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install nltk","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:25:44.468344Z","iopub.execute_input":"2025-09-16T09:25:44.468633Z","iopub.status.idle":"2025-09-16T09:25:48.007392Z","shell.execute_reply.started":"2025-09-16T09:25:44.468610Z","shell.execute_reply":"2025-09-16T09:25:48.006288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport string\ndef extract_indirect_features(df, text_col=\"comment_text\"):\n    # character-level\n    df['char_count'] = df[text_col].apply(len)\n    df['num_digits'] = df[text_col].apply(lambda x: sum(c.isdigit() for c in x))\n    df['num_spaces'] = df[text_col].apply(lambda x: x.count(\" \"))\n    \n    # word-level\n    df['word_count'] = df[text_col].apply(lambda x: len(x.split()))\n    df['unique_word_count'] = df[text_col].apply(lambda x: len(set(x.split())))\n    df['repetition_ratio'] = df['word_count'] / (df['unique_word_count'] + 1)\n    df['avg_word_length'] = df[text_col].apply(\n        lambda x: np.mean([len(w) for w in x.split()]) if len(x.split()) > 0 else 0\n    )\n    \n    # punctuation\n    df['num_punctuations'] = df[text_col].apply(lambda x: sum(c in string.punctuation for c in x))\n    df['num_exclamations'] = df[text_col].apply(lambda x: x.count(\"!\"))\n    df['num_questions'] = df[text_col].apply(lambda x: x.count(\"?\"))\n    df['num_quotes'] = df[text_col].apply(lambda x: x.count('\"') + x.count(\"'\"))\n    \n    # casing\n    df['num_upper_words'] = df[text_col].apply(lambda x: sum(1 for w in x.split() if w.isupper()))\n    df['upper_ratio'] = df[text_col].apply(lambda x: sum(c.isupper() for c in x) / (len(x) + 1))\n    \n    # special tokens\n    df['num_stopwords'] = df[text_col].apply(\n        lambda x: sum(1 for w in x.lower().split() if w in STOPWORDS)\n    )\n    df['stopword_ratio'] = df['num_stopwords'] / (df['word_count'] + 1)\n    \n    # line/sentence structure\n    df['num_lines'] = df[text_col].apply(lambda x: x.count(\"\\n\") + 1)\n    df['num_sentences'] = df[text_col].apply(lambda x: len(re.split(r\"[.!?]\", x)))\n    \n    return df\n\n\nimport nltk\nnltk.download('stopwords')\n\nfrom nltk.corpus import stopwords\nSTOPWORDS = set(stopwords.words('english'))\n\ntrain = extract_indirect_features(train, \"comment_text\")\ntest = extract_indirect_features(test, \"comment_text\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:25:48.008638Z","iopub.execute_input":"2025-09-16T09:25:48.008913Z","iopub.status.idle":"2025-09-16T09:26:31.128518Z","shell.execute_reply.started":"2025-09-16T09:25:48.008885Z","shell.execute_reply":"2025-09-16T09:26:31.127828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Select all numeric feature columns except ID, text, and target\nfeature_cols = [\n    col for col in train.columns \n    if col not in [\"id\", \"comment_text\", \"psychotic_depression\"]\n]\n\n# Plot all features as boxplots\nn_features = len(feature_cols)\nn_cols = 3  # number of plots per row\nn_rows = (n_features + n_cols - 1) // n_cols\n\nplt.figure(figsize=(18, 5 * n_rows))\n\nfor i, feature in enumerate(feature_cols, 1):\n    plt.subplot(n_rows, n_cols, i)\n    sns.boxplot(x=\"psychotic_depression\", y=feature, data=train)\n    plt.title(f\"{feature} by Target\")\n    plt.xlabel(\"psychotic_depression (0 or 1)\")\n    plt.ylabel(feature)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:26:31.129372Z","iopub.execute_input":"2025-09-16T09:26:31.129752Z","iopub.status.idle":"2025-09-16T09:26:35.177105Z","shell.execute_reply.started":"2025-09-16T09:26:31.129700Z","shell.execute_reply":"2025-09-16T09:26:35.176161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train.drop(['id', 'comment_text', 'psychotic_depression'], axis=1)\n\nX = train_df\ny = train['psychotic_depression']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:26:35.178628Z","iopub.execute_input":"2025-09-16T09:26:35.179182Z","iopub.status.idle":"2025-09-16T09:26:35.197190Z","shell.execute_reply.started":"2025-09-16T09:26:35.179145Z","shell.execute_reply":"2025-09-16T09:26:35.196362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:26:35.198100Z","iopub.execute_input":"2025-09-16T09:26:35.198357Z","iopub.status.idle":"2025-09-16T09:26:35.281101Z","shell.execute_reply.started":"2025-09-16T09:26:35.198337Z","shell.execute_reply":"2025-09-16T09:26:35.280192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install scikit-learn==1.2.2 imbalanced-learn==0.10.1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:27:29.537275Z","iopub.execute_input":"2025-09-16T09:27:29.538238Z","iopub.status.idle":"2025-09-16T09:27:36.633676Z","shell.execute_reply.started":"2025-09-16T09:27:29.538204Z","shell.execute_reply":"2025-09-16T09:27:36.632594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import SMOTE\n\nX_train, X_test, y_train, y_test = train_test_split(\n    X_scaled, y, test_size=0.25, stratify=y, random_state=42\n)\n\nfrom imblearn.over_sampling import BorderlineSMOTE\n\nsmote = BorderlineSMOTE(random_state=42, kind='borderline-1')\nX_train_res, y_train_res = smote.fit_resample(X_train, y_train)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module='threadpoolctl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:28:08.062108Z","iopub.execute_input":"2025-09-16T09:28:08.062759Z","iopub.status.idle":"2025-09-16T09:28:11.174424Z","shell.execute_reply.started":"2025-09-16T09:28:08.062724Z","shell.execute_reply":"2025-09-16T09:28:11.173554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install optuna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:28:17.693905Z","iopub.execute_input":"2025-09-16T09:28:17.694533Z","iopub.status.idle":"2025-09-16T09:28:21.414432Z","shell.execute_reply.started":"2025-09-16T09:28:17.694502Z","shell.execute_reply":"2025-09-16T09:28:21.413482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install catboost\n!pip install lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:28:21.417118Z","iopub.execute_input":"2025-09-16T09:28:21.417674Z","iopub.status.idle":"2025-09-16T09:28:28.705740Z","shell.execute_reply.started":"2025-09-16T09:28:21.417645Z","shell.execute_reply":"2025-09-16T09:28:28.704799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom catboost import CatBoostClassifier\n\ndef objective_cat(trial):\n    params = {\n        'iterations': trial.suggest_int('iterations', 100, 1000),\n        'depth': trial.suggest_int('depth', 4, 12),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1, 10),\n        'verbose': 0,\n        'random_state': 42\n    }\n    \n    model = CatBoostClassifier(**params)\n    score = cross_val_score(model, X_train_res, y_train_res, cv=3, scoring='f1_macro').mean()\n    return score\n\nstudy_cat = optuna.create_study(direction='maximize')\nstudy_cat.optimize(objective_cat, n_trials=15)\n\nbest_cat_params = study_cat.best_params\nprint(\"Best CatBoost params:\", best_cat_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T09:28:28.707425Z","iopub.execute_input":"2025-09-16T09:28:28.707680Z","iopub.status.idle":"2025-09-16T10:17:22.161885Z","shell.execute_reply.started":"2025-09-16T09:28:28.707657Z","shell.execute_reply":"2025-09-16T10:17:22.161103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import cross_val_score\n\ndef objective_lgb(trial):\n    params = {\n        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n        'max_depth': trial.suggest_int('max_depth', 4, 12),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n        'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n        'random_state': 42\n    }\n    \n    model = LGBMClassifier(**params)\n    score = cross_val_score(model, X_train_res, y_train_res, cv=3, scoring='f1_macro').mean()\n    return score\n\nstudy_lgb = optuna.create_study(direction='maximize')\nstudy_lgb.optimize(objective_lgb, n_trials=15)\n\nbest_lgb_params = study_lgb.best_params\nprint(\"Best LGBM params:\", best_lgb_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T10:17:22.163276Z","iopub.execute_input":"2025-09-16T10:17:22.163519Z","iopub.status.idle":"2025-09-16T10:22:23.934005Z","shell.execute_reply.started":"2025-09-16T10:17:22.163499Z","shell.execute_reply":"2025-09-16T10:22:23.933285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.metrics import classification_report, roc_auc_score\n\nbest_lgb = LGBMClassifier(**best_lgb_params)\nbest_cat = CatBoostClassifier(**best_cat_params)\n\nbest_lgb.fit(X_train_res, y_train_res)\nbest_cat.fit(X_train_res, y_train_res)\n\nensemble = VotingClassifier(\n    estimators=[\n        ('lgb', best_lgb),\n        ('cat', best_cat)\n    ],\n    voting='soft',\n    n_jobs=-1\n)\n\nensemble.fit(X_train_res, y_train_res)\n\ny_pred = ensemble.predict(X_test)\ny_proba = ensemble.predict_proba(X_test)[:, 1]  # for ROC-AUC (binary case)\n\nprint(\"=== Classification Report ===\")\nprint(classification_report(y_test, y_pred))\nprint(\"ROC-AUC Score:\", roc_auc_score(y_test, y_proba))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T10:28:40.821346Z","iopub.execute_input":"2025-09-16T10:28:40.821640Z","iopub.status.idle":"2025-09-16T10:34:44.839281Z","shell.execute_reply.started":"2025-09-16T10:28:40.821615Z","shell.execute_reply":"2025-09-16T10:34:44.838354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = ensemble.predict(X_test)\n\n# ROC-AUC handling binary vs multi-class\nif len(set(y_test)) == 2:\n    y_proba = ensemble.predict_proba(X_test)[:, 1]\n    roc_auc = roc_auc_score(y_test, y_proba)\nelse:\n    y_proba = ensemble.predict_proba(X_test)\n    roc_auc = roc_auc_score(y_test, y_proba, multi_class='ovr')\n\nprint(\"=== Classification Report ===\")\nprint(classification_report(y_test, y_pred))\nprint(f\"ROC-AUC Score: {roc_auc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T10:41:26.213118Z","iopub.execute_input":"2025-09-16T10:41:26.213789Z","iopub.status.idle":"2025-09-16T10:41:29.245043Z","shell.execute_reply.started":"2025-09-16T10:41:26.213748Z","shell.execute_reply":"2025-09-16T10:41:29.243904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = test.drop(['id', 'comment_text'], axis=1)\n\nfinal_pred = ensemble.predict(test_df)\n\n# Predict probabilities for class 1 (optional, if needed)\nfinal_proba = ensemble.predict_proba(test_df)[:, 1]\n\n# If you want to create a submission DataFrame\nimport pandas as pd\n\nsubmission = pd.DataFrame({\n    'ID': test['id'],      # Make sure test_df has the 'id' column\n    'psychotic_depression': final_pred\n})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-16T10:41:34.729994Z","iopub.execute_input":"2025-09-16T10:41:34.730737Z","iopub.status.idle":"2025-09-16T10:41:40.562216Z","shell.execute_reply.started":"2025-09-16T10:41:34.730698Z","shell.execute_reply":"2025-09-16T10:41:40.561247Z"}},"outputs":[],"execution_count":null}]}