{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":113366,"databundleVersionId":13605326,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import nltk\nimport numpy as np\nimport pandas as pd\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer, WordNetLemmatizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.linear_model import LogisticRegression, SGDClassifier\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score,classification_report\nfrom sklearn.metrics import f1_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:35.867650Z","iopub.execute_input":"2025-09-13T05:53:35.867977Z","iopub.status.idle":"2025-09-13T05:53:36.240237Z","shell.execute_reply.started":"2025-09-13T05:53:35.867953Z","shell.execute_reply":"2025-09-13T05:53:36.239288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stop_words = set(stopwords.words(\"english\"))\nnltk.download('omw-1.4')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:45.128402Z","iopub.execute_input":"2025-09-13T05:53:45.128772Z","iopub.status.idle":"2025-09-13T05:53:45.303476Z","shell.execute_reply.started":"2025-09-13T05:53:45.128744Z","shell.execute_reply":"2025-09-13T05:53:45.302558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/comments-classification/Dataset/train.csv')\ndf_test = pd.read_csv('/kaggle/input/comments-classification/Dataset/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:45.562970Z","iopub.execute_input":"2025-09-13T05:53:45.563281Z","iopub.status.idle":"2025-09-13T05:53:49.742816Z","shell.execute_reply.started":"2025-09-13T05:53:45.563259Z","shell.execute_reply":"2025-09-13T05:53:49.741409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:49.744791Z","iopub.execute_input":"2025-09-13T05:53:49.745178Z","iopub.status.idle":"2025-09-13T05:53:49.773310Z","shell.execute_reply.started":"2025-09-13T05:53:49.745146Z","shell.execute_reply":"2025-09-13T05:53:49.772381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.psychotic_depression.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:49.774297Z","iopub.execute_input":"2025-09-13T05:53:49.774580Z","iopub.status.idle":"2025-09-13T05:53:49.790549Z","shell.execute_reply.started":"2025-09-13T05:53:49.774552Z","shell.execute_reply":"2025-09-13T05:53:49.789241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:49.792786Z","iopub.execute_input":"2025-09-13T05:53:49.793898Z","iopub.status.idle":"2025-09-13T05:53:49.846878Z","shell.execute_reply.started":"2025-09-13T05:53:49.793852Z","shell.execute_reply":"2025-09-13T05:53:49.846020Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"execution":{"iopub.status.busy":"2025-09-10T13:40:55.122904Z","iopub.execute_input":"2025-09-10T13:40:55.123182Z","iopub.status.idle":"2025-09-10T13:40:55.137644Z","shell.execute_reply.started":"2025-09-10T13:40:55.123149Z","shell.execute_reply":"2025-09-10T13:40:55.136525Z"}}},{"cell_type":"code","source":"def stemming(text):\n    stemmer = SnowballStemmer(\"english\")\n    text = text.split()\n    text = [stemmer.stem(word) for word in text]\n    return \" \".join(text)\n\ndef remove_stop_words(text):\n    Text=[i for i in str(text).split() if i not in stop_words]\n    return \" \".join(Text)\n\ndef Removing_numbers(text):\n    text=''.join([i for i in text if not i.isdigit()])\n    return text\n\ndef lemmatization(text):\n    lemmatizer= WordNetLemmatizer()\n    text = text.split()\n    text=[lemmatizer.lemmatize(y) for y in text]\n    return \" \" .join(text)\n\ndef lower_case(text):\n    text = text.split()\n    text=[y.lower() for y in text]\n    return \" \" .join(text)\n            \ndef normalize_text(df):\n    #df.comment_text=df.comment_text.apply(lambda text : lower_case(text))\n    df.comment_text=df.comment_text.apply(lambda text : remove_stop_words(text))\n    #df.comment_text=df.comment_text.apply(lambda text : Removing_numbers(text))\n    #df.comment_text=df.comment_text.apply(lambda text : lemmatization(text))\n    #df.comment_text=df.comment_text.apply(lambda text : stemming(text))\n    return df\n\ndef normalized_sentence(sentence):\n    #sentence= lower_case(sentence)\n    sentence= remove_stop_words(sentence)\n    #sentence= Removing_numbers(sentence)\n    #sentence= lemmatization(sentence)\n    #sentence= stemming(sentence)\n    return sentence\n\n# even though theres all this preprocessing possibilities...pretty much everything reduced the f1 score...so just removing stop words","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:49.847728Z","iopub.execute_input":"2025-09-13T05:53:49.848075Z","iopub.status.idle":"2025-09-13T05:53:49.866358Z","shell.execute_reply.started":"2025-09-13T05:53:49.848029Z","shell.execute_reply":"2025-09-13T05:53:49.865060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"normalized_sentence(\"JUST SOME PSSYYYCCHOOO TESTTTT!!!@@456 __\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:50.770951Z","iopub.execute_input":"2025-09-13T05:53:50.771281Z","iopub.status.idle":"2025-09-13T05:53:50.778460Z","shell.execute_reply.started":"2025-09-13T05:53:50.771258Z","shell.execute_reply":"2025-09-13T05:53:50.777243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train= normalize_text(df_train)\ndf_test= normalize_text(df_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:51.809852Z","iopub.execute_input":"2025-09-13T05:53:51.810187Z","iopub.status.idle":"2025-09-13T05:53:55.939978Z","shell.execute_reply.started":"2025-09-13T05:53:51.810163Z","shell.execute_reply":"2025-09-13T05:53:55.939129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df_train[\"comment_text\"]\ny = df_train[\"psychotic_depression\"]\n\n# Stratified split\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y,\n    test_size=0.2,\n    stratify=y,\n    random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:55.941466Z","iopub.execute_input":"2025-09-13T05:53:55.941769Z","iopub.status.idle":"2025-09-13T05:53:56.021860Z","shell.execute_reply.started":"2025-09-13T05:53:55.941743Z","shell.execute_reply":"2025-09-13T05:53:56.020856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tfidf = TfidfVectorizer(\n    ngram_range=(1,2),\n    min_df=3,\n    max_features=10000\n)\n\nX_train_tfidf = tfidf.fit_transform(X_train)\nX_val_tfidf   = tfidf.transform(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T05:53:56.022996Z","iopub.execute_input":"2025-09-13T05:53:56.023449Z","iopub.status.idle":"2025-09-13T05:54:23.182550Z","shell.execute_reply.started":"2025-09-13T05:53:56.023410Z","shell.execute_reply":"2025-09-13T05:54:23.180863Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"code","source":"# 1. Logistic Regression\nmodelLR = LogisticRegression(max_iter=1000, class_weight=\"balanced\")\nmodelLR.fit(X_train_tfidf, y_train)\n\ny_val_pred_lr = modelLR.predict(X_val_tfidf)\nprint(\"Logistic Regression F1:\", f1_score(y_val, y_val_pred_lr))\nprint(classification_report(y_val, y_val_pred_lr))\n\n# 2. Linear SVM\nmodelSVM = SGDClassifier(loss=\"hinge\", class_weight=\"balanced\", max_iter=1000)\nmodelSVM.fit(X_train_tfidf, y_train)\n\ny_val_pred_svm = modelSVM.predict(X_val_tfidf)\nprint(\"Linear SVM F1:\", f1_score(y_val, y_val_pred_svm))\nprint(classification_report(y_val, y_val_pred_svm))\n\n# 3. Multinomial Naive Bayes\nmodelNB = MultinomialNB() \nmodelNB.fit(X_train_tfidf, y_train)\n\ny_val_pred_nb = modelNB.predict(X_val_tfidf)\nprint(\"Naive Bayes F1:\", f1_score(y_val, y_val_pred_nb))\nprint(classification_report(y_val, y_val_pred_nb))\n\n# 4. Random Forest\nmodelRF = RandomForestClassifier(\n    n_estimators=300,          \n    max_depth=10,             \n    class_weight='balanced',  \n    random_state=42,\n    n_jobs=-1\n)\nmodelRF.fit(X_train_tfidf, y_train)\n\ny_val_pred_rf = modelRF.predict(X_val_tfidf)\nprint(\"Random Forest F1:\", f1_score(y_val, y_val_pred_rf))\nprint(classification_report(y_val, y_val_pred_rf))\n\n# XGBoost\nneg = np.sum(y_val == 0)\npos = np.sum(y_val == 1)\nratio = neg / pos\n\nmodelXGB = XGBClassifier(\n    n_estimators=500,\n    max_depth=6,\n    learning_rate=0.05,\n    scale_pos_weight=ratio,  # for imbalaced class\n    random_state=42,\n    n_jobs=-1,\n)\nmodelXGB.fit(X_train_tfidf, y_train)\n\ny_val_pred_xg = modelXGB.predict(X_val_tfidf)\nprint(\"XGBoost F1:\", f1_score(y_val, y_val_pred_xg))\nprint(classification_report(y_val, y_val_pred_xg))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:00:16.439302Z","iopub.execute_input":"2025-09-13T06:00:16.440892Z","iopub.status.idle":"2025-09-13T06:07:34.836390Z","shell.execute_reply.started":"2025-09-13T06:00:16.440849Z","shell.execute_reply":"2025-09-13T06:07:34.835222Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"# Majority voting\nfinal_pred1 = np.round((y_val_pred_lr + y_val_pred_nb + y_val_pred_rf) / 3) # focusing on high recall for class \"1\"\nprint(\"Ensemble F1:\", f1_score(y_val, final_pred1))\nprint(classification_report(y_val, final_pred1)) \n\nfinal_pred2 = np.round((y_val_pred_lr + y_val_pred_svm + y_val_pred_nb + y_val_pred_rf + y_val_pred_xg) / 5)\nprint(\"Ensemble F1:\", f1_score(y_val, final_pred2))\nprint(classification_report(y_val, final_pred2))\n\n# Weighted voting\nfinal_pred = np.round((0.3*y_val_pred_lr + 0.2*y_val_pred_svm + 0.3*y_val_pred_nb + 0.2*y_val_pred_xg)) #Pretty random weights...didnt experiment much\nprint(\"Weighted Ensemble F1:\", f1_score(y_val, final_pred))\nprint(classification_report(y_val, final_pred)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:21:52.815766Z","iopub.execute_input":"2025-09-13T06:21:52.816111Z","iopub.status.idle":"2025-09-13T06:21:53.125479Z","shell.execute_reply.started":"2025-09-13T06:21:52.816085Z","shell.execute_reply":"2025-09-13T06:21:53.123838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/comments-classification/Dataset/test.csv')\ntest.isnull().sum()\ntest= normalize_text(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:22:44.474584Z","iopub.execute_input":"2025-09-13T06:22:44.475926Z","iopub.status.idle":"2025-09-13T06:22:47.545958Z","shell.execute_reply.started":"2025-09-13T06:22:44.475883Z","shell.execute_reply":"2025-09-13T06:22:47.544698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test[\"comment_text\"]\nX_test_tfidf= tfidf.transform(X_test)\ny_test_pred_LR = modelLR.predict(X_test_tfidf)\ny_test_pred_SVM = modelSVM.predict(X_test_tfidf)\ny_test_pred_NB = modelNB.predict(X_test_tfidf)\ny_test_pred_RF = modelRF.predict(X_test_tfidf)\ny_test_pred_XGB = modelXGB.predict(X_test_tfidf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:22:48.866849Z","iopub.execute_input":"2025-09-13T06:22:48.867174Z","iopub.status.idle":"2025-09-13T06:23:05.101295Z","shell.execute_reply.started":"2025-09-13T06:22:48.867152Z","shell.execute_reply":"2025-09-13T06:23:05.100540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Majority voting\nfinal_pred_maj_vote1 = np.round((y_test_pred_LR +y_test_pred_NB + y_test_pred_RF) / 3)\nfinal_pred_maj_vote2 = np.round((y_test_pred_LR + y_test_pred_SVM + y_test_pred_NB + y_test_pred_RF + y_test_pred_XGB) / 5)\n\n# Weighted voting\nfinal_pred_weigh_vote = np.round((0.3*y_test_pred_LR + 0.2*y_test_pred_SVM + 0.3*y_test_pred_NB + 0.2*y_test_pred_XGB))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:23:05.102267Z","iopub.execute_input":"2025-09-13T06:23:05.102550Z","iopub.status.idle":"2025-09-13T06:23:05.115150Z","shell.execute_reply.started":"2025-09-13T06:23:05.102528Z","shell.execute_reply":"2025-09-13T06:23:05.114143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ids = list(range(1, len(y_test_pred_LR) + 1))\nsubmission_LR = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": y_test_pred_LR\n})\n\nsubmission_SVM = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": y_test_pred_SVM\n})\n\nsubmission_NB = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": y_test_pred_NB\n})\n\nsubmission_rf = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": y_test_pred_RF\n})\n\nsubmission_xgb = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\":  y_test_pred_XGB\n})\n\nsubmission_maj_vote1 = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": final_pred_maj_vote1\n})\n\nsubmission_maj_vote2 = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": final_pred_maj_vote2\n})\n\nsubmission_weigh_vote = pd.DataFrame({\n    \"ID\": ids,\n    \"psychotic_depression\": final_pred_weigh_vote\n})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:23:07.440150Z","iopub.execute_input":"2025-09-13T06:23:07.440491Z","iopub.status.idle":"2025-09-13T06:23:07.764361Z","shell.execute_reply.started":"2025-09-13T06:23:07.440466Z","shell.execute_reply":"2025-09-13T06:23:07.762934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kaggle score for each model\n\nsubmission_LR = submission_LR.sort_values(by=\"ID\")\nsubmission_LR.to_csv(\"submission_LR.csv\", index=False) # 0.49\n\nsubmission_SVM = submission_SVM.sort_values(by=\"ID\")\nsubmission_SVM.to_csv(\"submission_SVM.csv\", index=False) # 0.49\n\nsubmission_NB = submission_NB.sort_values(by=\"ID\")\nsubmission_NB.to_csv(\"submission_NB.csv\", index=False) # 0.52\n\nsubmission_rf = submission_rf.sort_values(by=\"ID\")\nsubmission_rf.to_csv(\"submission_rf.csv\", index=False) # 0.57\n\nsubmission_xgb = submission_xgb.sort_values(by=\"ID\")\nsubmission_xgb.to_csv(\"submission_xgb.csv\", index=False) # 0.47\n\nsubmission_maj_vote1 = submission_maj_vote1.sort_values(by=\"ID\")\nsubmission_maj_vote1.to_csv(\"submission_maj_vote1.csv\", index=False) # 0.52\n\nsubmission_maj_vote2 = submission_maj_vote2.sort_values(by=\"ID\")\nsubmission_maj_vote2.to_csv(\"submission_maj_vote2.csv\", index=False) # 0.50\n\nsubmission_weigh_vote = submission_weigh_vote.sort_values(by=\"ID\")\nsubmission_weigh_vote.to_csv(\"submission_weigh_vote.csv\", index=False) # 0.49","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:23:09.992986Z","iopub.execute_input":"2025-09-13T06:23:09.993346Z","iopub.status.idle":"2025-09-13T06:23:11.074896Z","shell.execute_reply.started":"2025-09-13T06:23:09.993320Z","shell.execute_reply":"2025-09-13T06:23:11.073538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}