{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install textacy\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport os\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import accuracy_score\nimport nltk\nfrom nltk.stem import WordNetLemmatizer\nimport string\nfrom sklearn.model_selection import train_test_split\nimport re\nfrom textacy.viz.termite import draw_termite_plot\n# using binary relevance\nfrom skmultilearn.problem_transform import BinaryRelevance\nfrom sklearn.naive_bayes import GaussianNB# initialize binary relevance multi-label classifier\nfrom sklearn.svm import SVC\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bird View of Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_train['dataset_title'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_train['dataset_label'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Dataset Label and title which are not unique** "},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_train[df_train['dataset_title']!=df_train['dataset_label']].loc[:,['pub_title','dataset_title','dataset_label']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading JSON Files"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data(filename):\n    df_json = pd.read_json(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train/\"+str(filename)+\".json\")\n    text = \"\".join(row['text'] for _,row in df_json.iterrows())\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['json_text'] = df_train['Id'].apply(lambda x : data(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Know your Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['dataset_title'].value_counts().plot( kind='bar', figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['dataset_label'].value_counts().plot( kind='bar', figsize=(20,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = TfidfVectorizer( min_df =3, max_df=0.2, max_features=None, \n                    strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n                    ngram_range=(1, 1), use_idf=1,smooth_idf=1,sublinear_tf=1,\n                    stop_words = 'english', preprocessor=clean_text)\nvectorizer.fit(df_train['dataset_title'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"def create_tf_matrix(category):\n    return vectorizer.transform(df_train[df_train['dataset_title'] == category].json_text.apply(clean_text))\n\ndef create_term_freq(matrix, cat):\n    category_words = matrix.sum(axis=0)\n    category_words_freq = [(word, category_words[0, idx]) for word, idx in vectorizer.vocabulary_.items()]\n    return pd.DataFrame(list(sorted(category_words_freq, key = lambda x: x[1], reverse=True)),columns=['Terms', cat])\n\nfor cat in df_train.dataset_title.unique():\n    print(\"Top 10 terms for: \", cat)\n    df_right = create_term_freq(create_tf_matrix(cat), cat).head(5)\n    print(df_right)\n    print(\"###############\")\n    if cat != 'National Education Longitudinal Study':\n        df_top5_words = df_top5_words.merge(df_right, how='outer')\n    else:\n        df_top5_words = df_right.copy()\n    print(df_top5_words.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_top5_words.fillna(0, inplace=True )\ndf_top5_words.set_index('Terms', inplace=True)\ndf_top5_words.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_top5_words.copy()\ndf_norm = (df) / (df.max() - df.min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw_termite_plot(np.array(df_norm.values),df_top5_words.columns,df_top5_words.index, highlight_cols=[0, 4, 12,20,30,36] )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = pd.get_dummies(df_train['dataset_title'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = TfidfVectorizer( min_df =3, max_df=0.2, max_features=10000, \n                    strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n                    ngram_range=(1, 1), use_idf=1,smooth_idf=1,sublinear_tf=1,\n                    stop_words = 'english', preprocessor=clean_text)\nvectorizer.fit(df_train['json_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = vectorizer.transform(df_train.json_text.apply(clean_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.20, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# with a SVC base classifier\nclassifier = BinaryRelevance(classifier=SVC(), require_dense=[False,True])# train\nclassifier.fit(X_train, y_train)\npredictions = classifier.predict(X_test)\nprint(\"Accuracy = \",accuracy_score(y_test,predictions))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save model"},{"metadata":{"trusted":true},"cell_type":"code","source":"Pkl_Filename = \"model.pkl\"  \n\nwith open(Pkl_Filename, 'wb') as file:  \n    pickle.dump(classifier, file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyse Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"multilabel_confusion_matrix(y_test,predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def datatest(filename):\n    df_json = pd.read_json(\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/\"+str(filename)+\".json\")\n    text = \"\".join(row['text'] for _,row in df_json.iterrows())\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['json_text'] = df_submission['Id'].apply(lambda x : datatest(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = vectorizer.transform(df_submission.json_text.apply(clean_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = classifier.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"loading the model from pickle just to avoid re run as working just on below part."},{"metadata":{"trusted":true},"cell_type":"code","source":"Pkl_Filename = \"model.pkl\"  \nwith open(Pkl_Filename, 'rb') as file:\n    pickle_model = pickle.load(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = pickle_model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = pd.DataFrame(predictions.toarray(),columns=Y.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def updateSubmission(Id):\n    return df1.loc[:,df1.loc[Id] == 1].columns[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['PredictionString'] = df_submission.index.map(lambda x: df1.loc[:,df1.loc[x] == 1].columns[0]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['PredictionString'] = df_submission['PredictionString'].apply(clean_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.drop(columns =['json_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"next from dataset title we need to find matching phrases in the json text to get our final output"},{"metadata":{},"cell_type":"markdown","source":"**Work In progress - please upvote comment if you find anything interesting**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}