{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.preprocessing import LabelEncoder\nimport string\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_classif\nimport tensorflow as tf\nfrom tensorflow.python.keras import models\nfrom tensorflow.python.keras.layers import Dense\nfrom tensorflow.python.keras.layers import Dropout\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n\n!pip install hazm\nfrom hazm import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_path = \"/kaggle/input/computational-intelligence-course-final-project/train.csv\"\ntest_path = \"/kaggle/input/computational-intelligence-course-final-project/test.csv\"\n\ntrain = pd.read_csv(train_path, index_col='Unnamed: 0')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(test_path)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.Text\nle = LabelEncoder()\nY = le.fit_transform(train.Category)\nprint(Y.shape)\nY","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalizer = Normalizer()\nlemmatizer = Lemmatizer()\nstop_words = stopwords_list()\n\ndef remove_hyperlink(word):\n    word = re.sub(r\"(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\\\".,<>?«»“”‘’]))\", \"\", word[::-1])\n    word = re.sub(r\"(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\\\".,<>?«»“”‘’]))\", \"\", word[::-1])\n    return word\n\ndef to_lower(word):\n    return word.lower()\n\ndef remove_number(word):\n    return re.sub(r'\\d+', '', word)\n\ndef remove_punctuation(word):\n    word = word.translate(str.maketrans(dict.fromkeys(string.punctuation)))\n    return word.replace(\"،\", \"\")\n    \ndef remove_whitespace(word):\n    return word.strip()\n\ndef replace_newline(word):\n    return word.replace('\\n','')\n\ndef remove_stop_words_and_lammatize(word):\n    tokens = word_tokenize(word)\n    new_tokens = [lemmatizer.lemmatize(x) for x in tokens]\n    new_word = ' '.join(new_tokens)\n    return new_word\n\ndef normalize(word):\n    return normalizer.normalize(word)\n\ndef clean_up_pipeline(sentence):\n    cleaning_utils = [to_lower,\n                      remove_hyperlink,\n                      replace_newline,\n                      remove_number,\n                      remove_punctuation,\n                      remove_whitespace,\n                      normalize,\n#                       remove_stop_words_and_lammatize\n                     ]\n    for o in cleaning_utils:\n        sentence = o(sentence)\n    return sentence\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = [clean_up_pipeline(o) for o in X]\nprint(X[1])\nprint(x_train[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(x_train, Y, test_size = 0.2, random_state=42) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text_clf = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),\n#                     ('tfidf', TfidfTransformer()),\n#                     ('clf', MultinomialNB())])\n\n# count_vect = CountVectorizer()\n# X_train_counts = count_vect.fit_transform(train_x)\n# X_train_counts.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(len(set(Y)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text_clf = text_clf.fit(train_x, train_y)\n# predicted = text_clf.predict(test_x)\n# np.mean(predicted == test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text_clf_svm = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),\n#                      ('tfidf', TfidfTransformer()),\n#                      ('clf-svm', SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3, random_state=42)),\n# ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _ = text_clf_svm.fit(train_x, train_y)\n# predicted_svm = text_clf_svm.predict(test_x)\n# np.mean(predicted_svm == test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.model_selection import GridSearchCV\n\n# # parameters = {'vect__ngram_range': [(1, 1), (1, 2)],\n# #               'tfidf__use_idf': (True, False),\n# #               'clf__alpha': (1e-2, 1e-3),\n# # }\n\n# parameters = {'vect__ngram_range': [(1, 2)],\n#               'tfidf__use_idf': [True],\n#               'clf__alpha': [1e-3],\n# }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)\n# gs_clf = gs_clf.fit(train_x, train_y)\n# print(gs_clf.best_score_)\n# print(gs_clf.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predicted = gs_clf.predict(test_x)\n# np.mean(predicted == test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _ = text_clf_svm.fit(x_train, Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_test = [clean_up_pipeline(o) for o in test.Text]\n# predicted_svm = text_clf_svm.predict(x_test)\n# pred_test = le.inverse_transform(predicted_svm)\n# pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# output = pd.DataFrame({'Id': test.Id,\n#                        'Category': pred_test})\n# output.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import svm\n\ntext_clf_svc = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),\n                     ('tfidf', TfidfTransformer()),\n#                      ('select-K-best', SelectKBest(f_classif, k=200000)),\n                     ('clf-svm', LinearSVC(C=0.5, multi_class=\"crammer_singer\", max_iter=5000)),\n])\n\ntext_clf_svc = text_clf_svc.fit(x_train, Y)\npredicted = text_clf_svc.predict(test_x)\nnp.mean(predicted == test_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text_clf_svc = text_clf_svc.fit(x_train, Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = [clean_up_pipeline(o) for o in test.Text]\npredicted_svc = text_clf_svc.predict(x_test)\npred_test = le.inverse_transform(predicted_svc)\npred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.DataFrame({'Id': test.Id,\n                       'Category': pred_test})\noutput.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def _get_last_layer_units_and_activation(num_classes):\n#     if num_classes == 2:\n#         activation = 'sigmoid'\n#         units = 1\n#     else:\n#         activation = 'softmax'\n#         units = num_classes\n#     return units, activation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# def mlp_model(layers=2, units=128, dropout_rate=0.2, input_shape=(120076, 377014), num_classes=len(set(Y))):\n#     op_units, op_activation = _get_last_layer_units_and_activation(num_classes)\n#     model = models.Sequential()\n#     model.add(Dropout(rate=dropout_rate, input_shape=input_shape))\n\n#     for _ in range(layers-1):\n#         model.add(Dense(units=units, activation='relu'))\n#         model.add(Dropout(rate=dropout_rate))\n\n#     model.add(Dense(units=op_units, activation=op_activation))\n    \n#     # Compile model with learning parameters.\n#     if num_classes == 2:\n#         loss = 'binary_crossentropy'\n#     else:\n#         loss = 'sparse_categorical_crossentropy'\n#     optimizer = tf.keras.optimizers.Adam(lr=1e-3)\n#     model.compile(optimizer=optimizer, loss=loss, metrics=['acc'])\n    \n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class tf_sparse:\n#     def fit(x):\n#         return tf.sparse.reorder(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras.wrappers.scikit_learn import KerasRegressor\n\n# def train_ngram_model(learning_rate=1e-3, epochs=1000, batch_size=128, layers=2, units=128, dropout_rate=0.2):\n\n#     # Verify that validation labels are in the same range as training labels.\n#     num_classes = len(set(Y))\n\n#     # Create model instance.\n# #     model = mlp_model(layers=layers,\n# #                                   units=units,\n# #                                   dropout_rate=dropout_rate,\n# #                                   input_shape=train_x.shape[1:],\n# #                                   num_classes=num_classes)\n\n#     # Create callback for early stopping on validation loss. If the loss does\n#     # not decrease in two consecutive tries, stop training.\n#     callbacks = [tf.keras.callbacks.EarlyStopping(\n#         monitor='val_loss', patience=2)]\n    \n    \n    \n#     text_clf_keras = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),\n#                 ('tfidf', TfidfTransformer()),\n#                 ('tf-sparse', tf_sparse()),\n#                 ('clf-svm', KerasRegressor(build_fn=mlp_model, epochs=epochs, callbacks=callbacks, validation_data=(test_x, test_y), verbose=2, batch_size=batch_size)),\n#     ])\n    \n#     # Train and validate model.\n# #     history = model.fit(\n# #             train_x,\n# #             train_y,\n# #             epochs=epochs,\n# #             callbacks=callbacks,\n# #             validation_data=(test_x, test_y),\n# #             verbose=2,  # Logs once per epoch.\n# #             batch_size=batch_size)\n\n#     # Print results.\n# #     history = history.history\n# #     print('Validation accuracy: {acc}, loss: {loss}'.format(\n# #             acc=history['val_acc'][-1], loss=history['val_loss'][-1]))\n\n#     # Save model.\n# #     model.save('ci.h5')\n\n#     text_clf_keras = text_clf_keras.fit(train_x, train_y)\n#     predicted = text_clf_keras.predict(test_x)\n#     print(np.mean(predicted == test_y))\n# #     return history['val_acc'][-1], history['val_loss'][-1]\n\n# train_ngram_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# vectorizer = CountVectorizer(ngram_range=(1, 2))\n# train_x = vectorizer.fit_transform(train_x)\n# test_x = vectorizer.transform(test_x)\n# print(\"1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainsformer = TfidfTransformer()\n# train_x = trainsformer.fit_transform(train_x)\n# test_x = trainsformer.transform(test_x)\n# print(\"2\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# selector = SelectKBest(f_classif, k=50000)\n# selector.fit(train_x, train_y)\n# train_x = selector.transform(train_x)\n# test_x = selector.transform(test_x)\n# print(\"3\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# rtm = train_ngram_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}