{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#from tqdm import tqdm\n\ndf = pd.read_csv( '../input/clickbait-news-detection/train.csv')\n\ndft = pd.read_csv( '../input/clickbait-news-detection/test.csv')\n\n\n\n#tqdm.pandas(desc=\"my bar!\")\n#df.progress_apply(lambda x: x)\n\n#pd.set_option('display.max_rows',24871)\ndf\n\ndft","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df.shape)\nprint(dft.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['title'].fillna('missing', inplace=True)\ndft['title'].fillna('missing', inplace=True)\n\n\n\ndf\ndft","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nfrom string import punctuation\n\ndef process_text2(title):\n    \n    result = title.replace('/','').replace('\\n','')\n    result = re.sub(r'[0-9]+','number', result)   # we are substituting all kinds of no. with word number\n    result = re.sub(r'(\\w)(\\1{2,})', r'\\1', result)  # \\w matches one word/non word character\n    result = re.sub(r'(?x)\\b(?=\\w*\\d)\\w+\\s*', '', result)\n    \n    result = ''.join(word for word in result if word not in punctuation)  # removes all characters such as \"!\"#$%&'()*+, -./:;<=>?@[\\]^_`{|}~\"\n    result = re.sub(r' +', ' ', result).lower().strip()\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing the stopwords\nfrom nltk.corpus import stopwords\n\nstop = stopwords.words(\"english\")\n\ndef cnt_stopwords(title):\n    \n    result1 = title.split()\n    num1 =  len([word for word in result1 if word in stop])\n    \n    return num1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions = ['tis', 'aint', 'amnt', 'arent', 'cant', 'couldve', 'couldnt', 'couldntve',\n                'didnt', 'doesnt', 'dont', 'gonna', 'gotta', 'hadnt', 'hadntve', 'hasnt',\n                'havent', 'hed', 'hednt', 'hedve', 'hell', 'hes', 'hesnt', 'howd', 'howll',\n                'hows', 'id', 'idnt', 'idntve', 'idve', 'ill', 'im', 'ive', 'ivent', 'isnt',\n                'itd', 'itdnt', 'itdntve', 'itdve', 'itll', 'its', 'itsnt', 'mightnt',\n                'mightve', 'mustnt', 'mustntve', 'mustve', 'neednt', 'oclock', 'ol', 'oughtnt',\n                'shant', 'shed', 'shednt', 'shedntve', 'shedve', 'shell', 'shes', 'shouldve',\n                'shouldnt', 'shouldntve', 'somebodydve', 'somebodydntve', 'somebodys',\n                'someoned', 'someonednt', 'someonedntve', 'someonedve', 'someonell', 'someones',\n                'somethingd', 'somethingdnt', 'somethingdntve', 'somethingdve', 'somethingll',\n                'somethings', 'thatll', 'thats', 'thatd', 'thered', 'therednt', 'theredntve',\n                'theredve', 'therere', 'theres', 'theyd', 'theydnt', 'theydntve', 'theydve',\n                'theydvent', 'theyll', 'theyontve', 'theyre', 'theyve', 'theyvent', 'wasnt',\n                'wed', 'wedve', 'wednt', 'wedntve', 'well', 'wontve', 'were', 'weve', 'werent',\n                'whatd', 'whatll', 'whatre', 'whats', 'whatve', 'whens', 'whered', 'wheres',\n                'whereve', 'whod', 'whodve', 'wholl', 'whore', 'whos', 'whove', 'whyd', 'whyre',\n                'whys', 'wont', 'wontve', 'wouldve', 'wouldnt', 'wouldntve', 'yall', 'yalldve',\n                'yalldntve', 'yallll', 'yallont', 'yallllve', 'yallre', 'yallllvent', 'yaint',\n                'youd', 'youdve', 'youll', 'youre', 'yourent', 'youve', 'youvent']\n\ndef cnt_contract(title):\n    \n    result2 = title.split()\n    num2 = len([word for word in result2 if word in contractions])\n    return num2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_words = ['who', 'whos', 'whose', 'what', 'whats', 'whatre', 'when', 'whenre', 'whens', 'couldnt',\n        'where', 'wheres', 'whered', 'why', 'whys', 'can', 'cant', 'could', 'will', 'would', 'is',\n        'isnt', 'should', 'shouldnt', 'you', 'your', 'youre', 'youll', 'youd', 'here', 'heres',\n        'how', 'hows', 'howd', 'this', 'are', 'arent', 'which', 'does', 'doesnt']\n\ndef question_word(title):\n    \n    result3 = title.lower().split()\n    \n    if result3[0] in question_words:\n        return 1\n    else:\n        return 0\n    #return result3    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pos_tags(title):\n    \n    result4 = title.split()\n    \n    non_stop = [word for word in result4 if word not in stopwords.words(\"english\")]\n    pos = [part[1] for part in nltk.pos_tag(non_stop)]\n    pos = \" \".join(pos)\n    return pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\n\n# progress bar\nfrom tqdm import tqdm_notebook,tqdm\n\n# instantiate\ntqdm.pandas(tqdm_notebook)\n\ndf['processed_headline']     = df['title'].progress_apply(process_text2)\ndf['question'] = df['title'].progress_apply(question_word)\n\ndf['num_words']       = df['title'].progress_apply(lambda x: len(x.split()))\ndf['part_speech']     = df['title'].progress_apply(pos_tags)\ndf['num_contract']    = df['title'].progress_apply(cnt_contract)\ndf['num_stop_words']  = df['title'].progress_apply(cnt_stopwords)\ndf['stop_word_ratio'] = df['num_stop_words']/df['num_words']\ndf['contract_ratio']  = df['num_contract']/df['num_words']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook,tqdm\n\n# instantiate\ntqdm.pandas(tqdm_notebook)\n\ndft['processed_headline']     = dft['title'].progress_apply(process_text2)\ndft['question'] = dft['title'].progress_apply(question_word)\n\ndft['num_words']       = dft['title'].progress_apply(lambda x: len(x.split()))\ndft['part_speech']     = dft['title'].progress_apply(pos_tags)\ndft['num_contract']    = dft['title'].progress_apply(cnt_contract)\ndft['num_stop_words']  = dft['title'].progress_apply(cnt_stopwords)\ndft['stop_word_ratio'] = dft['num_stop_words']/dft['num_words']\ndft['contract_ratio']  = dft['num_contract']/dft['num_words']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.drop(columns = ['num_contract','num_stop_words'])\n\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft = dft.drop(columns = ['num_contract','num_stop_words'])\n\ndft","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\ntfidf = TfidfVectorizer(min_df=3, max_features=None, strip_accents='unicode',analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(1,5),use_idf=1, smooth_idf=1, sublinear_tf=1)\n\nX_train_headline = tfidf.fit_transform(df['processed_headline'])#1\nX_test_headline = tfidf.transform(dft['processed_headline'])\n\n\n\n\n\nprint(X_train_headline.shape)\nprint(X_train_headline.ndim)\n\nprint(X_test_headline.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.preprocessing import StandardScaler\ncv = CountVectorizer()\nsc = StandardScaler(with_mean = False)\n\nX_train_pos = cv.fit_transform(df['part_speech'])\nX_train_pos_sc = sc.fit_transform(X_train_pos)   #2\n\nX_test_pos = cv.transform(dft['part_speech'])\nX_test_pos_sc = sc.transform(X_test_pos)\n\nprint(X_test_pos_sc.shape)\n\nprint(X_train_pos_sc.shape)\nprint(X_train_pos_sc.ndim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_val = df.drop( columns = ['title','label','processed_headline','part_speech','text']).values\nX_test_val = dft.drop( columns = ['title','processed_headline','part_speech','text']).values\n\n\nsc = StandardScaler()\n\nX_train_val_sc = sc.fit_transform(X_train_val) #3\nX_test_val_sc = sc.transform(X_test_val)\n\n\nprint(X_train_val_sc.shape)\nprint(X_train_val_sc.ndim)\n\nprint(X_test_val_sc.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import LabelEncoder\n\nlabel_encoder = LabelEncoder()\nY_train = label_encoder.fit_transform(df[['label']])\n\n#onehot_encoder = OneHotEncoder(sparse=False)\n#Y_train = onehot_encoder.fit_transform(df[['label']])\n\nY_train\n\nprint(Y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy import sparse\n\nX_train = sparse.hstack([X_train_val_sc, X_train_headline, X_train_pos_sc]).tocsr()\nX_test = sparse.hstack([X_test_val_sc, X_test_headline, X_test_pos_sc]).tocsr()\n\n\nprint(X_train.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import LogisticRegression\n\n\n#param_grid  = [{'C': np.linspace(90,100,20)}]\n#grid_cv = GridSearchCV(LogisticRegression(), param_grid, scoring='accuracy', cv=5, verbose=1)\n\n#grid_cv.fit(X_train, Y_train)\n\n#print(grid_cv.best_params_)\n#print(grid_cv.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\n\nmodel = LogisticRegression(penalty='l2', C=93.684210526315795)\nmodel = model.fit(X_train, Y_train)\nY_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_predres = []\nfor x in Y_pred:\n    if x == 0 :\n        x = 'clickbait'\n        \n    if x == 1 :\n        x = 'news'\n    if x == 2:\n        x = 'other'\n    Y_predres.append(x)\n    \nY_predres = np.array(Y_predres)\n\nY_predres","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft.id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_submission = pd.DataFrame({'id': dft.id, 'label': Y_predres })\n\nmy_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}