{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \npd.set_option('display.max_colwidth', 1)\nimport re \nimport string\nimport os\nimport emoji\nfrom pprint import pprint\nimport collections\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set(style=\"darkgrid\")\nsns.set(font_scale=2.0)\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.metrics import classification_report\nfrom sklearn .metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import word_tokenize\nimport warnings\nwarnings.filterwarnings('ignore')\nnp.random.seed(37)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/commonlitreadabilityprize/train.csv\")\ntest = pd.read_csv(\"../input/commonlitreadabilityprize/test.csv\")\ndf = df.reindex(np.random.permutation(df.index))\nprint(df.columns)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[['excerpt','target']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,10))\nsns.kdeplot(x=\"target\", data=df,shade= True, palette=\"PuBuGn_d\")\nplt.show()","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TextCounts(BaseEstimator, TransformerMixin):\n    \n    def count_regex(self, pattern, text):\n        return len(re.findall(pattern, text))\n    \n    def fit(self, X, y=None, **fit_params):\n        # fit method is used when specific operations need to be done on the train data, but not on the test data\n        return self\n    \n    def transform(self, X, **transform_params):\n        count_words = X.apply(lambda x: self.count_regex(r'\\w+', x)) \n        count_mentions = X.apply(lambda x: self.count_regex(r'@\\w+', x))\n        count_hashtags = X.apply(lambda x: self.count_regex(r'#\\w+', x))\n        count_capital_words = X.apply(lambda x: self.count_regex(r'\\b[A-Z]{2,}\\b', x))\n        count_excl_quest_marks = X.apply(lambda x: self.count_regex(r'!|\\?', x))\n        count_urls = X.apply(lambda x: self.count_regex(r'http.?://[^\\s]+[\\s]?', x))\n        #We will replace the emoji symbols with a description, which makes using a regex for counting easier\n       # Moreover, it will result in having more words in the tweet\n        count_emojis = X.apply(lambda x: emoji.demojize(x)).apply(lambda x: self.count_regex(r':[a-z_&]+:', x))\n        \n        df = pd.DataFrame({'count_words': count_words\n                           , 'count_mentions': count_mentions\n                           , 'count_hashtags': count_hashtags\n                           , 'count_capital_words': count_capital_words\n                           , 'count_excl_quest_marks': count_excl_quest_marks\n                           , 'count_urls': count_urls\n                           , 'count_emojis': count_emojis\n                          })\n        \n        return df\ntc = TextCounts()\ndf_eda = tc.fit_transform(df.excerpt)\ndf_eda['target'] = df.target\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def show_dist(df, col):\n#     print('Descriptive stats for {}'.format(col))\n#     print('-'*(len(col)+22))\n#     print(df.groupby('target')[col].describe())\n#     bins = np.arange(df[col].min(), df[col].max() + 1)\n#     g = sns.FacetGrid(df, col='target', hue='target', size=5, palette=\"PuBuGn_d\")\n#     g = g.map(sns.distplot, col, kde=False, norm_hist=True, bins=bins)\n#     plt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show_dist(df_eda ,'count_words')\n# show_dist(df_eda , 'count_emojis')\n# show_dist(df_eda , 'count_mentions')\n# show_dist(df_eda , 'count_capital_words')\n# show_dist(df_eda , 'count_excl_quest_marks')\n# show_dist(df_eda , 'count_urls')\n# show_dist(df_eda , 'count_hashtags')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CleanText(BaseEstimator, TransformerMixin):\n    def remove_mentions(self, input_text):\n        return re.sub(r'@\\w+', '', input_text)\n    \n    def remove_urls(self, input_text):\n        return re.sub(r'http.?://[^\\s]+[\\s]?', '', input_text)\n    \n    def emoji_oneword(self, input_text):\n        # By compressing the underscore, the emoji is kept as one word\n        return input_text.replace('_','')\n    \n    def remove_punctuation(self, input_text):\n        # Make translation table\n        punct = string.punctuation\n        trantab = str.maketrans(punct, len(punct)*' ')\n        # Every punctuation symbol will be replaced by a space\n        return input_text.translate(trantab)\n    def remove_digits(self, input_text):\n        return re.sub('\\d+', '', input_text)\n    \n    def to_lower(self, input_text):\n        return input_text.lower()\n    \n    def remove_stopwords(self, input_text):\n        stopwords_list = stopwords.words('english')\n        # Some words which might indicate a certain sentiment are kept via a whitelist\n        whitelist = [\"n't\", \"not\", \"no\"]\n        words = input_text.split() \n        clean_words = [word for word in words if (word not in stopwords_list or word in whitelist) and len(word) > 1] \n        return \" \".join(clean_words) \n    \n    def stemming(self, input_text):\n        porter = PorterStemmer()\n        words = input_text.split() \n        stemmed_words = [porter.stem(word) for word in words]\n        return \" \".join(stemmed_words)\n    \n    def fit(self, X, y=None, **fit_params):\n        return self\n    \n    def transform(self, X, **transform_params):\n        clean_X = X.apply(self.remove_mentions).apply(self.remove_urls).apply(self.emoji_oneword).apply(self.remove_punctuation).apply(self.remove_digits).apply(self.to_lower).apply(self.remove_stopwords).apply(self.stemming)\n        return clean_X\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ct = CleanText()\nsr_clean = ct.fit_transform(df.excerpt)\nsr_clean.head(5)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_clean = sr_clean == ''\nprint('{} records have no words left after text cleaning'.format(sr_clean[empty_clean].count()))\nsr_clean.loc[empty_clean] = '[no_text]'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = CountVectorizer()\nbow = cv.fit_transform(sr_clean)\nword_freq = dict(zip(cv.get_feature_names(), np.asarray(bow.sum(axis=0)).ravel()))\nword_counter = collections.Counter(word_freq)\nword_counter_df = pd.DataFrame(word_counter.most_common(20), columns = ['word', 'freq'])\nfig, ax = plt.subplots(figsize=(30, 10))\nsns.barplot(x=\"word\", y=\"freq\", data=word_counter_df, palette=\"hot\", ax=ax)\nplt.show();","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_model = df_eda\ndf_model['clean_text'] = sr_clean\ndf_model.columns.tolist()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ColumnExtractor(TransformerMixin, BaseEstimator):\n    def __init__(self, cols):\n        self.cols = cols\n    def transform(self, X, **transform_params):\n        return X[self.cols]\n    def fit(self, X, y=None, **fit_params):\n        return self\nX_train, X_test, y_train, y_test = train_test_split(df_model.drop('target', axis=1),\n                                    df_model.target, test_size=0.1, random_state=37)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def grid_vect(clf, parameters_clf, X_train, X_test, parameters_text=None, vect=None, is_w2v=False):\n    \n    textcountscols = ['count_capital_words','count_emojis','count_excl_quest_marks','count_hashtags'\n                      ,'count_mentions','count_urls','count_words']   \n    if is_w2v:\n        w2vcols = []\n        for i in range(SIZE):\n            w2vcols.append(i)\n        features = FeatureUnion([('textcounts', ColumnExtractor(cols=textcountscols))\n                                 , ('w2v', ColumnExtractor(cols=w2vcols))]\n                                , n_jobs=-1)\n    else:\n        features = FeatureUnion([('textcounts', ColumnExtractor(cols=textcountscols))\n                                 , ('pipe', Pipeline([('cleantext', ColumnExtractor(cols='clean_text')), ('vect', vect)]))]\n                                , n_jobs=-1)  \n    pipeline = Pipeline([\n        ('features', features)\n        , ('clf', clf)\n    ])\n    parameters = dict()\n    if parameters_text:\n        parameters.update(parameters_text)\n    parameters.update(parameters_clf)\n    \n    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, cv=5)\n    \n    print(\"Performing grid search...\")\n    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n    \n    grid_search.fit(X_train, y_train)\n    \n    best_parameters = grid_search.best_estimator_.get_params()\n    print(\"RMSE\")\n    print(mean_squared_error(y_test, grid_search.best_estimator_.predict(X_test)), squared=False)                  \n    return grid_search","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters_vect = {\n    'features__pipe__vect__max_df': (0.25, 0.5, 0.75),\n    'features__pipe__vect__ngram_range': ((1, 1), (1, 2)),\n    'features__pipe__vect__min_df': (1,2)\n}\nparameters_logreg = {\n    'clf__C': (0.25, 0.5, 1.0),\n    'clf__penalty': ('l1', 'l2')\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}