{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv('../input/congressionaltweetcompetitionspring2022/congressional_tweet_training_data.csv')","metadata":{"id":"xllgMmlEKCFS","execution":{"iopub.status.busy":"2022-03-23T14:41:57.226506Z","iopub.execute_input":"2022-03-23T14:41:57.226869Z","iopub.status.idle":"2022-03-23T14:42:00.856166Z","shell.execute_reply.started":"2022-03-23T14:41:57.226761Z","shell.execute_reply":"2022-03-23T14:42:00.855205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"id":"GvmgVskvLAn_","outputId":"da1be875-8d30-4de9-be0e-a024da13a909","execution":{"iopub.status.busy":"2022-03-23T14:42:05.02523Z","iopub.execute_input":"2022-03-23T14:42:05.025522Z","iopub.status.idle":"2022-03-23T14:42:05.050073Z","shell.execute_reply.started":"2022-03-23T14:42:05.025487Z","shell.execute_reply":"2022-03-23T14:42:05.049287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport nltk\nfrom nltk import sent_tokenize\nfrom nltk import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import stopwords\n\nnltk.download('averaged_perceptron_tagger')\nnltk.download('wordnet')\nnltk.download('punkt')\nnltk.download('stopwords')\nfrom nltk.corpus import wordnet","metadata":{"id":"4q56UoyVLEre","outputId":"0b7fcbdd-3595-4678-f618-d7c2a9d74189","execution":{"iopub.status.busy":"2022-03-23T14:42:06.902041Z","iopub.execute_input":"2022-03-23T14:42:06.90234Z","iopub.status.idle":"2022-03-23T14:42:08.752768Z","shell.execute_reply.started":"2022-03-23T14:42:06.902308Z","shell.execute_reply":"2022-03-23T14:42:08.751623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatizer = WordNetLemmatizer()\nstop = stopwords.words('english')\npat = re.compile(r'\\b\\w{1,2}\\b')\ndf['post_tagged_text'] = df['full_text'].astype('str').str.replace(r'http\\S+', '', regex=True).str.replace(r'https\\S+', '', regex=True).str.replace('[^a-zA-Z0-9]', ' ', regex=True).str.strip().apply(lambda s: pat.sub('', s)).apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)])).apply(word_tokenize).apply(nltk.pos_tag)\ndf.head()","metadata":{"id":"2jqmoeZPLWkW","outputId":"dacf62ac-4eab-4c0b-be95-0bb67b1aaa86","execution":{"iopub.status.busy":"2022-03-23T14:42:12.738675Z","iopub.execute_input":"2022-03-23T14:42:12.738968Z","iopub.status.idle":"2022-03-23T14:58:49.952106Z","shell.execute_reply.started":"2022-03-23T14:42:12.738935Z","shell.execute_reply":"2022-03-23T14:58:49.95121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_wordnet_pos(nltk_tag):\n    if nltk_tag.startswith('J'):\n        return wordnet.ADJ\n    elif nltk_tag.startswith('V'):\n        return wordnet.VERB\n    elif nltk_tag.startswith('N'):\n        return wordnet.NOUN\n    elif nltk_tag.startswith('R'):\n        return wordnet.ADV\n    else:\n        return None \n\nwhole_set=[]\nfor x in df['post_tagged_text']:\n    lemmetized_sentences=[]\n    for tagged in x:\n        tag=get_wordnet_pos(tagged[1])\n        if tag:\n            lemmetized_sentences.append(lemmatizer.lemmatize(tagged[0],tag))\n        else:\n            lemmetized_sentences.append(tagged[0])\n    whole_set.append(lemmetized_sentences)","metadata":{"id":"3R0LWSn3Le9l","execution":{"iopub.status.busy":"2022-03-23T14:58:49.953828Z","iopub.execute_input":"2022-03-23T14:58:49.954103Z","iopub.status.idle":"2022-03-23T14:59:56.362437Z","shell.execute_reply.started":"2022-03-23T14:58:49.954055Z","shell.execute_reply":"2022-03-23T14:59:56.361275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['clean_data']=whole_set\ndf['text_clean']=df.clean_data.apply(lambda x: \" \".join(map(str,x)))\ndf.head()","metadata":{"id":"-L4zgopQLn-7","outputId":"e34b2f08-0e83-40c9-d16e-fd99b5679887","execution":{"iopub.status.busy":"2022-03-23T14:59:56.363702Z","iopub.execute_input":"2022-03-23T14:59:56.36395Z","iopub.status.idle":"2022-03-23T14:59:58.71892Z","shell.execute_reply.started":"2022-03-23T14:59:56.36392Z","shell.execute_reply":"2022-03-23T14:59:58.717969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.drop(labels=['full_text','post_tagged_text','clean_data','hashtags'],axis=1)\ndf.head()","metadata":{"id":"9e36BWfLP-Hd","outputId":"e1ebb989-c79c-4477-fdf0-94debdf211d7","execution":{"iopub.status.busy":"2022-03-23T14:59:58.721243Z","iopub.execute_input":"2022-03-23T14:59:58.72158Z","iopub.status.idle":"2022-03-23T14:59:59.090108Z","shell.execute_reply.started":"2022-03-23T14:59:58.721533Z","shell.execute_reply":"2022-03-23T14:59:59.089155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('cleaned_data.csv')","metadata":{"id":"ZFIh9MSHQJ_C","execution":{"iopub.status.busy":"2022-03-23T15:43:41.596338Z","iopub.execute_input":"2022-03-23T15:43:41.596919Z","iopub.status.idle":"2022-03-23T15:43:41.69284Z","shell.execute_reply.started":"2022-03-23T15:43:41.596828Z","shell.execute_reply":"2022-03-23T15:43:41.691237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#XGboost-No Text\nimport pandas as pd\ndf=pd.read_csv('cleaned_data.csv')\ndf=df.drop(labels=['hashtags','text_clean'],axis=1)\ndf['year'].fillna((df['year'].median()), inplace=True)\ndf.party_id=df.party_id.astype('category').cat.codes\ndf.head()","metadata":{"id":"SvvPfq9QQOPo","outputId":"58e378d1-dc81-4512-eafa-41013e48fc73","execution":{"iopub.status.busy":"2022-03-23T15:43:41.694158Z","iopub.status.idle":"2022-03-23T15:43:41.694513Z","shell.execute_reply.started":"2022-03-23T15:43:41.694329Z","shell.execute_reply":"2022-03-23T15:43:41.694356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\nimport pandas as pd\nimport numpy as np\nX, y = df.iloc[:,:-1],df.iloc[:,-1]\n","metadata":{"id":"RS9mg_RqiPp1","execution":{"iopub.status.busy":"2022-03-23T15:43:41.695642Z","iopub.status.idle":"2022-03-23T15:43:41.695994Z","shell.execute_reply.started":"2022-03-23T15:43:41.695817Z","shell.execute_reply":"2022-03-23T15:43:41.695837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate xgboost algorithm for classification\nfrom numpy import mean\nfrom numpy import std\nfrom sklearn.datasets import make_classification\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom xgboost import XGBClassifier\n# define dataset\n# define the model\nmodel = XGBClassifier()\n# evaluate the model\ncv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\nn_scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n# report performance\nprint('Accuracy: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))","metadata":{"id":"8eszjobQjgte","outputId":"9b77b6cd-6327-405a-b481-956a25f5db69","execution":{"iopub.status.busy":"2022-03-23T15:43:41.697459Z","iopub.status.idle":"2022-03-23T15:43:41.697942Z","shell.execute_reply.started":"2022-03-23T15:43:41.697681Z","shell.execute_reply":"2022-03-23T15:43:41.697725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make predictions using xgboost for classification\nfrom numpy import asarray\nfrom sklearn.datasets import make_classification\nfrom xgboost import XGBClassifier\n# define dataset\nX, y = make_classification(n_samples=1000, n_features=20, n_informative=15, n_redundant=5, random_state=7)\n# define the model\nmodel = XGBClassifier()\n# fit the model on the whole dataset\nmodel.fit(X, y)\n# make a single prediction\nrow = [0.2929949,-4.21223056,-1.288332,-2.17849815,-0.64527665,2.58097719,0.28422388,-7.1827928,-1.91211104,2.73729512,0.81395695,3.96973717,-2.66939799,3.34692332,4.19791821,0.99990998,-0.30201875,-4.43170633,-2.82646737,0.44916808]\nrow = asarray([row])\nyhat = model.predict(row)\nprint('Predicted Class: %d' % yhat[0])","metadata":{"id":"w0dIjlQEr3Gw","execution":{"iopub.status.busy":"2022-03-23T15:43:41.699393Z","iopub.status.idle":"2022-03-23T15:43:41.69993Z","shell.execute_reply.started":"2022-03-23T15:43:41.699733Z","shell.execute_reply":"2022-03-23T15:43:41.699753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#h2o-- Cleaned text.\n!pip install h2o","metadata":{"id":"xOk6Eir3rhVt","outputId":"475b07c6-ec30-45e1-b289-b94bcf8754c6","execution":{"iopub.status.busy":"2022-03-23T15:43:41.700924Z","iopub.status.idle":"2022-03-23T15:43:41.701476Z","shell.execute_reply.started":"2022-03-23T15:43:41.701293Z","shell.execute_reply":"2022-03-23T15:43:41.701313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h2o\nfrom h2o.estimators.random_forest import H2ORandomForestEstimator\nfrom h2o.estimators.gbm import H2OGradientBoostingEstimator\nfrom h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator\nfrom h2o.grid.grid_search import H2OGridSearch\nfrom __future__ import print_function\nh2o.init()\n","metadata":{"id":"E_TcnLaP43LP","outputId":"c0c1f222-87c3-4315-8746-a7b882a0a6e1","execution":{"iopub.status.busy":"2022-03-23T15:43:41.702522Z","iopub.status.idle":"2022-03-23T15:43:41.703029Z","shell.execute_reply.started":"2022-03-23T15:43:41.702853Z","shell.execute_reply":"2022-03-23T15:43:41.702873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dataset prep for h2o\nimport pandas as pd\ndf1=pd.read_csv('cleaned_data.csv')\n#df1=df1.drop(labels=['hashtags'],axis=1)\ntraindata = df1.sample(frac = 0.75)\ntestdata = df1.drop(traindata.index)\ntrain = h2o.H2OFrame(traindata)\ntest = h2o.H2OFrame(testdata)","metadata":{"id":"3-LPSGMSDkPb","outputId":"6b3065a7-aab4-4f63-aac1-ffc11c0a7999","execution":{"iopub.status.busy":"2022-03-23T15:43:41.704112Z","iopub.status.idle":"2022-03-23T15:43:41.704525Z","shell.execute_reply.started":"2022-03-23T15:43:41.704361Z","shell.execute_reply":"2022-03-23T15:43:41.704377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n # Import a sample binary outcome train/test set into H2O\n#train = h2o.import_file('traint.csv')\n#test = h2o.import_file('traintest.csv')\n\n#train = h2o.H2OFrame(traindata)\n#test = h2o.H2OFrame(testdata)\n# Identify predictors and response\nx = train.columns\ny = \"party_id\"\nx.remove(y)\n\n# For binary classification, response should be a factor\ntrain[y] = train[y].asfactor()\ntest[y] = test[y].asfactor()\n\n# Number of CV folds (to generate level-one data for stacking)\nnfolds = 5\n\n# There are a few ways to assemble a list of models to stack together:\n# 1. Train individual models and put them in a list\n# 2. Train a grid of models\n# 3. Train several grids of models\n# Note: All base models must have the same cross-validation folds and\n# the cross-validated predicted values must be kept.\n\n\n# 1. Generate a 2-model ensemble (GBM + RF)\n\n# Train and cross-validate a GBM\nmy_gbm = H2OGradientBoostingEstimator(distribution=\"bernoulli\",\n                                      ntrees=10,\n                                      max_depth=3,\n                                      min_rows=2,\n                                      learn_rate=0.1,\n                                      nfolds=nfolds,\n                                      fold_assignment=\"Modulo\",\n                                      keep_cross_validation_predictions=True,\n                                      seed=1)\nmy_gbm.train(x=x, y=y, training_frame=train)\n\n\n# Train and cross-validate a RF\nmy_rf = H2ORandomForestEstimator(ntrees=50,\n                                 nfolds=nfolds,\n                                 fold_assignment=\"Modulo\",\n                                 keep_cross_validation_predictions=True,\n                                 seed=1)\nmy_rf.train(x=x, y=y, training_frame=train)\n\n\n# Train a stacked ensemble using the GBM and GLM above\nensemble = H2OStackedEnsembleEstimator(model_id=\"my_ensemble_binomial\",\n                                       base_models=[my_gbm, my_rf])\nensemble.train(x=x, y=y, training_frame=train)\n\n# Eval ensemble performance on the test data\nperf_stack_test = ensemble.model_performance(test)\n\n# Compare to base learner performance on the test set\nperf_gbm_test = my_gbm.model_performance(test)\nperf_rf_test = my_rf.model_performance(test)\nbaselearner_best_auc_test = max(perf_gbm_test.auc(), perf_rf_test.auc())\nstack_auc_test = perf_stack_test.auc()\nprint(\"Best Base-learner Test AUC:  {0}\".format(baselearner_best_auc_test))\nprint(\"Ensemble Test AUC:  {0}\".format(stack_auc_test))\n\n# Generate predictions on a test set (if neccessary)\npred = ensemble.predict(test)\n\n\n# 2. Generate a random grid of models and stack them together\n\n# Specify GBM hyperparameters for the grid\nhyper_params = {\"learn_rate\": [0.01, 0.03],\n                \"max_depth\": [3, 4, 5, 6, 9],\n                \"sample_rate\": [0.7, 0.8, 0.9, 1.0],\n                \"col_sample_rate\": [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}\nsearch_criteria = {\"strategy\": \"RandomDiscrete\", \"max_models\": 3, \"seed\": 1}\n\n# Train the grid\ngrid = H2OGridSearch(model=H2OGradientBoostingEstimator(ntrees=10,\n                                                        seed=1,\n                                                        nfolds=nfolds,\n                                                        fold_assignment=\"Modulo\",\n                                                        keep_cross_validation_predictions=True),\n                     hyper_params=hyper_params,\n                     search_criteria=search_criteria,\n                     grid_id=\"gbm_grid_binomial\")\ngrid.train(x=x, y=y, training_frame=train)\n\n# Train a stacked ensemble using the GBM grid\nensemble = H2OStackedEnsembleEstimator(model_id=\"my_ensemble_gbm_grid_binomial\",\n                                       base_models=grid.model_ids)\nensemble.train(x=x, y=y, training_frame=train)\n\n# Eval ensemble performance on the test data\nperf_stack_test = ensemble.model_performance(test)\n\n# Compare to base learner performance on the test set\nbaselearner_best_auc_test = max([h2o.get_model(model).model_performance(test_data=test).auc() for model in grid.model_ids])\nstack_auc_test = perf_stack_test.auc()\nprint(\"Best Base-learner Test AUC:  {0}\".format(baselearner_best_auc_test))\nprint(\"Ensemble Test AUC:  {0}\".format(stack_auc_test))\n\n# Generate predictions on a test set (if neccessary)\npred = ensemble.predict(test)","metadata":{"id":"9HsXiTSTDtfo","outputId":"261c434a-4172-4bd0-939a-1a1e64470b8b","execution":{"iopub.status.busy":"2022-03-23T15:43:41.705411Z","iopub.status.idle":"2022-03-23T15:43:41.705826Z","shell.execute_reply.started":"2022-03-23T15:43:41.70566Z","shell.execute_reply":"2022-03-23T15:43:41.705676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#BERT- Combined single column text\nimport pandas as pd\ndf1=pd.read_csv('cleaned_data.csv')\ndf1=df1.drop(labels=['hashtags','party_id'],axis=1)\ndf1['year'].fillna((df1['year'].median()), inplace=True)\ndf1['combined'] = df1[df1.columns[1:]].apply(\n    lambda x: ','.join(x.dropna().astype(str)),\n    axis=1\n)\ndf1=df1.drop(labels=['favorite_count','retweet_count','year','text_clean'],axis=1)\ndf1.head()","metadata":{"id":"gGoVBjxvJw38","outputId":"fc6ec552-51a4-4080-ef0b-f8238e76d6ad","execution":{"iopub.status.busy":"2022-03-23T15:43:41.706621Z","iopub.status.idle":"2022-03-23T15:43:41.707021Z","shell.execute_reply.started":"2022-03-23T15:43:41.706864Z","shell.execute_reply":"2022-03-23T15:43:41.706881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lemmatizer = WordNetLemmatizer()\nstop = stopwords.words('english')\npat = re.compile(r'\\b\\w{1,2}\\b')\ndf1['combined'] = df1['combined'].astype('str').str.replace(r'http\\S+', '', regex=True).str.replace(r'https\\S+', '', regex=True).str.replace('[^a-zA-Z0-9]', ' ', regex=True).str.strip().apply(lambda s: pat.sub('', s)).apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)])).apply(word_tokenize).apply(nltk.pos_tag)\ndef get_wordnet_pos(nltk_tag):\n    if nltk_tag.startswith('J'):\n        return wordnet.ADJ\n    elif nltk_tag.startswith('V'):\n        return wordnet.VERB\n    elif nltk_tag.startswith('N'):\n        return wordnet.NOUN\n    elif nltk_tag.startswith('R'):\n        return wordnet.ADV\n    else:\n        return None \n\nwhole_set=[]\nfor x in df1['combined']:\n    lemmetized_sentences=[]\n    for tagged in x:\n        tag=get_wordnet_pos(tagged[1])\n        if tag:\n            lemmetized_sentences.append(lemmatizer.lemmatize(tagged[0],tag))\n        else:\n            lemmetized_sentences.append(tagged[0])\n    whole_set.append(lemmetized_sentences)\ndf1['combined']=whole_set\ndf1['combined']=df1.combined.apply(lambda x: \" \".join(map(str,x)))\n","metadata":{"id":"ig_LHIk2zPdx","outputId":"4b3086d7-4944-40ad-d9aa-38d47be8f1dc","execution":{"iopub.status.busy":"2022-03-23T15:43:41.708516Z","iopub.execute_input":"2022-03-23T15:43:41.708827Z","iopub.status.idle":"2022-03-23T15:43:41.733744Z","shell.execute_reply.started":"2022-03-23T15:43:41.708799Z","shell.execute_reply":"2022-03-23T15:43:41.732354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head()","metadata":{"id":"96YlBkT41Qmp","outputId":"c584f5d9-86f8-4da8-e505-4b67dacb727c","execution":{"iopub.status.busy":"2022-03-23T15:43:41.734865Z","iopub.status.idle":"2022-03-23T15:43:41.735353Z","shell.execute_reply.started":"2022-03-23T15:43:41.735167Z","shell.execute_reply":"2022-03-23T15:43:41.735187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\n# If there's a GPU available...\nif torch.cuda.is_available():    \n\n    # Tell PyTorch to use the GPU.    \n    device = torch.device(\"cuda\")\n\n    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n\n    print('We will use the GPU:', torch.cuda.get_device_name(0))\n\n# If not...\nelse:\n    print('No GPU available, using the CPU instead.')\n    device = torch.device(\"cpu\")","metadata":{"id":"zt72wYtX4TBZ","outputId":"ff7be015-2a22-4305-dfc5-8fdd2d951fbb","execution":{"iopub.status.busy":"2022-03-23T15:43:41.736296Z","iopub.status.idle":"2022-03-23T15:43:41.736782Z","shell.execute_reply.started":"2022-03-23T15:43:41.736608Z","shell.execute_reply":"2022-03-23T15:43:41.736627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"7_98WdfT1rEX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install transformers","metadata":{"id":"0NmMdkZO8R6q","outputId":"c3180191-d7ba-4508-a113-d3dacafa0115","execution":{"iopub.status.busy":"2022-03-23T15:43:41.737915Z","iopub.status.idle":"2022-03-23T15:43:41.738431Z","shell.execute_reply.started":"2022-03-23T15:43:41.738252Z","shell.execute_reply":"2022-03-23T15:43:41.738273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizer\n\n# Load the BERT tokenizer.\nprint('Loading BERT tokenizer...')\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)","metadata":{"id":"sCCo5NRr12b8","outputId":"bb65b19a-0777-42d9-b60d-3e360b22430d","execution":{"iopub.status.busy":"2022-03-23T15:43:41.739578Z","iopub.status.idle":"2022-03-23T15:43:41.740158Z","shell.execute_reply.started":"2022-03-23T15:43:41.739923Z","shell.execute_reply":"2022-03-23T15:43:41.739944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"rl4U7tVt2JOo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertForSequenceClassification\n\n# Load BertForSequenceClassification, the pretrained BERT model with a single \n# linear classification layer on top. \nmodel = BertForSequenceClassification.from_pretrained(\n    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n    num_labels = 2, # The number of output labels--2 for binary classification.\n)\n\n# Tell pytorch to run this model on the GPU.\ndesc = model.cuda()","metadata":{"id":"gFsCTp_mporB","outputId":"ca8d8e6b-745d-4ade-b2fa-0986426c08d9","execution":{"iopub.status.busy":"2022-03-23T15:43:41.741309Z","iopub.status.idle":"2022-03-23T15:43:41.741704Z","shell.execute_reply.started":"2022-03-23T15:43:41.741516Z","shell.execute_reply":"2022-03-23T15:43:41.741533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#glove=150","metadata":{"id":"O4h6rRjQ23XO","execution":{"iopub.status.busy":"2022-03-23T15:43:41.742971Z","iopub.status.idle":"2022-03-23T15:43:41.743513Z","shell.execute_reply.started":"2022-03-23T15:43:41.743326Z","shell.execute_reply":"2022-03-23T15:43:41.743346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Larger batch sizes tend to be better, and we can fit this in memory.\nbatch_size = 32\n\n# I used a smaller learning rate to combat over-fitting that I was seeing in the\n# validation loss. I could probably try even smaller.\nlearning_rate = 1e-5\n\n# Number of training epochs. \nepochs = 4","metadata":{"id":"g3DxsSO0474t","execution":{"iopub.status.busy":"2022-03-23T15:43:41.744492Z","iopub.status.idle":"2022-03-23T15:43:41.745004Z","shell.execute_reply.started":"2022-03-23T15:43:41.744811Z","shell.execute_reply":"2022-03-23T15:43:41.744844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 0\n\n# For every sentence...\nfor sent in sen_w_feats:\n\n    # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n    input_ids = tokenizer.encode(sent, add_special_tokens=True)\n\n    # Update the maximum sentence length.\n    max_len = max(max_len, len(input_ids))\n\nprint('Max sentence length: ', max_len)","metadata":{"id":"mzwk0Xc-261F","execution":{"iopub.status.busy":"2022-03-23T15:43:41.745998Z","iopub.status.idle":"2022-03-23T15:43:41.746516Z","shell.execute_reply.started":"2022-03-23T15:43:41.746325Z","shell.execute_reply":"2022-03-23T15:43:41.746345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}