{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"Published on May 17, 2023. By Marília Prata, mpwolke","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-18T01:13:17.718574Z","iopub.execute_input":"2023-05-18T01:13:17.719187Z","iopub.status.idle":"2023-05-18T01:13:20.182810Z","shell.execute_reply.started":"2023-05-18T01:13:17.719154Z","shell.execute_reply":"2023-05-18T01:13:20.181687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from plotly.subplots import make_subplots\nfrom plotly import graph_objects as go\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\n\nfrom xgboost import XGBClassifier , XGBRegressor\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neural_network import MLPClassifier\n# from sklearn.neighbors import KNeighborsClassifier\n# from sklearn.naive_bayes import GaussianNB\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom xgboost import XGBClassifier , XGBRegressor\n\nimport shap","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:03:33.963693Z","iopub.execute_input":"2023-05-18T01:03:33.964135Z","iopub.status.idle":"2023-05-18T01:03:40.852725Z","shell.execute_reply.started":"2023-05-18T01:03:33.964104Z","shell.execute_reply":"2023-05-18T01:03:40.851160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -n ../input/billion-word-imputation/train_v2.txt.zip","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:03:45.812780Z","iopub.execute_input":"2023-05-18T01:03:45.813716Z","iopub.status.idle":"2023-05-18T01:04:41.465148Z","shell.execute_reply.started":"2023-05-18T01:03:45.813669Z","shell.execute_reply":"2023-05-18T01:04:41.463937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -n ../input/billion-word-imputation/test_v2.txt.zip","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:04:46.211607Z","iopub.execute_input":"2023-05-18T01:04:46.212775Z","iopub.status.idle":"2023-05-18T01:04:48.429136Z","shell.execute_reply.started":"2023-05-18T01:04:46.212692Z","shell.execute_reply":"2023-05-18T01:04:48.427897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Extract zip files","metadata":{}},{"cell_type":"code","source":"import zipfile\ntraining= zipfile.ZipFile('../input/billion-word-imputation/train_v2.txt.zip')\ntraining.extractall()\ntest= zipfile.ZipFile('../input/billion-word-imputation/test_v2.txt.zip')\ntest.extractall()","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:04:53.614043Z","iopub.execute_input":"2023-05-18T01:04:53.616737Z","iopub.status.idle":"2023-05-18T01:05:42.829636Z","shell.execute_reply.started":"2023-05-18T01:04:53.616645Z","shell.execute_reply":"2023-05-18T01:05:42.828491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data = pd.read_csv( '../input/billion-word-imputation/train_v2.txt', error_bad_lines= False )\n#Data_itemid = Data[['itemid']]#Original Data_Event_ID = Data[['EventId']]\n#Data_Label = Data[['Label']]","metadata":{"execution":{"iopub.status.busy":"2023-05-18T00:37:55.664655Z","iopub.execute_input":"2023-05-18T00:37:55.665144Z","iopub.status.idle":"2023-05-18T00:37:56.579941Z","shell.execute_reply.started":"2023-05-18T00:37:55.665094Z","shell.execute_reply":"2023-05-18T00:37:56.577079Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nfilepath = \"/kaggle/input/billion-word-imputation/\"\nfor f in glob.glob(filepath+\"*.txt\"):\n    os.system(\"cat \"+f+\" >> test_v2.txt\")","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:05:49.662405Z","iopub.execute_input":"2023-05-18T01:05:49.662847Z","iopub.status.idle":"2023-05-18T01:05:49.674695Z","shell.execute_reply.started":"2023-05-18T01:05:49.662817Z","shell.execute_reply":"2023-05-18T01:05:49.673079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\n\n#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nData = pd.read_csv( '../input/billion-word-imputation/train_v2.txt', sep='\\t', error_bad_lines= False,quoting=csv.QUOTE_NONE )\n#Data_itemid = Data[['itemid']]#Original Data_Event_ID = Data[['EventId']]\n#Data_Label = Data[['Label']]\n\nData_Test = pd.read_csv( '../input/billion-word-imputation/test_v2.txt',  sep='\\t', error_bad_lines= False,quoting=csv.QUOTE_NONE)\n#Data_Test_itemid = Data_Test[['itemid']]\n\nData.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-18T00:23:16.478554Z","iopub.execute_input":"2023-05-18T00:23:16.479023Z","iopub.status.idle":"2023-05-18T00:23:17.334330Z","shell.execute_reply.started":"2023-05-18T00:23:16.478989Z","shell.execute_reply":"2023-05-18T00:23:17.332832Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data = pd.read_csv( '../input/billion-word-imputation/train_v2.txt', error_bad_lines= False )\n#Data_itemid = Data[['itemid']]#Original Data_Event_ID = Data[['EventId']]\n#Data_Label = Data[['Label']]","metadata":{"execution":{"iopub.status.busy":"2023-05-18T00:31:59.271928Z","iopub.execute_input":"2023-05-18T00:31:59.272999Z","iopub.status.idle":"2023-05-18T00:31:59.412642Z","shell.execute_reply.started":"2023-05-18T00:31:59.272864Z","shell.execute_reply":"2023-05-18T00:31:59.408747Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp = open(\"../input/billion-word-imputation/train_v2.txt\", \"r\").read()\nprint (imp[:53])\n\ntext = imp","metadata":{"execution":{"iopub.status.busy":"2023-05-18T00:26:48.330176Z","iopub.execute_input":"2023-05-18T00:26:48.330610Z","iopub.status.idle":"2023-05-18T00:26:48.369632Z","shell.execute_reply.started":"2023-05-18T00:26:48.330577Z","shell.execute_reply":"2023-05-18T00:26:48.367029Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Since train_v2 is huge and tried to allocated more memory. To avoid the Cloud I opened test_v2 with \n\n[:53]","metadata":{}},{"cell_type":"code","source":"# Load Dataset\nfilename    = 'test_v2.txt'\ntext        = open(filename, encoding='utf-8').read()\ntext        = text.lower()\nprint('corpus length:', len(text))\n\n# Find all the unique characters\nchars        = sorted(list(set(text)))\nchar_indices = dict((c, i) for i, c in enumerate(chars))\nindices_char = dict((i, c) for i, c in enumerate(chars))\nvocab_size   = len(chars)\n\nprint(\"List of unique characters : \\n\", chars)\nprint(\"Number of unique characters : \\n\", vocab_size)\nprint(\"Character to integer mapping : \\n\", char_indices)\nprint (filename[:53])","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:01.239435Z","iopub.execute_input":"2023-05-18T01:06:01.239928Z","iopub.status.idle":"2023-05-18T01:06:03.007772Z","shell.execute_reply.started":"2023-05-18T01:06:01.239894Z","shell.execute_reply":"2023-05-18T01:06:03.006153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#All work for that? How can someone impute that above? I barely could open it! ","metadata":{}},{"cell_type":"code","source":"# Preprocessing Dataset\n#Original below was 64, though it rendered: dic instead of dickens\nmax_seq_len = 68 # cut text in semi-redundant sequences of max_seq_len characters\nstep = 3 \nsentences = [] # list_X\nnext_chars= [] # list_Y\n\nfor i in range(0, len(text) - max_seq_len, step):\n    sentences.append(text[i: i + max_seq_len])\n    next_chars.append(text[i + max_seq_len])\nprint('nb sequences:', len(sentences))\n\nnum_sequences  = len(sentences)\nprint(\"Number of sequences: \", num_sequences)\nprint(sentences[0])","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:09.807262Z","iopub.execute_input":"2023-05-18T01:06:09.807788Z","iopub.status.idle":"2023-05-18T01:06:22.740286Z","shell.execute_reply.started":"2023-05-18T01:06:09.807752Z","shell.execute_reply":"2023-05-18T01:06:22.738872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Finally, one line: \"He added that people should not mess with mother","metadata":{}},{"cell_type":"markdown","source":"#Vectorization","metadata":{}},{"cell_type":"code","source":"print('Vectorization...')\ntrain_X = np.zeros((len(sentences), max_seq_len, len(chars)), dtype=np.bool)\ntrain_Y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n#for i, sentence in enumerate(sentences):\n #   for t, char in enumerate(sentence):\n  #      train_X[i, t, char_indices[char]] = 1\n   # train_Y[i, char_indices[next_chars[i]]] = 1\n\nprint(train_X.shape)\nprint(train_Y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:27.852879Z","iopub.execute_input":"2023-05-18T01:06:27.853344Z","iopub.status.idle":"2023-05-18T01:06:27.862319Z","shell.execute_reply.started":"2023-05-18T01:06:27.853314Z","shell.execute_reply":"2023-05-18T01:06:27.861305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (max_seq_len, vocab_size)\nprint(input_shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:33.624832Z","iopub.execute_input":"2023-05-18T01:06:33.625227Z","iopub.status.idle":"2023-05-18T01:06:33.631288Z","shell.execute_reply.started":"2023-05-18T01:06:33.625200Z","shell.execute_reply":"2023-05-18T01:06:33.630091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, LSTM, Input, Activation, concatenate, Bidirectional\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:40.338980Z","iopub.execute_input":"2023-05-18T01:06:40.339452Z","iopub.status.idle":"2023-05-18T01:06:52.170476Z","shell.execute_reply.started":"2023-05-18T01:06:40.339419Z","shell.execute_reply":"2023-05-18T01:06:52.169211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build Model\nmodel = Sequential()\nmodel.add(Bidirectional(LSTM(64, return_sequences=True), input_shape=input_shape))\nmodel.add(Bidirectional(LSTM(64)))\nmodel.add(Dense(len(chars), activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-18T01:06:55.954212Z","iopub.execute_input":"2023-05-18T01:06:55.955401Z","iopub.status.idle":"2023-05-18T01:06:57.563446Z","shell.execute_reply.started":"2023-05-18T01:06:55.955325Z","shell.execute_reply":"2023-05-18T01:06:57.562278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Model\nnum_epochs = 3 #Original is 30\nbatch_size = 128\n#model_path = \"BiLSTM-DonaldTrumpRallies .h5\"\n#checkpoint = ModelCheckpoint(model_path, monitor='loss', save_best_only=True, verbose=1, mode='min')\n#callbacks_list = [checkpoint]\n\nmodel.fit(train_X, train_Y, epochs = num_epochs, batch_size = batch_size, verbose=1) #, callbacks=callbacks_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#After so many Starts and errors. Imputation: No Way!\n\nAfter the snippet above, the Notebook returned to \"Draft Session off (run a cell to start)\" ","metadata":{}},{"cell_type":"markdown","source":"![](https://media.tenor.com/HWPyQXm5aL8AAAAM/no-way-no-way-jose.gif)tenor","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nMPWolke https://www.kaggle.com/code/mpwolke/g-expectations-not-enough-memory-pip\n\nMPWolke https://www.kaggle.com/code/mpwolke/prohibited-content-zip","metadata":{}}]}