{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.234118Z","iopub.execute_input":"2022-04-03T08:44:21.234362Z","iopub.status.idle":"2022-04-03T08:44:21.246681Z","shell.execute_reply.started":"2022-04-03T08:44:21.234338Z","shell.execute_reply":"2022-04-03T08:44:21.245843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import SimpleRNN\nfrom keras.layers import Embedding\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.269190Z","iopub.execute_input":"2022-04-03T08:44:21.269494Z","iopub.status.idle":"2022-04-03T08:44:21.275404Z","shell.execute_reply.started":"2022-04-03T08:44:21.269466Z","shell.execute_reply":"2022-04-03T08:44:21.274789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = \"\"\" Jack and Jill went up the hill .\\n To fetch a pail of water .\\n Jack fell down and broke his crown .\\n And Jill came tumbling after .\"\"\"\nprint(data)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.294599Z","iopub.execute_input":"2022-04-03T08:44:21.295310Z","iopub.status.idle":"2022-04-03T08:44:21.300438Z","shell.execute_reply.started":"2022-04-03T08:44:21.295266Z","shell.execute_reply":"2022-04-03T08:44:21.299688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_splitted=data.split('\\n')\ntokenizer=Tokenizer(filters='!\"#$%&()*+,-/:;<=>?@[\\]^_`{|}~')\ntokenizer.fit_on_texts(data_splitted)\n# +1 for 0' character at starting \nvocab_size = len(tokenizer.word_index) + 1\nsequences=tokenizer.texts_to_sequences(data_splitted)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.340139Z","iopub.execute_input":"2022-04-03T08:44:21.340869Z","iopub.status.idle":"2022-04-03T08:44:21.346941Z","shell.execute_reply.started":"2022-04-03T08:44:21.340811Z","shell.execute_reply":"2022-04-03T08:44:21.345906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('','\\t\\t\\tVocab Dictionary:',tokenizer.word_index,'',sep='\\n'+('-'*100)+'\\n')\nprint('',\"\\t\\tSentences:\", *data_splitted,'',sep='\\n'+('-'*50)+'\\n')\nprint('',\"\\tSequences of words in Sentences:\",*sequences,'',sep='\\n'+('-'*50)+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.369133Z","iopub.execute_input":"2022-04-03T08:44:21.369386Z","iopub.status.idle":"2022-04-03T08:44:21.378835Z","shell.execute_reply.started":"2022-04-03T08:44:21.369355Z","shell.execute_reply":"2022-04-03T08:44:21.377609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=list()\ny=list()\n\nfor i in range(len(sequences)):\n  X.insert(i,sequences[i][:-1])\n  y.insert(i,sequences[i])\n\nprint('',\"\\t\\tTraining X:\", *X,'', \"\\t\\tGround Truth Y: \",*y,'',sep='\\n'+('-'*50)+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.404374Z","iopub.execute_input":"2022-04-03T08:44:21.404666Z","iopub.status.idle":"2022-04-03T08:44:21.414725Z","shell.execute_reply.started":"2022-04-03T08:44:21.404635Z","shell.execute_reply":"2022-04-03T08:44:21.413825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Padding X train to make it equal length/ symmetric\nlength = max([len(sequence) for sequence in X])\n# length+1 to have 0 as the first input\nX=pad_sequences(X,maxlen=length+1,padding='pre')\ny=pad_sequences(y,maxlen=length+1,padding='pre')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.439476Z","iopub.execute_input":"2022-04-03T08:44:21.439740Z","iopub.status.idle":"2022-04-03T08:44:21.445973Z","shell.execute_reply.started":"2022-04-03T08:44:21.439711Z","shell.execute_reply":"2022-04-03T08:44:21.444438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('',\"\\t\\t Training X:\",X,f'Shape: {X.shape}','',sep='\\n'+('-'*50)+'\\n')\nprint('',\"\\t\\tGround Truth Y:\",y,f'Shape: {y.shape}','',sep='\\n'+('-'*50)+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.479467Z","iopub.execute_input":"2022-04-03T08:44:21.479789Z","iopub.status.idle":"2022-04-03T08:44:21.488657Z","shell.execute_reply.started":"2022-04-03T08:44:21.479760Z","shell.execute_reply":"2022-04-03T08:44:21.487655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One Hot Encodeing /convert Y to one hot vector/s\ny=keras.utils.to_categorical(y,num_classes=vocab_size) ","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.514349Z","iopub.execute_input":"2022-04-03T08:44:21.514629Z","iopub.status.idle":"2022-04-03T08:44:21.520092Z","shell.execute_reply.started":"2022-04-03T08:44:21.514601Z","shell.execute_reply":"2022-04-03T08:44:21.519403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Sequential()\nmodel.add(Embedding(input_dim=vocab_size, output_dim=10))\nmodel.add(SimpleRNN(units=100, return_sequences=True))\nmodel.add(Dense(units=vocab_size,activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:21.909717Z","iopub.execute_input":"2022-04-03T08:44:21.909992Z","iopub.status.idle":"2022-04-03T08:44:21.997729Z","shell.execute_reply.started":"2022-04-03T08:44:21.909961Z","shell.execute_reply":"2022-04-03T08:44:21.996203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:22.001751Z","iopub.execute_input":"2022-04-03T08:44:22.002023Z","iopub.status.idle":"2022-04-03T08:44:22.009831Z","shell.execute_reply.started":"2022-04-03T08:44:22.001994Z","shell.execute_reply":"2022-04-03T08:44:22.008780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(str(model.layers[0].input_shape))","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:22.011228Z","iopub.execute_input":"2022-04-03T08:44:22.012734Z","iopub.status.idle":"2022-04-03T08:44:22.023444Z","shell.execute_reply.started":"2022-04-03T08:44:22.012682Z","shell.execute_reply":"2022-04-03T08:44:22.022132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('',\"\\tInput Shape of all Layers:\",'',sep='\\n'+(50*'=')+'\\n')\nfor i in model.layers:\n    l=i.name\n    k=i.input_shape\n    print(f'{l}:',(25-len(l))*' ',k,end='\\n'+(50*'=')+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:22.024954Z","iopub.execute_input":"2022-04-03T08:44:22.025165Z","iopub.status.idle":"2022-04-03T08:44:22.039444Z","shell.execute_reply.started":"2022-04-03T08:44:22.025137Z","shell.execute_reply":"2022-04-03T08:44:22.038626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=\"rmsprop\", loss=\"categorical_crossentropy\", metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:22.040700Z","iopub.execute_input":"2022-04-03T08:44:22.040914Z","iopub.status.idle":"2022-04-03T08:44:22.053127Z","shell.execute_reply.started":"2022-04-03T08:44:22.040886Z","shell.execute_reply":"2022-04-03T08:44:22.051969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=model.fit(X, y, epochs=200, verbose=0)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:22.055004Z","iopub.execute_input":"2022-04-03T08:44:22.055367Z","iopub.status.idle":"2022-04-03T08:44:24.037198Z","shell.execute_reply.started":"2022-04-03T08:44:22.055339Z","shell.execute_reply":"2022-04-03T08:44:24.036133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"historydf=pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:24.038817Z","iopub.execute_input":"2022-04-03T08:44:24.039273Z","iopub.status.idle":"2022-04-03T08:44:24.043941Z","shell.execute_reply.started":"2022-04-03T08:44:24.039235Z","shell.execute_reply":"2022-04-03T08:44:24.042851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"historydf.plot()","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:24.045140Z","iopub.execute_input":"2022-04-03T08:44:24.045511Z","iopub.status.idle":"2022-04-03T08:44:24.266891Z","shell.execute_reply.started":"2022-04-03T08:44:24.045481Z","shell.execute_reply":"2022-04-03T08:44:24.265986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sample_all_seq_wo_seed(model, tokenizer, n_words, vocab_size): #all the words are sampled\n  encoded=list()\n  in_text = ''\n  # generate a fixed number of words = n_words\n  for i in range(n_words):\n    # encode the text as integer\n    encoded = tokenizer.texts_to_sequences([in_text])[0] # for words not in the vocab it returns []\n    #print(\"i:\", i, \"Encoded:\",encoded)\n    encoded.insert(0,0)\n    encoded=np.array(encoded)\n    encoded=np.reshape(encoded,newshape=(1,-1))\n    #print(\"Encoded:\",encoded, encoded.shape)\n    # predict probability and sample a word from vocab\n    if i == 0:\n      prob = model.predict(encoded, verbose=0)\n      #print(\"Prob:\", prob, prob.shape)\n      yhat=0\n    # Choose Random Sample for y^0 (i.e. for starting case only)\n      while yhat == 0:\n        yhat=np.random.choice(range(vocab_size),p=prob.ravel())\n      yhat=np.array([yhat]).reshape((1,-1))\n      #print(f\"Y\\u0302{i}:\", yhat, yhat.shape)\n    else:\n      prob = model.predict(encoded, verbose=0)\n      #print(\"i=\", i, \"Prob:\", prob, prob.shape)\n      yhat=np.append(yhat,0)  #just creating space for the next yhat\n      yhat=np.reshape(yhat,newshape=(1,-1))\n      while yhat[0,i] == 0:\n        yhat[0,i]=np.random.choice(range(vocab_size),p=prob[0,i].ravel())\n      #print(f\"Y\\u0302{i}:\", yhat, yhat.shape)\n        \n    \n    # map predicted word index to word\n    out_word = ''\n    for word, index in tokenizer.word_index.items():\n      if index == yhat[0,i]:\n        out_word = word\n        #print(\"index:\", index, \"out_word:\", out_word)\n        break\n    \n    # append to input\n    in_text = in_text + out_word + ' '\n    # for loop ends here\n  return in_text","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:24.268884Z","iopub.execute_input":"2022-04-03T08:44:24.269116Z","iopub.status.idle":"2022-04-03T08:44:24.284303Z","shell.execute_reply.started":"2022-04-03T08:44:24.269088Z","shell.execute_reply":"2022-04-03T08:44:24.282999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('','\\t\\tSampling without seed',f'Generate 8 word sentence:\\n {sample_all_seq_wo_seed(model, tokenizer, 8, vocab_size)}','',sep='\\n'+(50*'-')+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:24.285701Z","iopub.execute_input":"2022-04-03T08:44:24.285925Z","iopub.status.idle":"2022-04-03T08:44:25.555444Z","shell.execute_reply.started":"2022-04-03T08:44:24.285897Z","shell.execute_reply":"2022-04-03T08:44:25.554663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('','\\t\\tSampling without seed',f'Generate 8 word sentence:\\n {sample_all_seq_wo_seed(model, tokenizer, 7, vocab_size)}','',sep='\\n'+(50*'-')+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:25.556760Z","iopub.execute_input":"2022-04-03T08:44:25.556974Z","iopub.status.idle":"2022-04-03T08:44:26.012884Z","shell.execute_reply.started":"2022-04-03T08:44:25.556945Z","shell.execute_reply":"2022-04-03T08:44:26.012028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('','\\t\\tSampling without seed',f'Generate 8 word sentence:\\n {sample_all_seq_wo_seed(model, tokenizer, 12, vocab_size)}','',sep='\\n'+(50*'-')+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:26.015044Z","iopub.execute_input":"2022-04-03T08:44:26.015372Z","iopub.status.idle":"2022-04-03T08:44:26.725348Z","shell.execute_reply.started":"2022-04-03T08:44:26.015334Z","shell.execute_reply":"2022-04-03T08:44:26.723894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('','\\t\\tSampling without seed',f'Generate 8 word sentence:\\n {sample_all_seq_wo_seed(model, tokenizer, 15, vocab_size)}','',sep='\\n'+(50*'-')+'\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:45:39.821427Z","iopub.execute_input":"2022-04-03T08:45:39.821717Z","iopub.status.idle":"2022-04-03T08:45:40.821531Z","shell.execute_reply.started":"2022-04-03T08:45:39.821687Z","shell.execute_reply":"2022-04-03T08:45:40.820466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prob_of_input_sentence(model, tokenizer, sentence):\n  print(\"Input Sentence:\", sentence)\n  encoded=tokenizer.texts_to_sequences([sentence])[0]\n  print(\"encoded before insert:\", encoded)\n  encoded.insert(0,0)\n  encoded=np.array(encoded)\n  encoded=np.reshape(encoded,newshape=(1,-1))\n  print(\"Encoded:\", encoded, encoded.shape)\n  prob=model.predict(encoded, verbose=0)\n  print(\"Prob:\", prob, prob.shape)\n  probability=1\n  for i in range(prob.shape[1]-1):\n    probability = probability * prob[0,i,encoded[0,i+1]]\n  print(\"Probability of Sentence\", \"\\\"\", sentence, \"\\\"\", \"is:\", probability)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:44:27.342053Z","iopub.execute_input":"2022-04-03T08:44:27.342297Z","iopub.status.idle":"2022-04-03T08:44:27.351598Z","shell.execute_reply.started":"2022-04-03T08:44:27.342262Z","shell.execute_reply":"2022-04-03T08:44:27.350440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"-------------------Probability of Input Sentence------------------------------\")  \nprob_of_input_sentence(model, tokenizer, \"Jack and Jill Went up the hill\")\nprob_of_input_sentence(model, tokenizer, \"jack and jill went up the hill .\")\n\nprob_of_input_sentence(model, tokenizer, \"jack fell down and broke\")\nprob_of_input_sentence(model, tokenizer, \"and jill came tumbling after .\")\nprint(\"-------------------------------------------------------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2022-04-03T08:47:18.579142Z","iopub.execute_input":"2022-04-03T08:47:18.579421Z","iopub.status.idle":"2022-04-03T08:47:18.853109Z","shell.execute_reply.started":"2022-04-03T08:47:18.579389Z","shell.execute_reply":"2022-04-03T08:47:18.851809Z"},"trusted":true},"execution_count":null,"outputs":[]}]}