{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on May 28, 2023. By Marília Prata, mpwolke.","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\nfrom pathlib import Path\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-28T22:58:24.968449Z","iopub.execute_input":"2023-05-28T22:58:24.969398Z","iopub.status.idle":"2023-05-28T22:58:27.113690Z","shell.execute_reply.started":"2023-05-28T22:58:24.969360Z","shell.execute_reply":"2023-05-28T22:58:27.112515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Connectomics\n\n\"Connectomics is the study of the connections in the nervous system. Connectomics explores how neurons in the brain and nervous system are connected and interact with each other to produce behavior and cognition. This includes both the structural connections between neurons, as well as the functional connections and communication between them.\"\n\nhttps://www.o8t.com/blog/connectomics","metadata":{}},{"cell_type":"markdown","source":"#What is Connectomics?\n\n<iframe width=\"684\" height=\"385\" src=\"https://www.youtube.com/embed/ZRlt_s02tps\" title=\"What is Connectomics?\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" allowfullscreen></iframe>\n\nhttps://www.youtube.com/watch?v=ZRlt_s02tps","metadata":{}},{"cell_type":"code","source":"import tarfile\n\ntar = tarfile.open(\"../input/connectomics/normal-1.tgz\", \"r:gz\")\nfor member in tar.getmembers():\n     f = tar.extractfile(member)\n     if f is not None:\n         content = f.read()","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:58:39.573859Z","iopub.execute_input":"2023-05-28T22:58:39.574272Z","iopub.status.idle":"2023-05-28T22:58:55.664395Z","shell.execute_reply.started":"2023-05-28T22:58:39.574238Z","shell.execute_reply":"2023-05-28T22:58:55.663218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Tarfile module\n\ngetnames(self): Return the members of the archive as a list of their names. It has the same order as the list returned by getmembers().\n\nhttps://stackoverflow.com/questions/2018512/reading-tar-file-contents-without-untarring-it-in-python-script","metadata":{}},{"cell_type":"code","source":"tar.getnames()","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:01.938385Z","iopub.execute_input":"2023-05-28T22:59:01.938788Z","iopub.status.idle":"2023-05-28T22:59:01.946431Z","shell.execute_reply.started":"2023-05-28T22:59:01.938755Z","shell.execute_reply":"2023-05-28T22:59:01.945497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tar.getmembers()","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:06.603981Z","iopub.execute_input":"2023-05-28T22:59:06.604413Z","iopub.status.idle":"2023-05-28T22:59:06.612229Z","shell.execute_reply.started":"2023-05-28T22:59:06.604379Z","shell.execute_reply":"2023-05-28T22:59:06.611128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tar.list(verbose=True, members=None)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:11.705325Z","iopub.execute_input":"2023-05-28T22:59:11.705725Z","iopub.status.idle":"2023-05-28T22:59:11.712359Z","shell.execute_reply.started":"2023-05-28T22:59:11.705690Z","shell.execute_reply":"2023-05-28T22:59:11.711224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Extractall\n\ntar.extractall(path=\".\", members=None, *, numeric_owner=False)\n\nhttps://docs.python.org/3.7/library/tarfile.html","metadata":{}},{"cell_type":"code","source":"tar.extractall(path=\".\", members=None, numeric_owner=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:22.628255Z","iopub.execute_input":"2023-05-28T22:59:22.628717Z","iopub.status.idle":"2023-05-28T22:59:30.186114Z","shell.execute_reply.started":"2023-05-28T22:59:22.628686Z","shell.execute_reply":"2023-05-28T22:59:30.185082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tar.getmembers()","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:32.155297Z","iopub.execute_input":"2023-05-28T22:59:32.155663Z","iopub.status.idle":"2023-05-28T22:59:32.161722Z","shell.execute_reply.started":"2023-05-28T22:59:32.155636Z","shell.execute_reply":"2023-05-28T22:59:32.160883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nfrom pandas import DataFrame\nfrom nltk.stem import WordNetLemmatizer\nimport operator\nfrom nltk import pos_tag ","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:37.708526Z","iopub.execute_input":"2023-05-28T22:59:37.708893Z","iopub.status.idle":"2023-05-28T22:59:38.510341Z","shell.execute_reply.started":"2023-05-28T22:59:37.708865Z","shell.execute_reply":"2023-05-28T22:59:38.509345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If you would like to work with the raw text you can use 'moby_raw'\nwith open('../input/connectomics/datasets.txt', 'r', encoding='latin-1') as f:\n    moby_raw = f.read()\n    \n# If you would like to work with the novel in nltk.Text format you can use 'text1'\nmoby_tokens = nltk.word_tokenize(moby_raw)\ntext1 = nltk.Text(moby_tokens)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:47.079572Z","iopub.execute_input":"2023-05-28T22:59:47.079973Z","iopub.status.idle":"2023-05-28T22:59:47.106316Z","shell.execute_reply.started":"2023-05-28T22:59:47.079943Z","shell.execute_reply":"2023-05-28T22:59:47.105199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = sorted([(freq, token) for token, freq in text1.vocab().items() if len(token) > 5 and freq > 11 ], key=operator.itemgetter(0), reverse=True) \nprint(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:52.275396Z","iopub.execute_input":"2023-05-28T22:59:52.275922Z","iopub.status.idle":"2023-05-28T22:59:52.287406Z","shell.execute_reply.started":"2023-05-28T22:59:52.275878Z","shell.execute_reply":"2023-05-28T22:59:52.286050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Frequent words","metadata":{}},{"cell_type":"code","source":"df = DataFrame (x,columns=['freq','words'])\nprint (df)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T22:59:58.322959Z","iopub.execute_input":"2023-05-28T22:59:58.323371Z","iopub.status.idle":"2023-05-28T22:59:58.347966Z","shell.execute_reply.started":"2023-05-28T22:59:58.323338Z","shell.execute_reply":"2023-05-28T22:59:58.346422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tomic = open(\"../input/connectomics/datasets.txt\", \"r\").read()\nprint (tomic[:80])\n\n# http://www.gutenberg.org/cache/epub/730/pg730.txt\n#oliver_twist = open(\"../input/dickens/dickens/pg730.txt\", \"r\").read()\n#print (oliver_twist[:64])\n\n#christmas_carol = open(\"../input/dickens/dickens/pg19337.txt\", \"r\").read()\n#print (christmas_carol[:69])\n\ntext = tomic","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:05:14.054041Z","iopub.execute_input":"2023-05-28T23:05:14.054419Z","iopub.status.idle":"2023-05-28T23:05:14.061696Z","shell.execute_reply.started":"2023-05-28T23:05:14.054391Z","shell.execute_reply":"2023-05-28T23:05:14.060670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Carlos Aguayo https://www.kaggle.com/code/carlosaguayo/predicting-the-next-word-using-lstm\n\nimport string\n\n# turn a doc into clean tokens\ndef clean_doc(doc):\n    # replace '--' with a space ' '\n    doc = doc.replace('--', ' ')\n    # split into tokens by white space\n    tokens = doc.split()\n    # remove punctuation from each token\n    tokens = [' ' if w in string.punctuation else w for w in tokens]\n    # remove remaining tokens that are not alphabetic\n    tokens = [word for word in tokens if word.isalpha()]\n    # make lower case\n    tokens = [word.lower() for word in tokens]\n    return tokens\n \ntokens = clean_doc(text)\n\nnumber_of_unique_tokens = len(set(tokens))\n\nprint('Total Tokens: %d' % len(tokens))\nprint('Unique Tokens: %d' % number_of_unique_tokens)\nprint('These are the first 200 tokens: %s' % tokens[:200])\n\n# A key design decision is how long the input sequences should be. \n# They need to be long enough to allow the model to learn the context for the words to predict. \n# This input length will also define the length of seed text used to generate new sequences \n# when we use the model.\n# There is no correct answer. With enough time and resources, we could explore the ability of \n# the model to learn with differently sized input sequences.\n\nsequence_length = 2\n\n# organize into sequences of tokens of input words plus one output word\nlength = sequence_length + 1\nsequences = list()\nfor i in range(length, len(tokens)):\n    # select sequence of tokens\n    seq = tokens[i-length:i]\n    # convert into a line\n    line = ' '.join(seq)\n    # store\n    sequences.append(line)\n\nprint ('Total Sequences: %d' % len(sequences))\nprint ('This is the first sequence: {0}'.format(sequences[0]))","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:05:57.215329Z","iopub.execute_input":"2023-05-28T23:05:57.215727Z","iopub.status.idle":"2023-05-28T23:05:57.226756Z","shell.execute_reply.started":"2023-05-28T23:05:57.215695Z","shell.execute_reply":"2023-05-28T23:05:57.225547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import LSTM\nfrom keras.layers import Embedding","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:06:54.377856Z","iopub.execute_input":"2023-05-28T23:06:54.378231Z","iopub.status.idle":"2023-05-28T23:07:01.795653Z","shell.execute_reply.started":"2023-05-28T23:06:54.378201Z","shell.execute_reply":"2023-05-28T23:07:01.794676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Carlos Aguayo https://www.kaggle.com/code/carlosaguayo/predicting-the-next-word-using-lstm\n\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(sequences)\nsequences = tokenizer.texts_to_sequences(sequences)\n# vocab_size = len(tokenizer.word_index) + 1\nvocab_size = number_of_unique_tokens + 1\n \nsequences0 = np.array(sequences)\nX, y = sequences0[:,:-1], sequences0[:,-1]\ny = to_categorical(y, num_classes=vocab_size)\n\n# The learned embedding needs to know how many dimensions will be used to represent each word. \n# That is, the size of the embedding vector space. That is, the size of the embedding vector space.\n# Common values are 50, 100, and 300. Consider testing smaller or larger values.\ndimensions_to_represent_word = 100\n \nmodel = Sequential()\nmodel.add(Embedding(vocab_size, sequence_length, input_length=sequence_length))\n# We will use a two LSTM hidden layers with 100 memory cells each. \n# More memory cells and a deeper network may achieve better results.\nmodel.add(LSTM(100, return_sequences=True))\nmodel.add(LSTM(100))\nmodel.add(Dense(100, activation='relu'))\nmodel.add(Dense(vocab_size, activation='softmax'))\nprint(model.summary())\n\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n# Training may take a few hours on modern hardware without GPUs. \n# You can speed it up with a larger batch size and/or fewer training epochs.\n# model.fit(X, y, batch_size=128, epochs=100)\nmodel.fit(X, y, batch_size=128, epochs=100)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:07:19.458658Z","iopub.execute_input":"2023-05-28T23:07:19.459587Z","iopub.status.idle":"2023-05-28T23:07:29.960271Z","shell.execute_reply.started":"2023-05-28T23:07:19.459552Z","shell.execute_reply":"2023-05-28T23:07:29.959359Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#That was fast!","metadata":{}},{"cell_type":"code","source":"#Carlos Aguayo https://www.kaggle.com/code/carlosaguayo/predicting-the-next-word-using-lstm\n\nprint (X.shape)\nprediction = model.predict(X[0].reshape(1,sequence_length))\nprint (prediction.shape)\nprint (prediction)","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:07:56.089912Z","iopub.execute_input":"2023-05-28T23:07:56.090270Z","iopub.status.idle":"2023-05-28T23:07:57.022417Z","shell.execute_reply.started":"2023-05-28T23:07:56.090242Z","shell.execute_reply":"2023-05-28T23:07:57.021442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Below (test) Must use the tokens found on input 17!","metadata":{}},{"cell_type":"code","source":"#Carlos Aguayo https://www.kaggle.com/code/carlosaguayo/predicting-the-next-word-using-lstm\n\ntest = ['for', 'six', 'small', 'networks', 'with', 'each', 'network', 'has', 'the', 'same', 'connectivity', 'degree', 'but', 'different', 'levels', 'of', 'clustering', 'intended', 'for', 'fast', 'checks', 'of', 'the', 'positional', 'data', 'and', 'truth']\n\nfor t in test:\n    example = tokenizer.texts_to_sequences([t])\n    prediction = model.predict(np.array(example))\n    predicted_word = np.argmax(prediction)\n    reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))  # https://stackoverflow.com/a/43927939/246508\n    print (\"{0} -> {1}\".format(t, reverse_word_map[predicted_word]))","metadata":{"execution":{"iopub.status.busy":"2023-05-28T23:19:52.562707Z","iopub.execute_input":"2023-05-28T23:19:52.563114Z","iopub.status.idle":"2023-05-28T23:19:54.582826Z","shell.execute_reply.started":"2023-05-28T23:19:52.563083Z","shell.execute_reply":"2023-05-28T23:19:54.581743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Acknowledgements:\n\nCarlos Aguayo https://www.kaggle.com/code/carlosaguayo/predicting-the-next-word-using-lstm\n\nMPWolke https://www.kaggle.com/code/mpwolke/cry-baby-with-that-lstm\n\nMPWolke https://www.kaggle.com/code/mpwolke/read-tgz-file","metadata":{}}]}