{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U git+https://github.com/huggingface/transformers.git\n!pip install -U git+https://github.com/huggingface/accelerate.git","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-23T15:54:45.867242Z","iopub.execute_input":"2023-06-23T15:54:45.868195Z","iopub.status.idle":"2023-06-23T15:56:08.088520Z","shell.execute_reply.started":"2023-06-23T15:54:45.868160Z","shell.execute_reply":"2023-06-23T15:56:08.087549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Credits to https://stackoverflow.com/questions/76363436/cannot-import-name-partialstate-from-accelerate-when-using-huggingface-pipel for this fix.","metadata":{}},{"cell_type":"code","source":"# standard\nimport numpy as np\nimport pandas as pd\n\n# NLP stuff\nfrom bs4 import BeautifulSoup\nimport textwrap\nimport nltk\nfrom more_itertools import windowed\nfrom transformers import pipeline, AutoTokenizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-23T16:08:36.262580Z","iopub.execute_input":"2023-06-23T16:08:36.263033Z","iopub.status.idle":"2023-06-23T16:08:36.269270Z","shell.execute_reply.started":"2023-06-23T16:08:36.262996Z","shell.execute_reply":"2023-06-23T16:08:36.268044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file overview\n!ls -l '../input/2023-kaggle-ai-report'","metadata":{"execution":{"iopub.status.busy":"2023-06-23T15:56:26.784728Z","iopub.execute_input":"2023-06-23T15:56:26.785460Z","iopub.status.idle":"2023-06-23T15:56:27.904022Z","shell.execute_reply.started":"2023-06-23T15:56:26.785418Z","shell.execute_reply":"2023-06-23T15:56:27.902893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load Kaggle writeups data\ndf_writeups = pd.read_csv('../input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')\ndf_writeups.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T15:56:27.906937Z","iopub.execute_input":"2023-06-23T15:56:27.907871Z","iopub.status.idle":"2023-06-23T15:56:28.319835Z","shell.execute_reply.started":"2023-06-23T15:56:27.907829Z","shell.execute_reply":"2023-06-23T15:56:28.318700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overview\ndf_writeups.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T16:01:25.396503Z","iopub.execute_input":"2023-06-23T16:01:25.397400Z","iopub.status.idle":"2023-06-23T16:01:25.417012Z","shell.execute_reply.started":"2023-06-23T16:01:25.397362Z","shell.execute_reply":"2023-06-23T16:01:25.415556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert Writeup column to string\ndf_writeups.Writeup = df_writeups.Writeup.astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T16:03:05.308036Z","iopub.execute_input":"2023-06-23T16:03:05.308455Z","iopub.status.idle":"2023-06-23T16:03:05.315548Z","shell.execute_reply.started":"2023-06-23T16:03:05.308423Z","shell.execute_reply":"2023-06-23T16:03:05.314337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate lengths of writeups\ndf_writeups['length'] = df_writeups.Writeup.apply(len)\ndf_writeups['length'].describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T16:03:52.674499Z","iopub.execute_input":"2023-06-23T16:03:52.674909Z","iopub.status.idle":"2023-06-23T16:03:52.691093Z","shell.execute_reply.started":"2023-06-23T16:03:52.674876Z","shell.execute_reply":"2023-06-23T16:03:52.689934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# init summarizer\nMODEL_PATH = 'sshleifer/distilbart-cnn-12-6'\nsummarizer = pipeline('summarization', model=MODEL_PATH)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T15:56:28.321478Z","iopub.execute_input":"2023-06-23T15:56:28.322166Z","iopub.status.idle":"2023-06-23T15:56:43.418397Z","shell.execute_reply.started":"2023-06-23T15:56:28.322133Z","shell.execute_reply":"2023-06-23T15:56:43.417071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function for cleaning text\n# credits: https://medium.com/@lokaregns/text-summarization-with-hugging-face-transformers-a-beginners-guide-9e6c319bb5ed\ndef wrap(x):\n  return textwrap.fill(x, replace_whitespace = True, \n                       fix_sentence_endings = True)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T15:56:43.420224Z","iopub.execute_input":"2023-06-23T15:56:43.420646Z","iopub.status.idle":"2023-06-23T15:56:43.431138Z","shell.execute_reply.started":"2023-06-23T15:56:43.420604Z","shell.execute_reply":"2023-06-23T15:56:43.427746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Select a specific competition:","metadata":{}},{"cell_type":"code","source":"# select competition via name\nselect_comp = 'M5 Forecasting - Accuracy'\ndf_select = df_writeups[df_writeups['Title of Competition']==select_comp].reset_index()\ndf_select.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-23T16:31:33.036267Z","iopub.execute_input":"2023-06-23T16:31:33.037014Z","iopub.status.idle":"2023-06-23T16:31:33.058337Z","shell.execute_reply.started":"2023-06-23T16:31:33.036964Z","shell.execute_reply":"2023-06-23T16:31:33.057014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Select a specific write-up:","metadata":{}},{"cell_type":"code","source":"# select specific item via index\nindex_select = 10\n# wrap in HTML format using BeautifulSoup\nselect_content = BeautifulSoup(df_select.Writeup[index_select])\nselect_content","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-23T17:16:35.626758Z","iopub.execute_input":"2023-06-23T17:16:35.627180Z","iopub.status.idle":"2023-06-23T17:16:35.649225Z","shell.execute_reply.started":"2023-06-23T17:16:35.627150Z","shell.execute_reply":"2023-06-23T17:16:35.648060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract pure text\nselect_text = wrap(select_content.text)\nprint(select_text)","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:16:39.524100Z","iopub.execute_input":"2023-06-23T17:16:39.524479Z","iopub.status.idle":"2023-06-23T17:16:39.538359Z","shell.execute_reply.started":"2023-06-23T17:16:39.524451Z","shell.execute_reply":"2023-06-23T17:16:39.537200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check tokenizer\ntokenizer(select_text)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-23T17:16:43.515701Z","iopub.execute_input":"2023-06-23T17:16:43.516144Z","iopub.status.idle":"2023-06-23T17:16:43.533033Z","shell.execute_reply.started":"2023-06-23T17:16:43.516110Z","shell.execute_reply":"2023-06-23T17:16:43.531764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize\nif len(select_text)<4000:\n    summary = summarizer(select_text)\n    # show result\n    print(summary[0]['summary_text'])\nelse:\n    # split in parts\n    # credits: https://stackoverflow.com/questions/4576077/how-can-i-split-a-text-into-sentences\n    print('> Text too long, need to split...')\n    split_length = 20 # number of sentences per part\n    summaries = ''\n    elements = nltk.tokenize.sent_tokenize(select_text)\n    segments = windowed(elements, n=split_length, step=split_length)\n    text_splits = []\n    for seg in segments:\n              txt = ' '.join([t for t in seg if t])\n              if len(txt) > 0:\n                    text_splits.append(txt)\n                    \n    for ts in text_splits:\n        # summarize each part\n        summary_part = summarizer(ts)\n        if summaries == '':\n            summaries = summary_part[0]['summary_text']\n        else: \n            summaries = summaries + '\\n\\n' + summary_part[0]['summary_text']\n            \n    print('\\n> Concatenation of partial summaries:\\n')\n    print(summaries)\n    \n    print('\\n> Summary of partial summaries:')\n    summary_of_summaries = summarizer(summaries)\n    print(summary_of_summaries[0]['summary_text'])","metadata":{"execution":{"iopub.status.busy":"2023-06-23T17:16:46.779483Z","iopub.execute_input":"2023-06-23T17:16:46.779902Z","iopub.status.idle":"2023-06-23T17:17:23.325303Z","shell.execute_reply.started":"2023-06-23T17:16:46.779869Z","shell.execute_reply":"2023-06-23T17:17:23.324087Z"},"trusted":true},"execution_count":null,"outputs":[]}]}