{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Using LLMs to Extract Structured Data\n\nThis notebook demonstrate how to extract structured data from a vast collection of unstructured Kaggle writeups. The process is described in [this article](https://wandb.ai/darek/llmapps/reports/Using-LLMs-to-Extract-Structured-Data-OpenAI-Function-Calling-in-Action--Vmlldzo0Nzc0MzQ3).\n\nThe full dataset obtained with this process is available [here](https://www.kaggle.com/datasets/thedrcat/kaggle-winning-solutions-methods).","metadata":{}},{"cell_type":"markdown","source":"# Scraping \n\nThere's already a wealth of information collected on the amazing [Kaggle Solutions](https://farid.one/kaggle-solutions/) website, so let's start there - we'll scrape all the top solution description links along with the competition data. ","metadata":{}},{"cell_type":"code","source":"from bs4 import BeautifulSoup\nimport requests\n\nurl = \"https://farid.one/kaggle-solutions/\"\n\nresponse = requests.get(url)\nhtml = response.text\n\nsoup = BeautifulSoup(html, 'html.parser')\n\nlinks = []\nfor li_tag in soup.find_all('li', class_='secondary'):\n    try:\n        link = li_tag.find('a')['href']\n        if 'discussion' in link:\n            place = li_tag.find('img').next_sibling.strip().split(' ')[0]\n            competition_name = li_tag.find_previous('b').text.split('. ')[-1].strip()\n\n            # Find the competition details element\n            details_element = li_tag.find_previous('td', class_='td-details')\n\n            # Find the prize element within the competition details\n            prize_element = details_element.find('p', string=lambda s: s and 'prize' in s.lower())\n            prize = prize_element.text.split(':')[-1].strip() if prize_element else ''\n\n            # Find the team element within the competition details\n            team_element = details_element.find('p', string=lambda s: s and 'team' in s.lower())\n            team = team_element.text.split(':')[-1].strip() if team_element else ''\n\n            # Find the kind element within the competition details\n            kind_element = details_element.find('p', string=lambda s: s and 'kind' in s.lower())\n            kind = kind_element.text.split(':')[-1].strip() if kind_element else ''\n\n            # Find the metric element within the competition details\n            metric_element = details_element.find('p', string=lambda s: s and 'metric' in s.lower())\n            metric = metric_element.text.split(':')[-1].strip() if metric_element else ''\n\n            # Find the year element within the competition details\n            year_element = details_element.find('p', string=lambda s: s and 'year' in s.lower())\n            year = year_element.text.split(':')[-1].strip() if year_element else ''\n\n            info_tuple = (link, int(place[:-2]), competition_name, prize, team, kind, metric, year)\n            links.append(info_tuple)\n    except:\n        pass\n\nprint(len(links))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-03T09:29:45.307173Z","iopub.execute_input":"2023-07-03T09:29:45.307629Z","iopub.status.idle":"2023-07-03T09:29:49.184622Z","shell.execute_reply.started":"2023-07-03T09:29:45.307595Z","shell.execute_reply":"2023-07-03T09:29:49.183208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nscraped = pd.DataFrame(data=links, columns=['link', 'place', 'competition_name', 'prize', 'team', 'kind', 'metric', 'year'])","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:30:05.798686Z","iopub.execute_input":"2023-07-03T09:30:05.799555Z","iopub.status.idle":"2023-07-03T09:30:05.818920Z","shell.execute_reply.started":"2023-07-03T09:30:05.799509Z","shell.execute_reply":"2023-07-03T09:30:05.817392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining with Kaggle Writeups\n\nKaggle doesn't like scraping discussion posts, but fortunately they provided us a bunch of writeups in the competition data. Let's merge it with the scraped data. ","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:30:41.889726Z","iopub.execute_input":"2023-07-03T09:30:41.890320Z","iopub.status.idle":"2023-07-03T09:30:42.558194Z","shell.execute_reply.started":"2023-07-03T09:30:41.890261Z","shell.execute_reply":"2023-07-03T09:30:42.556923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the URLs follow different conventions so let's use discussion number to join both tables\nimport numpy as np\n\nscraped['nm'] = scraped.link.apply(lambda x: x.split('/')[-1])\ndf['nm'] = df['Writeup URL'].apply(lambda x: x.split('/')[-1])\nwriteup_mapping = dict(zip(df.nm, df['Writeup']))\nscraped['writeup'] = scraped.nm.apply(lambda x: writeup_mapping[x] if x in writeup_mapping else np.nan)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:32:18.052262Z","iopub.execute_input":"2023-07-03T09:32:18.052777Z","iopub.status.idle":"2023-07-03T09:32:18.082820Z","shell.execute_reply.started":"2023-07-03T09:32:18.052738Z","shell.execute_reply":"2023-07-03T09:32:18.081696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# not all the scraped solutions are listed in the writeups csv\nlen(scraped), len(scraped.dropna())","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:32:47.550805Z","iopub.execute_input":"2023-07-03T09:32:47.551353Z","iopub.status.idle":"2023-07-03T09:32:47.570277Z","shell.execute_reply.started":"2023-07-03T09:32:47.551310Z","shell.execute_reply":"2023-07-03T09:32:47.568693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writeups = scraped.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:33:06.915749Z","iopub.execute_input":"2023-07-03T09:33:06.916432Z","iopub.status.idle":"2023-07-03T09:33:06.940969Z","shell.execute_reply.started":"2023-07-03T09:33:06.916351Z","shell.execute_reply":"2023-07-03T09:33:06.939063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using LLM's for parsing unstructured writeups\n\nWe will first use OpenAI Function Calling to identify ML methods in solution descriptions. You will need OpenAI API key saved in the notebook secrets for this. Here, we'll do it for a small sample only. ","metadata":{}},{"cell_type":"code","source":"# let's install tiktoken to count number of tokens and openai\n!pip install tiktoken openai -qqq","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:43.656933Z","iopub.execute_input":"2023-07-03T09:38:43.657445Z","iopub.status.idle":"2023-07-03T09:38:58.313853Z","shell.execute_reply.started":"2023-07-03T09:38:43.657404Z","shell.execute_reply":"2023-07-03T09:38:58.312370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tiktoken\nGPT_MODEL = \"gpt-3.5-turbo-0613\"\nencoding = tiktoken.encoding_for_model(GPT_MODEL)\ncount_tokens = lambda x: len(encoding.encode(x))\nwriteups['num_tokens'] = writeups.writeup.apply(count_tokens)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:34:04.051888Z","iopub.execute_input":"2023-07-03T09:34:04.052346Z","iopub.status.idle":"2023-07-03T09:34:08.414855Z","shell.execute_reply.started":"2023-07-03T09:34:04.052304Z","shell.execute_reply":"2023-07-03T09:34:08.413598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writeups.num_tokens.hist();","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:34:08.417232Z","iopub.execute_input":"2023-07-03T09:34:08.417643Z","iopub.status.idle":"2023-07-03T09:34:09.136630Z","shell.execute_reply.started":"2023-07-03T09:34:08.417609Z","shell.execute_reply":"2023-07-03T09:34:09.134571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like most of the writeups should fit in the model context window. Let's save the outputs, but for the following demo we will use a small sample of the writeups only. Full dataset is available [here]().","metadata":{}},{"cell_type":"code","source":"writeups.to_csv('kaggle_writeups_expanded.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:35:00.403708Z","iopub.execute_input":"2023-07-03T09:35:00.404246Z","iopub.status.idle":"2023-07-03T09:35:00.815566Z","shell.execute_reply.started":"2023-07-03T09:35:00.404209Z","shell.execute_reply":"2023-07-03T09:35:00.814399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ws = writeups.sample(n=5)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:35:07.448459Z","iopub.execute_input":"2023-07-03T09:35:07.448983Z","iopub.status.idle":"2023-07-03T09:35:07.457165Z","shell.execute_reply.started":"2023-07-03T09:35:07.448939Z","shell.execute_reply":"2023-07-03T09:35:07.455791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we need to set wandb and openai api keys in secrets\nfrom kaggle_secrets import UserSecretsClient\nimport os\nuser_secrets = UserSecretsClient()\nos.environ['OPENAI_API_KEY'] = user_secrets.get_secret(\"OPENAI_API_KEY\")\nos.environ['WANDB_API_KEY'] = user_secrets.get_secret(\"wandb_api\")","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:37:34.975495Z","iopub.execute_input":"2023-07-03T09:37:34.976076Z","iopub.status.idle":"2023-07-03T09:37:35.247069Z","shell.execute_reply.started":"2023-07-03T09:37:34.976032Z","shell.execute_reply":"2023-07-03T09:37:35.245548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"functions  = [\n    {\n        'name': 'MethodsList',\n        'description': 'List of machine learning methods.',\n        'parameters': {\n            'type': 'object',\n            'properties': {\n                'methods': {\n                    'title': 'Methods', \n                    'type': 'array', \n                    'items': {'type': 'string'},\n            },\n        }\n        }, 'required': ['methodName', 'methodDescription']\n    }\n]\n\nfunction_call = {\"name\": \"MethodsList\"}","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:37:56.627526Z","iopub.execute_input":"2023-07-03T09:37:56.628034Z","iopub.status.idle":"2023-07-03T09:37:56.636192Z","shell.execute_reply.started":"2023-07-03T09:37:56.627994Z","shell.execute_reply":"2023-07-03T09:37:56.634732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"system_prompt = \"\"\"You are a machine learning engineer analyzing Kaggle competition solutions.\nYour goal is to come up with the ultimate set of proven machine learning methods.\nReview the winning solution description provided by user, and identify methods that workd.\nOmit methods that didn't work. \nDon't add any methods not mentioned in the solution description.\nCall the MethodsList function to save a list of methods that worked.\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:02.895217Z","iopub.execute_input":"2023-07-03T09:38:02.896155Z","iopub.status.idle":"2023-07-03T09:38:02.903378Z","shell.execute_reply.started":"2023-07-03T09:38:02.896105Z","shell.execute_reply":"2023-07-03T09:38:02.902076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_TOKENS = 3500\nencoding = tiktoken.encoding_for_model(GPT_MODEL)\ndef shorten(method_description):\n    tokens = encoding.encode(method_description)\n    return encoding.decode(tokens[:MAX_TOKENS])","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:10.783890Z","iopub.execute_input":"2023-07-03T09:38:10.784361Z","iopub.status.idle":"2023-07-03T09:38:10.791411Z","shell.execute_reply.started":"2023-07-03T09:38:10.784328Z","shell.execute_reply":"2023-07-03T09:38:10.789953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tenacity import retry, wait_random_exponential, stop_after_attempt\nimport json\nimport openai\n\n@retry(wait=wait_random_exponential(min=1, max=40), stop=stop_after_attempt(3))\ndef chat_completion_request(system_prompt, method_description, functions=None, function_call=None, model=GPT_MODEL):\n    short_prompt = shorten(method_description)\n    messages = []\n    messages.append({\"role\": \"system\", \"content\": system_prompt})\n    messages.append({\"role\": \"user\", \"content\": short_prompt})\n    response = openai.ChatCompletion.create(\n        model=model,\n        messages=messages,\n        functions=functions,\n        function_call=function_call\n    )\n    try:\n        res = response.choices[0].message.function_call.arguments\n        data = json.loads(res)\n        methods = data[\"methods\"]\n        return methods\n    except:\n        return response.choices[0].message","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:38:58.743910Z","iopub.execute_input":"2023-07-03T09:38:58.744476Z","iopub.status.idle":"2023-07-03T09:38:58.982193Z","shell.execute_reply.started":"2023-07-03T09:38:58.744433Z","shell.execute_reply":"2023-07-03T09:38:58.980970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nws['methods'] = ws.writeup.apply(lambda x: chat_completion_request(system_prompt, x, functions=functions, function_call=function_call))","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:39:24.496163Z","iopub.execute_input":"2023-07-03T09:39:24.497706Z","iopub.status.idle":"2023-07-03T09:39:41.602506Z","shell.execute_reply.started":"2023-07-03T09:39:24.497657Z","shell.execute_reply":"2023-07-03T09:39:41.600771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ws.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:39:53.171473Z","iopub.execute_input":"2023-07-03T09:39:53.172010Z","iopub.status.idle":"2023-07-03T09:39:53.219927Z","shell.execute_reply.started":"2023-07-03T09:39:53.171972Z","shell.execute_reply":"2023-07-03T09:39:53.217956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning up and standardizing the method names\n\nThe initial pass will contain a lot of synonyms and duplicates, because LLM is looking at one solution at the time. Let's use LLM again to clean this up and map dirty to clean categories. ","metadata":{}},{"cell_type":"code","source":"methods = ws.methods.explode().value_counts()\nmethods.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:40:48.015973Z","iopub.execute_input":"2023-07-03T09:40:48.016527Z","iopub.status.idle":"2023-07-03T09:40:48.029160Z","shell.execute_reply.started":"2023-07-03T09:40:48.016477Z","shell.execute_reply":"2023-07-03T09:40:48.027832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"methods_string = '\\n'.join(methods.index.tolist())","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:40:53.394735Z","iopub.execute_input":"2023-07-03T09:40:53.395248Z","iopub.status.idle":"2023-07-03T09:40:53.401088Z","shell.execute_reply.started":"2023-07-03T09:40:53.395206Z","shell.execute_reply":"2023-07-03T09:40:53.399802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restructure_prompt = \"\"\"You are a machine learning engineer analyzing Kaggle competition solutions. \nYou are provided with a granular, unstructured list of machine learning methods that worked in various competitions.\nYour goal is to restructure the list of methods to avoid duplicates or messy categories.\nDo NOT use \"Other\" as a category. Always come up with a specific category. You can also decide to keep the original category as is.\nReturn a mapping of dirty method to cleaned up category, in csv format, for example:\nXGBoost;XGBoost\nxgb;XGBoost\nRandom Forest;Random Forest\nCNN;CNNs\nEfficientNet-B3;CNNs\nSAINT;Transformers\nDistillationLoss;DistillationLoss\nBCE(smoothing=0.01);BCE\nHere's the list of methods. Start your response with the mapping, don't add any extra description or text. Begin!\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:46:13.338823Z","iopub.execute_input":"2023-07-03T09:46:13.339427Z","iopub.status.idle":"2023-07-03T09:46:13.346113Z","shell.execute_reply.started":"2023-07-03T09:46:13.339381Z","shell.execute_reply":"2023-07-03T09:46:13.344660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we will divide methods_string into chunks of 7000 tokens each and process each of them separately\nmethods_tokens = encoding.encode(methods_string)\ncompletions = []\n\nfor i in range(0, len(methods_tokens), 7000):\n    methods_chunk = encoding.decode(methods_tokens[i:i+7000])\n    completion = openai.ChatCompletion.create(\n      model='gpt-4',\n      messages=[\n        {\"role\": \"system\", \"content\": restructure_prompt},\n        {\"role\": \"user\", \"content\": methods_chunk},\n      ]\n    )\n    completions.append(completion)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:46:18.916499Z","iopub.execute_input":"2023-07-03T09:46:18.916987Z","iopub.status.idle":"2023-07-03T09:46:55.224825Z","shell.execute_reply.started":"2023-07-03T09:46:18.916943Z","shell.execute_reply":"2023-07-03T09:46:55.223187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"restructured_methods = '\\n'.join([c.choices[0].message.content for c in completions])\nlen(restructured_methods)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:46:55.227706Z","iopub.execute_input":"2023-07-03T09:46:55.228340Z","iopub.status.idle":"2023-07-03T09:46:55.238356Z","shell.execute_reply.started":"2023-07-03T09:46:55.228284Z","shell.execute_reply":"2023-07-03T09:46:55.236981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove the lines that don't have a single comma\nrestructured_methods = '\\n'.join([l for l in restructured_methods.split('\\n') if ';' in l])","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:46:55.256740Z","iopub.execute_input":"2023-07-03T09:46:55.257209Z","iopub.status.idle":"2023-07-03T09:46:55.266872Z","shell.execute_reply.started":"2023-07-03T09:46:55.257171Z","shell.execute_reply":"2023-07-03T09:46:55.265381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a StringIO object to mimic a file-like object from the string\nfrom io import StringIO\ncsv_file = StringIO(restructured_methods)\n\n# Read the CSV-formatted string into a DataFrame\ndf = pd.read_csv(csv_file, sep=';', header=None, names=['method', 'category'])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:47:24.923990Z","iopub.execute_input":"2023-07-03T09:47:24.924469Z","iopub.status.idle":"2023-07-03T09:47:24.943424Z","shell.execute_reply.started":"2023-07-03T09:47:24.924436Z","shell.execute_reply":"2023-07-03T09:47:24.941940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_mapping = dict(zip(df.method, df.category))\nws['cleaned_methods'] = ws.methods.apply(lambda x: [category_mapping[m] for m in x if m in category_mapping])","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:48:00.736160Z","iopub.execute_input":"2023-07-03T09:48:00.736622Z","iopub.status.idle":"2023-07-03T09:48:00.745536Z","shell.execute_reply.started":"2023-07-03T09:48:00.736586Z","shell.execute_reply":"2023-07-03T09:48:00.744028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ws = ws.explode('cleaned_methods')\nws = ws.drop_duplicates(subset=['link', 'cleaned_methods'])\nws = ws.dropna(subset=['cleaned_methods']).reset_index(drop=True)\nlen(ws)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:48:50.062733Z","iopub.execute_input":"2023-07-03T09:48:50.063252Z","iopub.status.idle":"2023-07-03T09:48:50.085981Z","shell.execute_reply.started":"2023-07-03T09:48:50.063216Z","shell.execute_reply":"2023-07-03T09:48:50.084578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ws.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:48:59.698990Z","iopub.execute_input":"2023-07-03T09:48:59.700566Z","iopub.status.idle":"2023-07-03T09:48:59.727220Z","shell.execute_reply.started":"2023-07-03T09:48:59.700500Z","shell.execute_reply":"2023-07-03T09:48:59.726136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ws.to_csv('kaggle_writeups_sample_processed.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:49:45.106591Z","iopub.execute_input":"2023-07-03T09:49:45.107142Z","iopub.status.idle":"2023-07-03T09:49:45.128200Z","shell.execute_reply.started":"2023-07-03T09:49:45.107098Z","shell.execute_reply":"2023-07-03T09:49:45.127013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore in W&B Table\n\n[W&B Tables](https://wandb.ai/darek/llmapps/reports/Using-LLMs-to-Extract-Structured-Data-OpenAI-Function-Calling-in-Action--Vmlldzo0Nzc0MzQ3) is a great tool to analyze the data we scraped, find out for yourself! You'll need to set WANDB_API_KEY in user secrets for this code to work. ","metadata":{}},{"cell_type":"code","source":"import wandb\nwith wandb.init(project='llmapps'):\n    wandb.log({\"methods\": wandb.Table(dataframe=ws, allow_mixed_types=True)})","metadata":{"execution":{"iopub.status.busy":"2023-07-03T09:49:59.460135Z","iopub.execute_input":"2023-07-03T09:49:59.461610Z","iopub.status.idle":"2023-07-03T09:51:03.106744Z","shell.execute_reply.started":"2023-07-03T09:49:59.461547Z","shell.execute_reply":"2023-07-03T09:51:03.105453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}