{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50947,"databundleVersionId":5741538,"sourceType":"competition"},{"sourceId":7714632,"sourceType":"datasetVersion","datasetId":4505365},{"sourceId":11372,"sourceType":"modelInstanceVersion","modelInstanceId":5388}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U keras-nlp\n!pip install -U keras\n!pip install markdown \n!pip install pdfkit \n!pip install pygments\n!pip install weasyprint","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-04-10T14:40:57.245733Z","iopub.execute_input":"2024-04-10T14:40:57.246472Z","iopub.status.idle":"2024-04-10T14:42:44.169681Z","shell.execute_reply.started":"2024-04-10T14:40:57.246436Z","shell.execute_reply":"2024-04-10T14:42:44.167878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nimport keras_nlp\nimport pandas as pd\nimport json\nimport re\nfrom IPython.display import Markdown, display\nfrom pygments import highlight\nfrom pygments.lexers import PythonLexer\nfrom pygments.formatters import HtmlFormatter\nfrom markdown import markdown\nfrom weasyprint import HTML","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:42:44.172476Z","iopub.execute_input":"2024-04-10T14:42:44.173143Z","iopub.status.idle":"2024-04-10T14:43:04.814585Z","shell.execute_reply.started":"2024-04-10T14:42:44.173098Z","shell.execute_reply":"2024-04-10T14:43:04.812860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nos.environ[\"KERAS_BACKEND\"] = \"jax\"\nos.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\"1.00\"","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:43:04.816615Z","iopub.execute_input":"2024-04-10T14:43:04.817703Z","iopub.status.idle":"2024-04-10T14:43:04.824758Z","shell.execute_reply.started":"2024-04-10T14:43:04.817657Z","shell.execute_reply":"2024-04-10T14:43:04.823261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(\"gemma_instruct_2b_en\")\ngemma_lm.summary()","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:43:04.828170Z","iopub.execute_input":"2024-04-10T14:43:04.828740Z","iopub.status.idle":"2024-04-10T14:44:53.382449Z","shell.execute_reply.started":"2024-04-10T14:43:04.828697Z","shell.execute_reply":"2024-04-10T14:44:53.381349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_solution_summary_prompt = \"\"\"\nOutput ONLY JSON array with no additional information. \nDO NOT include anything more, don't add any markdown syntax:\n[\n    \"value1\",\n    \"value2\"\n]\n\nList at least 20 basic Data Science, Machine Learning, and Deep Learning concepts used in Kaggle solutions. \nOrder them by difficulty, from the easiest to the more complex, to create a learning roadmap.\nEach concept should be general and introductory, avoiding specific technologies or implementations.\nAvoid repetition and ensure each concept is distinct.\n\nOutput ONLY JSON array with no additional information. \nDO NOT format text as question, only main title.\nDO NOT include anything more, don't add any markdown syntax: \n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:53.384235Z","iopub.execute_input":"2024-04-10T14:44:53.385075Z","iopub.status.idle":"2024-04-10T14:44:53.391476Z","shell.execute_reply.started":"2024-04-10T14:44:53.385034Z","shell.execute_reply":"2024-04-10T14:44:53.390165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nconcepts_response = gemma_lm.generate(kaggle_solution_summary_prompt, max_length=500)\nprint(concepts_response)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:45:10.706050Z","iopub.execute_input":"2024-04-10T14:45:10.706490Z","iopub.status.idle":"2024-04-10T14:47:18.308959Z","shell.execute_reply.started":"2024-04-10T14:45:10.706458Z","shell.execute_reply":"2024-04-10T14:47:18.308026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_response = concepts_response.replace(kaggle_solution_summary_prompt, \"\")\nprint(json_response)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:18.310760Z","iopub.execute_input":"2024-04-10T14:47:18.311127Z","iopub.status.idle":"2024-04-10T14:47:18.316899Z","shell.execute_reply.started":"2024-04-10T14:47:18.311095Z","shell.execute_reply":"2024-04-10T14:47:18.315871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concepts_array = json.loads(json_response)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:18.318940Z","iopub.execute_input":"2024-04-10T14:47:18.319841Z","iopub.status.idle":"2024-04-10T14:47:18.330418Z","shell.execute_reply.started":"2024-04-10T14:47:18.319743Z","shell.execute_reply":"2024-04-10T14:47:18.329001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explain_concept_prompt = \"\"\"\nYou are tasked with summarise Kaggle solution write-ups, emphasizing important facts, numbers and strategy, if given\nSummarize the provided Kaggle solution write-up {}, \nExtract facts, numbers and strategies used.Summarize the Extracted facts,numbers and strategies\nFocus on facts, numbers and strategies used. Divide the summary in points, be impersonal and use bullet points:\nAvoid any introductory phrases or additional context. \nDO NOT include any 'Sure, here's a detailed explanation of...'\nDO NOT start with any 'You are tasked with summarise Kaggle solution write-ups, emphasizing conceptualal details.\nSummarize the provided Kaggle solution write-up '\nRespond with Markdown Code.\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:53.393297Z","iopub.execute_input":"2024-04-10T14:44:53.393986Z","iopub.status.idle":"2024-04-10T14:44:53.407939Z","shell.execute_reply.started":"2024-04-10T14:44:53.393938Z","shell.execute_reply":"2024-04-10T14:44:53.406372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explain_concept_prompt2 = \"\"\"\nYou are professional who works on AI and ML solutions in academia and big tech and summarise given topic.\nExplain {} concept in detail so that begginer that just started learning Data Science will understand.\nMake sure explanation contains enought details.\nRespond just with an explanation.\nDO NOT include any 'Sure, here's a detailed explanation of...'\nRespond with Markdown Code.\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nconcept_prompt = explain_concept_prompt2.format(concepts_array[0])\nconcept_response = gemma_lm.generate(concept_prompt, max_length=1000)\nconcept_response_without_prompt = concept_response.replace(concept_prompt, '').replace('```', '')\ndisplay(Markdown(concept_response_without_prompt))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_explanation_prompt_template = \"\"\"\nInstruction:\nCreate example code for {}.\nOutput ONLY Python code nothing else.\n\nResponse:\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nexample_concept = concepts_array[9]\nexample_code_explanation_prompt = code_explanation_prompt_template.format(example_concept)\nexample_code_response = gemma_lm.generate(example_code_explanation_prompt, max_length=1500)\nexample_code_response_without_prompt = example_code_response.replace(example_code_explanation_prompt, '')\n\nprint(\"Example Concept: \", example_concept)\nprint(example_code_response_without_prompt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gemma_lm.backbone.enable_lora(rank=16)\ngemma_lm.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeup = pd.read_csv('/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')\nkaggle_writeup.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:53.409371Z","iopub.execute_input":"2024-04-10T14:44:53.409758Z","iopub.status.idle":"2024-04-10T14:44:53.851294Z","shell.execute_reply.started":"2024-04-10T14:44:53.409727Z","shell.execute_reply":"2024-04-10T14:44:53.849210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"template = \"Kaggle Write up:\\n{}\\n\\n\"","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:53.853118Z","iopub.execute_input":"2024-04-10T14:44:53.853560Z","iopub.status.idle":"2024-04-10T14:44:53.859951Z","shell.execute_reply.started":"2024-04-10T14:44:53.853529Z","shell.execute_reply":"2024-04-10T14:44:53.857898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_rows = kaggle_writeup[\n    kaggle_writeup['Writeup'].fillna('').str.split().apply(len).between(350, 700)\n]\nfiltered_rows.reset_index(drop=True, inplace=True)\nfiltered_rows.to_csv('long_writeups.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:53.862345Z","iopub.execute_input":"2024-04-10T14:44:53.862830Z","iopub.status.idle":"2024-04-10T14:44:54.346699Z","shell.execute_reply.started":"2024-04-10T14:44:53.862797Z","shell.execute_reply":"2024-04-10T14:44:54.345620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_rows.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:44:54.350302Z","iopub.execute_input":"2024-04-10T14:44:54.350646Z","iopub.status.idle":"2024-04-10T14:44:54.376206Z","shell.execute_reply.started":"2024-04-10T14:44:54.350617Z","shell.execute_reply":"2024-04-10T14:44:54.374259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_rows = filtered_rows[filtered_rows['Writeup'].str.contains('|'.join(concepts_array), case=False, na=False)]\nfiltered_rows['num_words'] = filtered_rows['Writeup'].str.split().apply(len)\nfiltered_rows_sorted = filtered_rows.sort_values(by='num_words', ascending=False)\ntop_rows = filtered_rows_sorted.head(120)\ntop_rows.reset_index(drop = True,inplace = True)\ntop_rows.to_csv('top_120_writeups.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:18.333737Z","iopub.execute_input":"2024-04-10T14:47:18.334228Z","iopub.status.idle":"2024-04-10T14:47:20.451725Z","shell.execute_reply.started":"2024-04-10T14:47:18.334194Z","shell.execute_reply":"2024-04-10T14:47:20.450413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_rows.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.456261Z","iopub.execute_input":"2024-04-10T14:47:20.456664Z","iopub.status.idle":"2024-04-10T14:47:20.470788Z","shell.execute_reply.started":"2024-04-10T14:47:20.456633Z","shell.execute_reply":"2024-04-10T14:47:20.469319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_rows['Writeup'].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.472778Z","iopub.execute_input":"2024-04-10T14:47:20.473208Z","iopub.status.idle":"2024-04-10T14:47:20.483004Z","shell.execute_reply.started":"2024-04-10T14:47:20.473171Z","shell.execute_reply":"2024-04-10T14:47:20.481834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = top_rows['Writeup'][:100]","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.486399Z","iopub.execute_input":"2024-04-10T14:47:20.486819Z","iopub.status.idle":"2024-04-10T14:47:20.493376Z","shell.execute_reply.started":"2024-04-10T14:47:20.486788Z","shell.execute_reply":"2024-04-10T14:47:20.491848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[0]","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.495193Z","iopub.execute_input":"2024-04-10T14:47:20.497374Z","iopub.status.idle":"2024-04-10T14:47:20.507639Z","shell.execute_reply.started":"2024-04-10T14:47:20.497324Z","shell.execute_reply":"2024-04-10T14:47:20.506090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(data)):\n    data[i] = template.format(data[i])","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.509171Z","iopub.execute_input":"2024-04-10T14:47:20.509575Z","iopub.status.idle":"2024-04-10T14:47:20.521033Z","shell.execute_reply.started":"2024-04-10T14:47:20.509544Z","shell.execute_reply":"2024-04-10T14:47:20.519718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.525127Z","iopub.execute_input":"2024-04-10T14:47:20.525618Z","iopub.status.idle":"2024-04-10T14:47:20.539720Z","shell.execute_reply.started":"2024-04-10T14:47:20.525586Z","shell.execute_reply":"2024-04-10T14:47:20.538501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writeup = top_rows['Writeup'].iloc[105]\nwriteup = template.format(writeup)\nwriteup","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.540860Z","iopub.execute_input":"2024-04-10T14:47:20.541310Z","iopub.status.idle":"2024-04-10T14:47:20.552639Z","shell.execute_reply.started":"2024-04-10T14:47:20.541267Z","shell.execute_reply":"2024-04-10T14:47:20.551271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nconcept_prompt = explain_concept_prompt.format(writeup)\nconcept_response = gemma_lm.generate(concept_prompt, max_length=1000)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:47:20.555028Z","iopub.execute_input":"2024-04-10T14:47:20.556048Z","iopub.status.idle":"2024-04-10T14:48:21.186781Z","shell.execute_reply.started":"2024-04-10T14:47:20.555999Z","shell.execute_reply":"2024-04-10T14:48:21.184833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concept_response_without_prompt = concept_response.split(\"Summarize the provided Kaggle solution write-up Kaggle Write up:\", 1)[-1].strip()\nconcept_response_without_prompt = concept_response_without_prompt.replace(concept_prompt, '').replace('```', '')\ndisplay(Markdown(concept_response_without_prompt))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Limit the input sequence length to 512 (to control memory usage).\ngemma_lm.preprocessor.sequence_length = 512\n# Use AdamW (a common optimizer for transformer models).\noptimizer = keras.optimizers.AdamW(\n    learning_rate=5e-5,\n    weight_decay=0.01,\n)\n# Exclude layernorm and bias terms from decay.\noptimizer.exclude_from_weight_decay(var_names=[\"bias\", \"scale\"])\n\ngemma_lm.compile(\n    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n    optimizer=optimizer,\n    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],\n)\ngemma_lm.fit(data, epochs=3, batch_size=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nconcept_prompt = explain_concept_prompt.format(writeup)\nconcept_response = gemma_lm.generate(concept_prompt, max_length=1000)\nconcept_response_without_prompt = concept_response.replace(concept_prompt, '').replace('```', '')\ndisplay(Markdown(concept_response_without_prompt))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"formatter = HtmlFormatter()\nlexer = PythonLexer()\n\nhtml_template = \"\"\"\n<html>\n<head>\n<style>\n{}\n</style>\n</head>\n<body>\n{}\n</body>\n</html>\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-10T15:03:16.367874Z","iopub.execute_input":"2024-04-10T15:03:16.369112Z","iopub.status.idle":"2024-04-10T15:03:16.376308Z","shell.execute_reply.started":"2024-04-10T15:03:16.369071Z","shell.execute_reply":"2024-04-10T15:03:16.374306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prompt = top_rows['Writeup'][100:]\ntest_prompt.reset_index(drop = True,inplace = True)\nfor i in range(len(test_prompt)):\n    test_prompt[i] = template.format(test_prompt[i])","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:48:32.302069Z","iopub.execute_input":"2024-04-10T14:48:32.303372Z","iopub.status.idle":"2024-04-10T14:48:32.312079Z","shell.execute_reply.started":"2024-04-10T14:48:32.303322Z","shell.execute_reply":"2024-04-10T14:48:32.310655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concept_code = dict()\nfor prompt in test_prompt:\n    code_explanation_prompt = explain_concept_prompt.format(prompt)\n    code_response = gemma_lm.generate(code_explanation_prompt, max_length=1500)\n    code_response_cleaned = code_response.replace(code_explanation_prompt, '').replace(\"python\", \"\").replace(\"```\", \"\")\n    concept_code[prompt] = code_response_cleaned","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:48:35.294954Z","iopub.execute_input":"2024-04-10T14:48:35.295971Z","iopub.status.idle":"2024-04-10T14:53:50.840060Z","shell.execute_reply.started":"2024-04-10T14:48:35.295924Z","shell.execute_reply":"2024-04-10T14:53:50.838920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_name_list = [f\"kaggle_writeup_summary{i+1}\" for i in range(len(test_prompt))]","metadata":{"execution":{"iopub.status.busy":"2024-04-10T14:53:50.842028Z","iopub.execute_input":"2024-04-10T14:53:50.842590Z","iopub.status.idle":"2024-04-10T14:53:50.847660Z","shell.execute_reply.started":"2024-04-10T14:53:50.842558Z","shell.execute_reply":"2024-04-10T14:53:50.846769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i =0\nfor concept in concept_code:\n    file_name = file_name_list[i]  # Use i instead of always selecting the first element\n    explanation = concept_code[concept]\n\n    html_explanation = markdown(explanation)\n\n    clean_html = re.sub(r'<style.*?</style>', '', html_explanation, flags=re.DOTALL)\n    clean_html = re.sub(r'<pre.*?</pre>', '', clean_html, flags=re.DOTALL)\n\n    # Generate the formatted HTML with the cleaned content\n    formatted_html = html_template.format(formatter.get_style_defs('.highlight'), clean_html)\n    # Generate the PDF\n    HTML(string=formatted_html).write_pdf(f\"{file_name}.pdf\")\n    i += 1  # Use shorthand for incrementing i","metadata":{"execution":{"iopub.status.busy":"2024-04-10T15:03:21.279974Z","iopub.execute_input":"2024-04-10T15:03:21.280393Z","iopub.status.idle":"2024-04-10T15:03:21.566350Z","shell.execute_reply.started":"2024-04-10T15:03:21.280363Z","shell.execute_reply":"2024-04-10T15:03:21.565113Z"},"trusted":true},"execution_count":null,"outputs":[]}]}