{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50947,"databundleVersionId":5741538,"sourceType":"competition"},{"sourceId":64148,"databundleVersionId":7669720,"sourceType":"competition"},{"sourceId":11371,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":5171}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on February 23, 2024, 17:03 PM GMT-3 . By Marília Prata, mpwolke.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-23T20:07:54.628011Z","iopub.execute_input":"2024-02-23T20:07:54.628393Z","iopub.status.idle":"2024-02-23T20:07:54.638470Z","shell.execute_reply.started":"2024-02-23T20:07:54.628367Z","shell.execute_reply":"2024-02-23T20:07:54.637615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Introducing Gemma models in Keras\n\nBy Martin Görner - February 21, 2024\n\n\"The Keras team is happy to announce that Gemma, a family of lightweight, state-of-the art open models built from the same research and technology that we used to create the Gemini models, is now available in the KerasNLP collection.\n\n\"Thanks to Keras 3, Gemma runs on JAX, PyTorch and TensorFlow. With this release, Keras is also introducing several new features specifically designed for large language models: a new LoRA API (Low Rank Adaptation) and large scale model-parallel training capabilities.\"\n\n![](https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjdWSrhXoP4nHcjFcmbTNjhKcd3LRLcIiL2uEp4_8ilX7h_zsMu_muQlLP52eEgxvq4ejAQKy0TQKNaFC07O4o9imxqDDKF8hgaLU-iYfwmcPYGpm64psp1WHyaJOZQPImAhCDpYtc4nWEvbM3hSERTA50n08rIhftkP0rK1ai9uB-o3nWx0TQMRWt1leQ/s1600/Keras-Gemma-GfD.png)https://developers.googleblog.com/2024/02/gemma-models-in-keras.html","metadata":{}},{"cell_type":"code","source":"import re\n# from https://www.kaggle.com/code/nilaychauhan/fine-tune-gemma-models-in-keras-using-lora\n\n# Install Keras 3 last. See https://keras.io/getting_started/ for more details.\n!pip install -q -U keras-nlp\n!pip install -q -U keras>=3\n\nimport os\n\nos.environ[\"KERAS_BACKEND\"] = \"jax\"  # Or \"torch\" or \"tensorflow\".\n# Avoid memory fragmentation on JAX backend.\nos.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\"0.9\"\n\nimport keras\nimport keras_nlp","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-23T20:08:01.273609Z","iopub.execute_input":"2024-02-23T20:08:01.273967Z","iopub.status.idle":"2024-02-23T20:08:40.946570Z","shell.execute_reply.started":"2024-02-23T20:08:01.273942Z","shell.execute_reply":"2024-02-23T20:08:40.945716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ngemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(\"gemma_2b_en\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-23T20:08:44.764187Z","iopub.execute_input":"2024-02-23T20:08:44.765349Z","iopub.status.idle":"2024-02-23T20:09:40.926198Z","shell.execute_reply.started":"2024-02-23T20:08:44.765317Z","shell.execute_reply":"2024-02-23T20:09:40.925283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"Which are the most frequent words in Kaggle Competitions Solutions?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T19:22:25.208973Z","iopub.execute_input":"2024-02-23T19:22:25.209696Z","iopub.status.idle":"2024-02-23T19:22:41.950570Z","shell.execute_reply.started":"2024-02-23T19:22:25.209666Z","shell.execute_reply":"2024-02-23T19:22:41.949592Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv', nrows =1000)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:39:16.374181Z","iopub.execute_input":"2024-02-23T20:39:16.374592Z","iopub.status.idle":"2024-02-23T20:39:16.433608Z","shell.execute_reply.started":"2024-02-23T20:39:16.374564Z","shell.execute_reply":"2024-02-23T20:39:16.432692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'black',\n                      colormap ='Set2',\n                      height =2000,\n                      width = 2000\n                     ).generate(str(df[\"Title of Writeup\"]))\nplt.rcParams['figure.figsize'] = (12,12)\nplt.axis(\"off\")\nplt.imshow(wordcloud)\nplt.title(\"Title of Kaggle Competitions Writeups\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:37:46.229378Z","iopub.execute_input":"2024-02-23T20:37:46.230111Z","iopub.status.idle":"2024-02-23T20:37:49.527675Z","shell.execute_reply.started":"2024-02-23T20:37:46.230083Z","shell.execute_reply":"2024-02-23T20:37:49.526778Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Title of Writeups","metadata":{}},{"cell_type":"code","source":"df[\"Title of Writeup\"].value_counts()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-23T18:24:28.622931Z","iopub.execute_input":"2024-02-23T18:24:28.623758Z","iopub.status.idle":"2024-02-23T18:24:28.643304Z","shell.execute_reply.started":"2024-02-23T18:24:28.623724Z","shell.execute_reply":"2024-02-23T18:24:28.642145Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Rob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nfig, ax = plt.subplots(figsize=(4, 4))\ndf[\"Title of Writeup\"].value_counts().head(10).sort_values(ascending=True).plot(\n    kind=\"barh\", color='r', ax=ax, title=\"Kaggle Competitions Writeups\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-23T17:54:52.356565Z","iopub.execute_input":"2024-02-23T17:54:52.356999Z","iopub.status.idle":"2024-02-23T17:54:52.632494Z","shell.execute_reply.started":"2024-02-23T17:54:52.356966Z","shell.execute_reply":"2024-02-23T17:54:52.631351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I dropped and made other stuff to avoid the XlaRuntimeError: \n\nRESOURCE_EXHAUSTED: Out of memory ","metadata":{}},{"cell_type":"code","source":"cols_to_drop=['Competition Launch Date','Competition URL', 'Title of Competition', 'Writeup', 'Writeup URL']\ndf=df.drop(cols_to_drop,axis=1)\ndf.tail()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:09:50.880084Z","iopub.execute_input":"2024-02-23T20:09:50.880940Z","iopub.status.idle":"2024-02-23T20:09:50.895701Z","shell.execute_reply.started":"2024-02-23T20:09:50.880911Z","shell.execute_reply":"2024-02-23T20:09:50.894751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Carl McBride Ellis https://www.kaggle.com/code/carlmcbrideellis/gemma-i-am-replacing-myself-with-an-llm\n\nWUP_dataset = []\n    \nfor index, row in df.iterrows():\n    instruction, response = row['Title of Writeup'], row['Date of Writeup']\n    template = (f\"Title of Writeup:\\n{instruction}\\n\\nDate of Writeup:\\n{response}\")\n    WUP_dataset.append(template)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:10:04.971480Z","iopub.execute_input":"2024-02-23T20:10:04.971849Z","iopub.status.idle":"2024-02-23T20:10:05.040317Z","shell.execute_reply.started":"2024-02-23T20:10:04.971820Z","shell.execute_reply":"2024-02-23T20:10:05.039539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#LoRA fine-tuning","metadata":{}},{"cell_type":"code","source":"# Enable LoRA for the model and set the LoRA rank to 64.\ngemma_lm.backbone.enable_lora(rank=64)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:10:09.971084Z","iopub.execute_input":"2024-02-23T20:10:09.971457Z","iopub.status.idle":"2024-02-23T20:10:10.366397Z","shell.execute_reply.started":"2024-02-23T20:10:09.971429Z","shell.execute_reply":"2024-02-23T20:10:10.365416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Control memory usage).","metadata":{}},{"cell_type":"markdown","source":"#XlaRuntimeError: RESOURCE_EXHAUSTED: Out of memory while trying to allocate 8405385216 bytes.\n\n#Carl hide the snippet below and \"my code\" got Exhausted. ","metadata":{}},{"cell_type":"code","source":"#By Carl McBride Ellis https://www.kaggle.com/code/carlmcbrideellis/gemma-i-am-replacing-myself-with-an-llm\n\n# Limit the input sequence length to 512 (to control memory usage).\ngemma_lm.preprocessor.sequence_length = 512\n# Use AdamW (a common optimizer for transformer models).\noptimizer = keras.optimizers.AdamW(\n    learning_rate=5e-5,\n    weight_decay=0.01,\n)\n# Exclude layernorm and bias terms from decay.\noptimizer.exclude_from_weight_decay(var_names=[\"bias\", \"scale\"])\n\ngemma_lm.compile(\n    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n    optimizer=optimizer,\n    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:10:18.480958Z","iopub.execute_input":"2024-02-23T20:10:18.481680Z","iopub.status.idle":"2024-02-23T20:10:18.512401Z","shell.execute_reply.started":"2024-02-23T20:10:18.481640Z","shell.execute_reply":"2024-02-23T20:10:18.511449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ngemma_lm.fit(WUP_dataset, epochs=1, batch_size=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:10:26.582750Z","iopub.execute_input":"2024-02-23T20:10:26.583117Z","iopub.status.idle":"2024-02-23T20:23:04.596146Z","shell.execute_reply.started":"2024-02-23T20:10:26.583090Z","shell.execute_reply":"2024-02-23T20:23:04.595190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"Which are the most frequent words on Kaggle Writeup Titles?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:23:19.321783Z","iopub.execute_input":"2024-02-23T20:23:19.322176Z","iopub.status.idle":"2024-02-23T20:23:33.850983Z","shell.execute_reply.started":"2024-02-23T20:23:19.322146Z","shell.execute_reply":"2024-02-23T20:23:33.849976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"Which are the most common titles on Title of Writeup?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:24:26.041361Z","iopub.execute_input":"2024-02-23T20:24:26.041736Z","iopub.status.idle":"2024-02-23T20:24:33.050905Z","shell.execute_reply.started":"2024-02-23T20:24:26.041709Z","shell.execute_reply":"2024-02-23T20:24:33.049905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Post-Mortem was written 11 years ago\n\nBARRENWUFFET · 9TH IN THIS COMPETITION · POSTED 11 YEARS AGO\nThis post earned a bronze medal\nPost-Mortem\nAnybody do or find anything interesting in this dataset? Or find any good tools for working with large data sets?\n\n\"I found it barely manageable given the size.  It also took me awhile to wrap by head around what was in each of the files. Also the database restrictions (1 and 2 but not 3, 1 and 2 and 3, 2 and 3 but not 1, etc) made a really difficult task much harder and considerably less enjoyable.  I ended up having to write 12 large files (and accompanying SQL code) for each model to compensate for this.\"\n\n\"I started out with a ~2% sample of the training data in R but even this was rough.  I tried using the ff package without much luck.  I ended up doing most of the data manipulation using SQL Server 2008 R2 Express and the SSMS which I found to be a bright spot in the whole process as it performed really well given the data size.  I especially appreciated the data import manager which helps with wide data sets.\"\n\nhttps://www.kaggle.com/c/Raising-Money-to-Fund-an-Organizational-Mission/discussion/2712","metadata":{}},{"cell_type":"code","source":"print(gemma_lm.generate(\"When was written Post_Mortem Title of Writeup?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:30:13.196303Z","iopub.execute_input":"2024-02-23T20:30:13.196953Z","iopub.status.idle":"2024-02-23T20:30:13.984769Z","shell.execute_reply.started":"2024-02-23T20:30:13.196924Z","shell.execute_reply":"2024-02-23T20:30:13.983816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"When was the 1st Kaggle Writeup?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:23:49.489553Z","iopub.execute_input":"2024-02-23T20:23:49.489913Z","iopub.status.idle":"2024-02-23T20:23:50.373448Z","shell.execute_reply.started":"2024-02-23T20:23:49.489884Z","shell.execute_reply":"2024-02-23T20:23:50.372340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"How many times the word medal appeared on Title of Writeup?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:25:00.068452Z","iopub.execute_input":"2024-02-23T20:25:00.068817Z","iopub.status.idle":"2024-02-23T20:25:00.503482Z","shell.execute_reply.started":"2024-02-23T20:25:00.068790Z","shell.execute_reply":"2024-02-23T20:25:00.502510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \" \".join(titl for titl in df['Title of Writeup'])\n\nstopwords = set(STOPWORDS)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:48:01.091344Z","iopub.execute_input":"2024-02-23T20:48:01.092017Z","iopub.status.idle":"2024-02-23T20:48:01.097115Z","shell.execute_reply.started":"2024-02-23T20:48:01.091988Z","shell.execute_reply":"2024-02-23T20:48:01.096113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Chris X https://www.kaggle.com/code/docxian/cord-19-dexamethasone-articles\n\nwordcloud = WordCloud(stopwords=stopwords, max_font_size=50, max_words=500,\n                      width = 600, height = 400,\n                      colormap='Greens',\n                      background_color=\"white\").generate(text)\nplt.figure(figsize=(12,8))\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:48:21.827829Z","iopub.execute_input":"2024-02-23T20:48:21.828774Z","iopub.status.idle":"2024-02-23T20:48:22.582757Z","shell.execute_reply.started":"2024-02-23T20:48:21.828738Z","shell.execute_reply":"2024-02-23T20:48:22.581773Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"How many 1st place solution on Title of Writeup?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:24:40.428564Z","iopub.execute_input":"2024-02-23T20:24:40.428932Z","iopub.status.idle":"2024-02-23T20:24:43.627004Z","shell.execute_reply.started":"2024-02-23T20:24:40.428906Z","shell.execute_reply":"2024-02-23T20:24:43.626031Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(gemma_lm.generate(\"How many 1st place solutions on Kaggle competitions Writeup titles?\", max_length=256))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:28:41.313905Z","iopub.execute_input":"2024-02-23T20:28:41.314568Z","iopub.status.idle":"2024-02-23T20:28:43.375313Z","shell.execute_reply.started":"2024-02-23T20:28:41.314538Z","shell.execute_reply":"2024-02-23T20:28:43.374264Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Thank, Organisers, Hosting, Surprised, Congratulation, Fun and Kaggle: Words on Writeups\n\nCommon words on Writeups.","metadata":{}},{"cell_type":"code","source":"##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'lightblue',\n                      colormap ='Set3',\n                      height =2000,\n                      width = 2000\n                     ).generate(str(df[\"Writeup\"]))\nplt.rcParams['figure.figsize'] = (12,12)\nplt.axis(\"off\")\nplt.imshow(wordcloud)\nplt.title(\"Kaggle Competitions Writeups\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:39:27.430641Z","iopub.execute_input":"2024-02-23T20:39:27.431370Z","iopub.status.idle":"2024-02-23T20:39:29.778880Z","shell.execute_reply.started":"2024-02-23T20:39:27.431337Z","shell.execute_reply":"2024-02-23T20:39:29.777939Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Closing Arguments\n\nMy Notebooks on 2023 AI Report provided me better answers than I got here. Maybe I did something wrong with the snippets or my questions weren't good enough to Gemma interpret them too.\n\nmpwolke https://www.kaggle.com/code/mpwolke/kaggle-competitions-still-rock-spacy\n\nmpwolke https://www.kaggle.com/code/mpwolke/kaggle-writeups-revival","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nNilay Chauhan https://www.kaggle.com/code/nilaychauhan/fine-tune-gemma-models-in-keras-using-lora\n\nCarl McBride Ellis https://www.kaggle.com/code/carlmcbrideellis/gemma-i-am-replacing-myself-with-an-llm","metadata":{}}]}