{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is inspired by Y.Nakama's [notebook to identified top referenced Kagglers](https://www.kaggle.com/code/yasufuminakama/ai4code-most-common-kagglers/notebook). This notebook identifies all Kaggle sources used by the notebooks in AI4Code's train dataset. \n\nSince different competitions' texts can have somewhat similar content (and maybe relatively similar processing, etc), this feature can be used to cluster the notebooks or for stratification purposes. \n\nThere can still be false negatives using this approach. For instance, code cells like this `DATA_DIR = \"/kaggle/input\"; SOURCE = f\"{DATA_DIR}/{DATA_SOURCE}\"` still have a data source, but just not captured by the regex I'm using. Feel free to fork and fix it. ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd, numpy as np, json\nfrom tqdm import tqdm \ntqdm.pandas()\ndf = pd.read_csv(\"../input/json-files-to-csv/ai4code_train_flat.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:42:43.049822Z","iopub.execute_input":"2022-05-13T15:42:43.050651Z","iopub.status.idle":"2022-05-13T15:43:26.235489Z","shell.execute_reply.started":"2022-05-13T15:42:43.050550Z","shell.execute_reply":"2022-05-13T15:43:26.233969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef get_comp_sources(dat_str):\n    itr = re.finditer(\"/input/([a-z0-9\\-]+)/\", dat_str.lower())\n    return list(set([group.group(1) for group in itr]))\n    \ndf[\"data_source\"] = df[\"dat\"].progress_apply(get_comp_sources)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:43:26.239154Z","iopub.execute_input":"2022-05-13T15:43:26.239709Z","iopub.status.idle":"2022-05-13T15:43:29.541134Z","shell.execute_reply.started":"2022-05-13T15:43:26.239653Z","shell.execute_reply":"2022-05-13T15:43:29.540264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"data_source\"].apply(len).describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:43:29.542592Z","iopub.execute_input":"2022-05-13T15:43:29.542869Z","iopub.status.idle":"2022-05-13T15:43:29.613030Z","shell.execute_reply.started":"2022-05-13T15:43:29.542836Z","shell.execute_reply":"2022-05-13T15:43:29.612097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"total_sources\"] = df[\"data_source\"].apply(len)\npd.set_option(\"display.max_colwidth\", None)\ndisplay(df[df[\"total_sources\"]>1].sample(20)[[\"id\", \"data_source\", \"total_sources\"]])\npd.set_option(\"display.max_colwidth\", 60)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:47:30.694732Z","iopub.execute_input":"2022-05-13T15:47:30.695051Z","iopub.status.idle":"2022-05-13T15:47:30.760561Z","shell.execute_reply.started":"2022-05-13T15:47:30.695020Z","shell.execute_reply":"2022-05-13T15:47:30.759629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"data_source\"].astype(str).value_counts().head(30)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:43:29.774519Z","iopub.execute_input":"2022-05-13T15:43:29.774809Z","iopub.status.idle":"2022-05-13T15:43:29.899326Z","shell.execute_reply.started":"2022-05-13T15:43:29.774752Z","shell.execute_reply":"2022-05-13T15:43:29.898562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[[\"id\", \"data_source\"]].to_csv(\"ai4code_data_sources.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T15:43:29.900332Z","iopub.execute_input":"2022-05-13T15:43:29.900999Z","iopub.status.idle":"2022-05-13T15:43:30.194274Z","shell.execute_reply.started":"2022-05-13T15:43:29.900964Z","shell.execute_reply":"2022-05-13T15:43:30.193394Z"},"trusted":true},"execution_count":null,"outputs":[]}]}