{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on May 16, 2023. By Marilia Prata, mpwolke","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\nfrom pathlib import Path\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-16T22:33:50.215912Z","iopub.execute_input":"2023-05-16T22:33:50.216359Z","iopub.status.idle":"2023-05-16T22:33:50.230294Z","shell.execute_reply.started":"2023-05-16T22:33:50.216324Z","shell.execute_reply":"2023-05-16T22:33:50.228947Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path(\"/kaggle/input/2023-kaggle-ai-report/\")\ntrain_df = pd.read_csv(DATA_DIR / \"kaggle_writeups_20230510.csv\")\nsubmission = pd.read_csv(DATA_DIR / \"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:34:55.521479Z","iopub.execute_input":"2023-05-16T22:34:55.521902Z","iopub.status.idle":"2023-05-16T22:34:55.870998Z","shell.execute_reply.started":"2023-05-16T22:34:55.521864Z","shell.execute_reply":"2023-05-16T22:34:55.870094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Anthony Wynne https://www.kaggle.com/code/anthony35813/ai-report-a-first-look-at-the-data-for-text-data\n\narxiv_metadata = pd.read_json(\"/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json\", \n                              lines=True,\n                             convert_dates=True)\narxiv_metadata.head() ","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:39:23.252487Z","iopub.execute_input":"2023-05-16T22:39:23.252930Z","iopub.status.idle":"2023-05-16T22:42:18.773052Z","shell.execute_reply.started":"2023-05-16T22:39:23.252899Z","shell.execute_reply":"2023-05-16T22:42:18.771991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Roland Erdei https://www.kaggle.com/code/rolanderdei/predict-my-sleep-patterns-with-simple-ensembling\n\narxiv_metadata['update_date'] = pd.to_datetime(arxiv_metadata['update_date'])","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:47:48.958560Z","iopub.execute_input":"2023-05-16T22:47:48.959146Z","iopub.status.idle":"2023-05-16T22:47:51.888192Z","shell.execute_reply.started":"2023-05-16T22:47:48.959109Z","shell.execute_reply":"2023-05-16T22:47:51.886973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Above, Datetime worked for json too","metadata":{}},{"cell_type":"markdown","source":"#Split datetime column into date and time columns in pandas","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/code/mpwolke/science-denial-jsonl-split-date-time-displacy\n\n#By Roland Erdei https://www.kaggle.com/code/rolanderdei/predict-my-sleep-patterns-with-simple-ensembling\n\n#Code by https://stackoverflow.com/questions/68149847/split-datetime-column-into-date-and-time-columns-in-pandas\n#answered Jun 27, 2021 at 9:49 user avatar Mustafa Aydın\n\n# Get train\ntrain_df = pd.read_csv('/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')\ntrain_df['Dates'] = pd.to_datetime(train_df['Competition Launch Date']).dt.date\ntrain_df['Time'] = pd.to_datetime(train_df['Competition Launch Date']).dt.time","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:56:45.966175Z","iopub.execute_input":"2023-05-16T22:56:45.966755Z","iopub.status.idle":"2023-05-16T22:56:46.406007Z","shell.execute_reply.started":"2023-05-16T22:56:45.966719Z","shell.execute_reply":"2023-05-16T22:56:46.404664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:57:12.575953Z","iopub.execute_input":"2023-05-16T22:57:12.576478Z","iopub.status.idle":"2023-05-16T22:57:12.595262Z","shell.execute_reply.started":"2023-05-16T22:57:12.576435Z","shell.execute_reply":"2023-05-16T22:57:12.594273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Group by for having a desired dataframe","metadata":{}},{"cell_type":"code","source":"df_grp = train_df.groupby([\"Dates\",\"Title of Competition\"])[[\"Competition URL\",\"Title of Writeup\"]].sum().reset_index()\ndf_grp.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:03:27.505991Z","iopub.execute_input":"2023-05-16T23:03:27.506534Z","iopub.status.idle":"2023-05-16T23:03:27.543516Z","shell.execute_reply.started":"2023-05-16T23:03:27.506499Z","shell.execute_reply":"2023-05-16T23:03:27.542047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I don't know if it was the correct way to separate date from time, then I splited year/month/date. I hope it's right.  Very likely, there is a cleaner, elegant, simple way of doing that","metadata":{}},{"cell_type":"code","source":"#Code by HarshGupta57 https://www.kaggle.com/harshgupta57/freediving-world-records-eda/notebook\n\ndf_grp[['year','month','date']] = df_grp['Dates'].apply(lambda x : pd.Series(str(x).split('-')))","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:08:27.525023Z","iopub.execute_input":"2023-05-16T23:08:27.525619Z","iopub.status.idle":"2023-05-16T23:08:27.610591Z","shell.execute_reply.started":"2023-05-16T23:08:27.525573Z","shell.execute_reply":"2023-05-16T23:08:27.608756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by HarshGupta57 https://www.kaggle.com/harshgupta57/freediving-world-records-eda/notebook\n\ndf_grp.drop('Dates', inplace=True, axis =1)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:09:49.865451Z","iopub.execute_input":"2023-05-16T23:09:49.865865Z","iopub.status.idle":"2023-05-16T23:09:49.872710Z","shell.execute_reply.started":"2023-05-16T23:09:49.865831Z","shell.execute_reply":"2023-05-16T23:09:49.871213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Tejashvi14 https://www.kaggle.com/tejashvi14/casualty-analysis/notebook\n\n#After that snippet the float(Year) and the other objects become integers.\n\ndf_grp[\"year\"]=df_grp[\"year\"].apply(int)\ndf_grp[\"month\"]=df_grp[\"month\"].apply(int)\ndf_grp[\"date\"]=df_grp[\"date\"].apply(int)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:13:16.304636Z","iopub.execute_input":"2023-05-16T23:13:16.305160Z","iopub.status.idle":"2023-05-16T23:13:16.317361Z","shell.execute_reply.started":"2023-05-16T23:13:16.305123Z","shell.execute_reply":"2023-05-16T23:13:16.316117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_grp.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:13:21.405459Z","iopub.execute_input":"2023-05-16T23:13:21.405904Z","iopub.status.idle":"2023-05-16T23:13:21.421284Z","shell.execute_reply.started":"2023-05-16T23:13:21.405869Z","shell.execute_reply":"2023-05-16T23:13:21.419961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Competitions Launch ","metadata":{}},{"cell_type":"code","source":"#By Rob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nfig, ax = plt.subplots(figsize=(4, 4))\ndf_grp[\"year\"].value_counts().head(15).sort_values(ascending=True).plot(\n    kind=\"barh\", color='g', ax=ax, title=\"Competitions Launched Years\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:21:02.368020Z","iopub.execute_input":"2023-05-16T23:21:02.369228Z","iopub.status.idle":"2023-05-16T23:21:02.610124Z","shell.execute_reply.started":"2023-05-16T23:21:02.369179Z","shell.execute_reply":"2023-05-16T23:21:02.607363Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Rob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nfig, ax = plt.subplots(figsize=(4, 4))\ndf_grp[\"month\"].value_counts().head(15).sort_values(ascending=True).plot(\n    kind=\"barh\", color='r', ax=ax, title=\"Competitions Launched by Months\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:20:51.691149Z","iopub.execute_input":"2023-05-16T23:20:51.691617Z","iopub.status.idle":"2023-05-16T23:20:51.914931Z","shell.execute_reply.started":"2023-05-16T23:20:51.691586Z","shell.execute_reply":"2023-05-16T23:20:51.913518Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(df_grp, x=\"year\", y=\"Title of Competition\", color_discrete_sequence=['darksalmon'], \n              title=\"Competitions Launched\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:44:01.094747Z","iopub.execute_input":"2023-05-16T23:44:01.095317Z","iopub.status.idle":"2023-05-16T23:44:03.229719Z","shell.execute_reply.started":"2023-05-16T23:44:01.095278Z","shell.execute_reply":"2023-05-16T23:44:03.228678Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(df_grp, \n             x='year', y='Title of Competition',color_discrete_sequence=['blue'],\n             title='Competitions Launched', text='month')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:46:26.065604Z","iopub.execute_input":"2023-05-16T23:46:26.066201Z","iopub.status.idle":"2023-05-16T23:46:26.190598Z","shell.execute_reply.started":"2023-05-16T23:46:26.066160Z","shell.execute_reply":"2023-05-16T23:46:26.189047Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(df_grp, x=\"year\", y=\"Title of Competition\",color_discrete_sequence=['#4257f5'], title=\"Competitions Launched\" )\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T23:48:52.678671Z","iopub.execute_input":"2023-05-16T23:48:52.679185Z","iopub.status.idle":"2023-05-16T23:48:52.767856Z","shell.execute_reply.started":"2023-05-16T23:48:52.679151Z","shell.execute_reply":"2023-05-16T23:48:52.766446Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Dustin Ober https://www.kaggle.com/code/dustinober/kjv-eda\n\nnumerical_columns = ['year', 'month', 'date']\nfor col in numerical_columns:\n    print(df_grp[col].describe())\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2023-05-17T00:00:20.848446Z","iopub.execute_input":"2023-05-17T00:00:20.849101Z","iopub.status.idle":"2023-05-17T00:00:20.881933Z","shell.execute_reply.started":"2023-05-17T00:00:20.849057Z","shell.execute_reply":"2023-05-17T00:00:20.879976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Competitions Distributions ","metadata":{}},{"cell_type":"code","source":"#By Dustin Ober https://www.kaggle.com/code/dustinober/kjv-eda\n\nfor col in numerical_columns:\n    plt.figure(figsize=(12, 6))\n    sns.histplot(data=df_grp, x=col, kde=True, bins=30)\n    plt.title(f'Distribution of Competitions by {col}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-17T00:01:46.930341Z","iopub.execute_input":"2023-05-17T00:01:46.930753Z","iopub.status.idle":"2023-05-17T00:01:48.096668Z","shell.execute_reply.started":"2023-05-17T00:01:46.930723Z","shell.execute_reply":"2023-05-17T00:01:48.095390Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Dustin Ober https://www.kaggle.com/code/dustinober/kjv-eda\n\n# Scatter plots for numerical variables\nsns.pairplot(df_grp, vars=numerical_columns, corner=True)\nplt.title('Competitions Pairplot')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-17T00:03:37.146475Z","iopub.execute_input":"2023-05-17T00:03:37.146903Z","iopub.status.idle":"2023-05-17T00:03:38.902270Z","shell.execute_reply.started":"2023-05-17T00:03:37.146872Z","shell.execute_reply":"2023-05-17T00:03:38.901019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Closing Arguments\n\n\"In this analytics competition, participants are expected to describe what the community has learned over the past 2 years of working and experimenting with.\"\n\nWhat the community has learned I don't know, however what I learned over the 4 years isn't enough for any professional purpose. Albeit I had and still have a lot of fun during those Competition's years.","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nAnthony Wynne https://www.kaggle.com/code/anthony35813/ai-report-a-first-look-at-the-data-for-text-data\n\nRoland Erdei https://www.kaggle.com/code/rolanderdei/predict-my-sleep-patterns-with-simple-ensembling\n\nRob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nDustin Ober https://www.kaggle.com/code/dustinober/kjv-eda\n\nMPWolke https://www.kaggle.com/code/mpwolke/covid-19-nigeria\n\nMPWolke https://www.kaggle.com/code/mpwolke/science-denial-jsonl-split-date-time-displacy","metadata":{}}]}