{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-30T03:57:08.129943Z","iopub.execute_input":"2023-06-30T03:57:08.130400Z","iopub.status.idle":"2023-06-30T03:57:08.160366Z","shell.execute_reply.started":"2023-06-30T03:57:08.130365Z","shell.execute_reply":"2023-06-30T03:57:08.159517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- Codes by HTMLcodes.ws -->\n<h1 style = \"color:Blue;font-family:newtimeroman;font-size:250%;text-align:left;border-radius:15px 50px;\">2023 Kaggle AI Report</h1>\n\nhttps://www.kaggle.com/competitions/2023-kaggle-ai-report/overview","metadata":{}},{"cell_type":"markdown","source":"# **Import the required libraries**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:57:11.535310Z","iopub.execute_input":"2023-06-30T03:57:11.535817Z","iopub.status.idle":"2023-06-30T03:57:12.475969Z","shell.execute_reply.started":"2023-06-30T03:57:11.535775Z","shell.execute_reply":"2023-06-30T03:57:12.474710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load the dataset**","metadata":{}},{"cell_type":"code","source":"# Read the dataset into a pandas DataFrame.\nessay_df = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\")\nsubmission_df = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:57:55.577092Z","iopub.execute_input":"2023-06-30T03:57:55.577521Z","iopub.status.idle":"2023-06-30T03:57:55.934044Z","shell.execute_reply.started":"2023-06-30T03:57:55.577489Z","shell.execute_reply":"2023-06-30T03:57:55.933246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Examine the dataset structure and variables\nessay_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:58:10.986237Z","iopub.execute_input":"2023-06-30T03:58:10.986638Z","iopub.status.idle":"2023-06-30T03:58:11.018439Z","shell.execute_reply.started":"2023-06-30T03:58:10.986585Z","shell.execute_reply":"2023-06-30T03:58:11.017208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop rows with missing writeups\nessay_df = essay_df.dropna(subset=['Writeup'])","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:58:19.514494Z","iopub.execute_input":"2023-06-30T03:58:19.514879Z","iopub.status.idle":"2023-06-30T03:58:19.527746Z","shell.execute_reply.started":"2023-06-30T03:58:19.514848Z","shell.execute_reply":"2023-06-30T03:58:19.526582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(essay_df.info())","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:58:27.725353Z","iopub.execute_input":"2023-06-30T03:58:27.725738Z","iopub.status.idle":"2023-06-30T03:58:27.749163Z","shell.execute_reply.started":"2023-06-30T03:58:27.725701Z","shell.execute_reply":"2023-06-30T03:58:27.747976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform descriptive statistics on the dataset\nessay_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:58:34.288639Z","iopub.execute_input":"2023-06-30T03:58:34.289048Z","iopub.status.idle":"2023-06-30T03:58:34.335181Z","shell.execute_reply.started":"2023-06-30T03:58:34.289012Z","shell.execute_reply":"2023-06-30T03:58:34.334118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Competition Launch Date Analysis:**\n\n* Total number of competitions\n* Earliest competition launch date\n* Latest competition launch date","metadata":{}},{"cell_type":"code","source":"# Competition Launch Date Analysis\ntotal_competitions = len(essay_df['Competition URL'].unique())\nearliest_launch_date = essay_df['Competition Launch Date'].min()\nlatest_launch_date = essay_df['Competition Launch Date'].max()\n\n# Print the results\nprint(\"Competition Launch Date Analysis:\")\nprint(\"Total number of competitions:\", total_competitions)\nprint(\"Earliest launch date:\", earliest_launch_date)\nprint(\"Latest launch date:\", latest_launch_date)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:58:56.329435Z","iopub.execute_input":"2023-06-30T03:58:56.330075Z","iopub.status.idle":"2023-06-30T03:58:56.339395Z","shell.execute_reply.started":"2023-06-30T03:58:56.330040Z","shell.execute_reply":"2023-06-30T03:58:56.338232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Competition Analysis:**\n\n* Most common competition titles.\n* Competition with the most write-ups.\n* Number of write-ups per competition.","metadata":{}},{"cell_type":"code","source":"# Competition Analysis\nmost_common_titles = essay_df['Title of Competition'].value_counts().head(5)\ncompetition_with_most_writeups = essay_df['Title of Competition'].mode()[0]\nwriteups_per_competition = essay_df['Title of Competition'].value_counts()\n\n# Number of write-ups per competition\nfig = px.bar(writeups_per_competition,\n             x=writeups_per_competition.index,\n             y=writeups_per_competition.values,\n             labels={'x': 'Competition', 'y': 'Number of Write-ups'},\n             title='Number of Write-ups per Competition')\n\nfig.update_layout(xaxis_tickangle=-45, \n                  xaxis_title_font=dict(size=14),\n                  yaxis_title_font=dict(size=14),\n                  plot_bgcolor='rgba(0,0,0,0)',  # Transparent background\n                  paper_bgcolor='rgba(0,0,0,0)',  # Transparent background\n                  font_family='Arial',\n                  font_color='#333333')\n\nfig.show()\n\n# Print the results\nprint(\"Competition Analysis:\")\nprint(\"Most common competition titles:\")\nprint(most_common_titles)\nprint(\"Competition with the most write-ups:\", competition_with_most_writeups)\nprint(\"Number of write-ups per competition:\")\nprint(writeups_per_competition)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:59:21.791877Z","iopub.execute_input":"2023-06-30T03:59:21.792515Z","iopub.status.idle":"2023-06-30T03:59:23.370220Z","shell.execute_reply.started":"2023-06-30T03:59:21.792479Z","shell.execute_reply":"2023-06-30T03:59:23.369166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Write-up Analysis:**\n\n* Earliest write-up date\n* Latest write-up date\n* Most common write-up titles\n* Average word count per write-up\n* Word cloud visualization of write-up titles","metadata":{}},{"cell_type":"code","source":"# Write-up Analysis\nearliest_writeup_date = essay_df['Date of Writeup'].min()\nlatest_writeup_date = essay_df['Date of Writeup'].max()\nmost_common_writeup_titles = essay_df['Title of Writeup'].value_counts().head(5)\naverage_word_count = essay_df['Writeup'].apply(lambda x: len(str(x).split()) if pd.notnull(x) else 0).mean()\n\n# Word Cloud Visualization\nwordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(essay_df['Title of Writeup'].value_counts())\n\n# Display Word Cloud\nplt.figure(figsize=(10, 6))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis('off')\nplt.show()\n\n# Print the results\nprint(\"Write-up Analysis:\")\nprint(\"Earliest write-up date:\", earliest_writeup_date)\nprint(\"Latest write-up date:\", latest_writeup_date)\nprint(\"Most common write-up titles:\")\nprint(most_common_writeup_titles)\nprint(\"Average word count per write-up:\", average_word_count)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:59:40.656459Z","iopub.execute_input":"2023-06-30T03:59:40.657627Z","iopub.status.idle":"2023-06-30T03:59:41.489304Z","shell.execute_reply.started":"2023-06-30T03:59:40.657588Z","shell.execute_reply":"2023-06-30T03:59:41.488286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate average word count per write-up\nessay_df['Word Count'] = essay_df['Writeup'].apply(lambda x: len(str(x).split()))\naverage_word_count = essay_df['Word Count'].mean()\n\n# Print the average word count\nprint(\"The average word count per write-up in the 'Chess ratings - Elo versus the Rest of the World' competition is:\", average_word_count)\n\nimport plotly.graph_objects as go\nfrom wordcloud import WordCloud\n\n# Generate word frequencies\nword_frequencies = essay_df['Writeup'].str.split(expand=True).stack().value_counts()\n\n# Create a word cloud\nwordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(word_frequencies)\n\n# Create a figure\nfig = go.Figure(data=go.Image(z=wordcloud.to_array()))\n\n# Customize the layout\nfig.update_layout(\n    title=\"Word Cloud of Write-ups\",\n    xaxis=dict(visible=False),\n    yaxis=dict(visible=False),\n)\n\n# Show the plot\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T03:59:51.236392Z","iopub.execute_input":"2023-06-30T03:59:51.237365Z","iopub.status.idle":"2023-06-30T03:59:55.621221Z","shell.execute_reply.started":"2023-06-30T03:59:51.237316Z","shell.execute_reply":"2023-06-30T03:59:55.619671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Convert the relevant columns to the appropriate data types.\nessay_df['Competition Launch Date'] = pd.to_datetime(essay_df['Competition Launch Date'])\nessay_df['Date of Writeup'] = pd.to_datetime(essay_df['Date of Writeup'])","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:00:03.599707Z","iopub.execute_input":"2023-06-30T04:00:03.600085Z","iopub.status.idle":"2023-06-30T04:00:03.955434Z","shell.execute_reply.started":"2023-06-30T04:00:03.600056Z","shell.execute_reply":"2023-06-30T04:00:03.954385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculate the total number of competitions launched\ntotal_competitions = essay_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:00:21.804223Z","iopub.execute_input":"2023-06-30T04:00:21.804576Z","iopub.status.idle":"2023-06-30T04:00:21.809460Z","shell.execute_reply.started":"2023-06-30T04:00:21.804548Z","shell.execute_reply":"2023-06-30T04:00:21.808334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create a bar plot to visualize the number of competitions launched per year.\n\ncompetition_counts = essay_df['Competition Launch Date'].dt.year.value_counts().sort_index()\n\nfig = px.bar(x=competition_counts.index, y=competition_counts.values,\n             labels={\"x\": \"Year\", \"y\": \"Number of Competitions Launched\"},\n             title=\"Number of Competitions Launched Per Year\",\n             width=800, height=400)\nfig.update_layout(showlegend=False)\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:00:27.394423Z","iopub.execute_input":"2023-06-30T04:00:27.395073Z","iopub.status.idle":"2023-06-30T04:00:27.460336Z","shell.execute_reply.started":"2023-06-30T04:00:27.395039Z","shell.execute_reply":"2023-06-30T04:00:27.459321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculate the average number of writeups per competition\navg_writeups_per_competition = essay_df.shape[0] / total_competitions","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:01:02.351834Z","iopub.execute_input":"2023-06-30T04:01:02.352283Z","iopub.status.idle":"2023-06-30T04:01:02.357349Z","shell.execute_reply.started":"2023-06-30T04:01:02.352246Z","shell.execute_reply":"2023-06-30T04:01:02.356485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Display the total number of competitions and the average number of writeups per competition.\nprint(\"Total Competitions Launched:\", total_competitions)\nprint(\"Average Writeups per Competition:\", avg_writeups_per_competition)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:02:16.268769Z","iopub.execute_input":"2023-06-30T04:02:16.269185Z","iopub.status.idle":"2023-06-30T04:02:16.274449Z","shell.execute_reply.started":"2023-06-30T04:02:16.269155Z","shell.execute_reply":"2023-06-30T04:02:16.273421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this analysis, we read the dataset, convert the necessary columns to datetime format, calculate the total number of competitions launched, create a bar plot to visualize the number of competitions launched per year, and calculate the average number of writeups per competition. Finally, we display the total number of competitions and the average number of writeups per competition.","metadata":{}},{"cell_type":"code","source":"import json\nimport pandas as pd\n\ndata = []\nwith open('/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json', 'r') as f:\n    for line in f:\n        data_item = json.loads(line)\n        data.append(data_item)\n\ndf = pd.DataFrame([item['title'] for item in data], columns=['Title'])\nprint(df)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:02:46.457616Z","iopub.execute_input":"2023-06-30T04:02:46.458050Z","iopub.status.idle":"2023-06-30T04:04:25.230196Z","shell.execute_reply.started":"2023-06-30T04:02:46.458012Z","shell.execute_reply":"2023-06-30T04:04:25.229065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Distribution:**\n\n* Calculates the total number of records in the dataset.\n* Determines the number of unique titles.","metadata":{}},{"cell_type":"code","source":"# Data Distribution\ntotal_records = len(df)\nunique_titles = len(df['Title'].unique())\n\n# Print the results\nprint(\"Data Distribution:\")\nprint(\"Total records:\", total_records)\nprint(\"Unique titles:\", unique_titles)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:04:25.232215Z","iopub.execute_input":"2023-06-30T04:04:25.233232Z","iopub.status.idle":"2023-06-30T04:04:27.338555Z","shell.execute_reply.started":"2023-06-30T04:04:25.233195Z","shell.execute_reply":"2023-06-30T04:04:27.337391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Word Frequency:**\n\n* Counts the frequency of each word in the titles.\n* Selects the most common words based on the frequency.\n* Prints the most common words in the titles.","metadata":{}},{"cell_type":"code","source":"# Word Frequency\nword_freq = df['Title'].str.split(expand=True).stack().value_counts()\nmost_common_words = word_freq[:10]\n\n# Word Cloud Visualization\nwordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(word_freq)\n\n# Display Word Cloud\nplt.figure(figsize=(10, 6))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:04:41.173237Z","iopub.execute_input":"2023-06-30T04:04:41.173613Z","iopub.status.idle":"2023-06-30T04:05:26.642683Z","shell.execute_reply.started":"2023-06-30T04:04:41.173585Z","shell.execute_reply":"2023-06-30T04:05:26.641502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Text Length:**\n\n* Computes the length of each title.\n* Creates a histogram to visualize the distribution of title lengths.\n* Calculates the average title length.","metadata":{}},{"cell_type":"code","source":"# Text Length\ndf['Title_Length'] = df['Title'].apply(lambda x: len(x))\ntitle_length_distribution = px.histogram(df, x='Title_Length')\n\naverage_title_length = df['Title_Length'].mean()\n\n# Print the results\nprint(\"Data Distribution:\")\nprint(f\"Total number of records: {total_records}\")\nprint(f\"Number of unique titles: {unique_titles}\\n\")\n\nprint(\"Word Frequency:\")\nprint(\"Most common words in titles:\")\nprint(most_common_words)\n\nprint(\"\\nText Length:\")\nprint(\"Distribution of title lengths:\")\ntitle_length_distribution.show()\n\nprint(f\"\\nAverage title length: {average_title_length}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:05:26.644675Z","iopub.execute_input":"2023-06-30T04:05:26.645052Z","iopub.status.idle":"2023-06-30T04:05:28.080768Z","shell.execute_reply.started":"2023-06-30T04:05:26.645019Z","shell.execute_reply":"2023-06-30T04:05:28.079554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Submission**","metadata":{}},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/sample_submission.csv\")\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:05:28.082550Z","iopub.execute_input":"2023-06-30T04:05:28.083269Z","iopub.status.idle":"2023-06-30T04:05:28.099438Z","shell.execute_reply.started":"2023-06-30T04:05:28.083225Z","shell.execute_reply":"2023-06-30T04:05:28.098164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.loc[0]['value']='Kaggle Competition'\nsubmission_df.loc[1]['value']='https://www.kaggle.com/code/jocelyndumlao/ai-report-kaggle-competitions'\nsubmission_df.loc[2]['value']='https://www.kaggle.com/code/mpwolke/pandora-kaggle-s-box-disclosure/comments#2317744'\nsubmission_df.loc[3]['value']='https://www.kaggle.com/code/arunodhayan/kaggle-ai-report-birdclef-series-2019-2023/comments#2317746'\nsubmission_df.loc[4]['value']='https://www.kaggle.com/code/yeemeitsang/unreasonably-effective-ensemble-learning/comments#2317765'\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:05:28.101633Z","iopub.execute_input":"2023-06-30T04:05:28.102028Z","iopub.status.idle":"2023-06-30T04:05:28.115020Z","shell.execute_reply.started":"2023-06-30T04:05:28.101993Z","shell.execute_reply":"2023-06-30T04:05:28.114035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the submission to a CSV file\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-30T04:05:28.116241Z","iopub.execute_input":"2023-06-30T04:05:28.116584Z","iopub.status.idle":"2023-06-30T04:05:28.129232Z","shell.execute_reply.started":"2023-06-30T04:05:28.116555Z","shell.execute_reply":"2023-06-30T04:05:28.128039Z"},"trusted":true},"execution_count":null,"outputs":[]}]}