{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport json\nimport matplotlib.pyplot as plt\n\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\nfrom nltk.corpus import stopwords as sw\nimport nltk\nfrom bs4 import BeautifulSoup\n\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nimport nltk\nfrom nltk.tokenize import word_tokenize\nimport string\n\n\nnltk.download('stopwords')\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T12:07:10.924091Z","iopub.execute_input":"2023-07-05T12:07:10.924528Z","iopub.status.idle":"2023-07-05T12:07:14.231572Z","shell.execute_reply.started":"2023-07-05T12:07:10.924496Z","shell.execute_reply":"2023-07-05T12:07:14.230294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load , clean, and show Datasets","metadata":{}},{"cell_type":"code","source":"# Load a dataset into a Pandas Dataframe\nwriteups_df = pd.read_csv('/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')\nforummas_df = pd.read_csv('/kaggle/input/meta-kaggle/ForumMessages.csv')\nforumtop_df = pd.read_csv('/kaggle/input/meta-kaggle/ForumTopics.csv')\nwinsol_df = pd.read_csv('/kaggle/input/kaggle-winning-solutions-methods/kaggle_winning_solutions_methods_detail.csv')\nwinsolmap_df = pd.read_csv('/kaggle/input/kaggle-winning-solutions-methods/method_category_mapping.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T12:07:14.234107Z","iopub.execute_input":"2023-07-05T12:07:14.236269Z","iopub.status.idle":"2023-07-05T12:07:44.689847Z","shell.execute_reply.started":"2023-07-05T12:07:14.236188Z","shell.execute_reply":"2023-07-05T12:07:44.688739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_file = '/kaggle/input/arxiv/arxiv-metadata-oai-snapshot.json'\n\nwith open(data_file, 'r') as f:\n    arxiv_df = pd.DataFrame(json.loads(line) for line in f)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:19:26.438693Z","iopub.execute_input":"2023-07-05T03:19:26.439092Z","iopub.status.idle":"2023-07-05T03:22:03.065382Z","shell.execute_reply.started":"2023-07-05T03:19:26.439060Z","shell.execute_reply":"2023-07-05T03:22:03.064481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_html(text):\n    if isinstance(text, str):\n        return BeautifulSoup(text, \"html.parser\").get_text()\n    return text","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:03.069462Z","iopub.execute_input":"2023-07-05T03:22:03.070583Z","iopub.status.idle":"2023-07-05T03:22:03.094957Z","shell.execute_reply.started":"2023-07-05T03:22:03.070514Z","shell.execute_reply":"2023-07-05T03:22:03.093710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"writeups_df['Writeup'] = writeups_df['Writeup'].apply(clean_html)\nwriteups_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:03.102026Z","iopub.execute_input":"2023-07-05T03:22:03.102435Z","iopub.status.idle":"2023-07-05T03:22:11.503996Z","shell.execute_reply.started":"2023-07-05T03:22:03.102406Z","shell.execute_reply":"2023-07-05T03:22:11.502828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forummas_df['Message'] = forummas_df['Message'].apply(clean_html)\n# forummas_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:11.505300Z","iopub.execute_input":"2023-07-05T03:22:11.505761Z","iopub.status.idle":"2023-07-05T03:22:11.510799Z","shell.execute_reply.started":"2023-07-05T03:22:11.505728Z","shell.execute_reply":"2023-07-05T03:22:11.509586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"forumtop_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:11.512043Z","iopub.execute_input":"2023-07-05T03:22:11.512356Z","iopub.status.idle":"2023-07-05T03:22:11.540332Z","shell.execute_reply.started":"2023-07-05T03:22:11.512330Z","shell.execute_reply":"2023-07-05T03:22:11.539252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_df['update_date'] = pd.to_datetime(arxiv_df['update_date'])\narxiv_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:11.541667Z","iopub.execute_input":"2023-07-05T03:22:11.542020Z","iopub.status.idle":"2023-07-05T03:22:14.297554Z","shell.execute_reply.started":"2023-07-05T03:22:11.541991Z","shell.execute_reply":"2023-07-05T03:22:14.296384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"winsol_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T12:07:44.691563Z","iopub.execute_input":"2023-07-05T12:07:44.692577Z","iopub.status.idle":"2023-07-05T12:07:44.735952Z","shell.execute_reply.started":"2023-07-05T12:07:44.692525Z","shell.execute_reply":"2023-07-05T12:07:44.734600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"winsolmap_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T12:07:44.738654Z","iopub.execute_input":"2023-07-05T12:07:44.739599Z","iopub.status.idle":"2023-07-05T12:07:44.756637Z","shell.execute_reply.started":"2023-07-05T12:07:44.739563Z","shell.execute_reply":"2023-07-05T12:07:44.755104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract year from \"update_date\"\narxiv_df['Year'] = arxiv_df['update_date'].dt.year\narxiv_df['Month'] = arxiv_df['update_date'].dt.month\n\n# Group by year and count the number of papers\npapers_per_year = arxiv_df.groupby('Year').size()\n\n# Plot the number of papers per year\nplt.figure(figsize=(10,6))\npapers_per_year.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Number of Arxiv Papers Published per Year')\nplt.xlabel('Year')\nplt.ylabel('Number of Papers')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:29:50.663430Z","iopub.execute_input":"2023-07-05T03:29:50.663896Z","iopub.status.idle":"2023-07-05T03:29:51.650069Z","shell.execute_reply.started":"2023-07-05T03:29:50.663859Z","shell.execute_reply":"2023-07-05T03:29:51.648759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wordcloud","metadata":{}},{"cell_type":"markdown","source":"## Create function to generate the wordcloud","metadata":{}},{"cell_type":"code","source":"def generate_wordcloud(df, column_name):\n    # Combine all the text into one large string\n    all_text = \" \".join(text for text in df[column_name] if isinstance(text, str))\n\n    # Generate a word cloud\n    stop_words = set(sw.words('english'))\n    wordcloud = WordCloud(stopwords=stop_words, background_color=\"white\").generate(all_text)\n\n    # Display the generated image:\n    plt.imshow(wordcloud, interpolation='bilinear')\n    plt.axis(\"off\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:15.017807Z","iopub.execute_input":"2023-07-05T03:22:15.018641Z","iopub.status.idle":"2023-07-05T03:22:15.025806Z","shell.execute_reply.started":"2023-07-05T03:22:15.018607Z","shell.execute_reply":"2023-07-05T03:22:15.024568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wordcloud for winner's writeup","metadata":{}},{"cell_type":"code","source":"generate_wordcloud(writeups_df, 'Writeup')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:15.027615Z","iopub.execute_input":"2023-07-05T03:22:15.028067Z","iopub.status.idle":"2023-07-05T03:22:24.579440Z","shell.execute_reply.started":"2023-07-05T03:22:15.028029Z","shell.execute_reply":"2023-07-05T03:22:24.578646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wordcloud for forum discussion title","metadata":{}},{"cell_type":"code","source":"generate_wordcloud(forumtop_df, 'Title')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:24.580382Z","iopub.execute_input":"2023-07-05T03:22:24.580680Z","iopub.status.idle":"2023-07-05T03:22:34.693711Z","shell.execute_reply.started":"2023-07-05T03:22:24.580654Z","shell.execute_reply":"2023-07-05T03:22:34.692407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most Frequent Words","metadata":{}},{"cell_type":"markdown","source":"## Create function to generate the word bar graph","metadata":{}},{"cell_type":"code","source":"def generate_word_bargraph(df, column_name, num_words=10, title='Most Common Words'):\n    # Define the set of stopwords to filter out\n    stop_words = set(sw.words('english'))\n\n    words = []\n    for text in df[column_name]:\n        if isinstance(text, str):\n            # Tokenize the text\n            text_words = word_tokenize(text)\n            for word in text_words:\n                # Convert to lower case\n                word = word.lower()\n                # Remove punctuation\n                word = word.translate(str.maketrans('', '', string.punctuation))\n                if word and word not in stop_words:\n                    words.append(word)\n    \n    # Count word frequencies\n    word_counts = Counter(words)\n\n    # Get the most common words and their counts\n    common_words = word_counts.most_common(num_words)\n\n    # Create a bar chart to show the frequency of the most common words\n    words, counts = zip(*common_words)\n    plt.figure(figsize=(10,6))\n    plt.bar(words, counts, color='skyblue', edgecolor='black')\n    plt.title(title)\n    plt.xlabel('Words')\n    plt.ylabel('Count')\n    plt.xticks(rotation=45)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:34.695651Z","iopub.execute_input":"2023-07-05T03:22:34.696094Z","iopub.status.idle":"2023-07-05T03:22:34.709373Z","shell.execute_reply.started":"2023-07-05T03:22:34.696055Z","shell.execute_reply":"2023-07-05T03:22:34.707987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Word bar graph for winner's writeup","metadata":{}},{"cell_type":"code","source":"generate_word_bargraph(writeups_df, 'Writeup', title='Most Common Words in Winner Writeups')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:22:34.714333Z","iopub.execute_input":"2023-07-05T03:22:34.714860Z","iopub.status.idle":"2023-07-05T03:23:06.971822Z","shell.execute_reply.started":"2023-07-05T03:22:34.714818Z","shell.execute_reply":"2023-07-05T03:23:06.970819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Word bar graph for forum discussion title","metadata":{}},{"cell_type":"code","source":"generate_word_bargraph(forumtop_df, 'Title', title='Most Common Words in Forum Titles')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:23:06.973068Z","iopub.execute_input":"2023-07-05T03:23:06.973370Z","iopub.status.idle":"2023-07-05T03:23:51.197487Z","shell.execute_reply.started":"2023-07-05T03:23:06.973344Z","shell.execute_reply":"2023-07-05T03:23:51.196364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Arxiv Trends","metadata":{}},{"cell_type":"code","source":"# Define keywords related to data-centric AI\nkeywords = ['data-centric', 'data quality', 'data cleaning', 'data augmentation', \n            'data labeling', 'data governance', 'data preparation',\n            \"training data development\", \"data collection\", \"data labeling\", \"data preparation\",\n            \"data reduction\", \"data augmentation\", \"pipeline search\", \"inference data development\",\n            \"in-distribution evaluation\", \"out-of-distribution evaluation\", \"prompt engineering\",\n            \"data maintenance\", \"data understanding\", \"data quality assurance\",\n            \"data storage and retrieval\", \"data benchmark\", \"data-centric evaluation\",\n            \"data-centric methods\", \"data-centric ai development\", \"data-centric systems\",\n            \"data-centric machine learning\", \"active learning\", \"feature selection\",\n            \"feature extraction\", \"data discovery\", \"data integration\",\n            \"data curation\", \"data normalization\", \"data standardization\", \"imbalanced learning\",\n            \"synthetic data\", \"data valuation\", \"data quality verification\",\n            \"automated machine learning\", \"domain adaptation\",\n            \"data semantics\", \"feature engineering\", \"machine learning pipelines\"]\n\n# Function to check whether a paper is related to data-centric AI\ndef is_data_centric(text):\n    for keyword in keywords:\n        if keyword in text:\n            return True\n    return False\n\n# Apply the function to the 'title' and 'abstract' columns\narxiv_df['title'] = arxiv_df['title'].str.lower()\narxiv_df['abstract'] = arxiv_df['abstract'].str.lower()\narxiv_df['Data_Centric'] = arxiv_df['title'].apply(is_data_centric) | arxiv_df['abstract'].apply(is_data_centric)\n\n# Filter for the years 2017-2023\narxiv_df_filtered = arxiv_df[(arxiv_df['Year'] >= 2017) & (arxiv_df['Year'] <= 2023)]\n\n# Group by year and sum the new column\ndata_centric_papers_per_year = arxiv_df_filtered.groupby('Year')['Data_Centric'].sum()\n\n# Number of months for which we have data in 2023\nm = arxiv_df_filtered[arxiv_df_filtered['Year'] == 2023]['Month'].max()\n\n# Number of papers in 2023 for 'm' months\npapers_m_months = data_centric_papers_per_year.loc[2023]\n\n# Projected number of papers for 2023\nprojected_papers_2023 = (papers_m_months / m) * 12\n\n# Replace the original number with the projected number for 2023\ndata_centric_papers_per_year.loc[2023] = projected_papers_2023","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:45:55.385769Z","iopub.execute_input":"2023-07-05T03:45:55.386671Z","iopub.status.idle":"2023-07-05T03:47:46.267453Z","shell.execute_reply.started":"2023-07-05T03:45:55.386628Z","shell.execute_reply":"2023-07-05T03:47:46.266509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the number of data-centric papers per year\nplt.figure(figsize=(10,6))\ndata_centric_papers_per_year.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Number of Data-Centric Related Arxiv Papers Published per Year (2017-2023)')\nplt.xlabel('Year')\nplt.ylabel('Number of Papers')\n\n# Change the x-tick label for 2023 to '2023 (projected)'\nx_ticks_labels = [str(year) if year != 2023 else '2023 (proj)' for year in data_centric_papers_per_year.index]\nplt.xticks(ticks=np.arange(len(x_ticks_labels)), labels=x_ticks_labels, rotation=45)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T03:54:41.760579Z","iopub.execute_input":"2023-07-05T03:54:41.761133Z","iopub.status.idle":"2023-07-05T03:54:42.129496Z","shell.execute_reply.started":"2023-07-05T03:54:41.761097Z","shell.execute_reply":"2023-07-05T03:54:42.128478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Winning Solution Trend","metadata":{}},{"cell_type":"code","source":"# Assuming 'cleaned_methods' are separated by commas, if not you can replace ',' with the appropriate separator.\nmethods = winsol_df['cleaned_methods'].str.split(',', expand=True).stack().str.strip()\n\n# Count the frequency of each method\nmethod_counts = methods.value_counts()\n\n# Select the top 10 methods\ntop_10_methods = method_counts.head(10)\n\n# Plot the bar chart for top 10 methods\nplt.figure(figsize=(10,6))\ntop_10_methods.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Frequency of Top 10 Methods Used in Winning Solution')\nplt.xlabel('Method')\nplt.ylabel('Frequency')\n# plt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:11:10.926910Z","iopub.execute_input":"2023-07-05T13:11:10.927559Z","iopub.status.idle":"2023-07-05T13:11:11.423727Z","shell.execute_reply.started":"2023-07-05T13:11:10.927511Z","shell.execute_reply":"2023-07-05T13:11:11.422330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_top_10_methods(cleaned_method):\n    # Filter dataframe for rows where 'cleaned_methods' is exactly the cleaned_method\n    filtered_df = winsol_df[winsol_df['cleaned_methods'] == cleaned_method]\n    \n    # Split the 'methods' column and count the frequency of each method\n    methods = filtered_df['methods'].str.split(',', expand=True).stack().str.strip()\n    method_counts = methods.value_counts()\n    \n    # Select the top 10 methods\n    top_10_methods = method_counts.head(10)\n    \n    # Plot the bar chart for top 10 methods\n    plt.figure(figsize=(10,6))\n    top_10_methods.plot(kind='bar', color='skyblue', edgecolor='black')\n    plt.title(f'Frequency of Top 10 {cleaned_method} Methods')\n    plt.xlabel('Method')\n    plt.ylabel('Frequency')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:10:03.283110Z","iopub.execute_input":"2023-07-05T13:10:03.284131Z","iopub.status.idle":"2023-07-05T13:10:03.292942Z","shell.execute_reply.started":"2023-07-05T13:10:03.284089Z","shell.execute_reply":"2023-07-05T13:10:03.291507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_top_10_methods('Ensemble Methods')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:10:05.710117Z","iopub.execute_input":"2023-07-05T13:10:05.710657Z","iopub.status.idle":"2023-07-05T13:10:06.096639Z","shell.execute_reply.started":"2023-07-05T13:10:05.710614Z","shell.execute_reply":"2023-07-05T13:10:06.095473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_top_10_methods('Data Augmentation')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:10:10.143751Z","iopub.execute_input":"2023-07-05T13:10:10.144275Z","iopub.status.idle":"2023-07-05T13:10:10.516636Z","shell.execute_reply.started":"2023-07-05T13:10:10.144211Z","shell.execute_reply":"2023-07-05T13:10:10.515313Z"},"trusted":true},"execution_count":null,"outputs":[]}]}