{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-07T23:01:44.884522Z","iopub.execute_input":"2023-06-07T23:01:44.884913Z","iopub.status.idle":"2023-06-07T23:01:44.899485Z","shell.execute_reply.started":"2023-06-07T23:01:44.884884Z","shell.execute_reply":"2023-06-07T23:01:44.898233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:01:44.90125Z","iopub.execute_input":"2023-06-07T23:01:44.902062Z","iopub.status.idle":"2023-06-07T23:01:44.906358Z","shell.execute_reply.started":"2023-06-07T23:01:44.902033Z","shell.execute_reply":"2023-06-07T23:01:44.905566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:01:44.907336Z","iopub.execute_input":"2023-06-07T23:01:44.908245Z","iopub.status.idle":"2023-06-07T23:01:44.91819Z","shell.execute_reply.started":"2023-06-07T23:01:44.908189Z","shell.execute_reply":"2023-06-07T23:01:44.917278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_json('/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json', lines=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:01:44.990815Z","iopub.execute_input":"2023-06-07T23:01:44.991433Z","iopub.status.idle":"2023-06-07T23:03:35.585514Z","shell.execute_reply.started":"2023-06-07T23:01:44.991403Z","shell.execute_reply":"2023-06-07T23:03:35.584578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.abstract[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:35.610117Z","iopub.execute_input":"2023-06-07T23:03:35.610513Z","iopub.status.idle":"2023-06-07T23:03:35.659756Z","shell.execute_reply.started":"2023-06-07T23:03:35.610485Z","shell.execute_reply":"2023-06-07T23:03:35.65873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.to_csv('/kaggle/working/musicAI.csv', index=True, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:35.660847Z","iopub.execute_input":"2023-06-07T23:03:35.662212Z","iopub.status.idle":"2023-06-07T23:03:35.674884Z","shell.execute_reply.started":"2023-06-07T23:03:35.662166Z","shell.execute_reply":"2023-06-07T23:03:35.67376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:35.677928Z","iopub.execute_input":"2023-06-07T23:03:35.678786Z","iopub.status.idle":"2023-06-07T23:03:35.689522Z","shell.execute_reply.started":"2023-06-07T23:03:35.678751Z","shell.execute_reply":"2023-06-07T23:03:35.688574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PLOT Number of Papers by Year","metadata":{}},{"cell_type":"code","source":"dfVersions = pd.DataFrame(df['versions'].apply(lambda x: [version['created'][-17:-13] for version in x if version['version'] == 'v1'][0] if any(version['version'] == 'v1' for version in x) else None))","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:35.690921Z","iopub.execute_input":"2023-06-07T23:03:35.69129Z","iopub.status.idle":"2023-06-07T23:03:39.196879Z","shell.execute_reply.started":"2023-06-07T23:03:35.69126Z","shell.execute_reply":"2023-06-07T23:03:39.195814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfVersions","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:39.198647Z","iopub.execute_input":"2023-06-07T23:03:39.199006Z","iopub.status.idle":"2023-06-07T23:03:39.223664Z","shell.execute_reply.started":"2023-06-07T23:03:39.198979Z","shell.execute_reply":"2023-06-07T23:03:39.221875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfVersions['versions'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:39.225235Z","iopub.execute_input":"2023-06-07T23:03:39.225551Z","iopub.status.idle":"2023-06-07T23:03:39.437874Z","shell.execute_reply.started":"2023-06-07T23:03:39.225526Z","shell.execute_reply":"2023-06-07T23:03:39.436649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfVersions.max()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:39.439268Z","iopub.execute_input":"2023-06-07T23:03:39.439552Z","iopub.status.idle":"2023-06-07T23:03:39.89431Z","shell.execute_reply.started":"2023-06-07T23:03:39.439528Z","shell.execute_reply":"2023-06-07T23:03:39.893156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfVersions['versions'].value_counts().dtypes","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:39.895309Z","iopub.execute_input":"2023-06-07T23:03:39.895549Z","iopub.status.idle":"2023-06-07T23:03:40.082883Z","shell.execute_reply.started":"2023-06-07T23:03:39.895528Z","shell.execute_reply":"2023-06-07T23:03:40.08216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nyears = dfVersions['versions'].value_counts().index.tolist()\nno_of_papers = dfVersions['versions'].value_counts().values.tolist()\n\nindex_2023 = years.index('2023')\nyears.insert(0, years.pop(index_2023))\nno_of_papers.insert(0, no_of_papers.pop(index_2023))\n\ncolors = ['blue' if year in ['2022', '2023'] else 'lightblue' for year in years]\n\nfig = go.Figure(data=[go.Bar(x=years, y=no_of_papers, marker_color=colors)])\n\nfig.update_layout(\n    title='Number of Papers by Year',\n    xaxis_title='Years',\n    yaxis_title='No of papers',\n    xaxis_tickangle=-45,\n    plot_bgcolor='black',\n    paper_bgcolor='black',\n    font=dict(color='white'),\n)\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:40.086588Z","iopub.execute_input":"2023-06-07T23:03:40.086943Z","iopub.status.idle":"2023-06-07T23:03:40.780883Z","shell.execute_reply.started":"2023-06-07T23:03:40.086904Z","shell.execute_reply":"2023-06-07T23:03:40.7798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter AI ML papers","metadata":{}},{"cell_type":"code","source":"AI_keywords = ['AI', 'Artificial Intelligence', 'ML', 'Machine Learning', 'Transformers', 'LLMS', 'Large Language Models', 'Machine Vision', 'LSTM', 'CNN', 'Deep Learning', 'Neural Networks', 'Natural Language Processing', 'Computer Vision', 'Reinforcement Learning', 'Generative Models', 'Robotic Process Automation', 'Speech Recognition', 'Virtual Assistants', 'Recommendation Systems', 'Data Science', 'Data Mining', 'Big Data', 'Pattern Recognition', 'Predictive Analytics', 'Image Recognition', 'Supervised Learning', 'Unsupervised Learning', 'Semi-Supervised Learning', 'Transfer Learning', 'AutoML', 'Data Engineering', 'Data Preprocessing', 'Data Visualization', 'Feature Extraction', 'Dimensionality Reduction', 'Ensemble Learning', 'Cloud Computing', 'Internet of Things', 'Chatbots', 'Deep Reinforcement Learning', 'Self-Supervised Learning', 'Explainable AI', 'Bias in AI', 'Ethical AI', 'AI Governance', 'AI Ethics']","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:40.782408Z","iopub.execute_input":"2023-06-07T23:03:40.782777Z","iopub.status.idle":"2023-06-07T23:03:40.788454Z","shell.execute_reply.started":"2023-06-07T23:03:40.782745Z","shell.execute_reply":"2023-06-07T23:03:40.787351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['abstract'][0:2]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:40.78978Z","iopub.execute_input":"2023-06-07T23:03:40.790108Z","iopub.status.idle":"2023-06-07T23:03:40.808684Z","shell.execute_reply.started":"2023-06-07T23:03:40.790078Z","shell.execute_reply":"2023-06-07T23:03:40.807932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['abstract'] = df['abstract'].apply(lambda text: re.sub(r'[^a-zA-Z\\s]', '', text.replace(\"\\n\", \"\")).strip())","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:03:40.809744Z","iopub.execute_input":"2023-06-07T23:03:40.810561Z","iopub.status.idle":"2023-06-07T23:04:17.766114Z","shell.execute_reply.started":"2023-06-07T23:03:40.810532Z","shell.execute_reply":"2023-06-07T23:04:17.765032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['title'] = df['title'].apply(lambda text: re.sub(r'[^a-zA-Z\\s]', '', text.replace(\"\\n\", \"\")).strip())","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:17.767349Z","iopub.execute_input":"2023-06-07T23:04:17.768033Z","iopub.status.idle":"2023-06-07T23:04:23.162149Z","shell.execute_reply.started":"2023-06-07T23:04:17.768004Z","shell.execute_reply":"2023-06-07T23:04:23.161165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['abstract'][0:2]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:23.16327Z","iopub.execute_input":"2023-06-07T23:04:23.163545Z","iopub.status.idle":"2023-06-07T23:04:23.17122Z","shell.execute_reply.started":"2023-06-07T23:04:23.163521Z","shell.execute_reply":"2023-06-07T23:04:23.169888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['title'][0:2]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:23.172879Z","iopub.execute_input":"2023-06-07T23:04:23.173406Z","iopub.status.idle":"2023-06-07T23:04:23.192504Z","shell.execute_reply.started":"2023-06-07T23:04:23.173379Z","shell.execute_reply":"2023-06-07T23:04:23.191299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:23.194428Z","iopub.execute_input":"2023-06-07T23:04:23.194733Z","iopub.status.idle":"2023-06-07T23:04:23.236556Z","shell.execute_reply.started":"2023-06-07T23:04:23.194689Z","shell.execute_reply":"2023-06-07T23:04:23.235533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dfAI = df[df['title'].apply(lambda abstract: any(keyword in abstract for keyword in AI_keywords))|\n#     df['abstract'].apply(lambda abstract: any(keyword in abstract for keyword in AI_keywords))]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:23.238083Z","iopub.execute_input":"2023-06-07T23:04:23.238438Z","iopub.status.idle":"2023-06-07T23:04:23.242701Z","shell.execute_reply.started":"2023-06-07T23:04:23.238405Z","shell.execute_reply":"2023-06-07T23:04:23.241872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Filter df on 'title' and 'abstract' columns | ignore case\n# dfAI1 = df[\n#     df['title'].str.contains('|'.join(AI_keywords), case=False) |\n#     df['abstract'].str.contains('|'.join(AI_keywords), case=False)]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:04:23.243883Z","iopub.execute_input":"2023-06-07T23:04:23.244125Z","iopub.status.idle":"2023-06-07T23:06:06.526255Z","shell.execute_reply.started":"2023-06-07T23:04:23.244105Z","shell.execute_reply":"2023-06-07T23:06:06.521032Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['title'] = df['title']","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:15.654133Z","iopub.execute_input":"2023-06-07T23:06:15.654499Z","iopub.status.idle":"2023-06-07T23:19:46.987654Z","shell.execute_reply.started":"2023-06-07T23:06:15.654471Z","shell.execute_reply":"2023-06-07T23:19:46.986252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfAI1","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.526962Z","iopub.status.idle":"2023-06-07T23:06:06.527272Z","shell.execute_reply.started":"2023-06-07T23:06:06.527125Z","shell.execute_reply":"2023-06-07T23:06:06.52714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfAI = dfAI1","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.528829Z","iopub.status.idle":"2023-06-07T23:06:06.529139Z","shell.execute_reply.started":"2023-06-07T23:06:06.528993Z","shell.execute_reply":"2023-06-07T23:06:06.529008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfAI['abstract'][0:10]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.531293Z","iopub.status.idle":"2023-06-07T23:06:06.531695Z","shell.execute_reply.started":"2023-06-07T23:06:06.531529Z","shell.execute_reply":"2023-06-07T23:06:06.531547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories = dfAI['categories'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.532974Z","iopub.status.idle":"2023-06-07T23:06:06.533297Z","shell.execute_reply.started":"2023-06-07T23:06:06.533152Z","shell.execute_reply":"2023-06-07T23:06:06.533166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the unique categories\n# for category in categories:\n#     print(category)\ncategories","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.534381Z","iopub.status.idle":"2023-06-07T23:06:06.534675Z","shell.execute_reply.started":"2023-06-07T23:06:06.534541Z","shell.execute_reply":"2023-06-07T23:06:06.534554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the count of each category\ncategory_counts = dfAI['categories'].value_counts()\n\n# Create a DataFrame from the category counts\ncategory_table = pd.DataFrame({'Category': category_counts.index, 'Count': category_counts.values})\n\n# Display the category count table\nprint(category_table)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.535748Z","iopub.status.idle":"2023-06-07T23:06:06.536039Z","shell.execute_reply.started":"2023-06-07T23:06:06.535897Z","shell.execute_reply":"2023-06-07T23:06:06.53591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filtering Music Papers only","metadata":{}},{"cell_type":"code","source":"music_keywords = ['Music', 'Audio Processing', 'Music Information Retrieval', 'Music Generation',\n                  'Music Classification', 'Music Recommendation', 'Music Analysis', 'Audio Recognition',\n                  'Music Transcription', 'Music Synthesis', 'Music Emotion Recognition', 'Music Style Transfer',\n                  'Music Composition', 'Music Perception', 'Music Production']","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.537967Z","iopub.status.idle":"2023-06-07T23:06:06.538381Z","shell.execute_reply.started":"2023-06-07T23:06:06.538247Z","shell.execute_reply":"2023-06-07T23:06:06.538261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAI = dfAI[ dfAI['title'].apply(lambda abstract: any(keyword in abstract for keyword in music_keywords))|\n    dfAI['abstract'].apply(lambda abstract: any(keyword in abstract for keyword in music_keywords))]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.539055Z","iopub.status.idle":"2023-06-07T23:06:06.539599Z","shell.execute_reply.started":"2023-06-07T23:06:06.539463Z","shell.execute_reply":"2023-06-07T23:06:06.539478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAI","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.540743Z","iopub.status.idle":"2023-06-07T23:06:06.541349Z","shell.execute_reply.started":"2023-06-07T23:06:06.541193Z","shell.execute_reply":"2023-06-07T23:06:06.541209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAIyear = pd.DataFrame(dfMusicAI['versions'].apply(lambda x: [version['created'][-17:-13] for version in x if version['version'] == 'v1'][0] if any(version['version'] == 'v1' for version in x) else None))","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.542272Z","iopub.status.idle":"2023-06-07T23:06:06.542536Z","shell.execute_reply.started":"2023-06-07T23:06:06.542405Z","shell.execute_reply":"2023-06-07T23:06:06.542418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAIyear['versions'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.543501Z","iopub.status.idle":"2023-06-07T23:06:06.543789Z","shell.execute_reply.started":"2023-06-07T23:06:06.54364Z","shell.execute_reply":"2023-06-07T23:06:06.543652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nyears = dfMusicAIyear['versions'].value_counts().index.tolist()\nno_of_papers = dfMusicAIyear['versions'].value_counts().values.tolist()\n\n# Sort years and no_of_papers together\nsorted_years, sorted_no_of_papers = zip(*sorted(zip(years, no_of_papers)))\n\nfig = go.Figure(data=[go.Bar(x=sorted_years, y=sorted_no_of_papers, marker_color='blue')])\n\nfig.update_layout(\n    title='Number of Music Papers by Year',\n    xaxis_title='Years',\n    yaxis_title='No of Music Papers',\n    xaxis_tickangle=-45,\n    plot_bgcolor='black',\n    paper_bgcolor='black',\n    font=dict(color='white'),\n)\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.54453Z","iopub.status.idle":"2023-06-07T23:06:06.544808Z","shell.execute_reply.started":"2023-06-07T23:06:06.544661Z","shell.execute_reply":"2023-06-07T23:06:06.544673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_data = pd.DataFrame({\n    'Authors': dfMusicAI['authors'],\n    'Title': dfMusicAI['title'],\n    'Abstract': dfMusicAI['abstract'],\n    'Versions': dfMusicAI['versions'].apply(lambda x: [version['created'][-17:-13] for version in x if version['version'] == 'v1'][0] if any(version['version'] == 'v1' for version in x) else None)\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.545445Z","iopub.status.idle":"2023-06-07T23:06:06.545697Z","shell.execute_reply.started":"2023-06-07T23:06:06.545575Z","shell.execute_reply":"2023-06-07T23:06:06.545587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_data","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.547081Z","iopub.status.idle":"2023-06-07T23:06:06.547352Z","shell.execute_reply.started":"2023-06-07T23:06:06.547218Z","shell.execute_reply":"2023-06-07T23:06:06.547231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_data.to_csv('/kaggle/working/Music_Data.csv', index=True, header= True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.54814Z","iopub.status.idle":"2023-06-07T23:06:06.548398Z","shell.execute_reply.started":"2023-06-07T23:06:06.548268Z","shell.execute_reply":"2023-06-07T23:06:06.548282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filtering Music Composition","metadata":{}},{"cell_type":"code","source":"music_all_keywords1 = [ 'Composition',\n    'Compositional techniques',\n    'Compositional processes',\n    'Structure',\n    'Form',\n    'Melody',\n    'Harmony',\n    'Counterpoint',\n    'Improvisation',\n    'Arrangement',\n    'Notation',\n    'Analysis',\n    'Aesthetics',\n    'Computer-assisted composition',\n    'Algorithmic composition',\n    'Generative',\n    'Creative AI', \n    'Generating',\n    'Generation',\n    'mastering', 'mixing', 'mix','master','remixing', 'composing']","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.549326Z","iopub.status.idle":"2023-06-07T23:06:06.549575Z","shell.execute_reply.started":"2023-06-07T23:06:06.549452Z","shell.execute_reply":"2023-06-07T23:06:06.549463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAICompose = df_Music_data[df_Music_data['Title'].apply(lambda title: any(keyword in title for keyword in music_all_keywords1)) |\n                              df_Music_data['Abstract'].apply(lambda abstract: any(keyword in abstract for keyword in music_all_keywords1))]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.550316Z","iopub.status.idle":"2023-06-07T23:06:06.550567Z","shell.execute_reply.started":"2023-06-07T23:06:06.550438Z","shell.execute_reply":"2023-06-07T23:06:06.550452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAICompose","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.551323Z","iopub.status.idle":"2023-06-07T23:06:06.551565Z","shell.execute_reply.started":"2023-06-07T23:06:06.551446Z","shell.execute_reply":"2023-06-07T23:06:06.551457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusicAICompose['Versions'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.552285Z","iopub.status.idle":"2023-06-07T23:06:06.552529Z","shell.execute_reply.started":"2023-06-07T23:06:06.55241Z","shell.execute_reply":"2023-06-07T23:06:06.552421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_Compose_data = pd.DataFrame({\n    'Authors': dfMusicAICompose['Authors'],\n    'Title': dfMusicAICompose['Title'],\n    'Abstract': dfMusicAICompose['Abstract'],\n    'Year': dfMusicAICompose['Versions']\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.554001Z","iopub.status.idle":"2023-06-07T23:06:06.554272Z","shell.execute_reply.started":"2023-06-07T23:06:06.554136Z","shell.execute_reply":"2023-06-07T23:06:06.554149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_Compose_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.555023Z","iopub.status.idle":"2023-06-07T23:06:06.55528Z","shell.execute_reply.started":"2023-06-07T23:06:06.55515Z","shell.execute_reply":"2023-06-07T23:06:06.555164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Music_Compose_data.to_csv('/kaggle/working/Music_Compose_Data.csv', index=True, header= True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.556488Z","iopub.status.idle":"2023-06-07T23:06:06.556827Z","shell.execute_reply.started":"2023-06-07T23:06:06.556668Z","shell.execute_reply":"2023-06-07T23:06:06.556682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nyears = [2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014, 2013, 2012, 2011, 2010, 2023,\n2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000, 1999, 1998, 1997, 1996,\n1995, 1994, 1993, 1992, 1991, 1990, 1989, 1988, 1986]\n\nno_of_papers = [185976, 181599, 178275, 155917, 140377, 123781, 113440, 105130, 97590, 92875,\n84374, 76602, 70288, 64429, 64071, 58810, 55749, 50305, 46874, 43713, 39392,\n36105, 33140, 30669, 27700, 24170, 19610, 15872, 13006, 10078, 6729, 3190,\n353, 26, 6, 1, 1]\n\n# colors = np.random.rand(len(years))\n\n# plt.figure(figsize=(10, 6))\n# plt.scatter(years, no_of_papers, c=colors)\n# plt.xlabel('Years')\n# plt.ylabel('No of papers')\n# plt.title('Number of Papers by Year')\n# plt.xticks(rotation=45)\n# plt.tight_layout()\n# plt.show()\n# Generate unique colors for each year\ncolors = np.random.rand(len(years))\n\n# Fit a polynomial curve to the data\ncurve_fit = np.polyfit(years, no_of_papers, 3)  # Adjust the degree of the polynomial if needed\ncurve = np.poly1d(curve_fit)\ncurve_x = np.linspace(min(years), max(years), 100)\ncurve_y = curve(curve_x)\n\nplt.figure(figsize=(10, 6))\nplt.scatter(years, no_of_papers, c=colors)\nplt.plot(curve_x, curve_y, color='black', label='Curve Fit')\nplt.xlabel('Years')\nplt.ylabel('No of papers')\nplt.title('Number of Papers by Year')\nplt.xticks(rotation=45)\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.557737Z","iopub.status.idle":"2023-06-07T23:06:06.558077Z","shell.execute_reply.started":"2023-06-07T23:06:06.55793Z","shell.execute_reply":"2023-06-07T23:06:06.557959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic1000 = pd.read_csv('/kaggle/input/AIMusic1000/AIMusic1000.csv', delimiter=',')","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.560202Z","iopub.status.idle":"2023-06-07T23:06:06.560788Z","shell.execute_reply.started":"2023-06-07T23:06:06.560504Z","shell.execute_reply":"2023-06-07T23:06:06.560528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic1000","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.562756Z","iopub.status.idle":"2023-06-07T23:06:06.563235Z","shell.execute_reply.started":"2023-06-07T23:06:06.563016Z","shell.execute_reply":"2023-06-07T23:06:06.563035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic1000['Title'] = dfMusic1000['Title'].astype(str)\n\ndfMusic1000['Abstract'] = dfMusic1000['Abstract'].astype(str)\n\ndfMusic1000.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.564822Z","iopub.status.idle":"2023-06-07T23:06:06.565272Z","shell.execute_reply.started":"2023-06-07T23:06:06.565058Z","shell.execute_reply":"2023-06-07T23:06:06.565077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"music_all_keywords1","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.566624Z","iopub.status.idle":"2023-06-07T23:06:06.567089Z","shell.execute_reply.started":"2023-06-07T23:06:06.566873Z","shell.execute_reply":"2023-06-07T23:06:06.566893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023 = dfMusic1000[dfMusic1000['Title'].apply(lambda title: any(keyword in title for keyword in music_all_keywords1)) |\n                              dfMusic1000['Abstract'].apply(lambda abstract: any(keyword in abstract for keyword in music_all_keywords1))]","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.568473Z","iopub.status.idle":"2023-06-07T23:06:06.568927Z","shell.execute_reply.started":"2023-06-07T23:06:06.568693Z","shell.execute_reply":"2023-06-07T23:06:06.568729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.572775Z","iopub.status.idle":"2023-06-07T23:06:06.573135Z","shell.execute_reply.started":"2023-06-07T23:06:06.572984Z","shell.execute_reply":"2023-06-07T23:06:06.573001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023 = dfMusic2022and2023.dropna(subset=['Title', 'Authors'])\ndfMusic2022and2023","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.57446Z","iopub.status.idle":"2023-06-07T23:06:06.574774Z","shell.execute_reply.started":"2023-06-07T23:06:06.574619Z","shell.execute_reply":"2023-06-07T23:06:06.574632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.575871Z","iopub.status.idle":"2023-06-07T23:06:06.57616Z","shell.execute_reply.started":"2023-06-07T23:06:06.576031Z","shell.execute_reply":"2023-06-07T23:06:06.576044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023 = pd.DataFrame({\n    'Authors': dfMusic2022and2023['Authors'],\n    'Title': dfMusic2022and2023['Title'],\n    'Abstract': dfMusic2022and2023['Abstract'],\n    'Year': dfMusic2022and2023['Year']\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.577189Z","iopub.status.idle":"2023-06-07T23:06:06.577463Z","shell.execute_reply.started":"2023-06-07T23:06:06.577326Z","shell.execute_reply":"2023-06-07T23:06:06.577338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.578613Z","iopub.status.idle":"2023-06-07T23:06:06.57889Z","shell.execute_reply.started":"2023-06-07T23:06:06.578758Z","shell.execute_reply":"2023-06-07T23:06:06.57877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.57979Z","iopub.status.idle":"2023-06-07T23:06:06.580072Z","shell.execute_reply.started":"2023-06-07T23:06:06.579919Z","shell.execute_reply":"2023-06-07T23:06:06.579931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dfMusic2022and2023 = dfMusic2022and2023['Year'].astype(int) if None\ndfMusic2022and2023['Year'] = dfMusic2022and2023['Year'].fillna(0).astype(int)\n\ndfMusic2022and2023.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.580735Z","iopub.status.idle":"2023-06-07T23:06:06.581054Z","shell.execute_reply.started":"2023-06-07T23:06:06.580917Z","shell.execute_reply":"2023-06-07T23:06:06.58093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.58158Z","iopub.status.idle":"2023-06-07T23:06:06.581855Z","shell.execute_reply.started":"2023-06-07T23:06:06.581703Z","shell.execute_reply":"2023-06-07T23:06:06.581732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfMusic2022and2023.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.582665Z","iopub.status.idle":"2023-06-07T23:06:06.582974Z","shell.execute_reply.started":"2023-06-07T23:06:06.582832Z","shell.execute_reply":"2023-06-07T23:06:06.582843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfONE = dfMusic2022and2023.reset_index(drop=True)\n\ndfONE.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.584255Z","iopub.status.idle":"2023-06-07T23:06:06.584529Z","shell.execute_reply.started":"2023-06-07T23:06:06.584393Z","shell.execute_reply":"2023-06-07T23:06:06.584405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTWO = df_Music_Compose_data.reset_index(drop=True)\ndfTWO.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.585687Z","iopub.status.idle":"2023-06-07T23:06:06.58596Z","shell.execute_reply.started":"2023-06-07T23:06:06.585831Z","shell.execute_reply":"2023-06-07T23:06:06.585843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTWO['Year'] = dfTWO['Year'].fillna(0).astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.586506Z","iopub.status.idle":"2023-06-07T23:06:06.586832Z","shell.execute_reply.started":"2023-06-07T23:06:06.586694Z","shell.execute_reply":"2023-06-07T23:06:06.586706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTWO.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.587433Z","iopub.status.idle":"2023-06-07T23:06:06.587672Z","shell.execute_reply.started":"2023-06-07T23:06:06.587555Z","shell.execute_reply":"2023-06-07T23:06:06.587567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfONE['Dataset'] = 'ArXiV'\ndfTWO['Dataset'] = 'Music_2022_2023'","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.588352Z","iopub.status.idle":"2023-06-07T23:06:06.588604Z","shell.execute_reply.started":"2023-06-07T23:06:06.588482Z","shell.execute_reply":"2023-06-07T23:06:06.588495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfONE.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.589314Z","iopub.status.idle":"2023-06-07T23:06:06.58957Z","shell.execute_reply.started":"2023-06-07T23:06:06.589443Z","shell.execute_reply":"2023-06-07T23:06:06.589455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfTWO.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.591227Z","iopub.status.idle":"2023-06-07T23:06:06.591535Z","shell.execute_reply.started":"2023-06-07T23:06:06.591394Z","shell.execute_reply":"2023-06-07T23:06:06.591411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfFinal = pd.concat([dfONE, dfTWO]).drop_duplicates(subset='Title')","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.592478Z","iopub.status.idle":"2023-06-07T23:06:06.592776Z","shell.execute_reply.started":"2023-06-07T23:06:06.592613Z","shell.execute_reply":"2023-06-07T23:06:06.592627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfFinal","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.593658Z","iopub.status.idle":"2023-06-07T23:06:06.593968Z","shell.execute_reply.started":"2023-06-07T23:06:06.593825Z","shell.execute_reply":"2023-06-07T23:06:06.593839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install langdetect","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.594949Z","iopub.status.idle":"2023-06-07T23:06:06.595218Z","shell.execute_reply.started":"2023-06-07T23:06:06.595086Z","shell.execute_reply":"2023-06-07T23:06:06.5951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from langdetect import detect\n\n# Function to detect the language of a given text\ndef detect_language(text):\n    try:\n        return detect(text)\n    except:\n        return None\n\n# Filter papers that are only in English language\ndfFinalEnglishPapers = dfFinal[dfFinal['Abstract'].apply(lambda x: detect_language(x) == 'en')]\ndfFinalEnglishPapers","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.596064Z","iopub.status.idle":"2023-06-07T23:06:06.596327Z","shell.execute_reply.started":"2023-06-07T23:06:06.596197Z","shell.execute_reply":"2023-06-07T23:06:06.59621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Create separate dataframes for papers mentioning Bert and GPT\ndfBertPapers = dfFinalEnglishPapers[dfFinalEnglishPapers['Title'].str.contains(r'\\bBert\\b', case=False) | dfFinalEnglishPapers['Abstract'].str.contains(r'\\bBert\\b', case=False)]\ndfGptPapers = dfFinalEnglishPapers[dfFinalEnglishPapers['Title'].str.contains(r'\\bGPT\\b', case=False) | dfFinalEnglishPapers['Abstract'].str.contains(r'\\bGPT\\b', case=False)]\nprint(dfBertPapers.shape)\nprint(dfGptPapers.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.597543Z","iopub.status.idle":"2023-06-07T23:06:06.597873Z","shell.execute_reply.started":"2023-06-07T23:06:06.59768Z","shell.execute_reply":"2023-06-07T23:06:06.597694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfGptPapers","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.598997Z","iopub.status.idle":"2023-06-07T23:06:06.599277Z","shell.execute_reply.started":"2023-06-07T23:06:06.599143Z","shell.execute_reply":"2023-06-07T23:06:06.599157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfBertPapers","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.600626Z","iopub.status.idle":"2023-06-07T23:06:06.600918Z","shell.execute_reply.started":"2023-06-07T23:06:06.600781Z","shell.execute_reply":"2023-06-07T23:06:06.600794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n# Create a new column 'PrecedingWord' to store the word preceding \"learning\" in titles\ndfFinalEnglishPapers['PrecedingWord'] = dfFinalEnglishPapers['Title'].str.extract(r'(\\S+)\\s+learning', flags=re.IGNORECASE)\n\n# Get the top 10 words preceding \"learning\" and their counts\ntop_words = dfFinalEnglishPapers['PrecedingWord'].value_counts().head(10)\n\n# Print the top 10 words and their counts\nfor word, count in top_words.items():\n    print(word, count)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.602202Z","iopub.status.idle":"2023-06-07T23:06:06.602484Z","shell.execute_reply.started":"2023-06-07T23:06:06.602344Z","shell.execute_reply":"2023-06-07T23:06:06.602357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfFinalEnglishPapers.to_csv('/kaggle/working/Music_2022_2023_list_final1.csv', index=True, header= True)","metadata":{"execution":{"iopub.status.busy":"2023-06-07T23:06:06.603198Z","iopub.status.idle":"2023-06-07T23:06:06.603487Z","shell.execute_reply.started":"2023-06-07T23:06:06.603341Z","shell.execute_reply":"2023-06-07T23:06:06.603355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}