{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install -U git+https://github.com/huggingface/transformers.git\n! pip install -U git+https://github.com/huggingface/accelerate.git\n! pip install keybert","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:43:20.404650Z","iopub.execute_input":"2023-07-12T22:43:20.404969Z","iopub.status.idle":"2023-07-12T22:44:37.081390Z","shell.execute_reply.started":"2023-07-12T22:43:20.404943Z","shell.execute_reply":"2023-07-12T22:44:37.080195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport os\nimport re\nimport sys\nfrom tqdm import tqdm\nimport glob\nimport gzip\nimport unidecode\nimport unicodedata\nfrom itertools import groupby\nfrom collections import Counter\n\nfrom keybert import KeyBERT\nfrom arxiv_taxonomy import GROUPS, ARCHIVES, CATEGORIES\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\narXive_paper = \"/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json\"\narxiv_doi = \"https://doi.org/10.48550/arXiv.\"","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:44:37.082777Z","iopub.execute_input":"2023-07-12T22:44:37.083099Z","iopub.status.idle":"2023-07-12T22:44:53.658732Z","shell.execute_reply.started":"2023-07-12T22:44:37.083072Z","shell.execute_reply":"2023-07-12T22:44:53.657508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef gen_meta_csv(in_fp):\n    # input prep\n    in_path, in_fn = os.path.split(in_fp)\n    in_fn_base, ext = os.path.splitext(in_fn)\n\n    # output prep\n    arxiv_meta = {\n                'arX_id':[],\n                'year':[],\n                'month': [],\n                'update_date':[],\n                'title':[],\n                'categories':[]\n                 }\n    \n    cs_arxiv_meta = {\n                'arX_id':[],\n                'year':[],\n                'month': [],\n                'update_date':[],\n                'title':[],\n                'doi':[],\n                'arxiv_doi':[],\n                'abstract':[],\n                'categories':[]\n                }\n  \n    #re to extract date and month from Arxiv id\n    aid_patt = re.compile(r'^(.*\\/)?(\\d\\d)(\\d\\d).*$')\n\n    num_lines = sum(1 for i in open(in_fp, 'rb'))\n    print('filling meta csv')\n    with open(in_fp) as f:\n        for line in tqdm(f, total=num_lines):\n            ppr_meta = json.loads(line.strip())\n            \n\n            try:\n                aid_m = aid_patt.match(ppr_meta['id'])\n                assert aid_m is not None\n                arX_id = aid_m.group(0)\n                y = int(aid_m.group(2))\n                m = int(aid_m.group(3))\n                update_date = int(ppr_meta['update_date'].split('-')[0])\n                # collect all arxiv data\n                arxiv_meta['arX_id'].append(arX_id)\n                arxiv_meta['year'].append(y)\n                arxiv_meta['month'].append(m)\n                arxiv_meta['update_date'].append(update_date)\n                arxiv_meta['title'].append(ppr_meta['title'])\n                arxiv_meta['categories'].append(ppr_meta['categories'])\n                \n                #collect only computer science paper data\n                if  y in [21,22,23] and any(category.startswith(\"cs.\") for category in ppr_meta['categories'].split()):\n                    cs_arxiv_meta['arX_id'].append(arX_id)\n                    cs_arxiv_meta['year'].append(y)\n                    cs_arxiv_meta['month'].append(m)\n                    cs_arxiv_meta['update_date'].append(update_date)\n                    cs_arxiv_meta['title'].append(ppr_meta['title'])\n                    cs_arxiv_meta['doi'].append(ppr_meta['doi'])\n                    cs_arxiv_meta['arxiv_doi'].append(arxiv_doi+ppr_meta['id'])\n                    cs_arxiv_meta['abstract'].append(ppr_meta['abstract'])\n                    cs_arxiv_meta['categories'].append(ppr_meta['categories'])\n            except: pass        \n                \n                \n                \n    def remove_duplicate_papers(df):\n        #checking for duplicate the are different version of same paper\n        has_duplicate = df.duplicated(subset=['title','year']).any()\n        print(f'Did dataset has duplicate values? : {has_duplicate}')\n\n        # duplicate count\n        duplicate_df = df[df.duplicated(subset=['title'])]\n        print(f'Total number of duplicate rows : {len(duplicate_df)}')\n\n        # drop duplicate\n        df.drop_duplicates(subset=['title'], inplace=True)\n\n        return df\n\n    def get_arxiv_discipline(cat_id):\n        \"\"\" In arXiv taxonomy, go\n                category -> archive -> group.\n            e.g.\n                hep-th -> hep-th -> grp_physics\n                cs.CL -> cs -> grp_cs\n        \"\"\"\n\n        cat = CATEGORIES.get(cat_id, None)\n        if cat is not None:\n            archive_id = cat['in_archive']\n            archive = ARCHIVES.get(archive_id, None)\n            if archive is not None:\n                group_id = archive['in_group']\n                group = GROUPS.get(group_id, None)\n                if group is not None:\n                    return group['name']\n                    \n        return None\n\n\n\n    def get_arxiv_category_name(cat_id):\n        cat = CATEGORIES.get(cat_id, None)\n        if cat is not None:\n            return cat['name']\n        return None\n   \n        \n    \n    def process_dataframe(df):\n        df = pd.DataFrame(df)\n        df.sort_values(['year','title'], inplace=True)\n        remove_duplicate_papers(df)\n        df['categories'] = df['categories'].astype(str).str.split()\n        df['category_name'] = df['categories'].apply(lambda x: [get_arxiv_category_name(elem) for elem in x])\n        df['discipline'] = df['categories'].apply(lambda x: [get_arxiv_discipline(elem) for elem in x])\n        return df\n        \n    \n    #converting to dataframe\n    cs_arxiv_df = process_dataframe(cs_arxiv_meta)\n    arxiv_df = process_dataframe(arxiv_meta)\n    \n    return cs_arxiv_df,arxiv_df","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:44:53.660040Z","iopub.execute_input":"2023-07-12T22:44:53.660400Z","iopub.status.idle":"2023-07-12T22:44:53.678073Z","shell.execute_reply.started":"2023-07-12T22:44:53.660350Z","shell.execute_reply":"2023-07-12T22:44:53.677094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_arxiv_df,arxiv_df = gen_meta_csv(arXive_paper)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:44:53.680883Z","iopub.execute_input":"2023-07-12T22:44:53.681457Z","iopub.status.idle":"2023-07-12T22:46:34.010189Z","shell.execute_reply.started":"2023-07-12T22:44:53.681423Z","shell.execute_reply":"2023-07-12T22:46:34.009164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_arxiv_df.to_csv('cs_arxiv.csv')\narxiv_df.to_csv('arxiv_papers.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:34.011691Z","iopub.execute_input":"2023-07-12T22:46:34.012114Z","iopub.status.idle":"2023-07-12T22:46:55.161159Z","shell.execute_reply.started":"2023-07-12T22:46:34.012080Z","shell.execute_reply":"2023-07-12T22:46:55.160303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:55.162274Z","iopub.execute_input":"2023-07-12T22:46:55.162602Z","iopub.status.idle":"2023-07-12T22:46:55.176280Z","shell.execute_reply.started":"2023-07-12T22:46:55.162576Z","shell.execute_reply":"2023-07-12T22:46:55.174455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_arxiv_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:55.177802Z","iopub.execute_input":"2023-07-12T22:46:55.178345Z","iopub.status.idle":"2023-07-12T22:46:55.508351Z","shell.execute_reply.started":"2023-07-12T22:46:55.178308Z","shell.execute_reply":"2023-07-12T22:46:55.507175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_arxiv_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:55.509921Z","iopub.execute_input":"2023-07-12T22:46:55.510274Z","iopub.status.idle":"2023-07-12T22:46:55.536866Z","shell.execute_reply.started":"2023-07-12T22:46:55.510244Z","shell.execute_reply":"2023-07-12T22:46:55.535583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_arxiv_df['year'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:55.538636Z","iopub.execute_input":"2023-07-12T22:46:55.538957Z","iopub.status.idle":"2023-07-12T22:46:55.547271Z","shell.execute_reply.started":"2023-07-12T22:46:55.538934Z","shell.execute_reply":"2023-07-12T22:46:55.545780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extracting only time series data","metadata":{}},{"cell_type":"code","source":"time_series_subtasks = ['Time Series Analysis','Time Series Forecasting','Time Series Prediction','Time Series Anomaly Detection',\n                'Time Series Clustering', 'Time Series Regression','Time Series Averaging','Time Series Alignment',\n                'Semi-supervised time series classification', 'Time Series Streams','Time Series Denoising',\n                'Multivariate Time Series Forecasting','Probabilistic Time Series Forecasting',\n                'Univariate Time Series Forecasting','Multivariate Time Series Imputation','Time Series Classification',\n                'Dynamic Time Warping','Classification on Time Series with Missing Data'\n                'Stock Prediction','Stock Price Prediction','Stock Trend Prediction','Stock Market Prediction',\n                'Weather Forecasting','Solar Irradiance Forecasting','Change Point Detection','Spatio-Temporal Forecasting','Irregular Time Series',\n                'energy management','energy trading','Time Series']\n\n\n#KeyBERT parameters\nNGRAM_MIN = 1  # Min N-gram length\nNGRAM_MAX = 3  # Max N-gram length\nTOP_N = 5  # Number of keywords to return\nDIVERSITY_ALGO = 'mmr'  # or 'maxsum'\nDIVERSITY = 0.2  # For MMR (higher value = more dissimilar keywords)\nNR_CANDIDATES = 20  # For MaxSum (number of candidates)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:46:55.549095Z","iopub.execute_input":"2023-07-12T22:46:55.549465Z","iopub.status.idle":"2023-07-12T22:46:55.558896Z","shell.execute_reply.started":"2023-07-12T22:46:55.549437Z","shell.execute_reply":"2023-07-12T22:46:55.557596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_keybert(df: pd.DataFrame, ngram_min: int, ngram_max: int, diversity_algo: str, top_n: int, diversity: float, nr_candidates: int):\n    kw_model = KeyBERT(model='all-MiniLM-L6-v2')\n    use_mmr = diversity_algo.lower() == 'mmr'\n    use_maxsum = diversity_algo.lower() == 'maxsum'\n    \n    for i, row in df.iterrows():\n        abstract_text = row['abstract']\n        kw_output = kw_model.extract_keywords(abstract_text,\n                                              keyphrase_ngram_range=(ngram_min, ngram_max),\n                                              stop_words='english',\n                                              use_mmr=use_mmr,\n                                              use_maxsum=use_maxsum,\n                                              top_n=top_n,\n                                              diversity=diversity,\n                                              nr_candidates=nr_candidates)\n        df.at[i, 'keywords_and_scores'] = kw_output\n        df.at[i, 'keywords'] = [pair[0] for pair in kw_output]\n    return df\n\ndef TSA_add_features():\n    \n        df = cs_arxiv_df[cs_arxiv_df['title'].str.contains('|'.join(time_series_subtasks),case=False)]\n        \n        # Add a new column with the matched strings\n        df.loc[:,'TS_subtasks'] = df.apply(lambda row: [string for string in time_series_subtasks \n                                                                          if string.lower() in row['title'].lower()], axis=1)\n        \n        #Extract keywords using KeyBERT\n        \n        # Create empty column to store keyword and similarity scores\n        df['keywords_and_scores'] = ''\n\n        # Create empty column to store keyword texts only\n        df['keywords'] = ''\n        \n        df = run_keybert(df,NGRAM_MIN,NGRAM_MAX,DIVERSITY_ALGO,TOP_N,DIVERSITY,NR_CANDIDATES)\n        \n        return df\n        \n                                                                         \n                                                                         \n\n\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:47:21.787713Z","iopub.execute_input":"2023-07-12T22:47:21.788099Z","iopub.status.idle":"2023-07-12T22:47:21.799581Z","shell.execute_reply.started":"2023-07-12T22:47:21.788069Z","shell.execute_reply":"2023-07-12T22:47:21.797833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntime_series_df = TSA_add_features()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:47:23.979082Z","iopub.execute_input":"2023-07-12T22:47:23.979475Z","iopub.status.idle":"2023-07-12T22:57:14.944356Z","shell.execute_reply.started":"2023-07-12T22:47:23.979448Z","shell.execute_reply":"2023-07-12T22:57:14.943452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_series_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:57:44.903054Z","iopub.execute_input":"2023-07-12T22:57:44.903463Z","iopub.status.idle":"2023-07-12T22:57:44.923559Z","shell.execute_reply.started":"2023-07-12T22:57:44.903432Z","shell.execute_reply":"2023-07-12T22:57:44.921400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_series_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:57:45.507590Z","iopub.execute_input":"2023-07-12T22:57:45.507978Z","iopub.status.idle":"2023-07-12T22:57:45.544144Z","shell.execute_reply.started":"2023-07-12T22:57:45.507949Z","shell.execute_reply":"2023-07-12T22:57:45.543052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_series_df.groupby('year').agg({'title':'count'})","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:58:12.365071Z","iopub.execute_input":"2023-07-12T22:58:12.365549Z","iopub.status.idle":"2023-07-12T22:58:12.378954Z","shell.execute_reply.started":"2023-07-12T22:58:12.365515Z","shell.execute_reply":"2023-07-12T22:58:12.377390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_series_df.to_csv('time_series_arxivdf.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-12T22:58:12.909462Z","iopub.execute_input":"2023-07-12T22:58:12.909865Z","iopub.status.idle":"2023-07-12T22:58:12.966215Z","shell.execute_reply.started":"2023-07-12T22:58:12.909838Z","shell.execute_reply":"2023-07-12T22:58:12.964870Z"},"trusted":true},"execution_count":null,"outputs":[]}]}