{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T15:41:07.34088Z","iopub.execute_input":"2022-07-11T15:41:07.341327Z","iopub.status.idle":"2022-07-11T15:41:38.5104Z","shell.execute_reply.started":"2022-07-11T15:41:07.341291Z","shell.execute_reply":"2022-07-11T15:41:38.508767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_json(\"/kaggle/input/AI4Code/train/a3faba2871daaa.json\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:41:38.530534Z","iopub.execute_input":"2022-07-11T15:41:38.530938Z","iopub.status.idle":"2022-07-11T15:41:38.548774Z","shell.execute_reply.started":"2022-07-11T15:41:38.530905Z","shell.execute_reply":"2022-07-11T15:41:38.547169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_json(\"/kaggle/input/AI4Code/test/0009d135ece78d.json\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:41:38.565217Z","iopub.execute_input":"2022-07-11T15:41:38.566486Z","iopub.status.idle":"2022-07-11T15:41:38.577895Z","shell.execute_reply.started":"2022-07-11T15:41:38.566437Z","shell.execute_reply":"2022-07-11T15:41:38.576863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sk = df['source'][3].split('\\n')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:41:38.578989Z","iopub.execute_input":"2022-07-11T15:41:38.579701Z","iopub.status.idle":"2022-07-11T15:41:38.584313Z","shell.execute_reply.started":"2022-07-11T15:41:38.579669Z","shell.execute_reply":"2022-07-11T15:41:38.583358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#comment and code\ncomment = []\ncode=[]\nfor i in sk:\n    if i:\n        if i[0]=='#':\n            comment.append(i)\n        else:\n            code.append(i)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:41:38.585398Z","iopub.execute_input":"2022-07-11T15:41:38.585745Z","iopub.status.idle":"2022-07-11T15:41:38.596152Z","shell.execute_reply.started":"2022-07-11T15:41:38.585714Z","shell.execute_reply":"2022-07-11T15:41:38.595249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comment","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k=\" \".join(code)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:40:50.419182Z","iopub.execute_input":"2022-07-11T16:40:50.420249Z","iopub.status.idle":"2022-07-11T16:40:50.424889Z","shell.execute_reply.started":"2022-07-11T16:40:50.42021Z","shell.execute_reply":"2022-07-11T16:40:50.423843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Separate:\n    \n    def __init__(self,txt):\n        self.txt = txt\n    def comment_code(self):\n        comment = []\n        code = []\n        self.txt = self.txt.split(\"\\n\")\n        for i in self.txt:\n            if i:\n                if i[0]==\"#\":\n                    comment.append(i)\n                else:\n                    code.append(i)\n        return comment,code","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:43:05.151872Z","iopub.execute_input":"2022-07-11T15:43:05.152284Z","iopub.status.idle":"2022-07-11T15:43:05.16002Z","shell.execute_reply.started":"2022-07-11T15:43:05.15225Z","shell.execute_reply":"2022-07-11T15:43:05.158889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_1 = Separate (sk)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:43:10.916779Z","iopub.execute_input":"2022-07-11T15:43:10.917181Z","iopub.status.idle":"2022-07-11T15:43:10.922089Z","shell.execute_reply.started":"2022-07-11T15:43:10.917149Z","shell.execute_reply":"2022-07-11T15:43:10.921048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_1.txt=k","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:46:36.135543Z","iopub.execute_input":"2022-07-11T15:46:36.135988Z","iopub.status.idle":"2022-07-11T15:46:36.141437Z","shell.execute_reply.started":"2022-07-11T15:46:36.135951Z","shell.execute_reply":"2022-07-11T15:46:36.140134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def func(txt):\n    instance = Separate (txt)\n    instance.txt = txt\n    return instance.comment_code()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-11T15:51:14.476154Z","iopub.execute_input":"2022-07-11T15:51:14.476578Z","iopub.status.idle":"2022-07-11T15:51:14.482147Z","shell.execute_reply.started":"2022-07-11T15:51:14.476544Z","shell.execute_reply":"2022-07-11T15:51:14.481208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"comments\"]=[\" \".join(func(i)[0]) for i in df[\"source\"] if i]\ndf[\"code\"]=[\" \".join(func(i)[1]) for i in df[\"source\"]]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:08:26.695519Z","iopub.execute_input":"2022-07-11T16:08:26.696701Z","iopub.status.idle":"2022-07-11T16:08:26.704646Z","shell.execute_reply.started":"2022-07-11T16:08:26.696662Z","shell.execute_reply":"2022-07-11T16:08:26.703057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"code\"][6]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:09:37.410458Z","iopub.execute_input":"2022-07-11T16:09:37.411859Z","iopub.status.idle":"2022-07-11T16:09:37.419392Z","shell.execute_reply.started":"2022-07-11T16:09:37.411801Z","shell.execute_reply":"2022-07-11T16:09:37.418498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:36:50.394343Z","iopub.execute_input":"2022-07-11T16:36:50.394731Z","iopub.status.idle":"2022-07-11T16:36:50.399875Z","shell.execute_reply.started":"2022-07-11T16:36:50.394701Z","shell.execute_reply":"2022-07-11T16:36:50.398827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectoriser = TfidfVectorizer()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:39:11.691621Z","iopub.execute_input":"2022-07-11T16:39:11.692009Z","iopub.status.idle":"2022-07-11T16:39:11.696469Z","shell.execute_reply.started":"2022-07-11T16:39:11.691978Z","shell.execute_reply":"2022-07-11T16:39:11.695666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=vectoriser.fit_transform(k)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T16:41:17.00738Z","iopub.execute_input":"2022-07-11T16:41:17.007777Z","iopub.status.idle":"2022-07-11T16:41:17.02963Z","shell.execute_reply.started":"2022-07-11T16:41:17.007743Z","shell.execute_reply":"2022-07-11T16:41:17.028038Z"},"trusted":true},"execution_count":null,"outputs":[]}]}