{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-07T08:21:26.758463Z","iopub.execute_input":"2022-06-07T08:21:26.759392Z","iopub.status.idle":"2022-06-07T08:21:26.793037Z","shell.execute_reply.started":"2022-06-07T08:21:26.759251Z","shell.execute_reply":"2022-06-07T08:21:26.791946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport joblib\nimport pandas as pd\nimport sqlite3\nimport csv\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport re\nimport datetime as dt\n\nfrom IPython.display import Image\nfrom wordcloud import WordCloud\nfrom sqlalchemy import create_engine # Database connection\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.snowball import SnowballStemmer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score,precision_score,recall_score\nfrom sklearn import svm\nfrom sklearn.linear_model import LogisticRegression\nfrom skmultilearn.adapt import mlknn\nfrom skmultilearn.problem_transform import ClassifierChain\nfrom skmultilearn.problem_transform import BinaryRelevance\nfrom skmultilearn.problem_transform import LabelPowerset\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.model_selection import GridSearchCV\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2022-06-04T14:37:31.38443Z","iopub.execute_input":"2022-06-04T14:37:31.385053Z","iopub.status.idle":"2022-06-04T14:37:32.83725Z","shell.execute_reply.started":"2022-06-04T14:37:31.384943Z","shell.execute_reply":"2022-06-04T14:37:32.836328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learn SQL: https://www.w3schools.com/sql/default.asp\n\nif not os.path.isfile('train.db'):\n    start = datetime.now()\n    disk_engine = create_engine('sqlite:///train.db')\n    start = dt.datetime.now()\n    chunksize = 180000\n    j = 0\n    index_start = 1\n    for df in pd.read_csv(\n        '../input/facebook-recruiting-iii-keyword-extraction/Train.zip', \n        names=['Id', 'Title', 'Body', 'Tags'], chunksize=chunksize, \n        iterator=True, encoding='utf-8'\n    ):\n        df.index += index_start\n        j += 1\n        print('{} rows'.format(j*chunksize))\n        df.to_sql('data', disk_engine, if_exists = 'append')\n        index_start = df.index[-1] + 1\n    print(\"Time taken to run this cell: \", datetime.now() - start)","metadata":{"execution":{"iopub.status.busy":"2022-06-04T14:37:57.404864Z","iopub.execute_input":"2022-06-04T14:37:57.405327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile('train.db'):\n    start = datetime.now()\n    con = sqlite3.connect('train.db')\n    num_rows = pd.read_sql_query(\"\"\"SELECT count(*) FROM data\"\"\", con)\n    print(\"Number of rows in the database: \",\"\\n\",num_rows['count(*)'].values[0])\n    con.close()  # Always remember to close the database\n    print(\"Time taken to count the number of rows: \", datetime.now() - start)\nelse:\n    print(\"Please download the train.db file from drive or run the above cell to genarate train.db file\")","metadata":{"execution":{"iopub.status.busy":"2022-06-04T14:32:33.770233Z","iopub.execute_input":"2022-06-04T14:32:33.770671Z","iopub.status.idle":"2022-06-04T14:32:33.837587Z","shell.execute_reply.started":"2022-06-04T14:32:33.770634Z","shell.execute_reply":"2022-06-04T14:32:33.836676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\nif os.path.isfile(\"train.db\"):\n    con=sqlite3.connect('train.db')\n    no_row=pd.read_sql_query(\"select count(*) from data\",con)\n    con.close()\n    print(no_row)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-04T14:32:44.55076Z","iopub.execute_input":"2022-06-04T14:32:44.551211Z","iopub.status.idle":"2022-06-04T14:32:44.618488Z","shell.execute_reply.started":"2022-06-04T14:32:44.551178Z","shell.execute_reply":"2022-06-04T14:32:44.617428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"con=sqlite3.connect('train.db')\ndup_values=pd.read_sql_query(\"select id,title,body,tags, count(*) as dup  from data GROUP BY Title, Body, Tags\", con)\ncon.close()","metadata":{"execution":{"iopub.status.busy":"2022-06-04T14:32:49.989989Z","iopub.execute_input":"2022-06-04T14:32:49.990444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_values.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_vaue=dup_values['dup'].value_counts().sum()\nunique_vaue","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"no_duplicate=no_row[\"count(*)\"].values[0]-z\nno_duplicate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_values=dup_values.drop(np.where(dup_values.isnull())[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_values[\"Tags_count\"]=dup_values[\"Tags\"].apply(lambda x:len(x.split(\" \")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_values.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_values[\"Tags_count\"].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating a new database with no duplicates\nif not os.path.isfile('train_no_dup.db'):\n    disk_dup = create_engine(\"sqlite:///train_no_dup.db\")\n    no_dup = pd.DataFrame(dup_values, columns=['Title', 'Body', 'Tags'])\n    no_dup.to_sql('no_dup_train',disk_dup)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.stat(\"./train_no_dup.db\").st_size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile('train_no_dup.db'):\n    \n    con = sqlite3.connect('train_no_dup.db')\n    tags_df = pd.read_sql_query(\"SELECT Tags FROM no_dup_train\", con)\n    con.close()\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    tags_df.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tags_df.drop(tags_df.index[0], inplace=True)\ntags_df.head(2)\n \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}