{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-19T16:12:23.802269Z","iopub.execute_input":"2022-01-19T16:12:23.802641Z","iopub.status.idle":"2022-01-19T16:12:23.833796Z","shell.execute_reply.started":"2022-01-19T16:12:23.802545Z","shell.execute_reply":"2022-01-19T16:12:23.83275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install gensim==3.8","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:12:25.568156Z","iopub.execute_input":"2022-01-19T16:12:25.568452Z","iopub.status.idle":"2022-01-19T16:12:41.327895Z","shell.execute_reply.started":"2022-01-19T16:12:25.568422Z","shell.execute_reply":"2022-01-19T16:12:41.327001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom gensim.models import Word2Vec\nfrom scipy.stats import entropy","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:12:41.329884Z","iopub.execute_input":"2022-01-19T16:12:41.33014Z","iopub.status.idle":"2022-01-19T16:12:45.457636Z","shell.execute_reply.started":"2022-01-19T16:12:41.330111Z","shell.execute_reply":"2022-01-19T16:12:45.456688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def word2vec(sentences, emb_size=16, output_prefix='lalala'):\n    temp_sentences = []\n    for i in range(len(sentences)):\n        temp_sentences.append([str(x) for x in sentences[i]])\n    model = Word2Vec(temp_sentences, size=emb_size, window=5, min_count=2, sg=0, hs=1, seed=4399,workers=-1)\n    #对seq里的所有单词的embedding取均值\n    emb_matrix = []\n    for seq in sentences:\n        vec = []\n        for w in seq:\n            if w in model:\n                vec.append(model[w])\n        if len(vec) > 0:\n            emb_matrix.append(np.mean(vec, axis=0))\n        else:\n            emb_matrix.append([0] * emb_size)\n    emb_matrix = np.array(emb_matrix)\n    tmp = pd.DataFrame()\n    for i in range(emb_size):\n        tmp['{}_emb_{}'.format(output_prefix, i)] = emb_matrix[:,i]\n    del model, emb_matrix, sentences\n    return tmp","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:17:02.842861Z","iopub.execute_input":"2022-01-19T16:17:02.843481Z","iopub.status.idle":"2022-01-19T16:17:02.852692Z","shell.execute_reply.started":"2022-01-19T16:17:02.843441Z","shell.execute_reply":"2022-01-19T16:17:02.851724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/DeepShare_nlpstarter/train.csv')\ntest_df = pd.read_csv('/kaggle/input/DeepShare_nlpstarter/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:13:24.337052Z","iopub.execute_input":"2022-01-19T16:13:24.337368Z","iopub.status.idle":"2022-01-19T16:13:31.912083Z","shell.execute_reply.started":"2022-01-19T16:13:24.337334Z","shell.execute_reply":"2022-01-19T16:13:31.911063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['label'] = -1\ndf = pd.concat([train_df,test_df],axis=0).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:14:45.200071Z","iopub.execute_input":"2022-01-19T16:14:45.200356Z","iopub.status.idle":"2022-01-19T16:14:45.304853Z","shell.execute_reply.started":"2022-01-19T16:14:45.200326Z","shell.execute_reply":"2022-01-19T16:14:45.303984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w2v_df = word2vec(df['text'].apply(lambda x: x.split(' ')),emb_size=128,output_prefix='w2v')","metadata":{"execution":{"iopub.status.busy":"2022-01-19T16:17:53.573639Z","iopub.execute_input":"2022-01-19T16:17:53.573945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([df,w2v_df],axis=1).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df.query('label>-1').reset_index(drop=True)\ntest_df = df.query('label==-1').reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature = [x for x in df.columns if 'w2v' in x]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n#Lgb\nclf = LGBMClassifier()\nclf.fit(train_df[feature], train_df['label'].iloc[:].values)\nprint('Finish Lgb')\n\n#提交结果\nres = pd.DataFrame()\nres['id'] = test_df['id']\nres['label'] = clf.predict(test_df[feature])\nres.to_csv('submission.csv', index=None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}