{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nnp.random.seed(12345)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/avito-category-prediction/train.csv')\ntest = pd.read_csv('../input/avito-category-prediction/test.csv')\ntrain.loc[train.description.isna(), \"description\"] = \"\"\ntest.loc[test.description.isna(), \"description\"] = \"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"title&description\"] = train[\"title\"] + \" \" + train[\"description\"]\ntest[\"title&description\"] = test[\"title\"] + \" \" + test[\"description\"]\ntrain.drop(labels=[\"title\", \"description\"], axis=1, inplace=True)\ntest.drop(labels=[\"title\", \"description\"], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport unicodedata\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem.snowball import SnowballStemmer\nimport string\nstemmer = SnowballStemmer('russian')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"string.punctuation\n# все символы пунктуации я копирую и экранирую символ \", такое решение мне показалось самым простым\n# ещё по какой-то причине в string.punctuation нет « и » , добавил сам","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def CustomTokenizer(string : str) -> np.array:\n    word_tokenized = word_tokenize(string.lower())\n    word_filtered = [word for word in word_tokenized if (word not in stopwords.words(\"russian\"))\n                     and (not re.findall(\"[0-9]\", word))\n                     and (not re.findall(\"[—«»!\\\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~]\", word))]\n    return map(stemmer.stem, word_filtered)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_stemmed = CustomTokenizer(train[\"title&description\"][0])\nprint(\"До обработки: \\n\", train[\"title&description\"][0])\nprint(\"\\n\\nПосле обработки: \\n\", ' '.join(text_stemmed))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split  \nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler(with_mean=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(12345)\nfinal_vec = TfidfVectorizer(tokenizer=CustomTokenizer, max_df=0.9)\ntrain_sample = train.sample(500_000)\ntrain_final = scaler.fit_transform(final_vec.fit_transform(train_sample[\"title&description\"]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(12345)\nsgd = SGDClassifier()\nsgd.fit(train_final, train_sample[\"Category\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final = scaler.transform(final_vec.transform(test[\"title&description\"]))\ntest[\"Category\"] = sgd.predict(test_final)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.rename(columns = {'itemid':'Id'}, inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(\"title&description\", axis=1).to_csv(r'./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}