{"cells":[{"metadata":{},"cell_type":"markdown","source":"# ULMFiT - the first place solution "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import fastai\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from fastai.text import *\nfrom fastai.callbacks import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load the data "},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train = pd.read_csv(\"../input/train.csv\")\ndata_valid = pd.read_csv(\"../input/valid.csv\")\ndata_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some rows have missing article titles or texts - let's fix that."},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.fillna(\"xxempty\", inplace=True)\ndata_valid.fillna(\"xxempty\", inplace=True)\ndata_test.fillna(\"xxempty\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's join them together, putting titles at the end."},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train[\"full\"] = data_train[\"text\"].apply(lambda x: x + \" xxtitle \") + data_train[\"title\"]\ndata_valid[\"full\"] = data_valid[\"text\"].apply(lambda x: x + \" xxtitle \") + data_valid[\"title\"]\ndata_test[\"full\"] = data_test[\"text\"].apply(lambda x: x + \" xxtitle \") + data_test[\"title\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train[\"is_valid\"] = False\ndata_valid[\"is_valid\"] = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Language model fine-tuning "},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lm = (TextList.from_df(pd.concat([data_train, data_valid]), cols=[\"full\"])\n           .split_from_df(\"is_valid\")\n           .label_for_lm()\n           .databunch())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3, pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.fit_one_cycle(1, 4e-3, callbacks=[SaveModelCallback(lm, name=\"best_lm\")], moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.fit_one_cycle(5, 4e-3, callbacks=[SaveModelCallback(lm, name=\"best_lm\")], moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.load(\"best_lm\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lm.save_encoder(\"enc\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_clf = (TextList.from_df(pd.concat([data_train, data_valid]), vocab=data_lm.vocab, cols=[\"full\"]).\n           split_from_df(\"is_valid\").\n           label_from_df(\"label\").\n           add_test(data_test[\"full\"]).\n           databunch()\n          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = text_classifier_learner(data_clf, AWD_LSTM, drop_mult=0.3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A Kaggle kernel's memory is not infinite, so let's do some clean up."},{"metadata":{"trusted":true},"cell_type":"code","source":"del lm\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.load_encoder(\"enc\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.fit(3, 2e-3, callbacks=[SaveModelCallback(clf, name=\"best_clf\")])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.load(\"best_clf\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.fit(1, 3e-4, callbacks=[SaveModelCallback(clf, name=\"best_clf_ft1\")])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.fit(1, 3e-4, callbacks=[SaveModelCallback(clf, name=\"best_clf_ft2\")])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation "},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val = clf.get_preds(DatasetType.Valid, ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_l = pred_val[0].argmax(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(pred_val[1], pred_val_l))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make a submission "},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test, label_test = clf.get_preds(DatasetType.Test, ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_ = pred_test.argmax(1)\npred_test_l = [data_clf.train_ds.y.classes[n] for n in pred_test_]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = pd.Series(pred_test_l, index=data_test.index, name=\"label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res.index.name = \"id\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(res).to_csv(\"submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}