{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\ntqdm.pandas()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The description below is completely wrong. Please wait for revision.","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# load train data\ndf_train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\nprint(df_train.shape)\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\ndf_label = df_train.value_counts('cleaned_label').reset_index()\ndf_label.columns = ['label', 'count']\ndf_label['-count'] = -df_label['count']\ndf_label['label'] = df_label['label'].apply(lambda x: x.strip())\ndf_label = df_label.sort_values(['-count', 'label']).reset_index(drop=True)\ndf_label['target'] = np.arange(len(df_label))\nprint(df_label.shape)\ndf_label.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target'] = df_train['cleaned_label'].progress_apply(lambda x: \n    df_label['target'][df_label['label']==x.strip()].values[0]\n)\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aggregation\ndf_tmp = df_train.groupby('Id')['target'].agg(lambda x: np.array(x)).reset_index()\ndf_tmp.columns = ['Id', 'target']\ndf_train_agg = df_train[df_train['Id'].duplicated()==False].reset_index(drop=True)\ndf_train_agg = pd.merge(df_train_agg['Id'], df_tmp, on='Id')\ndf_train_agg['target'] = df_train_agg['target'].apply(lambda x: x.reshape(-1))\nprint(df_train_agg.shape)\ndf_train_agg.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric implementaion","metadata":{}},{"cell_type":"code","source":"def calc_score(y_true, y_pred, beta=0.5):\n    TP = 0\n    FP = 0\n    FN = 0\n    for i in range(len(y_true)):\n        y_true_i = y_true[i]\n        y_pred_i = y_pred[i]\n        FP += len(y_pred_i)\n        for j in range(len(y_true_i)):\n            if y_true_i[j] in y_pred_i:\n                TP += 1\n                FP -= 1\n            else:\n                FN += 1\n    F_beta = (1+beta**2)*TP/((1+beta**2)*TP + beta**2*FP + FN)\n    return F_beta","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's do math!","metadata":{}},{"cell_type":"markdown","source":"Let $T$ be the number of sample in the train data. Let $N$ be the number of positive labels in the train data. \nThen the average number of positive labels for each sample of the train data can be written as $N/T$.\n","metadata":{}},{"cell_type":"code","source":"T = len(df_train_agg)\nN = len(df_train)\nN_per_T = N/T\nprint(\"N/T: {:.6f}\".format(N_per_T))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The metric of this competition is $F_\\beta$, \n$$ F_{\\beta}(P) = \\frac{(1+\\beta^2)TP}{(1+\\beta^2)TP + \\beta^2FP + FN}, \\tag{1}$$\nwhere $P$ is the prediction and $TP$, $FP$ and $FN$ is the number of true positive, false positive and false negative respectively.  \nLet $n_0$ be the number of target 0 in the train data and let $P_0$ be the prediction that predicts all samples as 0. Then, \n\n$$ F_{\\beta}(P_0) = \\frac{(1+\\beta^2)n_0}{(1+\\beta^2)n_0 + \\beta^2(T-n_0) + (N-n_0)} = \\frac{(1+\\beta^2)n_0}{N+\\beta^2T } \\tag{2}$$\n\nAs same as above, let $n_1$ be the number of target 1 in the train data and let $P_1$ be the prediction that predicts all samples as 1. Then,\n\n$$ F_{\\beta}(P_1) = \\frac{(1+\\beta^2)n_1}{N+\\beta^2T} \\tag{3}$$\n\nMoreover, let $P_{0,1}$ be the prediction that predicts all samples as 0 and 1. Then,\n\n$$ F_{\\beta}(P_{0,1}) = \\frac{(1+\\beta^2)(n_0 + n_1)}{(1+\\beta^2)(n_0+n_1) + \\beta^2(T-n_0-n_1) + (2N-n_0-n_1)} = \\frac{(1+\\beta^2)(n_0+n_1)}{N+2\\beta^2T} \\tag{4}$$\n\n$F_{\\beta}(P_0)$, $F_{\\beta}(P_1)$ and $F_{\\beta}(P_{0,1}) $ are calcurated in the next cell.","metadata":{}},{"cell_type":"code","source":"# predict all data as 0\npred0 = np.ones([len(df_train_agg), 1])*0\nF_beta0 = calc_score(df_train_agg['target'].values, pred0)\nprint(\"F_beta0 : {:.6f}\".format(F_beta0))\n\n# predict all data as 1\npred1 = np.ones([len(df_train_agg), 1])*1\nF_beta1 = calc_score(df_train_agg['target'].values, pred1)\nprint(\"F_beta1 : {:.6f}\".format(F_beta1))\n\n\n# predict all data as [0,1]\npred01 = np.zeros([len(df_train_agg), 2])\npred01[:,1] = 1\nF_beta01 = calc_score(df_train_agg['target'].values, pred01)\nprint(\"F_beta01: {:.6f}\".format(F_beta01))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the equation (2), (3) and (4) we have, \n\n\n$$ F_{\\beta}(P_0)(N+\\beta^2T) = (1+\\beta^2)n_0 \\tag{5}$$\n\n$$ F_{\\beta}(P_1)(N+\\beta^2T) = (1+\\beta^2)n_1 \\tag{6}$$\n\n$$ F_{\\beta}(P_{0,1})(N+2\\beta^2T) = (1+\\beta^2)(n_0+n_1). \\tag{7}$$\n\nBy equation (5) + (6), we have, \n\n\n$$ (F_{\\beta}(P_0)+F_{\\beta}(P_1))(N+\\beta^2T) = (1+\\beta^2)(n_0+n_1) \\tag{8}$$\n\nFrom equation (7) and (8), we have,\n\n$$ (F_{\\beta}(P_0)+F_{\\beta}(P_1))(N+\\beta^2T) = F_{\\beta}(P_{0,1})(N+2\\beta^2T) \\tag{9}$$\n\nBy transforming (9), we have,\n\n\n$$ \\frac{N}{T} = \\beta^2\\frac{2F_{\\beta}(P_{0,1})-F_{\\beta}(P_0)-F_{\\beta}(P_1)}{F_{\\beta}(P_0)+F_{\\beta}(P_1)-F_{\\beta}(P_{0,1})} \\tag{10}$$\n\nOK. Using equation (10), we can calcurate $N/T$ with $F_{\\beta}(P_0)$, $F_{\\beta}(P_1)$ and $F_{\\beta}(P_{0,1})$. Let's check it.\n","metadata":{}},{"cell_type":"code","source":"def calc_N_per_T(F_beta0, F_beta1, F_beta01, beta=0.5):\n    return (beta**2)*(2*F_beta01-F_beta0-F_beta1)/(F_beta0+F_beta1-F_beta01)\n\ntmp = calc_N_per_T(F_beta0, F_beta1, F_beta01)\nprint('estimated N/T: {:.6f}'.format(tmp))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks Fine! We can do same calcuration on the test data by submitting $P_0$, $P_1$ and $P_{0,1}$. I submitted them and got results below. ","metadata":{}},{"cell_type":"code","source":"F_beta0_test = 0.022 # submission 1\nF_beta1_test = 0.014 # submission 2\nF_beta01_test = 0.026 # submission 3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's calcurate $N/T$ of the test data!","metadata":{}},{"cell_type":"code","source":"N_per_T_predicted = calc_N_per_T(F_beta0_test, F_beta1_test, F_beta01_test)\nprint('estimated N/T of the test data: {:.6f}'.format(N_per_T_predicted))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we have the average number of positive labels for each sample of the test data. But the result is surprising. It's much less than that of the train data.  \nAnyway, let's culculate $T$ and $N$! $T$ of private is about 8000 as discribed [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/data) and $T$ of public : $T$ of private is 12:88 as discribed [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/leaderboard). So that, ","metadata":{}},{"cell_type":"code","source":"T_public_predicted = 8000 * 12/88\nN_public_predicted = N_per_T_predicted * T_public_predicted\nprint('estimated T of the public test data: {:.6f}'.format(T_public_predicted))\nprint('estimated N of the public test data: {:.6f}'.format(N_public_predicted))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Another approach\nAs described [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/data), public test data includes train data. Therefore, we can make submission only with True positive labels. I made a submisstion with 200 true positive labels and submitted it. The score is ","metadata":{}},{"cell_type":"code","source":"F_beta_200TP = 0.164 # submission 4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"$F_{\\beta}(P_{200TP})$ can be written as \n\n$$ F_{\\beta}(P_{200TP}) = \\frac{(1+\\beta^2)200}{(1+\\beta^2)200 + (N-200)} \\tag{11}$$\n\nBy transforming (11), we have,\n\n\n$$ N = \\frac{200}{F_{\\beta}(P_{200TP})}(1+\\beta^2-\\beta^2F_{\\beta}(P_{200TP})) \\tag{12}$$\n\nOK. We've got another equation to get $N$. Let's calculate this!","metadata":{}},{"cell_type":"code","source":"beta = 0.5\nN_public_predicted2 = 200/F_beta_200TP*(1+beta**2-F_beta_200TP*beta**2)\nprint('estimated N of the public test data: {:.6f}'.format(N_public_predicted2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Oops, the estimated $N$ is very different from the one above! Assuming that my calculation is correct, these result suggest that $\\beta$ of Leaderboard scoring is wrong. Actually, if $\\beta = 1.2$ all the results becomes plausible like this.","metadata":{}},{"cell_type":"code","source":"beta = 1.2\nN_per_T_predicted = calc_N_per_T(F_beta0_test, F_beta1_test, F_beta01_test, beta=beta)\nN_public_predicted = N_per_T_predicted * T_public_predicted\nN_public_predicted2 = 200/F_beta_200TP*(1+beta**2-F_beta_200TP*beta**2)\nprint('estimated N/T of the test data: {:.6f}'.format(N_per_T_predicted))\nprint('estimated N of the public test data: {:.6f}'.format(N_public_predicted))\nprint('estimated N of the public test data (ver 2): {:.6f}'.format(N_public_predicted2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This conclusion well explains our situation. If really $\\beta = 0.5$, it must be so easy to make better submission than the string-matching baseline model because the false negative labels are lesser penalized. If $\\beta$ is bigger, the FNs wll heavily penalized and the model exploring the unseen datasets can not beat the baseline.","metadata":{}},{"cell_type":"markdown","source":"# make submissions discribed above","metadata":{}},{"cell_type":"code","source":"# make submisttion 1\ndf_sub1 = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub1['PredictionString'] = df_label['label'][0]\ndf_sub1.to_csv(\"submission.csv\", index=None)\ndf_sub1.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submisttion 2\ndf_sub2 = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub2['PredictionString'] = df_label['label'][1]\ndf_sub2.to_csv(\"submission.csv\", index=None)\ndf_sub2.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submission 3\ndf_sub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub['PredictionString'] = \"{}|{}\".format(df_label['label'][0], df_label['label'][1])\ndf_sub.to_csv(\"submission.csv\", index=None)\ndf_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # make submission 4\n# # aggregate train data\n# df_train_reduced = df_train[df_train['Id'].duplicated()==False].reset_index(drop=True)\n# df_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # aggregate labels\n# def agg_label(x):\n#     labels = df_train['cleaned_label'][df_train['Id']==x].values\n#     labels = np.sort(np.unique(labels))\n#     labels_str = []\n#     labels_str.append('|'.join(labels))\n#     labels_str = labels_str[0]\n#     return labels_str\n\n# df_train_reduced['cleaned_labels'] = df_train_reduced['Id'].progress_apply(lambda x: agg_label(x))\n# df_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # label encoding\n# def get_target(x):\n#     ans = []\n#     x_list = x.split('|')\n#     for i, item in enumerate(x_list):\n#         ans.append(df_label['target'][df_label['label']==item.strip()].values[0])\n#     return ans\n\n# df_train_reduced['targets'] = df_train_reduced['cleaned_labels'].progress_apply(get_target)\n# df_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # load text\n# import os, json\n\n# train_files_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\n\n# def read_append_return(filename, train_files_path=train_files_path, output='text'):\n#     \"\"\"\n#     Function to read json file and then return the text data from them and append to the dataframe\n#     \"\"\"\n#     json_path = os.path.join(train_files_path, (filename+'.json'))\n#     headings = []\n#     contents = []\n#     combined = []\n#     with open(json_path, 'r') as f:\n#         json_decode = json.load(f)\n#         for data in json_decode:\n#             headings.append(data.get('section_title'))\n#             contents.append(data.get('text'))\n#             combined.append(data.get('section_title'))\n#             combined.append(data.get('text'))\n    \n#     all_headings = ' '.join(headings)\n#     all_contents = ' '.join(contents)\n#     all_data = '. '.join(combined)\n    \n#     if output == 'text':\n#         return all_contents\n#     elif output == 'head':\n#         return all_headings\n#     else:\n#         return all_data\n    \n\n# df_train_reduced['text'] = df_train_reduced['Id'].progress_apply(lambda x: read_append_return(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # load test data and test text\n# df_test = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\n# test_files_path = \"../input/coleridgeinitiative-show-us-the-data/test\"\n# df_test['text'] = df_test['Id'].progress_apply(lambda x: read_append_return(x, train_files_path=test_files_path))\n# df_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # find train data in test data\n# def detect_duplicated(x):\n#     for i in range(len(df_train_reduced)):\n#         if x==df_train_reduced['text'][i]:\n#             return df_train_reduced['Id'][i]\n#     return 'no dup'\n\n# df_test['dup_id'] = df_test['text'].progress_apply(lambda x: detect_duplicated(x))\n# df_test['dup'] = df_test['dup_id']!='no dup'\n# df_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # use true label of train data\n# def pred_dup(x):\n#     df_tmp = df_train_reduced[df_train_reduced['text']==x]\n#     if len(df_tmp)>0:\n#         label_list = df_tmp['targets'].values[0]\n#     else:\n#         label_list = np.zeros(0, np.int64)\n#     return label_list\n\n# df_test['pred_dup'] = df_test['text'].apply(lambda x: pred_dup(x))\n# df_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # reduce true label to 200\n# REDUCE_THRESHOLD = 200\n# count = 0\n# new_preds = np.zeros([len(df_test), 0]).tolist()\n# # df_test['pred_det_rduced'] = df_test['pred_det_reduced'].apply(lambda x: [])\n# for i in range(len(df_test)):\n#     if count>=REDUCE_THRESHOLD: break\n#     tmp_pred = list(df_test['pred_dup'][i])\n#     new_pred = []\n#     for j in range(len(tmp_pred)):\n#         if count>=REDUCE_THRESHOLD: break\n#         new_pred.append(tmp_pred[j])\n#         count += 1\n#     new_preds[i] = new_pred\n# # \n# print(new_preds)\n# print(count)\n# df_test['pred_dup_reduced'] = new_preds\n# df_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # decode label encoding\n# def get_label(x, ref_label, ref_target):\n#     predict = []\n#     for i in range(len(x)):\n#         predict.append(ref_label[ref_target==x[i]][0])\n#     predict = np.unique(predict).tolist()\n#     tmp_list = []\n#     tmp_list.append('|'.join(predict))\n#     return tmp_list[0]\n\n# df_test['pred_dup_str'] = df_test['pred_dup_reduced'].progress_apply(lambda x: \n#         get_label(x, df_label['label'].values, df_label['target'].values)\n# )\n# df_sub4 = df_test[['Id', 'pred_dup_str']]\n# df_sub4.columns = ['Id', 'PredictionString']\n# df_sub4.to_csv(\"submission.csv\", index=None)\n# df_sub4.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}