{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\ntqdm.pandas()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Updated to deal with the rescoring on 2021/5/5**","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# load train data\ndf_train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\nprint(df_train.shape)\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\ndf_label = df_train.value_counts('cleaned_label').reset_index()\ndf_label.columns = ['label', 'count']\ndf_label['-count'] = -df_label['count']\ndf_label['label'] = df_label['label'].apply(lambda x: x.strip())\ndf_label = df_label.sort_values(['-count', 'label']).reset_index(drop=True)\ndf_label['target'] = np.arange(len(df_label))\nprint(df_label.shape)\ndf_label.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target'] = df_train['cleaned_label'].progress_apply(lambda x: \n    df_label['target'][df_label['label']==x.strip()].values[0]\n)\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aggregation\ndf_tmp = df_train.groupby('Id')['target'].agg(lambda x: np.array(x)).reset_index()\ndf_tmp.columns = ['Id', 'target']\ndf_train_agg = df_train[df_train['Id'].duplicated()==False].reset_index(drop=True)\ndf_train_agg = pd.merge(df_train_agg['Id'], df_tmp, on='Id')\ndf_train_agg['target'] = df_train_agg['target'].apply(lambda x: x.reshape(-1))\nprint(df_train_agg.shape)\ndf_train_agg.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric implementaion","metadata":{}},{"cell_type":"code","source":"def calc_score(y_true, y_pred, beta=0.5):\n    TP = 0\n    FP = 0\n    FN = 0\n    for i in range(len(y_true)):\n        y_true_i = y_true[i]\n        y_pred_i = y_pred[i]\n        FP += len(y_pred_i)\n        for j in range(len(y_true_i)):\n            if y_true_i[j] in y_pred_i:\n                TP += 1\n                FP -= 1\n            else:\n                FN += 1\n    F_beta = (1+beta**2)*TP/((1+beta**2)*TP + beta**2*FN + FP)\n    return F_beta","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's do math!","metadata":{}},{"cell_type":"markdown","source":"Let $T$ be the number of sample in the train data. Let $N$ be the number of positive labels in the train data. \nThen the average number of positive labels for each sample of the train data can be written as $N/T$.\n","metadata":{}},{"cell_type":"code","source":"T = len(df_train_agg)\nN = len(df_train)\nN_per_T = N/T\nprint(\"N/T: {:.6f}\".format(N_per_T))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The metric of this competition is $F_\\beta$, \n$$ F_{\\beta}(P) = \\frac{(1+\\beta^2)TP}{(1+\\beta^2)TP + \\beta^2FN + FP}, \\tag{1}$$\nwhere $P$ is the prediction and $TP$, $FP$ and $FN$ is the number of true positive, false positive and false negative respectively.  \nLet $n_0$ be the number of target 0 in the train data and let $P_0$ be the prediction that predicts all samples as 0. Then, \n\n$$ F_{\\beta}(P_0) = \\frac{(1+\\beta^2)n_0}{(1+\\beta^2)n_0 + \\beta^2(N-n_0) + (T-n_0)} = \\frac{(1+\\beta^2)n_0}{\\beta^2N+T } \\tag{2}$$\n\nAs same as above, let $n_1$ be the number of target 1 in the train data and let $P_1$ be the prediction that predicts all samples as 1. Then,\n\n$$ F_{\\beta}(P_1) = \\frac{(1+\\beta^2)n_1}{\\beta^2N+T} \\tag{3}$$\n\nMoreover, let $P_{0,1}$ be the prediction that predicts all samples as 0 and 1. Then,\n\n$$ F_{\\beta}(P_{0,1}) = \\frac{(1+\\beta^2)(n_0 + n_1)}{(1+\\beta^2)(n_0+n_1) + \\beta^2(N-n_0-n_1) + (2T-n_0-n_1)} = \\frac{(1+\\beta^2)(n_0+n_1)}{\\beta^2N+2T} \\tag{4}$$\n\n$F_{\\beta}(P_0)$, $F_{\\beta}(P_1)$ and $F_{\\beta}(P_{0,1}) $ are calcurated in the next cell.","metadata":{}},{"cell_type":"code","source":"# predict all data as 0\npred0 = np.ones([len(df_train_agg), 1])*0\nF_beta0 = calc_score(df_train_agg['target'].values, pred0)\nprint(\"F_beta0 : {:.6f}\".format(F_beta0))\n\n# predict all data as 1\npred1 = np.ones([len(df_train_agg), 1])*1\nF_beta1 = calc_score(df_train_agg['target'].values, pred1)\nprint(\"F_beta1 : {:.6f}\".format(F_beta1))\n\n\n# predict all data as [0,1]\npred01 = np.zeros([len(df_train_agg), 2])\npred01[:,1] = 1\nF_beta01 = calc_score(df_train_agg['target'].values, pred01)\nprint(\"F_beta01: {:.6f}\".format(F_beta01))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the equation (2), (3) and (4) we have, \n\n\n$$ F_{\\beta}(P_0)(\\beta^2N+T) = (1+\\beta^2)n_0 \\tag{5}$$\n\n$$ F_{\\beta}(P_1)(\\beta^2N+T) = (1+\\beta^2)n_1 \\tag{6}$$\n\n$$ F_{\\beta}(P_{0,1})(\\beta^2N+2T) = (1+\\beta^2)(n_0+n_1). \\tag{7}$$\n\nBy equation (5) + (6), we have, \n\n\n$$ (F_{\\beta}(P_0)+F_{\\beta}(P_1))(\\beta^2N+T) = (1+\\beta^2)(n_0+n_1) \\tag{8}$$\n\nFrom equation (7) and (8), we have,\n\n$$ (F_{\\beta}(P_0)+F_{\\beta}(P_1))(\\beta^2N+T) = F_{\\beta}(P_{0,1})(\\beta^2N+2T) \\tag{9}$$\n\nBy transforming (9), we have,\n\n\n$$ \\frac{N}{T} = \\frac{1}{\\beta^2}\\frac{2F_{\\beta}(P_{0,1})-F_{\\beta}(P_0)-F_{\\beta}(P_1)}{F_{\\beta}(P_0)+F_{\\beta}(P_1)-F_{\\beta}(P_{0,1})} \\tag{10}$$\n\nOK. Using equation (10), we can calcurate $N/T$ with $F_{\\beta}(P_0)$, $F_{\\beta}(P_1)$ and $F_{\\beta}(P_{0,1})$. Let's check it.\n","metadata":{}},{"cell_type":"code","source":"def calc_N_per_T(F_beta0, F_beta1, F_beta01, beta=0.5):\n    return 1/(beta**2)*(2*F_beta01-F_beta0-F_beta1)/(F_beta0+F_beta1-F_beta01)\n\ntmp = calc_N_per_T(F_beta0, F_beta1, F_beta01)\nprint('estimated N/T: {:.6f}'.format(tmp))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks Fine! We can do same calcuration on the test data by submitting $P_0$, $P_1$ and $P_{0,1}$. I submitted them and got results below. ","metadata":{}},{"cell_type":"code","source":"F_beta0_test = 0.016 # submission 1\nF_beta1_test = 0.011 # submission 2\nF_beta01_test = 0.021 # submission 3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's calcurate $N/T$ of the test data!","metadata":{}},{"cell_type":"code","source":"N_per_T_predicted = calc_N_per_T(F_beta0_test, F_beta1_test, F_beta01_test)\nprint('estimated N/T of the test data: {:.6f}'.format(N_per_T_predicted))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we have the average number of positive labels for each sample of the test data. But the result is surprising. It's much less than that of the train data.  \nAnyway, let's culculate $T$ and $N$! $T$ of private is about 8000 as discribed [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/data) and $T$ of public : $T$ of private is 12:88 as discribed [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/leaderboard). So that, ","metadata":{}},{"cell_type":"code","source":"T_public_predicted = 8000 * 12/88\nN_public_predicted = N_per_T_predicted * T_public_predicted\nprint('estimated T of the public test data: {:.6f}'.format(T_public_predicted))\nprint('estimated N of the public test data: {:.6f}'.format(N_public_predicted))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Another approach\nAs described [here](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/data), public test data includes train data. Therefore, we can make submission only with True positive labels. I made a submisstion with 200 true positive labels and submitted it. The score is ","metadata":{}},{"cell_type":"code","source":"F_beta_200TP = 0.107 # submission 4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"$F_{\\beta}(P_{200TP})$ can be written as \n\n$$ F_{\\beta}(P_{200TP}) = \\frac{(1+\\beta^2)200}{(1+\\beta^2)200 + \\beta^2(N-200)} \\tag{11}$$\n\nBy transforming (11), we have,\n\n\n$$ N = \\frac{200}{\\beta^2F_{\\beta}(P_{200TP})}(1+\\beta^2-F_{\\beta}(P_{200TP})) \\tag{12}$$\n\nOK. We've got another equation to get $N$. Let's calculate this!","metadata":{}},{"cell_type":"code","source":"beta = 0.5\nN_public_predicted2 = 200/(beta**2*F_beta_200TP)*(1+beta**2-F_beta_200TP)\nprint('estimated N of the public test data: {:.6f}'.format(N_public_predicted2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"OK. The estimated $N$ is similar enough to the one above considering the LB scores are rounded and the estimated T of the public test is not precise. The latter one must be more reliable.  \n\n# Conclusion\nThere are more positive labels for each sample in the test data than that in the train data.  \n- Train: 1.37\n- Test: about 10\n\nI may still have a mistake. Please point it out if you find it.","metadata":{}},{"cell_type":"markdown","source":"# make submissions discribed above","metadata":{}},{"cell_type":"code","source":"# make submisttion 1\ndf_sub1 = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub1['PredictionString'] = df_label['label'][0]\ndf_sub1.to_csv(\"submission.csv\", index=None)\ndf_sub1.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submisttion 2\ndf_sub2 = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub2['PredictionString'] = df_label['label'][1]\ndf_sub2.to_csv(\"submission.csv\", index=None)\ndf_sub2.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submission 3\ndf_sub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ndf_sub['PredictionString'] = \"{}|{}\".format(df_label['label'][0], df_label['label'][1])\ndf_sub.to_csv(\"submission.csv\", index=None)\ndf_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submission 4\n# aggregate train data\ndf_train_reduced = df_train[df_train['Id'].duplicated()==False].reset_index(drop=True)\ndf_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aggregate labels\ndef agg_label(x):\n    labels = df_train['cleaned_label'][df_train['Id']==x].values\n    labels = np.sort(np.unique(labels))\n    labels_str = []\n    labels_str.append('|'.join(labels))\n    labels_str = labels_str[0]\n    return labels_str\n\ndf_train_reduced['cleaned_labels'] = df_train_reduced['Id'].progress_apply(lambda x: agg_label(x))\ndf_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encoding\ndef get_target(x):\n    ans = []\n    x_list = x.split('|')\n    for i, item in enumerate(x_list):\n        ans.append(df_label['target'][df_label['label']==item.strip()].values[0])\n    return ans\n\ndf_train_reduced['targets'] = df_train_reduced['cleaned_labels'].progress_apply(get_target)\ndf_train_reduced.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load text\nimport os, json\n\ntrain_files_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\n\ndef read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\n    \n\ndf_train_reduced['text'] = df_train_reduced['Id'].progress_apply(lambda x: read_append_return(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load test data and test text\ndf_test = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntest_files_path = \"../input/coleridgeinitiative-show-us-the-data/test\"\ndf_test['text'] = df_test['Id'].progress_apply(lambda x: read_append_return(x, train_files_path=test_files_path))\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find train data in test data\ndef detect_duplicated(x):\n    for i in range(len(df_train_reduced)):\n        if x==df_train_reduced['text'][i]:\n            return df_train_reduced['Id'][i]\n    return 'no dup'\n\ndf_test['dup_id'] = df_test['text'].progress_apply(lambda x: detect_duplicated(x))\ndf_test['dup'] = df_test['dup_id']!='no dup'\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use true label of train data\ndef pred_dup(x):\n    df_tmp = df_train_reduced[df_train_reduced['text']==x]\n    if len(df_tmp)>0:\n        label_list = df_tmp['targets'].values[0]\n    else:\n        label_list = np.zeros(0, np.int64)\n    return label_list\n\ndf_test['pred_dup'] = df_test['text'].apply(lambda x: pred_dup(x))\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce true label to 200\nREDUCE_THRESHOLD = 200\ncount = 0\nnew_preds = np.zeros([len(df_test), 0]).tolist()\n# df_test['pred_det_rduced'] = df_test['pred_det_reduced'].apply(lambda x: [])\nfor i in range(len(df_test)):\n    if count>=REDUCE_THRESHOLD: break\n    tmp_pred = list(df_test['pred_dup'][i])\n    new_pred = []\n    for j in range(len(tmp_pred)):\n        if count>=REDUCE_THRESHOLD: break\n        new_pred.append(tmp_pred[j])\n        count += 1\n    new_preds[i] = new_pred\n# \nprint(new_preds)\nprint(count)\ndf_test['pred_dup_reduced'] = new_preds\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# decode label encoding\ndef get_label(x, ref_label, ref_target):\n    predict = []\n    for i in range(len(x)):\n        predict.append(ref_label[ref_target==x[i]][0])\n    predict = np.unique(predict).tolist()\n    tmp_list = []\n    tmp_list.append('|'.join(predict))\n    return tmp_list[0]\n\ndf_test['pred_dup_str'] = df_test['pred_dup_reduced'].progress_apply(lambda x: \n        get_label(x, df_label['label'].values, df_label['target'].values)\n)\ndf_sub4 = df_test[['Id', 'pred_dup_str']]\ndf_sub4.columns = ['Id', 'PredictionString']\ndf_sub4.to_csv(\"submission.csv\", index=None)\ndf_sub4.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}