{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Hi!\nThere is a lot of implementations of competition's metric: Group Mean of Log(MeanAbsoluteError).\n\nSome of them are claimed to be more fastest than other, so i've decided to benchmark them and compare to my own.\n## Upvote if you find it useful"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom sklearn import metrics\nfrom tqdm import tqdm\n\nimport numba\nfrom numba import jit, float32\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=numba.NumbaWarning)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics Definition"},{"metadata":{},"cell_type":"markdown","source":"## Implementation from that [script](https://www.kaggle.com/marcelotamashiro/lgb-public-kernels-plus-more-features)\n## short name: lgb_plus"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def comp_score (y_true, y_pred, jtype):\n    df = pd.DataFrame()\n    df['y_true'] , df['y_pred'], df['jtype'] = y_true , y_pred, jtype\n    score = 0 \n    for t in np.unique(jtype):\n        score_jtype = np.log(metrics.mean_absolute_error(df[df.jtype==t]['y_true'],df[df.jtype==t]['y_pred']))\n        score += score_jtype\n        #print(f'{t} : {score_jtype}')\n    score /= len(np.unique(jtype))\n    return score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Slightly modified version from [here](https://www.kaggle.com/abhishek/competition-metric)\n\n## short name: competition_metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(df, preds, verbose=False):\n    \n    if verbose:\n        iterator = lambda x: tqdm(x)\n    else:\n        iterator = list\n        \n    df[\"prediction\"] = list(preds)\n    maes = []\n    for t in iterator(df.type.unique()):\n        y_true = df[df.type==t].scalar_coupling_constant.values\n        y_pred = df[df.type==t].prediction.values\n        mae = np.log(metrics.mean_absolute_error(y_true, y_pred))\n        maes.append(mae)\n    return np.mean(maes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Version from [that kernel](https://www.kaggle.com/uberkinder/efficient-metric)\n## short name: efficent_metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):\n    \"\"\"\n    Fast metric computation for this competition: https://www.kaggle.com/c/champs-scalar-coupling\n    Code is from this kernel: https://www.kaggle.com/uberkinder/efficient-metric\n    \"\"\"\n    maes = (y_true-y_pred).abs().groupby(types).mean()\n    return np.log(maes.map(lambda x: max(x, floor))).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## short name: basic"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Basic version\ndef mean_log_mae(y_true, y_pred, types, verbose=False):\n    if verbose:\n        iterator = lambda x: tqdm(x)\n    else:\n        iterator = list\n    \n    per_type_data = {\n        t : {\n            'true': [],\n            'pred': []\n        } \n        for t in list(set(types))\n    }\n    for true, pred, t in iterator(zip(y_true, y_pred, types)):\n        per_type_data[t]['true'].append(true)\n        per_type_data[t]['pred'].append(pred)\n        \n    maes = []\n    for t in iterator(set(types)):\n        maes.append(np.log(metrics.mean_absolute_error(per_type_data[t]['true'], per_type_data[t]['pred'])))\n        \n    return np.mean(maes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## short name: partial_jit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compiling efficent log(mae) implementation\n@jit(float32(float32[:], float32[:]))\ndef jit_log_mae(y_true: np.ndarray, y_pred: np.ndarray):\n    n = y_true.shape[0]\n    return np.log(np.sum(np.absolute(y_true - y_pred))/n)\n\n\ndef speedup_mean_log_mae(y_true: np.ndarray, y_pred: np.ndarray, types: np.ndarray, verbose=False) -> np.float64:\n    if verbose:\n        iterator = lambda x: tqdm(x)\n    else:\n        iterator = list\n    \n    per_type_data = {\n        t : {\n            'true': [],\n            'pred': []\n        } \n        for t in list(set(types))\n    }\n    for true, pred, t in iterator(zip(y_true, y_pred, types)):\n        per_type_data[t]['true'].append(true)\n        per_type_data[t]['pred'].append(pred)\n        \n    maes = []\n    for t in iterator(set(types)):\n        maes.append(\n            jit_log_mae( ## that's the speedup\n                np.array(per_type_data[t]['true'], dtype=np.float32),\n                np.array(per_type_data[t]['pred'], dtype=np.float32)\n            )\n        )\n        \n    return np.mean(maes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## short name: full_jit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Trying to jit-compile all\n@jit\ndef jit_mean_log_mae(y_true: np.ndarray, y_pred: np.ndarray, types: np.ndarray) -> np.float64:\n    \n    uniq_types: np.ndarray = np.unique(types)\n    \n    per_type_data = dict()\n    for t in uniq_types:\n        per_type_data[t] = {\n            'true': [],\n            'pred': []\n        }\n    \n    for i in np.arange(len(types)):\n        per_type_data[types[i]]['true'].append(y_true[i])\n        per_type_data[types[i]]['pred'].append(y_pred[i])\n        \n    maes = []\n    for t in uniq_types:\n        maes.append(jit_log_mae(np.array(per_type_data[t]['true'], dtype=np.float32), np.array(per_type_data[t]['pred'], dtype=np.float32)))\n        \n    return np.mean(maes)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Benchmarking part"},{"metadata":{"trusted":true},"cell_type":"code","source":"from timeit import Timer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"general_train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Pre-Compiling jit functions\njit_mean_log_mae(general_train.scalar_coupling_constant.values, np.zeros(len(general_train)), general_train.type.values)\nspeedup_mean_log_mae(general_train.scalar_coupling_constant.values, np.zeros(len(general_train)), general_train.type.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"benchmarking_code = {\n    'lgb_plus': \"comp_score(train.scalar_coupling_constant.values, zeros, train.type.values)\",\n    'competition_metric': \"metric(train, zeros, verbose=False)\",\n    'efficent_metric': \"group_mean_log_mae(train.scalar_coupling_constant, zeros, train.type)\",\n    \"basic\": \"mean_log_mae(train.scalar_coupling_constant.values, zeros, train.type.values, verbose=False)\",\n    \"partial_jit\": \"speedup_mean_log_mae(train.scalar_coupling_constant.values, zeros, train.type.values, verbose=False)\",\n    \"full_jit\": \"jit_mean_log_mae(train.scalar_coupling_constant.values, zeros, train.type.values)\"\n}\n\nquality_code = {\n    'lgb_plus': lambda train, zero_arr: comp_score(train.scalar_coupling_constant.values, zero_arr, train.type.values),\n    'competition_metric': lambda train, zero_arr: metric(train, zero_arr, verbose=False),\n    'efficent_metric': lambda train, zero_arr: group_mean_log_mae(train.scalar_coupling_constant, zero_arr, train.type),\n    \"basic\": lambda train, zero_arr: mean_log_mae(train.scalar_coupling_constant.values, zero_arr, train.type.values, verbose=False),\n    \"partial_jit\": lambda train, zero_arr: speedup_mean_log_mae(train.scalar_coupling_constant.values, zero_arr, train.type.values, verbose=False),\n    \"full_jit\": lambda train, zero_arr: jit_mean_log_mae(train.scalar_coupling_constant.values, zero_arr, train.type.values)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## ensure that results are the same\ndef measure_quality(implementations, n_samples=1000, n_different_seeds=5):\n    np.random.seed(0) # for reproducible random seed generation\n    results = {key:[] for key in implementations.keys()}\n    for seed in np.random.randint(0, 100, size=n_different_seeds):\n        train = general_train.sample(n=n_samples, random_state=seed).reset_index(drop=True)\n        zeros = np.zeros(n_samples)\n        \n        for impl_name, impl in implementations.items():\n            value = impl(train, zeros)\n            results[impl_name].append(value)\n            \n    result = pd.DataFrame(results)\n    result['std'] = result.std(axis=1)\n    \n    mean_std = result['std'].mean()\n    return result, mean_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def measure_performance(implementations, n_samples=1000, n_different_seeds=5, n_iterations=1000, n_repeats=10):\n    np.random.seed(0) # for reproducible random seed generation\n    results = {key:[] for key in implementations.keys()}\n    for seed in np.random.randint(0, 100, size=n_different_seeds):\n        train = general_train.sample(n=n_samples, random_state=seed).reset_index(drop=True)\n        zeros = np.zeros(n_samples)\n        scope = dict(globals(), **locals())\n        for impl_name, impl_code in implementations.items():\n            eval_speed = Timer(impl_code, globals=scope).repeat(repeat=n_repeats, number=n_iterations)\n            results[impl_name] += eval_speed\n            \n    return pd.DataFrame(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nqdf, mstd = measure_quality(quality_code, 100000, 20)\nprint(f\"Mean standard deviation of metric values across different implementations: {mstd:.10f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf = measure_performance(benchmarking_code, 1000, 10, 1000, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Please share your thoughts in comments\n## Upvote if you find it useful\n## and share your metrics)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}