{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Example of weighted ensemble\nThis is an example to ensemble prediction results from different models, assuming following results there:\n- `predXX4valid.npy` - validation results for estimating ensembled score and calculating weight balance.\n- `predXX.npy` - test results for building final submission results.\n\nAll prediction results are predicted probability for all classes:\n- Shape of one model prediction results for validation set (1232, 15)\n- Shape of one model prediction results for test set (1500, 15)\n","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nimport pandas as pd\nimport numpy as np\nimport keras\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\nENSEM_TRY = 'E9' # Attempt Id\n\nensemble_list = ['D6', 'E', 'E4', 'H2']\n\ndatadir = \"../input/acoustic-scene-2018/\" # Original data\nexampledir = \"../input/tutasc-ensemble-example/\" # My examples\n\n# Make label lists\ny_labels_train = pd.read_csv(datadir + 'y_train.csv', sep=',')['scene_label'].tolist()\nlabels = sorted(list(set(y_labels_train)))\nlabel2int = {l:i for i, l in enumerate(labels)}\nint2label = {i:l for i, l in enumerate(labels)}\n\n# Load y_valid\ny_train_org = [label2int[l] for l in y_labels_train]\nsplitlist = pd.read_csv(datadir + 'crossvalidation_train.csv', sep=',')['set'].tolist()\ny_valid_ref = np.array([y for i, y in enumerate(y_train_org) if splitlist[i] == 'test'])\ny_valid_ref = keras.utils.to_categorical(y_valid_ref)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimize weight balance\nfrom scipy import optimize\n\nraw_valid_preds = [np.load(exampledir + 'preds%s4valid.npy' % e) for e in ensemble_list]\nref_valid_cls = [np.argmax(y) for y in y_valid_ref]\n\ndef f(weights):\n    valid_preds = np.average(raw_valid_preds, axis=0, weights=weights)\n    y_valid_pred_cls = [np.argmax(pred) for pred in valid_preds]\n    return y_valid_pred_cls\n\ndef loss_function(weights):\n    y_valid_pred_cls = f(weights)\n    n_lost = [result != ref for result, ref in zip(y_valid_pred_cls, ref_valid_cls)]\n    #print('loss', np.sum(n_lost) / len(y_valid_pred_cls), 'current weights', weights)\n    return np.sum(n_lost) / len(y_valid_pred_cls)\n\nopt_weights = optimize.minimize(loss_function,\n                                [1/len(ensemble_list)] * len(ensemble_list),\n                                constraints=({'type': 'eq','fun': lambda w: 1-sum(w)}),\n                                method= 'Nelder-Mead', #'SLSQP',\n                                bounds=[(0.0, 1.0)] * len(ensemble_list),\n                                options = {'ftol':1e-10},\n                            )['x']\n\nprint('Optimum weights = ', opt_weights, 'with loss', loss_function(opt_weights))\n\ndef acc_function(weights):\n    y_valid_pred_cls = f(weights)\n    n_eq = [result == ref for result, ref in zip(y_valid_pred_cls, ref_valid_cls)]\n    return np.sum(n_eq) / len(y_valid_pred_cls)\n\nprint('Ensembled Accuracy =', acc_function(opt_weights))\n\n# double check answers\nn = 5\ny_valid_pred_cls = f(opt_weights)\nfor result, ref in zip(y_valid_pred_cls[:n], ref_valid_cls[:n]):\n    print(result, '\\t', ref)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_preds = np.average(raw_valid_preds, axis=0, weights=[ 0.25499878,  0.25463257,  0.24666748,  0.25988159])\ny_valid_pred_cls = [np.argmax(pred) for pred in valid_preds]\nprint(y_valid_pred_cls[:5]) # --> [10, 11, 5, 14, 5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ensemble submission results\nraw_preds = [np.load(exampledir + 'preds%s.npy' % e) for e in ensemble_list]\ny_preds = np.average(raw_preds, axis=0, weights=opt_weights) # mean ensemble\n\nwith open('submit%s.csv' % ENSEM_TRY, 'w') as f:\n    f.writelines(['Id,Scene_label\\n'])\n    f.writelines(['%d,%s\\n' % (i, int2label[np.argmax(pred)]) for i, pred in enumerate(y_preds)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Shape of one model prediction results for validation set', raw_valid_preds[0].shape)\nprint('Shape of one model prediction results for test set', raw_preds[0].shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]}]}