{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **This Notebook is a baseline on how to handle the hdf5 data and correctly submit. There is no complex feature selection, catagorical data or multi-site data included, which you will most certianly need to obtain a good score. Good luck!**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport sys\nimport time\nimport sklearn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pheno_train = pd.read_csv('../input/abide/comp_data/pheno_train.csv')\npheno_test = pd.read_csv('../input/abide/comp_data/pheno_test.csv')\nss = pd.read_csv('../input/abide/comp_data/sample_submission.csv')\nfunc = h5py.File('../input/abide/comp_data/abide.hdf5', 'r')\n\n# adjust according to submission format\npheno_train['DX_GROUP'] = pheno_train['DX_GROUP'] - 1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_train(data, pheno, derivative):\n    X = []\n    y = []\n    i = 0\n    total = pheno.shape[0]\n    for row in pheno.iterrows():\n        file_id, dx_group = row[1]['FILE_ID'], row[1]['DX_GROUP']\n        connectivity = data['patients'][file_id][derivative][()]\n        X.append(connectivity)\n        y.append(dx_group)\n        sys.stdout.write(\"\\r{:.2f}%>\".format(i/total))\n        sys.stdout.flush()\n        i += 1\n        \n    X = np.array(X).astype(np.float32)\n    y = np.array(y).astype(np.float32)\n    return X, y\n\ndef get_data_test(data, pheno, derivative):\n    X_test = []\n    sub_ids = []\n    j = 0\n    total = pheno.shape[0]\n    for row in pheno.iterrows():\n        file_id, sub_id = row[1]['FILE_ID'], row[1]['SUB_ID']\n        connectivity = data['patients'][file_id][derivative][()]\n        X_test.append(connectivity)\n        sub_ids.append(sub_id)\n        sys.stdout.write(\"\\r{:.2f}%>\".format(j/total))\n        sys.stdout.flush()\n        j += 1\n        \n    X_test = np.array(X_test).astype(np.float32)\n    return X_test, sub_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, y = get_data_train(func, pheno_train, 'aal')\nX.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test, sub_ids = get_data_test(func, pheno_test, 'aal')\nX_test.shape, len(sub_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingClassifier as hgbc\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport gc\n\nN_SPLITS = 10\nkf = KFold(n_splits=N_SPLITS)\nfinal_preds = np.zeros((X_test.shape[0], N_SPLITS))\ni = 0\nfor tr_idx, val_idx in kf.split(X):\n    clf = hgbc(max_depth=6, max_leaf_nodes=25, verbose=0, max_iter=30)\n    clf.fit(X[tr_idx], y[tr_idx])\n    final_preds[:, i] = clf.predict_proba(X_test)[:, 1]\n    fold_preds = clf.predict_proba(X[val_idx])[:, 1]\n    print('Fold: {0}, score: {1}'.format(i, roc_auc_score(y[val_idx], fold_preds)))\n    del clf\n    gc.collect()\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = np.mean(final_preds, axis=1)\nsub = pd.DataFrame({'SUB_ID': sub_ids, 'preds': preds})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# just in case to assert order\nalmost_final_sub = ss.merge(sub, on='SUB_ID', how='left')\nss['DX_GROUP'] = almost_final_sub['preds']\nss # final sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dont forget correct submission type\nss['SUB_ID'] = ss['SUB_ID'].map(str)\nss.to_csv('first_submission_abide.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}