{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport tqdm\nimport gc\nimport sys\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Note: The dataset is too large in terms of features we will be using the dataset in form of chunks in the entire solution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"date = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip', nrows=10000)\nnumeric = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip', nrows=10000)\ncategory = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip', nrows=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"date","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numeric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"category","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FEATURE ENGINEERING","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### The list of numeric features is selected based on the other XGBOOST classifier check the numericclassifier notebook","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_feats = ['Id',\n       'L3_S30_F3514', 'L0_S9_F200', 'L3_S29_F3430', 'L0_S11_F314',\n       'L0_S0_F18', 'L3_S35_F3896', 'L0_S12_F350', 'L3_S36_F3918',\n       'L0_S0_F20', 'L3_S30_F3684', 'L1_S24_F1632', 'L0_S2_F48',\n       'L3_S29_F3345', 'L0_S18_F449', 'L0_S21_F497', 'L3_S29_F3433',\n       'L3_S30_F3764', 'L0_S1_F24', 'L3_S30_F3554', 'L0_S11_F322',\n       'L3_S30_F3564', 'L3_S29_F3327', 'L0_S2_F36', 'L0_S9_F180',\n       'L3_S33_F3855', 'L0_S0_F4', 'L0_S21_F477', 'L0_S5_F114',\n       'L0_S6_F122', 'L1_S24_F1122', 'L0_S9_F165', 'L0_S18_F439',\n       'L1_S24_F1490', 'L0_S6_F132', 'L3_S29_F3379', 'L3_S29_F3336',\n       'L0_S3_F80', 'L3_S30_F3749', 'L1_S24_F1763', 'L0_S10_F219',\n 'Response']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"length = date.drop('Id', axis=1).count()\ndate_cols = length.reset_index().sort_values(by=0, ascending=False)\nstations = sorted(date_cols['index'].str.split('_',expand=True)[1].unique().tolist())\ndate_cols['station'] = date_cols['index'].str.split('_',expand=True)[1]\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = None\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',usecols=['Id'] + date_cols,chunksize=50000,low_memory=False):\n\n    chunk.columns = ['Id'] + stations\n    chunk['start_station'] = -1\n    chunk['end_station'] = -1\n    \n    for s in stations:\n        chunk[s] = 1 * (chunk[s] >= 0)\n        id_not_null = chunk[chunk[s] == 1].Id\n        chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n        chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n    data = pd.concat([data, chunk])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for chunk in pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=['Id'] + date_cols,chunksize=50000,low_memory=False):\n    \n    chunk.columns = ['Id'] + stations\n    chunk['start_station'] = -1\n    chunk['end_station'] = -1\n    for s in stations:\n        chunk[s] = 1 * (chunk[s] >= 0)\n        id_not_null = chunk[chunk[s] == 1].Id\n        chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n        chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n    data = pd.concat([data, chunk])\ndel chunk\ngc.collect()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data[['Id','start_station','end_station']]\nusefuldatefeatures = ['Id']+date_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"minmaxfeatures = None\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',usecols=usefuldatefeatures,chunksize=50000,low_memory=False):\n    features = chunk.columns.values.tolist()\n    features.remove('Id')\n    df_mindate_chunk = chunk[['Id']].copy()\n    df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n    df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n    df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    minmaxfeatures = pd.concat([minmaxfeatures, df_mindate_chunk])\n\ndel chunk\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for chunk in pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=usefuldatefeatures,chunksize=50000,low_memory=False):\n    features = chunk.columns.values.tolist()\n    features.remove('Id')\n    df_mindate_chunk = chunk[['Id']].copy()\n    df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n    df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n    df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    minmaxfeatures = pd.concat([minmaxfeatures, df_mindate_chunk])\n\ndel chunk\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"minmaxfeatures.sort_values(by=['mindate', 'Id'], inplace=True)\nminmaxfeatures['min_Id_rev'] = -minmaxfeatures.Id.diff().shift(-1)\nminmaxfeatures['min_Id'] = minmaxfeatures.Id.diff()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [['Id']+date_cols,num_feats]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindata = None\ntestdata = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainfiles = ['train_date.csv.zip','train_numeric.csv.zip']\ntestfiles = ['test_date.csv.zip','test_numeric.csv.zip']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i,f in enumerate(trainfiles):\n    \n    subset = None\n    \n    for chunk in pd.read_csv('../input/bosch-production-line-performance/' + f,usecols=cols[i],chunksize=100000,low_memory=False):\n        subset = pd.concat([subset, chunk])\n    \n    if traindata is None:\n        traindata = subset.copy()\n    else:\n        traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n        \ndel subset,chunk\ngc.collect()\ndel cols[1][-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, f in enumerate(testfiles):\n    subset = None\n    \n    for chunk in pd.read_csv('../input/bosch-production-line-performance/' + f,usecols=cols[i],chunksize=100000,low_memory=False):\n        subset = pd.concat([subset, chunk])\n        \n    if testdata is None:\n        testdata = subset.copy()\n    else:\n        testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n    \ndel subset,chunk\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindata = traindata.merge(minmaxfeatures, on='Id')\ntraindata = traindata.merge(data, on='Id')\ntestdata = testdata.merge(minmaxfeatures, on='Id')\ntestdata = testdata.merge(data, on='Id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del minmaxfeatures,data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = traindata[::2]\nvalid = traindata[1::2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del traindata\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mcc(tp, tn, fp, fn):\n    num = tp * tn - fp * fn\n    den = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if den == 0:\n        return 0\n    else:\n        return num / np.sqrt(den)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_mcc(y_true, y_prob):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true) \n    numn = n - nump \n    tp,fp = nump,numn\n    tn,fn = 0.0,0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    return best_mcc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return 'MCC', best_mcc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODELLING","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nparams = {'objective':\"binary:logistic\",\n          'max_depth':25,\n          'base_score':0.005,\n          'eval_metric':'auc',\n          'n_jobs':-1\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainm = xgb.DMatrix(train.drop(['Response','Id'],axis=1),train['Response'])\nvalidm = xgb.DMatrix(valid.drop(['Response','Id'],axis=1),valid['Response'])\n\ntest = xgb.DMatrix(testdata.drop(['Id'],axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del train,valid,testdata\n#gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"watchlist = [(trainm, 'train'), (validm, 'val')]\nclf = xgb.train(params, trainm,\n                num_boost_round=100,\n                evals=watchlist,\n                early_stopping_rounds=20,\n                feval=mcc_eval,\n                maximize=True\n                )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predictions = clf.predict(validm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.metrics import matthews_corrcoef\nthresholds = np.linspace(0.01, 0.99, 50)\nmcc = np.array([matthews_corrcoef(valid.Response, predictions>threshold) for threshold in thresholds])\n\nplt.plot(thresholds, mcc)\nbest_prob = thresholds[mcc.argmax()]\nbest_prob","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,18))\nxgb.plot_importance(clf,ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test = clf.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"testdata['Response'] = (test>best_prob).astype(int)\ntestdata[['Id','Response']].to_csv(\"submitwoId.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"!gzip submitwoId.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}