{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport tqdm\nimport gc\nimport sys\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Note: The dataset is too large in terms of features we will be using the dataset in form of chunks in the entire solution","metadata":{}},{"cell_type":"code","source":"date = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip', nrows=10000)\nnumeric = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip', nrows=10000)\ncategory = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip', nrows=10000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FEATURE ENGINEERING","metadata":{}},{"cell_type":"markdown","source":"### The list of numeric features is selected based on the other XGBOOST classifier check the numericclassifier notebook","metadata":{}},{"cell_type":"code","source":"num_feats = ['Id',\n       'L3_S30_F3514', 'L0_S9_F200', 'L3_S29_F3430', 'L0_S11_F314',\n       'L0_S0_F18', 'L3_S35_F3896', 'L0_S12_F350', 'L3_S36_F3918',\n       'L0_S0_F20', 'L3_S30_F3684', 'L1_S24_F1632', 'L0_S2_F48',\n       'L3_S29_F3345', 'L0_S18_F449', 'L0_S21_F497', 'L3_S29_F3433',\n       'L3_S30_F3764', 'L0_S1_F24', 'L3_S30_F3554', 'L0_S11_F322',\n       'L3_S30_F3564', 'L3_S29_F3327', 'L0_S2_F36', 'L0_S9_F180',\n       'L3_S33_F3855', 'L0_S0_F4', 'L0_S21_F477', 'L0_S5_F114',\n       'L0_S6_F122', 'L1_S24_F1122', 'L0_S9_F165', 'L0_S18_F439',\n       'L1_S24_F1490', 'L0_S6_F132', 'L3_S29_F3379', 'L3_S29_F3336',\n       'L0_S3_F80', 'L3_S30_F3749', 'L1_S24_F1763', 'L0_S10_F219',\n 'Response']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length = date.drop('Id', axis=1).count()\ndate_cols = length.reset_index().sort_values(by=0, ascending=False)\nstations = sorted(date_cols['index'].str.split('_',expand=True)[1].unique().tolist())\ndate_cols['station'] = date_cols['index'].str.split('_',expand=True)[1]\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = None\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',usecols=['Id'] + date_cols,chunksize=50000,low_memory=False):\n\n    chunk.columns = ['Id'] + stations\n    chunk['start_station'] = -1\n    chunk['end_station'] = -1\n    \n    for s in stations:\n        chunk[s] = 1 * (chunk[s] >= 0)\n        id_not_null = chunk[chunk[s] == 1].Id\n        chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n        chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n    data = pd.concat([data, chunk])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for chunk in pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=['Id'] + date_cols,chunksize=50000,low_memory=False):\n    \n    chunk.columns = ['Id'] + stations\n    chunk['start_station'] = -1\n    chunk['end_station'] = -1\n    for s in stations:\n        chunk[s] = 1 * (chunk[s] >= 0)\n        id_not_null = chunk[chunk[s] == 1].Id\n        chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n        chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n    data = pd.concat([data, chunk])\ndel chunk\ngc.collect()   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[['Id','start_station','end_station']]\nusefuldatefeatures = ['Id']+date_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"minmaxfeatures = None\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',usecols=usefuldatefeatures,chunksize=50000,low_memory=False):\n    features = chunk.columns.values.tolist()\n    features.remove('Id')\n    df_mindate_chunk = chunk[['Id']].copy()\n    df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n    df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n    df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    minmaxfeatures = pd.concat([minmaxfeatures, df_mindate_chunk])\n\ndel chunk\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for chunk in pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',usecols=usefuldatefeatures,chunksize=50000,low_memory=False):\n    features = chunk.columns.values.tolist()\n    features.remove('Id')\n    df_mindate_chunk = chunk[['Id']].copy()\n    df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n    df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n    df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n    minmaxfeatures = pd.concat([minmaxfeatures, df_mindate_chunk])\n\ndel chunk\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"minmaxfeatures.sort_values(by=['mindate', 'Id'], inplace=True)\nminmaxfeatures['min_Id_rev'] = -minmaxfeatures.Id.diff().shift(-1)\nminmaxfeatures['min_Id'] = minmaxfeatures.Id.diff()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [['Id']+date_cols,num_feats]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindata = None\ntestdata = None","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainfiles = ['train_date.csv.zip','train_numeric.csv.zip']\ntestfiles = ['test_date.csv.zip','test_numeric.csv.zip']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,f in enumerate(trainfiles):\n    \n    subset = None\n    \n    for chunk in pd.read_csv('../input/bosch-production-line-performance/' + f,usecols=cols[i],chunksize=100000,low_memory=False):\n        subset = pd.concat([subset, chunk])\n    \n    if traindata is None:\n        traindata = subset.copy()\n    else:\n        traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n        \ndel subset,chunk\ngc.collect()\ndel cols[1][-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, f in enumerate(testfiles):\n    subset = None\n    \n    for chunk in pd.read_csv('../input/bosch-production-line-performance/' + f,usecols=cols[i],chunksize=100000,low_memory=False):\n        subset = pd.concat([subset, chunk])\n        \n    if testdata is None:\n        testdata = subset.copy()\n    else:\n        testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n    \ndel subset,chunk\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindata = traindata.merge(minmaxfeatures, on='Id')\ntraindata = traindata.merge(data, on='Id')\ntestdata = testdata.merge(minmaxfeatures, on='Id')\ntestdata = testdata.merge(data, on='Id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del minmaxfeatures,data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindata.fillna(value=0,inplace=True)\ntestdata.fillna(value=0,inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mcc(tp, tn, fp, fn):\n    num = tp * tn - fp * fn\n    den = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if den == 0:\n        return 0\n    else:\n        return num / np.sqrt(den)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_mcc(y_true, y_prob):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true) \n    numn = n - nump \n    tp,fp = nump,numn\n    tn,fn = 0.0,0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    return best_mcc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return 'MCC', best_mcc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(suppress=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = traindata[traindata['Response']==0].sample(frac=1).head(400000)\ntotal = pd.concat([total,traindata[traindata['Response']==1]]).sample(frac=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X,y = total.drop(['Response','Id'],axis=1),total['Response']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42,stratify=y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELLING","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=500,n_jobs=-1,verbose=1,random_state=11)\nmodel.fit(X_train,y_train)\npred = model.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import recall_score,precision_score,plot_precision_recall_curve,confusion_matrix,classification_report,matthews_corrcoef","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(pred,y_test))\nprint(matthews_corrcoef(y_test,pred))\nconfusion_matrix(y_test,pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(recall_score(y_test,pred))\nprecision_score(y_test,pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_precision_recall_curve(model,X_test,y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = model.predict(testdata.drop(['Id'],axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata['Response'] = test\ntestdata[['Id','Response']].to_csv(\"submit.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!gzip submit.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm submit.csv.gz","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}