{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport timeit\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.\n\n# Load the training data set\ndata = pd.read_csv(\"../input/train.csv\")\n\n#== DATA EXPLORATION\ndta = data.copy()\ndta['freq'] = dta.groupby('place_id')['x'].transform('count')\n\ndel data\nprint(dta[:5])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# import relevant general libraries\nimport math\nimport itertools\n\n# import libraries for data visualization\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom plotly import tools\n\n# import libraries for classification algorithms\nfrom sklearn import tree\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.multiclass import OneVsOneClassifier\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.svm import LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier\n    \n# create grid and split data points assign cell index to each data point\ndef create_grid(a, N, datapoints, start_index=1):\n    n = N*N\n    modVal = N - start_index + 1 # modval is used to combine the x and y values of grid into a unique cell index\n    \n    celldata = np.zeros((n,), dtype=[('gridcell_index', 'i4'), ('row', 'i4'),('col', 'i4')])\n    gridcell = pd.DataFrame(celldata)\n\n    cellValuesCombo = np.asarray(list(itertools.product(range(1, int(N)+1),range(1, int(N)+1))))\n    gridcell['row'] = cellValuesCombo[:,0]\n    gridcell['col'] = cellValuesCombo[:,1]\n    gridcell['gridcell_index'] = gridcell['row'] + (modVal*gridcell['col'])\n    gridcell['gridcell_index'] = gridcell['gridcell_index'].astype(int)\n    \n    datapoints['gridcell_x'] = (datapoints['x']*N/a) + 1\n    datapoints['gridcell_x'] = datapoints['gridcell_x'].astype(int)\n    datapoints['gridcell_y'] = (datapoints['y']*N/a) + 1\n    datapoints['gridcell_y'] = datapoints['gridcell_y'].astype(int)\n    datapoints['gridcell_index'] = datapoints['gridcell_x'] + (modVal*datapoints['gridcell_y'])\n    datapoints['gridcell_index'] = datapoints['gridcell_index'].astype(int)\n    datapoints = datapoints.drop(['gridcell_x', 'gridcell_y'], axis=1, inplace=True)\n    \n    return gridcell\n\"\"\"\n# filter data points by grid cell index and frequency\ndef filter_data(x, y, datapoints, datapoints_full, gridcell):\n    filtered_data_index = datapoints[datapoints['gridcell_index'] == gridcell.iloc[x,y]]\n    filtered_data = datapoints_full[datapoints_full['row_id'].isin(filtered_data_index['row_id'])].copy()\n    if('place_id' in filtered_data.columns):\n        filtered_data['freq_grid'] = filtered_data.groupby('place_id')['x'].transform('count')\n        filtered_data = filtered_data[filtered_data['freq_grid'] > 3].copy()\n    \n    return filtered_data\n\"\"\"\ndef filter_data(index, datapoints, place_frequency, calcFreq=False, enablePrints=False):\n    t0 = timeit.default_timer()\n    filtered_data = datapoints[datapoints['gridcell_index'] == index].copy()\n    if(enablePrints): print( \"filtered_data_index time interval: \", (timeit.default_timer()-t0) )\n    \n    \"\"\"\n    t0 = timeit.default_timer()\n    filtered_data = datapoints_full[datapoints_full['row_id'].isin(filtered_data_index['row_id'])].copy()\n    print( \"filtered_data time interval: \", (timeit.default_timer()-t0) )\n    \"\"\"\n    \n    filtered_data_used = None\n    filtered_data_unused = None\n    if(calcFreq):\n        t0 = timeit.default_timer()\n        filtered_data['freq_grid'] = filtered_data.groupby('place_id')['x'].transform('count')\n        if(enablePrints): print( \"filtered_data-freq_grid time interval: \", (timeit.default_timer()-t0) )\n    \n        t0 = timeit.default_timer()\n        filtered_data_used = filtered_data[filtered_data['freq_grid'] > place_frequency].copy()\n        if(enablePrints): print( \"filtered_data-filtered_data time interval: \", (timeit.default_timer()-t0) )\n    \n        t0 = timeit.default_timer()\n        filtered_data_unused = filtered_data[filtered_data['freq_grid'] <= place_frequency].copy()\n        if(enablePrints): print( \"filtered_data-filtered_data time interval: \", (timeit.default_timer()-t0) )\n    else:\n        filtered_data_used = filtered_data.copy()\n    \n    del filtered_data\n    return filtered_data_used, filtered_data_unused\n\n# split time value into smaller group sets\ndef split_time(datapoints):\n    datapoints['hour'] = (datapoints['time'] / 60) % 24\n    datapoints['hour'] = datapoints['hour'].astype(int)\n    datapoints['weekday'] = (datapoints['time'] / (60*24)) % 7\n    datapoints['weekday'] = datapoints['weekday'].astype(int)\n    datapoints['month'] = (datapoints['time'] / (60*24*30)) % 12\n    datapoints['month'] = datapoints['month'].astype(int)\n    datapoints['year'] = datapoints['time'] / (60*24*365)\n    datapoints['year'] = datapoints['year'].astype(int)\n    datapoints['day'] = (datapoints['time'] / (60*24)) % 365\n    datapoints['day'] = datapoints['day'].astype(int)\n\n# visualize data\ndef init_visualization_params(plot3d=False, size=1):\n    fig = None\n    axarr = None\n    halfSize = 0\n    \n    if plot3d:\n        fig = plt.figure(figsize=(16,10))\n    else:\n        halfSize = int(size / 2)\n        fig, axarr = plt.subplots(halfSize, halfSize) if (halfSize>1) else plt.subplots(1, 1)\n        fig.tight_layout()\n        \n    return fig, axarr, halfSize\n    \ndef visualize_data(datapoints, count, figure, axesInfo, plot3d=False, halfSize=0):\n    #-- plotting x against y with colors for each place\n    if plot3d:\n        projCount = 221 + count\n        ax = figure.add_subplot(projCount, projection='3d')\n        ax.scatter(datapoints['x'], datapoints['y'], datapoints['hour'], c=datapoints['place_id'], linewidth=0.0)\n    else:\n        datapoints_sub = datapoints[['x', 'y']].copy()\n\n        #-- reduce dimensionality of features\n        x_and_y = PCA(n_components=1).fit_transform(datapoints_sub)\n        datapoints['x_and_y'] = x_and_y\n        #tu_std = StandardScaler().fit_transform(tu)\n        #x_and_y_and_time = PCA(n_components=2).fit_transform(tu_std)\n\n        if (halfSize>0):\n            xIndex = (count % halfSize) #remainder\n            yIndex = (count / halfSize) #quotient\n            axesInfo[xIndex, yIndex].scatter(datapoints['x_and_y'], datapoints['time'], c=datapoints['place_id'], linewidth=0.0)#\n        else:\n            axesInfo.scatter(datapoints['x_and_y'], datapoints['hour'], c=datapoints['place_id'], linewidth=0.0)#\n        \n    del datapoints_sub\n    plt.show()\n\ndef train_model(datapoints, datapoints_unused, model_type, valOnFull=False, enablePrints=False):\n    X = datapoints[['x', 'y', 'hour', 'accuracy', 'day']].copy()\n    y = datapoints['place_id'].copy()\n    X_unused = datapoints_unused[['x', 'y', 'hour', 'accuracy', 'day']].copy()\n    y_unused = datapoints_unused['place_id'].copy()\n\n    #split sample data into test and training sets\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=50)\n    \n    X_test_full = X_test.append(X_unused)\n    y_test_full = y_test.append(y_unused)\n\n    if(enablePrints): print( \"X_train Length: \", len(X_train) )\n    if(enablePrints): print( \"y_train Length: \", len(y_train) )\n    if(enablePrints): print( \"X_test Length: \", len(X_test) )\n    if(enablePrints): print( \"y_test Length: \", len(y_test) )\n    if(enablePrints): print( \"X_full Length: \", len(X_unused) )\n    if(enablePrints): print( \"y_full Length: \", len(y_unused) )\n\n    classifier_model = None\n    score_model = 0\n    useLogReg = True if (model_type == \"Logistic Regression\") else False\n    useDTClf = True if (model_type == \"Decision Tree\") else False\n    useRFClf = True if (model_type == \"Random Forest\") else False\n    if useLogReg:\n        logReg = LogisticRegression()\n        logReg.fit(X_train, y_train)\n        score_model = logReg.score(X_test_full, y_test_full) if (valOnFull) else logReg.score(X_test, y_test)\n        if(enablePrints): print (\"\\n\\nlog reg score: %.3f\", score_model)\n        \n        \"\"\"\n        OVR = OneVsRestClassifier(LogisticRegression()).fit(X_train, y_train)\n        if( displayPrint ):\n            print (\"OVR accuracy score: %.3f\", OVR.score(X_test, y_test))\n        \n        #X = features[:500]\n        #y = places[:500]\n        OVO = OneVsOneClassifier(LogisticRegression()).fit(X_train, y_train)\n        if( displayPrint ):\n            print (\"OVO accuracy score: %.3f\", OVO.score(X_test, y_test))\n        \"\"\"\n        \n        classifier_model = logReg\n        \n    elif useDTClf:\n        dtClf_model = tree.DecisionTreeClassifier()\n        dtClf_model.fit(X_train, y_train)\n        score_model = dtClf_model.score(X_test_full, y_test_full) if (valOnFull) else dtClf_model.score(X_test, y_test)\n        if(enablePrints): print (\"\\n\\ndecision tree classifier score: %.3f\", score_model)\n        \n        classifier_model = dtClf_model\n        \n    elif useRFClf:\n        rfClf_model = RandomForestClassifier(n_estimators=40, # Number of trees\n                                             max_features=2,    # Num features considered\n                                             oob_score=True)    # Use OOB scoring*\n        rfClf_model.fit(X_train, y_train)\n        score_model = rfClf_model.score(X_test_full, y_test_full) if (valOnFull) else rfClf_model.score(X_test, y_test)\n        #score_model = rfClf_model.oob_score_\n        if(enablePrints): print (\"\\n\\nrandom forest classifier score: %.3f\", score_model)\n        if(enablePrints): print (\"\\n\\nrandom forest classifier oob score: %.3f\", rfClf_model.oob_score_)\n        \n        classifier_model = rfClf_model\n    \n    if(enablePrints): print (\"\\n\\n score on test val: %.3f\", classifier_model.score(X_test, y_test))\n    if(enablePrints): print (\"\\n\\n score on test_full val: %.3f\", classifier_model.score(X_test_full, y_test_full))\n    \n    del X\n    del y\n    del X_unused\n    del y_unused\n    del X_train\n    del y_train\n    del X_test\n    del y_test\n    del X_test_full\n    del y_test_full\n    return classifier_model, score_model\n\ndef filter_gridcells(gridcell, gridcell_density, gridcell_distribution, enablePrints=False):\n    gridcell_used_update = None\n    gridcell_unused_update = None\n\t\n    if('gc_density' in gridcell.columns):\n        t0 = timeit.default_timer()\n        gridcell_used = gridcell[gridcell['gc_density'] > gridcell_density].copy()\n        if(enablePrints): print( \"gc_density-gridcell_used time interval: \", (timeit.default_timer()-t0) )\n    \n        t0 = timeit.default_timer()\n        gridcell_unused = gridcell[gridcell['gc_density'] <= gridcell_density].copy()\n        if(enablePrints): print( \"gc_density-gridcell_unused time interval: \", (timeit.default_timer()-t0) )\n\t\t\n        if('gc_dist' in gridcell.columns):\n            t0 = timeit.default_timer()\n            gridcell_used_update = gridcell_used[gridcell_used['gc_dist'] > gridcell_distribution].copy()\n            if(enablePrints): print( \"gc_dist-gridcell_used time interval: \", (timeit.default_timer()-t0) )\n    \n            t0 = timeit.default_timer()\n            gridcell_unused_update = gridcell_used[gridcell_used['gc_dist'] <= gridcell_distribution].copy()\n            if(enablePrints): print( \"gc_dist-gridcell_unused time interval: \", (timeit.default_timer()-t0) )\n\t\t\n            gridcell_unused_update = gridcell_unused.append(gridcell_unused_update)\n    \n    del gridcell_used\n    del gridcell_unused\n    return gridcell_used_update, gridcell_unused_update\n\ndef train_partial_data(num_parts, part_to_train, gridcell, datapoints_train, classifiers, scores, model_name, freq, validate_on_full=True, enablePrints=False):\n    n = len(gridcell)\n    parts = int(n / num_parts)\n    min_range = (part_to_train - 1) * parts\n    max_range = part_to_train * parts\n    #n = 1\n\n    loading_str = \"...\"\n    # create an array of classifier models for each grid cell index\n    for i in range(min_range, max_range):\n        gc_index = gridcell.iloc[i]['gridcell_index']\n        fData, fData_unused = filter_data(index=gc_index, datapoints=datapoints_train, place_frequency=freq, calcFreq=True)\n        split_time(fData)\n        split_time(fData_unused)\n\n        #print( \"i: \", i )\n        #print( \"gc_index: \", gc_index )\n        #print( fData['place_id'].unique() )\n        #print( fData[:5] )\n\n        clf, sc = train_model(datapoints=fData, datapoints_unused=fData_unused, model_type=model_name, valOnFull=validate_on_full)\n        #classifiers.append(clf)\n        #scores.append(sc)\n        classifiers[i] = clf\n        scores[i] = sc\n\t\t\n        if((i % 10) == 0): print( \".\", end=\"\" )\n\t\t\n        del fData\n        del fData_unused\n\n    #return datapoints_predict\n\t\ndef predict_partial_data(num_parts, part_to_predict, gridcell, datapoints_predict, classifiers, enablePrints=False):\n    n = len(gridcell)\n    parts = int(n / num_parts)\n    min_range = (part_to_predict - 1) * parts\n    max_range = part_to_predict * parts\n    #n = 1\n\n    loading_str = \"...\"\n    # create an array of classifier models for each grid cell index\n    for i in range(min_range, max_range):\n        t1 = timeit.default_timer()\n        \n        gc_index = gridcell.iloc[i]['gridcell_index']\n        if(enablePrints): print( \"gc_index: \", gc_index )\n        \n        t0 = timeit.default_timer()\n        p_fData, p_fData_unused = filter_data(index=gc_index, datapoints=datapoints_predict, place_frequency=0, calcFreq=False)\n        split_time(p_fData)\n        if(enablePrints): print( \"filter_data && split_time time interval: \", (timeit.default_timer()-t0) )\n\t\t\n        p_X = p_fData[['x', 'y', 'hour', 'accuracy', 'day']].copy()\n        \n        t0 = timeit.default_timer()\n        probs_y = classifiers[i].predict_proba(p_X)\n        if(enablePrints): print( \"predict_proba time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        probs_order = np.argsort(probs_y, axis=1)\n        if(enablePrints): print( \"argsort time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        best_n = classifiers[i].classes_[probs_order[:, -3:]]\n        p_fData['place_id'] = best_n.tolist()\n        #p_dta_pre_merge = p_fData[['row_id', 'place_id']].copy()\n        if(enablePrints): print( \"misc time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        myindex = datapoints_predict['gridcell_index'] == gc_index\n        datapoints_predict.ix[myindex, 'place_id'] = p_fData['place_id']\n        #datapoints_predict.loc[myindex, 'place_id'] = p_fData['place_id']\n        #datapoints_predict = pd.merge(datapoints_predict, p_dta_pre_merge, how='left', on='row_id')\n        if(enablePrints): print( \"merge time interval: \", (timeit.default_timer()-t0) )\n            \n        if(enablePrints): print( \"loop: {} ## time interval: {}\", i, (timeit.default_timer()-t1) )\n\t\t\n        if((i % 10) == 0): print( \".\", end=\"\" )\n\t\t\n        del p_fData\n        del p_X\n        del probs_y\n        del probs_order\n        del best_n\n        del myindex\n\n    #return datapoints_predict\n\ndef train_predict_partial_data(num_parts, part_to_process, gridcell, datapoints_train, datapoints_predict, scores, model_name, freq, classifiers=None, validate_on_full=True, enablePrints=False):\n    n = len(gridcell)\n    parts = int(n / num_parts)\n    min_range = (part_to_process - 1) * parts\n    max_range = part_to_process * parts\n    #n = 1\n\n    loading_str = \"...\"\n    # create an array of classifier models for each grid cell index\n    for i in range(min_range, max_range):\n        gc_index = gridcell.iloc[i]['gridcell_index']\n        fData, fData_unused = filter_data(index=gc_index, datapoints=datapoints_train, place_frequency=freq, calcFreq=True)\n        split_time(fData)\n        split_time(fData_unused)\n\n        clf, sc = train_model(datapoints=fData, datapoints_unused=fData_unused, model_type=model_name, valOnFull=validate_on_full)\n        #classifiers[i] = clf\n        scores[i] = sc\n\t\t\n\t\t\n\t\t\n        t0 = timeit.default_timer()\n        p_fData, p_fData_unused = filter_data(index=gc_index, datapoints=datapoints_predict, place_frequency=0, calcFreq=False)\n        split_time(p_fData)\n        if(enablePrints): print( \"filter_data && split_time time interval: \", (timeit.default_timer()-t0) )\n\t\t\n        p_X = p_fData[['x', 'y', 'hour', 'accuracy', 'day']].copy()\n        \n        t0 = timeit.default_timer()\n        probs_y = clf.predict_proba(p_X)\n        if(enablePrints): print( \"predict_proba time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        probs_order = np.argsort(probs_y, axis=1)\n        if(enablePrints): print( \"argsort time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        best_n = clf.classes_[probs_order[:, -3:]]\n        p_fData['place_id'] = best_n.tolist()\n        #p_dta_pre_merge = p_fData[['row_id', 'place_id']].copy()\n        if(enablePrints): print( \"misc time interval: \", (timeit.default_timer()-t0) )\n        \n        t0 = timeit.default_timer()\n        myindex = datapoints_predict['gridcell_index'] == gc_index\n        datapoints_predict.ix[myindex, 'place_id'] = p_fData['place_id']\n        #datapoints_predict.loc[myindex, 'place_id'] = p_fData['place_id']\n        #datapoints_predict = pd.merge(datapoints_predict, p_dta_pre_merge, how='left', on='row_id')\n        if(enablePrints): print( \"merge time interval: \", (timeit.default_timer()-t0) )\n\t\t\n        if((i % 10) == 0): print( \".\", end=\"\" )\n\t\t\n        del clf\n        del fData\n        del fData_unused\n        del p_fData\n        del p_X\n        del probs_y\n        del probs_order\n        del best_n\n        del myindex\n\n    #return datapoints_predict"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print( dta[:5] )\n\nt0 = timeit.default_timer()\ngCell = create_grid(a=10, N=100, datapoints=dta)\nprint( \"gCell create_grid time interval: \", (timeit.default_timer()-t0) )\n\np_dta = pd.read_csv(\"../input/test.csv\")\n\nt0 = timeit.default_timer()\np_gCell = create_grid(a=10, N=100, datapoints=p_dta)\nprint( \"p_gCell create_grid time interval: \", (timeit.default_timer()-t0) )\n\nprint( dta[:5] )\nprint( \"dta length: \", len(dta) )\nprint( p_dta[:5] )\nprint( \"p_dta length: \", len(p_dta) )\n\np_dta['place_id'] = np.nan\nclf_models = [0] * len(gCell)\nsc_models = [0] * len(gCell)\n\nprint( len(sc_models) )"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "start_time = timeit.default_timer()\n\n#train_partial_data(num_parts=25, part_to_train=1, gridcell=gCell, datapoints_train=dta, classifiers=clf_models, scores=sc_models, model_name=\"Decision Tree\", freq=3)\ntrain_predict_partial_data(num_parts=50, part_to_process=23, gridcell=gCell, datapoints_train=dta, datapoints_predict=p_dta, scores=sc_models, model_name=\"Decision Tree\", freq=3)\n#predict_partial_data(num_parts=16, part_to_predict=3, gridcell=gCell, datapoints_predict=p_dta, classifiers=clf_models)\n\nend_time = timeit.default_timer()\nprint(\"\\n\\ntime interval: \", (end_time-start_time) )\nprint(\"average score: \", np.mean(sc_models))\nprint(\"average positive score: \", np.mean([i for i in sc_models if i > 0]))\nprint(\"gCell len: \", len(gCell))\n#print(\"clf_models len: \", len(clf_models))\nprint(\"sc_models len: \", len(sc_models))\n\n\n\n#print(p_dta[p_dta['gc_index'] == 101][:5])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "del clf_models"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}