{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import cross_validation\nfrom sklearn.metrics import jaccard_similarity_score\n\n\ndef prepare_data(df_train, df_test, n_cell_x, n_cell_y):\n    \"\"\"\n    Some feature engineering (mainly with the time feature) + normalization \n    of all features (substracting the mean and dividing by std) +  \n    computation of a grid (size = n_cell_x * n_cell_y), which is included\n    as a new column (grid_cell) in the dataframes.\n    \n    Parameters:\n    ----------    \n    df_train: pandas DataFrame\n              Training data\n    df_test : pandas DataFrame\n              Test data\n    n_cell_x: int\n              Number of grid cells on the x axis\n    n_cell_y: int\n              Number of grid cells on the y axis\n    \n    Returns:\n    -------    \n    df_train, df_test: pandas DataFrame\n                       Modified training and test datasets.\n    \"\"\"  \n    print('Feature engineering...')\n    print('    Computing some features from x and y ...')\n    ##x, y, and accuracy remain the same\n        ##New feature x/y\n    eps = 0.00001  #required to avoid some divisions by zero.\n    df_train['x_d_y'] = df_train.x.values / (df_train.y.values + eps) \n    df_test['x_d_y'] = df_test.x.values / (df_test.y.values + eps) \n        ##New feature x*y\n    df_train['x_t_y'] = df_train.x.values * df_train.y.values  \n    df_test['x_t_y'] = df_test.x.values * df_test.y.values\n    \n    print('    Creating datetime features ...')\n    ##time related features (assuming the time = minutes)\n    initial_date = np.datetime64('2014-01-01T01:01',   #Arbitrary decision\n                                 dtype='datetime64[m]') \n        #working on df_train  \n    d_times = pd.DatetimeIndex(initial_date + np.timedelta64(int(mn), 'm') \n                               for mn in df_train.time.values)    \n    df_train['hour'] = d_times.hour\n    df_train['weekday'] = d_times.weekday\n    df_train['day'] = d_times.day\n    df_train['month'] = d_times.month\n    df_train['year'] = d_times.year\n    df_train = df_train.drop(['time'], axis=1)\n        #working on df_test    \n    d_times = pd.DatetimeIndex(initial_date + np.timedelta64(int(mn), 'm') \n                               for mn in df_test.time.values)    \n    df_test['hour'] = d_times.hour\n    df_test['weekday'] = d_times.weekday\n    df_test['day'] = d_times.day\n    df_test['month'] = d_times.month\n    df_test['year'] = d_times.year\n    df_test = df_test.drop(['time'], axis=1)\n    \n    print('Computing the grid ...')\n    #Creating a new colum with grid_cell id  (there will be \n    #n = (n_cell_x * n_cell_y) cells enumerated from 0 to n-1)\n    size_x = 10. / n_cell_x\n    size_y = 10. / n_cell_y\n        #df_train\n    xs = np.where(df_train.x.values < eps, 0, df_train.x.values - eps)\n    ys = np.where(df_train.y.values < eps, 0, df_train.y.values - eps)\n    pos_x = (xs / size_x).astype(np.int)\n    pos_y = (ys / size_y).astype(np.int)\n    df_train['grid_cell'] = pos_y * n_cell_x + pos_x\n            #df_test\n    xs = np.where(df_test.x.values < eps, 0, df_test.x.values - eps)\n    ys = np.where(df_test.y.values < eps, 0, df_test.y.values - eps)\n    pos_x = (xs / size_x).astype(np.int)\n    pos_y = (ys / size_y).astype(np.int)\n    df_test['grid_cell'] = pos_y * n_cell_x + pos_x \n    \n    ##Normalization\n    print('Normalizing the data: (X - mean(X)) / std(X) ...')\n    cols = ['x', 'y', 'accuracy', 'x_d_y', 'x_t_y', 'hour', \n            'weekday', 'day', 'month', 'year']\n    for cl in cols:\n        ave = df_train[cl].mean()\n        std = df_train[cl].std()\n        df_train[cl] = (df_train[cl].values - ave ) / std\n        df_test[cl] = (df_test[cl].values - ave ) / std\n        \n    #Returning the modified dataframes\n    return df_train, df_test\n\n\ndef process_one_cell(df_train, df_test, grid_id, th):\n    \"\"\"\n    Does all the processing inside a single grid cell: Computes the training\n    and test sets inside the cell. Fits a classifier to the training data\n    and predicts on the test data. Selects the top 3 predictions.\n    \n    Parameters:\n    ----------    \n    df_train: pandas DataFrame\n              Training set\n    df_test: pandas DataFrame\n             Test set\n    grid_id: int\n             The id of the grid to be analyzed\n    th: int\n       Threshold for place_id. Only samples with place_id with at least th\n       occurrences are kept in the training set.\n    \n    Return:\n    ------    \n    pred_labels: numpy ndarray\n                 Array with the prediction of the top 3 labels for each sample\n    row_ids: IDs of the samples in the submission dataframe \n    \"\"\"   \n    #Working on df_train\n    df_cell_train = df_train.loc[df_train.grid_cell == grid_id]\n    place_counts = df_cell_train.place_id.value_counts()\n    mask = place_counts[df_cell_train.place_id.values] >= th\n    df_cell_train = df_cell_train.loc[mask.values]\n    \n    #Working on df_test\n    df_cell_test = df_test.loc[df_test.grid_cell == grid_id]\n    row_ids = df_cell_test.index\n    \n    le = LabelEncoder()\n    y = le.fit_transform(df_cell_train.place_id.values)\n    X = df_cell_train.drop(['place_id', 'grid_cell'], axis = 1).values\n    \n    X_train, X_test, y_train, y_test = \\\n        cross_validation.train_test_split(X, y, random_state=1301, stratify=y, test_size=0.4)\n    \n    \n    #print X_train.shape\n    \n    #print X_test.shape\n    \n    #print y_train.shape\n    \n    #print y_test.shape\n    \n    clf = RandomForestClassifier(n_estimators = 30, n_jobs = -1,random_state=0);\n        \n    clf.fit(X_train, y_train)\n\n    y_pred1 = clf.predict(X_test)\n    \n    bhai = jaccard_similarity_score(y_test, y_pred1)\n    \n    print (bhai)\n    \n    #Training Classifier\n    #clf = SGDClassifier(loss='modified_huber', n_iter=1, random_state=0, n_jobs=-1)  \n    #clf.fit(X, y)\n    X_test1 = df_cell_test.drop(['grid_cell'], axis = 1).values\n    y_pred = clf.predict_proba(X_test1)\n\n    pred_labels = le.inverse_transform(np.argsort(y_pred, axis=1)[:,::-1][:,:3])    \n    return pred_labels, row_ids\n   \n   \ndef process_grid(df_train, df_test, df_sub, th, n_cells):\n    \"\"\"\n    Iterates over all grid cells and aggregates the results of individual cells\n    \"\"\"    \n    for g_id in range(n_cells):\n        if g_id % 10 == 0:\n            print('iteration: %s' %(g_id))\n        \n        #Applying classifier to one grid cell\n        pred_labels, row_ids = process_one_cell(df_train, df_test, g_id, th)\n        #Converting the prediction to the submission format\n        str_labels = np.apply_along_axis(lambda x: ' '.join(x.astype(str)), \n                                         1, pred_labels)\n        #Updating submission file\n        df_sub.loc[row_ids] = str_labels.reshape(-1,1)\n        \n    return df_sub       \n                 \n\nif __name__ == '__main__':\n\n    print('Loading data ...')\n    df_train = pd.read_csv('../input/train.csv', dtype={'x':np.float32, \n                                               'y':np.float32, \n                                               'accuracy':np.int16,\n                                               'time':np.int,\n                                               'place_id':np.int}, \n                                               index_col = 0)\n    df_test = pd.read_csv('../input/test.csv', dtype={'x':np.float32,\n                                              'y':np.float32, \n                                              'accuracy':np.int16,\n                                              'time':np.int,\n                                              'place_id':np.int}, \n                                              index_col = 0)\n    #df_sub = pd.read_csv('Downloads/sample_submission.csv (3)/sample_submissionsubset.csv', index_col = 0)   \n    \n    #Defining the size of the grid\n    n_cell_x = 10\n    n_cell_y = 10\n    df_train, df_test = prepare_data(df_train, df_test, n_cell_x, n_cell_y)\n    \n    #Solving classification problems inside each grid cell\n    th = 500 #Threshold on place_id inside each cell. Only place_ids with at \n            #least th occurrences inside each grid_cell are considered. This\n            #is to avoid classes with very few samples and speed-up the \n            #computation.\n    \n    #df_submission  = process_grid(df_train, df_test, df_sub, th, \n    #                              n_cell_x * n_cell_y)                                 \n    #creating the submission\n    #print('Generating submission file ...')\n\n    #df_submission.to_csv(\"sub.csv\", index=True)  \n    "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}