# coding: utf-8 __author__ = 'Sandro Vega Pons : https://www.kaggle.com/svpons' '''Partially based on grid_plus_classifier script: https://www.kaggle.com/svpons/facebook-v-predicting-check-ins/grid-plus-classifier ''' import numpy as np import pandas as pd from sklearn.preprocessing import LabelEncoder from sklearn.neighbors import KNeighborsClassifier def prepare_data(df, n_cell_x, n_cell_y, isTraining): """ Feature engineering and computation of the grid. """ #Creating the grid size_x = 10. / n_cell_x size_y = 10. / n_cell_y eps = 0.00001 xs = np.where(df.x.values < eps, 0, df.x.values - eps) ys = np.where(df.y.values < eps, 0, df.y.values - eps) pos_x = (xs / size_x).astype(np.int) pos_y = (ys / size_y).astype(np.int) df['grid_cell'] = pos_y * n_cell_x + pos_x #Feature engineering fw = [500, 1000, 4, 3, 2, 10, 10.] #feature weights (black magic here) df.x = df.x.values * fw[0] df.y = df.y.values * fw[1] initial_date = np.datetime64('2014-01-01T01:01', dtype='datetime64[m]') d_times = pd.DatetimeIndex(initial_date + np.timedelta64(int(mn), 'm') for mn in df.time.values) df['hour'] = d_times.hour + (d_times.minute / 60.) df['weekday'] = (d_times.weekday + (df['hour'] / 24.)) * fw[3] df['month'] = d_times.month * fw[4] df['year'] = (d_times.year - 2013) * fw[5] df['accuracy'] = np.log10(df['accuracy']) * fw[6] if isTraining: df_after_midnight = df[df['hour'] < 3] df_after_midnight.hour = df_after_midnight.hour + 24 df = df.append(df_after_midnight) del df_after_midnight df_before_midnight = df[df['hour'] > 22] df_before_midnight.hour = df_before_midnight.hour - 24 df = df.append(df_before_midnight) del df_before_midnight df['hour'] = df['hour'] * fw[2] df = df.drop(['time'], axis=1) return df def cell_neighbours(grid_id, n_cell_x, n_cell_y): neighbours = [grid_id, grid_id - n_cell_x, grid_id + n_cell_x] if grid_id % n_cell_x > 0: neighbours += [grid_id - 1] neighbours += [grid_id - 1 - n_cell_x] neighbours += [grid_id - 1 + n_cell_x] if grid_id % n_cell_x < n_cell_x - 1: neighbours += [grid_id + 1] neighbours += [grid_id + 1 - n_cell_x] neighbours += [grid_id + 1 + n_cell_x] neighbours = [x for x in neighbours if (x >= 0) & (x < n_cell_x*n_cell_y)] return neighbours def process_one_cell(df_train, df_test, grid_id, th, n_cell_x, n_cell_y): """ Classification inside one grid cell. """ #Working on df_train df_cell_train = df_train.loc[df_train.grid_cell.isin(cell_neighbours(grid_id, n_cell_x, n_cell_y))] place_counts = df_cell_train.place_id.value_counts() mask = (place_counts[df_cell_train.place_id.values] >= th).values df_cell_train = df_cell_train.loc[mask] #Working on df_test df_cell_test = df_test.loc[df_test.grid_cell == grid_id] row_ids = df_cell_test.index #Preparing data le = LabelEncoder() y = le.fit_transform(df_cell_train.place_id.values) X = df_cell_train.drop(['place_id', 'grid_cell'], axis=1).values.astype(int) X_test = df_cell_test.drop(['grid_cell'], axis = 1).values.astype(int) #Applying the classifier clf = KNeighborsClassifier(n_neighbors=25, weights='distance', metric='manhattan') clf.fit(X, y) y_pred = clf.predict_proba(X_test) pred_labels = le.inverse_transform(np.argsort(y_pred, axis=1)[:,::-1][:,:3]) return pred_labels, row_ids def process_grid(df_train, df_test, th, n_cell_x, n_cell_y): """ Iterates over all grid cells, aggregates the results and makes the submission. """ preds = np.zeros((df_test.shape[0], 3), dtype=int) n_cells = n_cell_x * n_cell_y for g_id in range(n_cells): if g_id % 100 == 0: print('\rProcessed ' + str(g_id + 1) + ' / ' + str(n_cells) + ' cells.', end='') #Applying classifier to one grid cell pred_labels, row_ids = process_one_cell(df_train, df_test, g_id, th, n_cell_x, n_cell_y) #Updating predictions preds[row_ids] = pred_labels return preds if __name__ == '__main__': """ """ # Change to False if you want to generate a submission file and run on the full dataset evaluation = True pd.options.mode.chained_assignment = None print('Loading data ...') df_train = pd.read_csv('../input/train.csv', usecols=['row_id','x','y','time','place_id', 'accuracy'], index_col = 0) df_test = pd.read_csv('../input/test.csv', usecols=['row_id','x','y','time', 'accuracy'], index_col = 0) #Defining the size of the grid n_cell_x = 20 n_cell_y = 20 if evaluation: n_train = 2000000 n_test = 500000 df_test = df_train.iloc[n_train : n_train + n_test, :].copy() df_train = df_train.iloc[0 : n_train, :] df_test.index = range(0, n_test) df_test.name = 'row_id' target = df_test.place_id.copy() df_test = df_test.drop(['place_id'], axis = 1) print('Preparing train data') df_train = prepare_data(df_train, n_cell_x, n_cell_y, isTraining = True) print('Preparing test data') df_test = prepare_data(df_test, n_cell_x, n_cell_y, isTraining = False) #Solving classification problems inside each grid cell th = 8 #Keeping place_ids with more than th samples. preds = process_grid(df_train, df_test, th, n_cell_x, n_cell_y) if evaluation: df_aux = pd.DataFrame(preds, dtype=int, columns=['l1', 'l2', 'l3']) correct = (df_aux.l1 == target) | (df_aux.l2 == target) | (df_aux.l3 == target) print("\nCV score: {0:.4f}".format(np.sum(correct.values)/df_test.shape[0])) else: df_aux = pd.DataFrame(preds, dtype=str, columns=['l1', 'l2', 'l3']) #Concatenating the 3 predictions for each sample ds_sub = df_aux.l1.str.cat([df_aux.l2, df_aux.l3], sep=' ') #Writting to csv ds_sub.name = 'place_id' ds_sub.to_csv('fb_kaggle_submission.csv', index=True, header=True, index_label='row_id')