# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load in import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the "../input/" directory. # For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory from subprocess import check_output print(check_output(["ls", "../input"]).decode("utf8")) # Any results you write to the current directory are saved as output. import numpy as np import pandas as pd from sklearn.preprocessing import LabelEncoder from sklearn.ensemble import RandomForestClassifier from sklearn import cross_validation from sklearn.metrics import jaccard_similarity_score def prepare_data(df_train, df_test, n_cell_x, n_cell_y): """ Some feature engineering (mainly with the time feature) + normalization of all features (substracting the mean and dividing by std) + computation of a grid (size = n_cell_x * n_cell_y), which is included as a new column (grid_cell) in the dataframes. Parameters: ---------- df_train: pandas DataFrame Training data df_test : pandas DataFrame Test data n_cell_x: int Number of grid cells on the x axis n_cell_y: int Number of grid cells on the y axis Returns: ------- df_train, df_test: pandas DataFrame Modified training and test datasets. """ print('Feature engineering...') print(' Computing some features from x and y ...') ##x, y, and accuracy remain the same ##New feature x/y eps = 0.00001 #required to avoid some divisions by zero. df_train['x_d_y'] = df_train.x.values / (df_train.y.values + eps) df_test['x_d_y'] = df_test.x.values / (df_test.y.values + eps) ##New feature x*y df_train['x_t_y'] = df_train.x.values * df_train.y.values df_test['x_t_y'] = df_test.x.values * df_test.y.values print(' Creating datetime features ...') ##time related features (assuming the time = minutes) initial_date = np.datetime64('2014-01-01T01:01', #Arbitrary decision dtype='datetime64[m]') #working on df_train d_times = pd.DatetimeIndex(initial_date + np.timedelta64(int(mn), 'm') for mn in df_train.time.values) df_train['hour'] = d_times.hour df_train['weekday'] = d_times.weekday df_train['day'] = d_times.day df_train['month'] = d_times.month df_train['year'] = d_times.year df_train = df_train.drop(['time'], axis=1) #working on df_test d_times = pd.DatetimeIndex(initial_date + np.timedelta64(int(mn), 'm') for mn in df_test.time.values) df_test['hour'] = d_times.hour df_test['weekday'] = d_times.weekday df_test['day'] = d_times.day df_test['month'] = d_times.month df_test['year'] = d_times.year df_test = df_test.drop(['time'], axis=1) print('Computing the grid ...') #Creating a new colum with grid_cell id (there will be #n = (n_cell_x * n_cell_y) cells enumerated from 0 to n-1) size_x = 10. / n_cell_x size_y = 10. / n_cell_y #df_train xs = np.where(df_train.x.values < eps, 0, df_train.x.values - eps) ys = np.where(df_train.y.values < eps, 0, df_train.y.values - eps) pos_x = (xs / size_x).astype(np.int) pos_y = (ys / size_y).astype(np.int) df_train['grid_cell'] = pos_y * n_cell_x + pos_x #df_test xs = np.where(df_test.x.values < eps, 0, df_test.x.values - eps) ys = np.where(df_test.y.values < eps, 0, df_test.y.values - eps) pos_x = (xs / size_x).astype(np.int) pos_y = (ys / size_y).astype(np.int) df_test['grid_cell'] = pos_y * n_cell_x + pos_x ##Normalization print('Normalizing the data: (X - mean(X)) / std(X) ...') cols = ['x', 'y', 'accuracy', 'x_d_y', 'x_t_y', 'hour', 'weekday', 'day', 'month', 'year'] for cl in cols: ave = df_train[cl].mean() std = df_train[cl].std() df_train[cl] = (df_train[cl].values - ave ) / std df_test[cl] = (df_test[cl].values - ave ) / std #Returning the modified dataframes return df_train, df_test def process_one_cell(df_train, df_test, grid_id, th): """ Does all the processing inside a single grid cell: Computes the training and test sets inside the cell. Fits a classifier to the training data and predicts on the test data. Selects the top 3 predictions. Parameters: ---------- df_train: pandas DataFrame Training set df_test: pandas DataFrame Test set grid_id: int The id of the grid to be analyzed th: int Threshold for place_id. Only samples with place_id with at least th occurrences are kept in the training set. Return: ------ pred_labels: numpy ndarray Array with the prediction of the top 3 labels for each sample row_ids: IDs of the samples in the submission dataframe """ #Working on df_train df_cell_train = df_train.loc[df_train.grid_cell == grid_id] place_counts = df_cell_train.place_id.value_counts() mask = place_counts[df_cell_train.place_id.values] >= th df_cell_train = df_cell_train.loc[mask.values] #Working on df_test df_cell_test = df_test.loc[df_test.grid_cell == grid_id] row_ids = df_cell_test.index le = LabelEncoder() y = le.fit_transform(df_cell_train.place_id.values) X = df_cell_train.drop(['place_id', 'grid_cell'], axis = 1).values X_train, X_test, y_train, y_test = \ cross_validation.train_test_split(X, y, random_state=1301, stratify=y, test_size=0.4) #print X_train.shape #print X_test.shape #print y_train.shape #print y_test.shape clf = RandomForestClassifier(n_estimators = 30, n_jobs = -1,random_state=0); clf.fit(X_train, y_train) y_pred1 = clf.predict(X_test) bhai = jaccard_similarity_score(y_test, y_pred1) print (bhai) #Training Classifier #clf = SGDClassifier(loss='modified_huber', n_iter=1, random_state=0, n_jobs=-1) #clf.fit(X, y) X_test1 = df_cell_test.drop(['grid_cell'], axis = 1).values y_pred = clf.predict_proba(X_test1) pred_labels = le.inverse_transform(np.argsort(y_pred, axis=1)[:,::-1][:,:3]) return pred_labels, row_ids def process_grid(df_train, df_test, df_sub, th, n_cells): """ Iterates over all grid cells and aggregates the results of individual cells """ for g_id in range(n_cells): if g_id % 10 == 0: print('iteration: %s' %(g_id)) #Applying classifier to one grid cell pred_labels, row_ids = process_one_cell(df_train, df_test, g_id, th) #Converting the prediction to the submission format str_labels = np.apply_along_axis(lambda x: ' '.join(x.astype(str)), 1, pred_labels) #Updating submission file df_sub.loc[row_ids] = str_labels.reshape(-1,1) return df_sub if __name__ == '__main__': print('Loading data ...') df_train = pd.read_csv('../input/train.csv', dtype={'x':np.float32, 'y':np.float32, 'accuracy':np.int16, 'time':np.int, 'place_id':np.int}, index_col = 0) df_test = pd.read_csv('../input/test.csv', dtype={'x':np.float32, 'y':np.float32, 'accuracy':np.int16, 'time':np.int, 'place_id':np.int}, index_col = 0) df_sub = pd.read_csv('../input/sample_submission.csv', index_col = 0) #Defining the size of the grid n_cell_x = 20 n_cell_y = 20 df_train, df_test = prepare_data(df_train, df_test, n_cell_x, n_cell_y) #Solving classification problems inside each grid cell th = 750 #Threshold on place_id inside each cell. Only place_ids with at #least th occurrences inside each grid_cell are considered. This #is to avoid classes with very few samples and speed-up the #computation. df_submission = process_grid(df_train, df_test, df_sub, th, n_cell_x * n_cell_y) #creating the submission #print('Generating submission file ...') #df_submission.to_csv("sub.csv", index=True)