{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pandas.read_csv)\nimport seaborn as sns\nimport random\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.tree import DecisionTreeRegressor\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.model_selection import train_test_split\nimport warnings\nfrom tqdm import tqdm_notebook\n\nwarnings.filterwarnings('ignore')\n\nimport gc\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This reduces the memory usage!\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Reading all the data\ndef read_data(data_map, location):\n    for key in data_map.keys():\n        if key == location[39:-4]:\n            data = pd.read_csv(location)\n            print(key)\n            print(\"Size:\",data.shape)\n            print(\"-----------------------------------------------------\")\n            data_map[key] = reduce_mem_usage(data)\n\nlist_of_loc = [\"/kaggle/input/ashrae-energy-prediction/train.csv\",\n               \"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\",\n               \"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",\n               \"/kaggle/input/ashrae-energy-prediction/weather_test.csv\",\n               \"/kaggle/input/ashrae-energy-prediction/weather_train.csv\",\n               \"/kaggle/input/ashrae-energy-prediction/test.csv\"]\n\ndata_map = {'train':None, 'building_metadata': None, 'sample_submission': None, 'weather_test':None, 'weather_train': None, 'test':None}\nfor loc in list_of_loc:\n    read_data(data_map, loc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This function will show a distplot of all numerical features of a dataframe.\n\ndef show_dist(data_map, key):\n    df = data_map[key].select_dtypes('number')\n    list_of_col = df.columns\n    print(list_of_col)\n\n    fig, list_of_axis = plt.subplots(ncols = 2, nrows=int(np.ceil(len(df.columns)/2)), figsize = (15,12))\n\n    for i,col in enumerate(list_of_col):\n        sns.distplot(df.loc[~df[col].isna(), col], ax = list_of_axis[int(i/2),i%2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let's analyze the distribution of numerical values of train and test sets in order to ensure that they're having a \n#compatible distrbution!\n#Let's try to define a function for this\n\ndef compare_dist(data_map, train, test):\n    df_train = data_map[train].select_dtypes('number')\n    df_test = data_map[test].select_dtypes('number')\n    list_of_col = set(df_train.columns).intersection(set(df_test.columns))\n    #print(list_of_col)\n\n    fig, list_of_axis = plt.subplots(ncols = 2, nrows=len(list_of_col), figsize = (15,4*len(list_of_col)))\n    \n    for i,col in enumerate(list_of_col):\n        train_plot = sns.distplot(df_train.loc[~df_train[col].isna(), col], ax = list_of_axis[i,0])\n        test_plot = sns.distplot(df_test.loc[~df_test[col].isna(), col], ax = list_of_axis[i,1])\n        \n        if i == 0:\n            train_plot.set_title(\"Train data\")\n            test_plot.set_title(\"Test data\")\n            \n#compare_dist(data_map, 'train', 'test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_map['weather_train'].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Handling Missing Data [It's best to finish it here]"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Alright! We performed a little preprocessing, ensured that both test and train data belong to the same distribution.\n#As of now, I believe that the problem has provided the new weather conditions and all we must do is predict, the power used.\n#So let's tackle the missing values now.\n\ndef show_missing(data_map, keys):\n    for key in keys:\n        list_of_missing_col = []\n        df = data_map[key]\n        for col in df.columns:\n            if df[col].isna().sum().any() >0:\n                list_of_missing_col.append((col, data_map[key][col].isna().sum()/data_map[key].shape[0]))\n        \n        if list_of_missing_col:\n            print(key)\n            print(list_of_missing_col)\n        \nshow_missing(data_map, data_map.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let us drop year_built and floor_count. Pretty useless!\n#data_map['building_metadata'].drop(['year_built', 'floor_count'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_id = random.randint(0,1448)\nfig, ax_l = plt.subplots(nrows=2, ncols=2, figsize=(10,10))\nmeter_map = {0:'Electricity', 1: 'Chilledwater', 2:'Steam', 3:'Hotwater'}\nfor i in range(4):\n    data = data_map['train'][(data_map['train']['building_id'] == building_id) & (data_map['train']['meter'] == i)]\n    sns.scatterplot(range(data.shape[0]), data['meter_reading'], ax=ax_l[int(i/2), int(i%2)]).set_title(\"Meter for %s\"%meter_map[i])\n    \n#Looks like each meter has their own trend. We should make 4 different meters to boost the working.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We intend to create models to fill some of the numerical values in the column. Though not very effective, it is definitely bettwr than simply filling with mode or mean.\n#Initially I thought of dropping the columns with large number of missing values, however we'll run a column selection test later and use XGB to choose best combination of columns.\n\ndef extract_datetime(df):\n    if 'timestamp' in df.columns:\n        time_df = pd.to_datetime(df['timestamp'])\n        df['year'] = time_df.dt.year\n        df['month'] = time_df.dt.month\n        df['day'] = time_df.dt.day\n        df['hour'] = time_df.dt.hour\n        \n        #Finally, dropping timestamp as we don't need it anymore\n        #df.drop('timestamp', axis=1, inplace=True)\n    return df\n\ndef build_weather_regressor(df, num_features, cat_features, target):\n    print(\"Building regressor for\",target)\n    regressor = DecisionTreeRegressor(random_state=0)\n    \n    df = pd.get_dummies(df[num_features+cat_features+[target]], columns=cat_features, prefix=cat_features)[~df[target].isna()]\n    #We gotta use timestamp too!\n    extract_datetime(df)\n    scores = cross_val_score(regressor, df.drop(labels=target, axis=1), df[target], cv=5)\n    print(\"Mean Cross validation score:\", np.mean(scores))\n\n    regressor.fit(df.drop(labels=target, axis=1), df[target])\n    score = regressor.score(df.drop(labels=target, axis=1), df[target])\n    print(\"Regressor Accuracy(On train data itself)\", score)\n    \n    return regressor\n\ndef fill_missing_weather(train,test):\n\n    #Let's fill the mode of air_temperature, wind_direction and dew_temperature for their respective values\n    missing_features = ['air_temperature', 'wind_direction', 'dew_temperature']\n    for feature in missing_features:\n        data_map[train].loc[data_map[train][feature].isna(), feature] = data_map[train][feature].mode()[0]\n        data_map[test].loc[data_map[test][feature].isna(), feature] = data_map[test][feature].mode()[0]\n    \n    #Now, we use these to predict other missing values, which are 'cloud_coverage', 'sea_level_pressure', 'wind_direction', 'wind_speed'\n    missing_features = ['cloud_coverage', 'sea_level_pressure', 'wind_direction', 'wind_speed', 'precip_depth_1_hr' ]\n    num_features = ['air_temperature', 'wind_direction', 'dew_temperature']\n    cat_features = ['site_id'] \n    \n    regressors = {}\n    \n    for key in (train, test):\n        for target in missing_features:\n            if target not in regressors:\n                #Building and training the regressor. Might as well use both training and test data for this.\n                regressors[target] = build_weather_regressor(pd.concat([data_map[train], data_map[test]]), num_features, cat_features, target)\n\n            missing_data = pd.get_dummies(data_map[key][num_features+cat_features+[target]], columns=cat_features, prefix=cat_features)[data_map[key][target].isna()]\n            missing_data.drop(labels=target, axis=1, inplace = True)\n            #We gotta use timestamp too!\n            extract_datetime(missing_data)\n            \n            if missing_data.shape[0] > 0:\n                data_map[key].loc[data_map[key][target].isna(), target] = regressors[target].predict(missing_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fill missing values for weather data in here\nfill_missing_weather('weather_train', 'weather_test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_map['weather_train'].drop(['precip_depth_1_hr', 'cloud_coverage'], axis=1, inplace=True)\ndata_map['weather_test'].drop(['precip_depth_1_hr', 'cloud_coverage'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Cleaning "},{"metadata":{"trusted":true},"cell_type":"code","source":"#We gotta get rid of zeros, because they're a lot! Skewing and screwing our model!\n#filter_ = data_map['train']['meter_reading']>0\n#data_map['train'] = data_map['train'][filter_]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Since we're applying log, we better not have it as zero\nfilter_ = data_map['train']['meter_reading'] == 0\ndata_map['train'].loc[filter_,'meter_reading'] = 0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Some values are negative for precipitation, which is not possible.\n#data_map['weather_train']['precip_depth_1_hr'][data_map['weather_train']['precip_depth_1_hr'] < 0] = 0\n#data_map['weather_test']['precip_depth_1_hr'][data_map['weather_test']['precip_depth_1_hr'] < 0] = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Extraction"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now that's done, we need to work on some features. They are timestano, primary_use and site_id. \n#We need to convert the time of meter reading into timeseries and perform analysis. Let's go! Also, we just\n#Split the timestamp into year, month, day and hour. Let's go!\n\ndef extract_datetime(df):\n    if 'timestamp' in df.columns:\n        time_df = pd.to_datetime(df['timestamp'])\n        df['year'] = time_df.dt.year\n        df['month'] = time_df.dt.month\n        df['day'] = time_df.dt.day\n        df['hour'] = time_df.dt.hour\n        \n        #Finally, dropping timestamp as we don't need it anymore\n        #df.drop('timestamp', axis=1, inplace=True)\n    return df\n\n#Something like a pipeline.. here. Only extracts date and provides one_hot_encoding.\ndef preprocess(df, cat_features=None):\n    extract_datetime(df)\n    if cat_features:\n        df = pd.concat([pd.get_dummies(df, columns=cat_features, prefix=cat_features), df[cat_features]], axis=1)\n    gc.collect()\n\n    return df\n\ndata_map['weather_train'] = preprocess(data_map['weather_train'])\ndata_map['weather_test'] = preprocess(data_map['weather_test'])\ndata_map['building_metadata'] = preprocess(data_map['building_metadata'], ['primary_use']) #We exclude site id here, because weather data already has dummies of it!","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Combining Dataframes to create Merged Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let's join them to create one big chunk of data\ndef combine_dataframes(main_df, weather_df):\n    metadata_df = data_map['building_metadata'] #Static info\n    #First merge\n    df = pd.merge(left=main_df, right=metadata_df, how=\"left\", on=[\"building_id\"])\n    #Second merge\n    df = pd.merge(left=df, right=weather_df, how=\"left\", on=[\"site_id\", \"timestamp\"])\n\n    gc.collect()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['building_id', 'site_id', 'square_feet','primary_use_Education',\n       'primary_use_Entertainment/public assembly',\n       'primary_use_Food sales and service', 'primary_use_Healthcare',\n       'primary_use_Lodging/residential',\n       'primary_use_Manufacturing/industrial', 'primary_use_Office',\n       'primary_use_Other', 'primary_use_Parking',\n       'primary_use_Public services', 'primary_use_Religious worship',\n       'primary_use_Retail', 'primary_use_Services',\n       'primary_use_Technology/science', 'primary_use_Utility',\n       'primary_use_Warehouse/storage', 'air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed', 'year', 'month', 'day', 'hour']\n\ntarget = 'meter_reading'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = {}\nfor i in range(4):\n    models[i] = lgb.LGBMRegressor(reg_alpha=0.5, reg_lambda=0.5, random_state=0, n_jobs=4, subsample=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Okay, we've now successfully combined the data into test and train set after some analysis. Let's move on to train the model\n\ndef train_model(model, train_df, testing = False):\n    #train_df = data_map['train']\n    train = combine_dataframes(train_df, data_map['weather_train']).drop(labels=['timestamp', 'primary_use', 'meter'], axis=1)\n    if testing:\n        X_train, X_test, y_train, y_test = train_test_split(train.drop('meter_reading', axis=1), np.log(train['meter_reading']))\n    else:\n        X_train, y_train = train.drop('meter_reading', axis=1), np.log(train['meter_reading'])\n\n    model.fit(X_train,y_train,verbose=False)\n\n    if testing:\n        y_pred = model.predict(X_test)\n        error = np.sqrt(mean_squared_log_error( np.exp(y_test), np.exp(y_pred) ))\n        print(\"Test error\", error)\n        return error\n    gc.collect()\n\n#train_model(model, data_map['train'], testing = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#All features\nfor i in range(4):\n    train_data = data_map['train'][data_map['train']['meter'] == i]\n    train_model(models[i], train_data, testing = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_prediction(model, test_df, output, batch_size=100000, features=None):\n    #test_df = data_map['test']\n    row_ids = test_df['row_id'].reset_index(drop=True)\n    test = combine_dataframes(test_df, data_map['weather_test']).drop(labels=['timestamp', 'primary_use', 'row_id', 'meter'], axis=1)\n    if features != None:\n        test = test[features]\n\n    for i in tqdm_notebook(range(0,test.shape[0], batch_size)):\n        test_batch = test.iloc[i:i+batch_size,:]\n        selected_row_ids = row_ids[i:i+batch_size]\n        #print(model.predict(test_batch).shape, output.iloc[selected_row_ids, 'meter_Reading'].shape)\n        output.loc[selected_row_ids, 'meter_reading'] = np.exp(model.predict(test_batch))\n\n#result = make_prediction(model, data_map['test'], 100000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reading = pd.DataFrame(data_map['test']['row_id'])\nreading['meter_reading'] = 0\nfor i in range(4):\n    print(\"For meter %d\"%i)\n    test_data = data_map['test'][data_map['test']['meter'] == i]\n    make_prediction(models[i], test_data, reading)\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reading.to_csv('solution.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_best_parameters():\n    alpha_grid = np.linspace(0,1,1)\n    lambda_grid = np.linspace(0,1,1)\n    num_leaves = list(range(20,21,1))\n\n    result = []\n    \n    for alpha in alpha_grid:\n        for lambda_ in lambda_grid:\n            for nleaves in num_leaves:\n                model = lgb.LGBMRegressor(reg_alpha=alpha, reg_lambda=lambda_, random_state=0, n_jobs=4, subsample=0.9, num_leaves=nleaves,  learning_rate=0.01, )\n                error = train_model(model, data_map['train'], testing = True)\n                \n                result.append((alpha, lambda_,num_leaves, error))\n\n    gc.collect()\n    \n    return result\n    \n#pd.DataFrame(find_best_parameters()).to_csv('parameter_grid.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}