{"cells":[{"metadata":{"_uuid":"0d19ecee-2d50-4385-9b95-bc55b4727362","_cell_guid":"c6d5caad-ab39-4867-a632-f26eacd06274","trusted":true,"_execution_state":"idle","_kg_hide-output":true},"cell_type":"code","source":"## Importing packages\n\nlibrary(tidyverse) # metapackage with lots of helpful functions\nlibrary(caret)\nlibrary(glmnet)\nlibrary(lightgbm)\nlibrary(lubridate)\n#library(DataExplorer)\nlibrary(gridExtra)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2ffcb10-b0ff-4971-a58f-06671858a205","_cell_guid":"d13da3fb-9c4a-4b65-9592-f58e2d5a26f9","trusted":true},"cell_type":"code","source":"list.files(path = \"../input/ashrae-energy-prediction\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf4701a4-f7bd-480b-99aa-ceaeeeb16c5f","_cell_guid":"32647657-cf40-4c70-8205-9bbf1e324b89","trusted":true},"cell_type":"code","source":"tr <- read.csv(\"../input/ashrae-energy-prediction/train.csv\")\nweather_train <- read.csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\nbuilding <- read.csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"390f7b4f-429c-4f43-8b1d-a6de191db3ac","_cell_guid":"1e050f1f-3694-4b0a-9c19-1a20e1084cf1","trusted":true},"cell_type":"markdown","source":"# **Data overview**"},{"metadata":{"_uuid":"0eb4a6e0-7645-4659-9acd-aa6be86b745e","_cell_guid":"95f2cf7c-044a-496c-bff6-120996431589","trusted":true},"cell_type":"markdown","source":"## **Train data**"},{"metadata":{"_uuid":"76531f9b-d63d-4fd5-ba5a-100800daeb9b","_cell_guid":"ccc85d66-5f1c-4bf9-8654-e4af0b125d56","trusted":true},"cell_type":"raw","source":"str(tr)"},{"metadata":{"_uuid":"99eeee53-03b7-4d8f-a7f3-5656b6702d1b","_cell_guid":"9b9912c7-ee06-4326-af82-b8ba9a823bbc","trusted":true},"cell_type":"code","source":"# adding time features\ntr$time <- ymd_hms(tr$timestamp)\ntr <- tr %>% mutate(month = month(time),\n                   hour = hour(time),\n                   day = date(time))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0da68c2f-8685-4a8b-95c6-ae49675619c2","_cell_guid":"add0adce-5565-4d88-80ff-ee1de223ce65","trusted":true},"cell_type":"code","source":"options(repr.plot.width=12, repr.plot.height=7)\ngrid.arrange(\ntr %>% group_by(building_id) %>%\n    summarize(count = n()) %>%\n    ggplot(aes(building_id, count)) + geom_col(fill = \"steel blue\") + labs(title = \"building_id\"),\n\ntr %>% group_by(meter) %>%\n    summarize(count = n()) %>%\n    ggplot(aes(meter, count)) + geom_col(fill = \"steel blue\") + labs(title = \"meter\"),\n\ntr %>% group_by(day) %>%\n    summarize(count = n()) %>%\n    ggplot(aes(day, count)) + geom_col(fill = \"steel blue\") + labs(title = \"day\"),    \n    \nncol = 2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e85f0d40-6234-413c-a64a-3cb8477be508","_cell_guid":"45dfad63-a7f1-4a53-a994-e5675d43da5b","trusted":true},"cell_type":"code","source":"options(repr.plot.width=12, repr.plot.height=4)\ngrid.arrange(\n    tr %>% ggplot(aes(meter_reading)) + geom_histogram(fill = \"steel blue\") + geom_density(),\n    \n    tr %>% group_by(day) %>% summarize(avg_meter_reading = mean(meter_reading)) %>%\n        ggplot(aes(day, avg_meter_reading)) + geom_line(color = \"steel blue\"),\n    \n    ncol = 2\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Total energy consumption by time:"},{"metadata":{"trusted":true},"cell_type":"code","source":"options(repr.plot.width=12, repr.plot.height=4)\ngrid.arrange(\n    tr %>% group_by(day) %>% summarize(total_meter_reading = sum(meter_reading)) %>%\n        ggplot(aes(day, total_meter_reading)) + geom_line(color = \"steel blue\"),\n    \n    tr %>% group_by(hour) %>% summarize(avg_meter_reading = mean(meter_reading)) %>%\n        ggplot(aes(hour, avg_meter_reading)) + geom_line(color = \"steel blue\"),\n    \n    ncol = 2\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is one peak at November:"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr %>% filter(month == 11) %>% group_by(day) %>% summarize(total_meter_reading = sum(meter_reading)) %>%\n    filter(total_meter_reading > 2e8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Energy consumption of first 10 building:"},{"metadata":{"trusted":true},"cell_type":"code","source":"options(repr.plot.width=15, repr.plot.height=8)\ntr %>% filter(building_id <= 10) %>% \n    ggplot(aes(day, meter_reading, color = building_id)) + geom_line() + facet_wrap(.~building_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Building**"},{"metadata":{"_uuid":"4290ef7e-ae6e-47b8-a10f-67b674489da0","_cell_guid":"e7ddb3b2-b988-4fd9-92d1-51e4edcd3424","trusted":true},"cell_type":"code","source":"str(building)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2818857f-9e31-45c6-9b1a-367b9e887ec9","_cell_guid":"a73cf00e-e691-42c4-8720-a788fde39e10","trusted":true},"cell_type":"code","source":"building %>% group_by(primary_use) %>% summarize(count = n()) %>%\n    mutate(primary_use = reorder(primary_use, count)) %>%\n    ggplot(aes(primary_use, count, fill = count)) + geom_col() + coord_flip() + \n    theme_classic() + theme(legend.position = \"none\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd03d2b4-c459-4a1e-8be5-800b81995e6a","_cell_guid":"1ef56f52-28d6-4e08-92f0-85e975837fcf","trusted":true},"cell_type":"code","source":"options(repr.plot.width=12, repr.plot.height=7)\nbuilding %>% ggplot(aes(square_feet)) + geom_histogram(fill = \"steel blue\", color = \"white\") + theme_classic() +\nlabs(title = \"Distribution of building square feet\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"acb84894-e960-4635-b598-0f7c6ea0ebb6","_cell_guid":"3a634a4c-824b-4f57-8649-c266c3776cbc","trusted":true},"cell_type":"code","source":"summary(building)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae0f21d8-bc6e-4c0b-b753-5b4e2974a9b4","_cell_guid":"c5d03073-e692-4e7f-a027-f132b8635413","trusted":true},"cell_type":"code","source":"building$floor_count[is.na(building$floor_count)] <- median(building$floor_count, na.rm = T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_tr <- tr %>% left_join(building, by = \"building_id\")\nhead(d_tr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Weather_train**"},{"metadata":{"trusted":true},"cell_type":"code","source":"head(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"library(DataExplorer)\nDataExplorer::plot_histogram(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert to character\nd_tr$site_id <- as.character(d_tr$site_id)\nweather_train$site_id <- as.character(weather_train$site_id)\n\n# create new linked key\nd_tr <- d_tr %>% mutate(key_id = str_c(site_id,timestamp, sep = \"-\"))\nweather_train <- weather_train %>% mutate(key_id = str_c(site_id,timestamp, sep = \"-\"))\n\n# left join\nd_tr <- d_tr %>% left_join(weather_train, by = \"key_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"head(d_tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rm(tr, weather_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Combine data**"},{"metadata":{},"cell_type":"markdown","source":"Visualize the NA value"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.frame(variable = names(d_tr), na_value = colSums(is.na(d_tr))/nrow(d_tr)) %>%\n    mutate(variable = reorder(variable, na_value)) %>%\n    ggplot(aes(variable, na_value)) + geom_col(fill = \"steel blue\") + coord_flip() + geom_label(aes(label = round(na_value, 4)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then replace the NA value by median value or 0."},{"metadata":{"trusted":true},"cell_type":"code","source":"# replace na values\nd_tr$air_temperature[is.na(d_tr$air_temperature)] <- median(d_tr$air_temperature, na.rm = T)\nd_tr$dew_temperature[is.na(d_tr$dew_temperature)] <- median(d_tr$dew_temperature, na.rm = T)\nd_tr$wind_speed[is.na(d_tr$wind_speed)] <- 0\nd_tr$cloud_coverage[is.na(d_tr$cloud_coverage)] <- 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"names(d_tr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Sampling for EDA**"},{"metadata":{},"cell_type":"markdown","source":"Sampling helps a lot in research. It is one of the most important factors which determines the accuracy of your research/survey result. If anything goes wrong with your sample then it will be directly reflected in the final result. There are lot of techniques which help us to gather sample depending upon the need and situation. \n\nNow pick up"},{"metadata":{"trusted":true},"cell_type":"code","source":"building_id <- building %>% group_by(building_id) %>% pull(building_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set.seed(123)\nspl <- sample(building_id,20)\nspl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_tr %>% filter(building_id %in% spl) %>%\n    ggplot(aes(day, meter_reading, color = primary_use)) + geom_line() + facet_wrap(.~building_id, scales = \"free\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_tr %>% group_by(primary_use, day) %>%\n    summarize(avg_meter_reading = mean(meter_reading, na.rm = T)) %>%\n    ggplot(aes(day, avg_meter_reading, color = primary_use)) + geom_line() + facet_wrap(.~primary_use)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Correlation between temperature and meter_reading**"},{"metadata":{"trusted":true},"cell_type":"code","source":"d_tr %>% filter(building_id %in% spl) %>%\n    ggplot(aes(air_temperature, meter_reading, color = primary_use)) + geom_line() + geom_smooth(color = \"black\") + facet_wrap(.~building_id, scales = \"free\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Simple linear model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"d_tr <- d_tr %>% select(building_id, meter, month, hour, day, meter_reading) %>%\n    mutate(meter_reading = log1p(meter_reading))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_missing(d_tr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fit_lm <- lm(meter_reading ~., data = d_tr)\nsummary(fit_lm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#te <- read.csv(\"../input/ashrae-energy-prediction/test.csv\")\n#weather_test <- read.csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\n#sample_sub <- read.csv(\"../input/ashrae-energy-prediction/sample_submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}