{"cells":[{"metadata":{},"cell_type":"markdown","source":"# *** Explore the Power Plants Data ***"},{"metadata":{},"cell_type":"markdown","source":"# Import and Data Prep"},{"metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true},"cell_type":"code","source":"library(tidyverse)\nlibrary(jsonlite)\nlibrary(rworldmap)\nlist.files(path = \"../input/ds4g-environmental-insights-explorer/eie_data/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list.files(path = \"../input/ds4g-environmental-insights-explorer/eie_data/gppd\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_plants <- read.csv('../input/ds4g-environmental-insights-explorer/eie_data/gppd/gppd_120_pr.csv', sep=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"head(df_plants)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"summary(df_plants)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Several columns are not useful here => remove from data frame\ndf_plants$other_fuel1 <- NULL # only NAs\ndf_plants$other_fuel2 <- NULL # only NAs\ndf_plants$other_fuel3 <- NULL # only NAs\ndf_plants$country <- NULL # always USA\ndf_plants$country_long <- NULL # always United States of America\ndf_plants$geolocation_source <- NULL # always WRI\ndf_plants$generation_gwh_2013 <- NULL # always 0\ndf_plants$generation_gwh_2014 <- NULL # always 0\ndf_plants$generation_gwh_2015 <- NULL # always 0\ndf_plants$generation_gwh_2016 <- NULL # always 0\ndf_plants$generation_gwh_2017 <- NULL # always 0\ndf_plants$year_of_capacity_data <- NULL # always 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# conversions\ndf_plants$wepp_id <- as.factor(df_plants$wepp_id)\ndf_plants$geo <- as.character(df_plants$`.geo`)\ndf_plants$`.geo` <- NULL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract geo coordinates from JSON column\nextract_lon <- function(text) {\n    my_record <- jsonlite::fromJSON(text)\n    return(my_record$coordinates[1])\n}\n\nextract_lat <- function(text) {\n    my_record <- jsonlite::fromJSON(text)\n    return(my_record$coordinates[2])\n}\n\ndf_plants$geo_lat <- as.numeric(lapply(df_plants$geo, extract_lat))\ndf_plants$geo_lon <- as.numeric(lapply(df_plants$geo, extract_lon))\ndf_plants$geo <- NULL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# now look at the full table\ndf_plants","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Location Plots"},{"metadata":{"trusted":true},"cell_type":"code","source":"summary(df_plants$geo_lon)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# basic plot of locations; bubble area ~ capacity_mw\nnewmap <- getMap(resolution = \"high\")\nplot(newmap, xlim = c(-68, -64), ylim = c(17.8, 18.8), asp = 1)\nsymbols(df_plants$geo_lon, df_plants$geo_lat, circles=sqrt(df_plants$capacity_mw), inches=0.15, bg='#0000A040', \n        xlab='Longitude', ylab='Latitude', main='Power Plants', add=TRUE)\ngrid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# basic plot of locations; bubble area ~ estimated_generation_gwh\nnewmap <- getMap(resolution = \"high\")\nplot(newmap, xlim = c(-68, -64), ylim = c(17.8, 18.8), asp = 1)\nsymbols(df_plants$geo_lon, df_plants$geo_lat, circles=sqrt(df_plants$estimated_generation_gwh), inches=0.5, bg='#0000A040', \n        xlab='Longitude', ylab='Latitude', main='Power Plants', add=TRUE)\ngrid()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Other basic explorations"},{"metadata":{"trusted":true},"cell_type":"code","source":"bp <- ggplot(df_plants, aes(x=primary_fuel,fill=primary_fuel)) + geom_bar() + ggtitle('Primary Fuel')\nbp + theme(axis.text.x = element_text(angle = 90, hjust = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bp <- ggplot(df_plants, aes(x=source,fill=source)) + geom_bar() + ggtitle('Source')\nbp + theme(axis.text.x = element_text(angle = 90, hjust = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bp <- ggplot(df_plants, aes(x=owner,fill=owner)) + geom_bar() + ggtitle('Owner')\nbp + theme(axis.text.x = element_text(angle = 90, hjust = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist(df_plants$capacity_mw)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist(df_plants$estimated_generation_gwh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# w/o \"outlier\"\nhist(df_plants$estimated_generation_gwh[df_plants$estimated_generation_gwh<400000], main='Est.gen.GWh - Histogram excluding max value')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(ecdf(df_plants$estimated_generation_gwh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# w/o \"outlier\"\nplot(ecdf(df_plants$estimated_generation_gwh), xlim=c(0,10000))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"summary(as.factor(df_plants$commissioning_year))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(as.factor(df_plants$commissioning_year), las=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Correlations"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(df_plants$capacity_mw, df_plants$estimated_generation_gwh)\ngrid()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Again the \"outlier\" dominates the picture => let's remove it for the following"},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove \"outlier\"\ndf_plants_ex_out <- dplyr::filter(df_plants, estimated_generation_gwh < 450000)\n\nplot(df_plants_ex_out$capacity_mw, df_plants_ex_out$estimated_generation_gwh,\n     main='Estimated Generation vs Capacity (ex outlier)')\ngrid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cor_pearson <- cor(df_plants_ex_out$capacity_mw, df_plants_ex_out$estimated_generation_gwh, method='pearson')\ncor_spearman <- cor(df_plants_ex_out$capacity_mw, df_plants_ex_out$estimated_generation_gwh, method='spearman')\nprint(paste0('cor_pearson  = ', cor_pearson))\nprint(paste0('cor_spearman = ', cor_spearman))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Spearman correlation much weaker than Pearson correlation. Let's zoom into the smaller values by using log transform."},{"metadata":{"trusted":true},"cell_type":"code","source":"# in log scale we can add the \"outlier\" again\nplot(log10(df_plants$capacity_mw), log10(df_plants$estimated_generation_gwh),\n     main='Estimated Generation vs Capacity - Log/Log')\ngrid()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### That's interesting, we seem to have a group-wise linear structure (in log coordinates)!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# check dependency of primary_fuel by using color coding\nplot(log10(df_plants$capacity_mw), log10(df_plants$estimated_generation_gwh),\n     col = df_plants$primary_fuel, pch=16,\n     main='Estimated Generation vs Capacity - Log/Log')\ngrid()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Conclusion: For fixed primary_fuel the estimated generation is simply a power function of the capacity (actually it is even a linear function)!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Look at an example: Hydro\ndf_plants_hydro <- dplyr::filter(df_plants, primary_fuel=='Hydro')\nplot(log10(df_plants_hydro$capacity_mw), log10(df_plants_hydro$estimated_generation_gwh),\n     col = 'green', pch = 16,\n     main='Estimated Generation vs Capacity - Hydro - Log/Log')\ngrid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit linear model to logs\nx <- df_plants_hydro$capacity_mw\ny <- df_plants_hydro$estimated_generation_gwh\n\nxx <- log10(x)\nyy <- log10(y)\n\n# fit linear model to logs\nmy_mod <- lm(yy ~ xx)\nmy_mod","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt <- my_mod$coefficients[1]\nmm <- my_mod$coefficients[2]\n\nzz = mm * xx + tt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check fit\nzz - yy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# undo trafo\nz <- 10^zz\n# and check\nz - y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we have a slope of 1. After undoing the log transformations this will be the exponent.\n\nSo in this case we simply have\ny = x * 10^tt = 45.69318 * x"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Linearity holds also for the other primary_fuel types.\n# We can simply check by calculating the ratio of estimated_generation and capacity\ndf_plants$ratio <- df_plants$estimated_generation_gwh / df_plants$capacity_mw\n\ndf_plants[,c('primary_fuel','ratio')]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Group by primary_fuel"},{"metadata":{"trusted":true},"cell_type":"code","source":"stats_by_fuel <- dplyr::group_by(df_plants, primary_fuel) %>% dplyr::summarise(n=n(),\n                                                              mean_capacity=round(mean(capacity_mw),2),\n                                                              min_capacity=min(capacity_mw),\n                                                              max_capacity=max(capacity_mw),\n                                                              mean_est_generation=round(mean(estimated_generation_gwh),2),\n                                                              min_est_generation=round(min(estimated_generation_gwh),2),\n                                                              max_est_generation=round(max(estimated_generation_gwh),2))\nstats_by_fuel <- as.data.frame(stats_by_fuel)\n\nstats_by_fuel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Again our \"outlier\". It is the only coal power plant."}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":4}