diff --git a/DESCRIPTION b/DESCRIPTION
index e5cc969..4bba28b 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,6 +1,6 @@
Package: forecast-eval
Title: Forecast Evaluation Dashboard
-Version: 3
+Version: 3.1
Authors@R: person("Kate", "Harwood",
role = c("cre")),
person("Chris", "Scott",
diff --git a/dashboard/about.md b/dashboard/about.md
index 636edf6..8762dea 100644
--- a/dashboard/about.md
+++ b/dashboard/about.md
@@ -87,7 +87,9 @@ Though hospitalizations are forecasted on a daily basis, in keeping with the cas
The forecasts and scores are available as RDS files and are uploaded weekly to a publicly accessible AWS bucket.
You can use the url https://forecast-eval.s3.us-east-2.amazonaws.com/ + filename to download
-any of the files from the bucket. For instance: https://forecast-eval.s3.us-east-2.amazonaws.com/score_cards_nation_cases.rds to download scores for nation level case predictions.
+any of the files from the bucket.
+
+For instance: https://forecast-eval.s3.us-east-2.amazonaws.com/score_cards_nation_cases.rds to download scores for nation level case predictions.
The available files are:
* predictions_cards.rds (forecasts)
@@ -97,5 +99,29 @@ The available files are:
* score_cards_state_deaths.rds
* score_cards_state_hospitalizations.rds
* score_cards_nation_hospitalizations.rds
+
+You can also connect to AWS and retrieve the data in R. Example of retrieving state cases file:
+
+```
+library(aws.s3)
+Sys.setenv("AWS_DEFAULT_REGION" = "us-east-2")
+s3bucket = tryCatch(
+ {
+ get_bucket(bucket = 'forecast-eval')
+ },
+ error = function(e) {
+ e
+ }
+)
+
+stateCases = tryCatch(
+ {
+ s3readRDS(object = "score_cards_state_cases.rds", bucket = s3bucket)
+ },
+ error = function(e) {
+ e
+ }
+)
+```
diff --git a/dashboard/app.R b/dashboard/app.R
index f94fa1a..224ffbc 100644
--- a/dashboard/app.R
+++ b/dashboard/app.R
@@ -9,20 +9,10 @@ library(shinyjs)
library(tsibble)
library(aws.s3)
-COVERAGE_INTERVALS = c("10", "20", "30", "40", "50", "60", "70", "80", "90", "95", "98")
-DEATH_FILTER = "deaths_incidence_num"
-CASE_FILTER = "confirmed_incidence_num"
-HOSPITALIZATIONS_FILTER = "confirmed_admissions_covid_1d"
-HOSPITALIZATIONS_TARGET_DAY = "Wednesday"
-TOTAL_LOCATIONS = "Totaled Over States*"
-AHEAD_OPTIONS = c(1,2,3,4)
-
-# Num days to offset the forecast week by
-# Example: if HOSPITALIZATIONS_TARGET_DAY is Wednesday and HOSPITALIZATIONS_OFFSET is 2,
-# ahead 1 has to have forecast date of Monday or earlier,
-# ahead 2 has to have forecast date of Monday + 7 days or earlier (offset + 7 days or more), etc
-HOSPITALIZATIONS_OFFSET = 2
-HOSPITALIZATIONS_AHEAD_OPTIONS = c(HOSPITALIZATIONS_OFFSET, HOSPITALIZATIONS_OFFSET + 7, HOSPITALIZATIONS_OFFSET + 14, HOSPITALIZATIONS_OFFSET + 21)
+source('./common.R')
+
+# All data is fully loaded from AWS
+dataLoaded = FALSE
# Score explanations
wisExplanation = includeMarkdown("wis.md")
@@ -42,9 +32,12 @@ if(length(cssFiles)!=1){
cssFile = cssFiles[1]
cat(file=stderr(),"Loaded css file:",cssFile,"\n")
+source('./export_scores.R')
+
########
# Layout
########
+
ui <- fluidPage(padding=0,
tags$head(
tags$link(rel = "stylesheet", type = "text/css", href = cssFile)
@@ -58,7 +51,7 @@ ui <- fluidPage(padding=0,
)
),
div(id="title", class="col-sm-6",
- HTML("FORECAST EVALUATION DASHBOARD ",
+ HTML("FORECAST EVALUATION DASHBOARD ",
includeHTML("arrow-left.svg"), " Back"),
),
div(id="github-logo-container", class="col-sm-1",
@@ -103,12 +96,13 @@ ui <- fluidPage(padding=0,
),
tags$p(id="forecaster-disclaimer", "Some forecasters may not have data for the chosen location or scoring metric"),
checkboxGroupInput(
- "aheads",
+ "aheads",
"Forecast Horizon (Weeks)",
choices = AHEAD_OPTIONS,
selected = AHEAD_OPTIONS[1],
inline = TRUE
),
+ hidden(tags$p(id="horizon-disclaimer", "Forecasters submitted earlier than Mondays may have longer actual prediction horizons")),
conditionalPanel(condition = "input.scoreType == 'coverage'",
selectInput(
"coverageInterval",
@@ -128,11 +122,13 @@ ui <- fluidPage(padding=0,
)
),
tags$hr(),
+ export_scores_ui,
+ tags$hr(),
),
includeMarkdown("about-dashboard.md"),
width=3,
),
-
+
mainPanel(
width=9,
tabsetPanel(id = "tabset",
@@ -161,18 +157,19 @@ ui <- fluidPage(padding=0,
plotlyOutput(outputId = "summaryPlot", height="auto"),
fluidRow(
column(11, offset=1,
- div(id="refresh-colors", actionButton(inputId="refreshColors", label= "Recolor"))
+ hidden(div(id="refresh-colors", actionButton(inputId="refreshColors", label= "Recolor")))
)),
tags$br(),
plotlyOutput(outputId = "truthPlot", height="auto"),
fluidRow(
column(11, offset=1,
- div(id="notes", "About the Scores"),
+ div(id="loading-message", "DATA IS LOADING...(this may take a while)"),
+ hidden(div(id="notes", "About the Scores")),
hidden(div(id = "wisExplanation", wisExplanation)),
hidden(div(id = "sharpnessExplanation", sharpnessExplanation)),
hidden(div(id = "aeExplanation", aeExplanation)),
hidden(div(id = "coverageExplanation", coverageExplanation)),
- div(id = "scoringDisclaimer", scoringDisclaimer)
+ hidden(div(id = "scoringDisclaimer", scoringDisclaimer))
)
),
fluidRow(
@@ -183,7 +180,6 @@ ui <- fluidPage(padding=0,
tags$br()
)
)
-
)
),
),
@@ -203,7 +199,7 @@ server <- function(input, output, session) {
return(NULL)
}
)
-
+
# Get and prepare data
getData <- function(filename){
if(!is.null(s3bucket)) {
@@ -220,7 +216,7 @@ server <- function(input, output, session) {
getFallbackData(filename)
}
}
-
+
getFallbackData = function(filename) {
path = ifelse(
file.exists(filename),
@@ -229,13 +225,14 @@ server <- function(input, output, session) {
)
readRDS(path)
}
-
+
dfStateCases <- getData("score_cards_state_cases.rds")
dfStateDeaths <- getData("score_cards_state_deaths.rds")
dfNationCases = getData("score_cards_nation_cases.rds")
dfNationDeaths = getData("score_cards_nation_deaths.rds")
dfStateHospitalizations = getData("score_cards_state_hospitalizations.rds")
dfNationHospitalizations = getData("score_cards_nation_hospitalizations.rds")
+ dataLoaded = TRUE
# Pick out expected columns only
covCols = paste0("cov_", COVERAGE_INTERVALS)
@@ -253,45 +250,42 @@ server <- function(input, output, session) {
df <- rbind(dfStateCases, dfStateDeaths, dfNationCases, dfNationDeaths, dfStateHospitalizations, dfNationHospitalizations)
df <- df %>% rename("10" = cov_10, "20" = cov_20, "30" = cov_30, "40" = cov_40, "50" = cov_50, "60" = cov_60, "70" = cov_70, "80" = cov_80, "90" = cov_90, "95" = cov_95, "98" = cov_98)
-
+
# Prepare color palette
colorSeed = 100
-
+
# Prepare input choices
forecasterChoices = sort(unique(df$forecaster))
updateForecasterChoices(session, df, forecasterChoices, 'wis')
-
+
##################
# CREATE MAIN PLOT
##################
- summaryPlot = function(scoreDf, targetVariable, scoreType, forecasters,
- horizon, loc, coverageInterval = NULL, colorSeed, logScale, scaleByBaseline) {
+ summaryPlot = function(scoreDf, colorSeed) {
allLocations = FALSE
- if (loc == TOTAL_LOCATIONS) {
+ if (input$location == TOTAL_LOCATIONS) {
allLocations = TRUE
}
signalFilter = CASE_FILTER
- if (targetVariable == "Deaths") {
+ if (input$targetVariable == "Deaths") {
signalFilter = DEATH_FILTER
}
- if (targetVariable == "Hospitalizations") {
+ if (input$targetVariable == "Hospitalizations") {
signalFilter = HOSPITALIZATIONS_FILTER
}
- scoreDf = scoreDf %>%
+ filteredScoreDf = scoreDf %>%
filter(signal == signalFilter) %>%
- filter(forecaster %in% forecasters)
+ filter(forecaster %in% input$forecasters)
+
if (signalFilter == HOSPITALIZATIONS_FILTER) {
- scoreDf = filterHospitalizationsAheads(scoreDf)
+ filteredScoreDf = filterHospitalizationsAheads(filteredScoreDf)
}
- scoreDf = scoreDf %>% filter(ahead %in% horizon)
- filteredScoreDf <- scoreDf %>% rename(Forecaster = forecaster, Forecast_Date = forecast_date,
- Week_End_Date = target_end_date)
-
- if (scoreType == "wis" || scoreType == "sharpness") {
+ filteredScoreDf = filteredScoreDf %>% filter(ahead %in% input$aheads)
+ if (input$scoreType == "wis" || input$scoreType == "sharpness") {
# Only show WIS or Sharpness for forecasts that have all intervals
filteredScoreDf = filteredScoreDf %>% filter(!is.na(`50`)) %>% filter(!is.na(`80`)) %>% filter(!is.na(`95`))
- if (targetVariable == "Deaths") {
+ if (input$targetVariable == "Deaths") {
filteredScoreDf = filteredScoreDf %>% filter(!is.na(`10`)) %>% filter(!is.na(`20`)) %>% filter(!is.na(`30`)) %>%
filter(!is.na(`40`)) %>% filter(!is.na(`60`)) %>% filter(!is.na(`70`)) %>% filter(!is.na(`90`)) %>% filter(!is.na(`98`))
}
@@ -299,51 +293,33 @@ server <- function(input, output, session) {
output$renderWarningText <- renderText("The selected forecasters do not have enough data to display the selected scoring metric.")
return()
}
- if (scoreType == "wis") {
- filteredScoreDf <- filteredScoreDf %>% rename(Score = wis)
- title = "Weighted Interval Score"
+ if (input$scoreType == "wis") {
+ plotTitle = "Weighted Interval Score"
}
else {
- filteredScoreDf <- filteredScoreDf %>% rename(Score = sharpness)
- title = "Spread"
+ plotTitle = "Spread"
}
}
- if (scoreType == "ae") {
- filteredScoreDf <- filteredScoreDf %>% rename(Score = ae)
- title = "Absolute Error"
+ if (input$scoreType == "ae") {
+ plotTitle = "Absolute Error"
}
- if (scoreType == "coverage") {
- filteredScoreDf <- filteredScoreDf %>% rename(Score = !!coverageInterval)
- title = "Coverage"
+ if (input$scoreType == "coverage") {
+ plotTitle = "Coverage"
}
+ filteredScoreDf = renameScoreCol(filteredScoreDf, input$scoreType, input$coverageInterval)
# Totaling over all locations
- locationsIntersect = list()
- if (allLocations || scoreType == "coverage") {
- filteredScoreDf = filteredScoreDf %>% filter(!is.na(Score))
- # Create df with col for all locations across each unique date, ahead and forecaster combo
- locationDf = filteredScoreDf %>% group_by(Forecaster, Week_End_Date, ahead) %>%
- summarize(location_list = paste(sort(unique(geo_value)),collapse=","))
- locationDf = locationDf %>% filter(location_list != c('us'))
- # Create a list containing each row's location list
- locationList = sapply(locationDf$location_list, function(x) strsplit(x, ","))
- locationList = lapply(locationList, function(x) x[x != 'us'])
- # Get the intersection of all the locations in these lists
- locationsIntersect = unique(Reduce(intersect, locationList))
- filteredScoreDf = filteredScoreDf %>% filter(geo_value %in% locationsIntersect)
+ if (allLocations || input$scoreType == "coverage") {
+ filteredScoreDfAndIntersections = filterOverAllLocations(filteredScoreDf, input$scoreType)
+ filteredScoreDf = filteredScoreDfAndIntersections[[1]]
+ locationsIntersect = filteredScoreDfAndIntersections[[2]]
aggregateText = "*For fair comparison, all displayed forecasters on all displayed dates are compared across a common set of states and territories."
- if (scoreType == "coverage") {
+ if (input$scoreType == "coverage") {
aggregate = "Averaged"
- filteredScoreDf = filteredScoreDf %>%
- group_by(Forecaster, Forecast_Date, Week_End_Date, ahead) %>%
- summarize(Score = sum(Score)/length(locationsIntersect), actual = sum(actual))
output$renderAggregateText = renderText(paste(aggregateText," Some forecasters may not have any data for the coverage interval chosen. Locations inlcuded: "))
}
else {
aggregate = "Totaled"
- filteredScoreDf = filteredScoreDf %>%
- group_by(Forecaster, Forecast_Date, Week_End_Date, ahead) %>%
- summarize(Score = sum(Score), actual = sum(actual))
output$renderAggregateText = renderText(paste(aggregateText, " Locations included: "))
}
if (length(locationsIntersect) == 0) {
@@ -361,28 +337,31 @@ server <- function(input, output, session) {
}
# Not totaling over all locations
} else {
- filteredScoreDf <- filteredScoreDf %>% filter(geo_value == tolower(loc)) %>%
- group_by(Forecaster, Forecast_Date, Week_End_Date, ahead) %>%
+ filteredScoreDf <- filteredScoreDf %>% filter(geo_value == tolower(input$location)) %>%
+ group_by(forecaster, forecast_date, target_end_date, ahead) %>%
summarize(Score = Score, actual = actual)
locationSubtitleText = paste0(', Location: ', input$location)
output$renderAggregateText = renderText("")
output$renderLocations <- renderText("")
output$renderWarningText <- renderText("")
}
-
+
+ # Rename columns that will be used as labels
+ filteredScoreDf = filteredScoreDf %>% rename(Forecaster = forecaster, Forecast_Date = forecast_date,
+ Week_End_Date = target_end_date)
# Render truth plot with observed values
showElement("truthPlot")
showElement("refresh-colors")
truthDf = filteredScoreDf
output$truthPlot <- renderPlotly({
- truthPlot(truthDf, targetVariable, locationsIntersect, allLocations || scoreType == "coverage")
+ truthPlot(truthDf, locationsIntersect, allLocations || input$scoreType == "coverage")
})
-
+
# Format and transform data
filteredScoreDf = filteredScoreDf[c("Forecaster", "Forecast_Date", "Week_End_Date", "Score", "ahead")]
filteredScoreDf = filteredScoreDf %>% mutate(across(where(is.numeric), ~ round(., 2)))
- if (scoreType != 'coverage') {
- if (scaleByBaseline) {
+ if (input$scoreType != 'coverage') {
+ if (input$scaleByBaseline && input$targetVariable != "Hospitalizations") {
baselineDf = filteredScoreDf %>% filter(Forecaster %in% 'COVIDhub-baseline')
filteredScoreDfMerged = merge(filteredScoreDf, baselineDf, by=c("Week_End_Date","ahead"))
# Scaling score by baseline forecaster
@@ -391,13 +370,13 @@ server <- function(input, output, session) {
rename(Forecaster = Forecaster.x, Score = Score.x, Forecast_Date = Forecast_Date.x) %>%
select(Forecaster, Forecast_Date, Week_End_Date, ahead, Score)
}
- if (logScale) {
+ if (input$logScale) {
filteredScoreDf$Score = log10(filteredScoreDf$Score)
}
}
- titleText = paste0('',title,'','
', '',
- 'Target Variable: ', targetVariable,
+ titleText = paste0('', plotTitle,'','
', '',
+ 'Target Variable: ', input$targetVariable,
locationSubtitleText, '
',
tags$span(id="drag-to-zoom", " Drag to zoom"),
'')
@@ -409,7 +388,7 @@ server <- function(input, output, session) {
# Set labels for faceted horizon plots
horizonOptions = AHEAD_OPTIONS
horizonLabels = lapply(AHEAD_OPTIONS, function (x) paste0("Horizon: ", x, " Week(s)"))
- if (targetVariable == 'Hospitalizations') {
+ if (input$targetVariable == 'Hospitalizations') {
horizonOptions = HOSPITALIZATIONS_AHEAD_OPTIONS
horizonLabels = lapply(HOSPITALIZATIONS_AHEAD_OPTIONS, function (x) paste0("Horizon: ", x, " Days"))
}
@@ -419,7 +398,7 @@ server <- function(input, output, session) {
set.seed(colorSeed)
forecasterRand <- sample(unique(df$forecaster))
colorPalette = setNames(object = viridis(length(unique(df$forecaster))), nm = forecasterRand)
-
+
p = ggplot(
filteredScoreDf,
aes(x = Week_End_Date, y = Score, color = Forecaster, shape = Forecaster, label = Forecast_Date)
@@ -430,43 +409,43 @@ server <- function(input, output, session) {
scale_x_date(date_labels = "%b %Y") +
facet_wrap(~ahead, ncol=1) +
scale_color_manual(values = colorPalette) +
- theme_bw() +
+ theme_bw() +
theme(panel.spacing=unit(0.5, "lines")) +
theme(legend.title = element_blank())
- if (scoreType == "coverage") {
- p = p + geom_hline(yintercept = .01 * as.integer(coverageInterval))
+ if (input$scoreType == "coverage") {
+ p = p + geom_hline(yintercept = .01 * as.integer(input$coverageInterval))
}
- if (logScale) {
+ if (input$logScale) {
p = p + scale_y_continuous(label = function(x) paste0("10^", x))
} else {
p = p + scale_y_continuous(limits = c(0,NA), labels = scales::comma)
}
- plotHeight = 550 + (length(horizon)-1)*100
+ plotHeight = 550 + (length(input$aheads)-1)*100
finalPlot <-
ggplotly(p, tooltip = c("x", "y", "shape", "label")) %>%
layout(
- height = plotHeight,
- legend = list(orientation = "h", y = -0.1),
- margin = list(t=90),
- height=500,
- hovermode = 'x unified',
+ height = plotHeight,
+ legend = list(orientation = "h", y = -0.1),
+ margin = list(t=90),
+ height=500,
+ hovermode = 'x unified',
xaxis = list(
- title = list(text = "Target Date",standoff = 8L),
+ title = list(text = "Target Date",standoff = 8L),
titlefont = list(size = 12))
) %>%
config(displayModeBar = F)
-
+
return(finalPlot)
}
-
+
###################
# CREATE TRUTH PLOT
###################
# Create the plot for target variable ground truth
- truthPlot = function(scoreDf = NULL, targetVariable = NULL, locationsIntersect = NULL, allLocations = FALSE) {
- observation = paste0('Incident ', targetVariable)
- if (targetVariable == "Hospitalizations") {
+ truthPlot = function(scoreDf = NULL, locationsIntersect = NULL, allLocations = FALSE) {
+ observation = paste0('Incident ', input$targetVariable)
+ if (input$targetVariable == "Hospitalizations") {
observation = paste0('Hospital Admissions')
}
titleText = paste0('Observed ', observation, '')
@@ -475,13 +454,13 @@ server <- function(input, output, session) {
}
scoreDf <- scoreDf %>%
group_by(Week_End_Date) %>% summarize(Reported_Incidence = actual)
-
+
return (ggplotly(ggplot(scoreDf, aes(x = Week_End_Date, y = Reported_Incidence)) +
geom_line() +
geom_point() +
labs(x = "", y = "", title = titleText) +
scale_y_continuous(limits = c(0,NA), labels = scales::comma) +
- scale_x_date(date_labels = "%b %Y") + theme_bw())
+ scale_x_date(date_labels = "%b %Y") + theme_bw())
%>% layout(hovermode = 'x unified')
%>% config(displayModeBar = F))
}
@@ -490,22 +469,20 @@ server <- function(input, output, session) {
# PLOT OUTPUT
#############
output$summaryPlot <- renderPlotly({
- summaryPlot(df, input$targetVariable, input$scoreType, input$forecasters,
- input$aheads, input$location, input$coverageInterval, colorSeed, input$logScale, input$scaleByBaseline)
+ summaryPlot(df, colorSeed)
})
###################
# EVENT OBSERVATION
###################
-
+
observeEvent(input$refreshColors, {
colorSeed = floor(runif(1, 1, 1000))
output$summaryPlot <- renderPlotly({
- summaryPlot(df, input$targetVariable, input$scoreType, input$forecasters,
- input$aheads, input$location, input$coverageInterval, colorSeed, input$logScale, input$scaleByBaseline)
+ summaryPlot(df, colorSeed)
})
})
-
+
# When the target variable changes, update available forecasters, locations, and CIs to choose from
observeEvent(input$targetVariable, {
if (input$targetVariable == 'Deaths') {
@@ -521,7 +498,7 @@ server <- function(input, output, session) {
updateLocationChoices(session, df, input$targetVariable, input$forecasters, input$location)
updateCoverageChoices(session, df, input$targetVariable, input$forecasters, input$coverageInterval, output)
})
-
+
observeEvent(input$scoreType, {
if (input$targetVariable == 'Deaths') {
df = df %>% filter(signal == DEATH_FILTER)
@@ -530,9 +507,9 @@ server <- function(input, output, session) {
} else {
df = df %>% filter(signal == HOSPITALIZATIONS_FILTER)
}
- # Only show forecasters that have data for the score chosen
+ # Only show forecasters that have data for the score chosen
updateForecasterChoices(session, df, input$forecasters, input$scoreType)
-
+
if (input$scoreType == "wis") {
show("wisExplanation")
hide("sharpnessExplanation")
@@ -574,9 +551,16 @@ server <- function(input, output, session) {
updateLocationChoices(session, df, input$targetVariable, input$forecasters, input$location)
updateCoverageChoices(session, df, input$targetVariable, input$forecasters, input$coverageInterval, output)
})
-
+
# Ensure the minimum necessary input selections
observe({
+ # Show data loading message and hide other messages until all data is loaded
+ if (dataLoaded) {
+ hide("loading-message")
+ show("refresh-colors")
+ show("notes")
+ show("scoringDisclaimer")
+ }
# Ensure there is always one ahead selected
if(length(input$aheads) < 1) {
if (input$targetVariable == 'Hospitalizations') {
@@ -596,7 +580,9 @@ server <- function(input, output, session) {
if(input$scaleByBaseline && !("COVIDhub-baseline" %in% input$forecasters)) {
updateSelectInput(session, "forecasters", selected = c(input$forecasters, "COVIDhub-baseline"))
}
- })
+ })
+
+ export_scores_server(input, output, df)
}
################
@@ -652,11 +638,14 @@ updateLocationChoices = function(session, df, targetVariable, forecasterChoices,
updateAheadChoices = function(session, df, targetVariable, forecasterChoices, aheads, targetVariableChange) {
df = df %>% filter(forecaster %in% forecasterChoices)
- aheadOptions = AHEAD_OPTIONS
- title = "Forecast Horizon (Weeks)"
if (targetVariable == 'Hospitalizations') {
aheadOptions = HOSPITALIZATIONS_AHEAD_OPTIONS
title = "Forecast Horizon (Days)"
+ show("horizon-disclaimer")
+ } else {
+ aheadOptions = AHEAD_OPTIONS
+ title = "Forecast Horizon (Weeks)"
+ hide("horizon-disclaimer")
}
aheadChoices = Filter(function(x) any(unique(df$ahead) %in% x), aheadOptions)
# Ensure previsouly selected options are still allowed
@@ -676,26 +665,4 @@ updateAheadChoices = function(session, df, targetVariable, forecasterChoices, ah
inline = TRUE)
}
-# Only use weekly aheads for hospitalizations
-# May change in the future
-filterHospitalizationsAheads = function(scoreDf) {
- scoreDf['weekday'] = weekdays(as.Date(scoreDf$target_end_date))
- scoreDf = scoreDf %>% filter(weekday == HOSPITALIZATIONS_TARGET_DAY)
-
- oneAheadDf = scoreDf %>% filter(ahead >= HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 7 + HOSPITALIZATIONS_OFFSET) %>%
- group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
- mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[1])
- twoAheadDf = scoreDf %>% filter(ahead >= 7 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 14 + HOSPITALIZATIONS_OFFSET) %>%
- group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
- mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[2])
- threeAheadDf = scoreDf %>% filter(ahead >= 14 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 21 + HOSPITALIZATIONS_OFFSET) %>%
- group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
- mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[3])
- fourAheadDf = scoreDf %>% filter(ahead >= 21 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 28 + HOSPITALIZATIONS_OFFSET) %>%
- group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
- mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[4])
-
- return(rbind(oneAheadDf, twoAheadDf, threeAheadDf, fourAheadDf))
-}
-
shinyApp(ui = ui, server = server)
diff --git a/dashboard/common.R b/dashboard/common.R
new file mode 100644
index 0000000..8d15859
--- /dev/null
+++ b/dashboard/common.R
@@ -0,0 +1,81 @@
+
+COVERAGE_INTERVALS = c("10", "20", "30", "40", "50", "60", "70", "80", "90", "95", "98")
+DEATH_FILTER = "deaths_incidence_num"
+CASE_FILTER = "confirmed_incidence_num"
+HOSPITALIZATIONS_FILTER = "confirmed_admissions_covid_1d"
+HOSPITALIZATIONS_TARGET_DAY = "Wednesday"
+TOTAL_LOCATIONS = "Totaled Over States*"
+AHEAD_OPTIONS = c(1,2,3,4)
+
+# Num days to offset the forecast week by
+# Example: if HOSPITALIZATIONS_TARGET_DAY is Wednesday and HOSPITALIZATIONS_OFFSET is 2,
+# ahead 1 has to have forecast date of Monday or earlier,
+# ahead 2 has to have forecast date of Monday + 7 days or earlier (offset + 7 days or more), etc
+HOSPITALIZATIONS_OFFSET = 2
+HOSPITALIZATIONS_AHEAD_OPTIONS = c(HOSPITALIZATIONS_OFFSET, HOSPITALIZATIONS_OFFSET + 7, HOSPITALIZATIONS_OFFSET + 14, HOSPITALIZATIONS_OFFSET + 21)
+
+
+renameScoreCol = function(filteredScoreDf, scoreType, coverageInterval) {
+ if (scoreType == "wis") {
+ filteredScoreDf <- filteredScoreDf %>% rename(Score = wis)
+ }
+ else if (scoreType == "sharpness") {
+ filteredScoreDf <- filteredScoreDf %>% rename(Score = sharpness)
+ }
+ else if (scoreType == "ae") {
+ filteredScoreDf <- filteredScoreDf %>% rename(Score = ae)
+ }
+ else if (scoreType == "coverage") {
+ filteredScoreDf <- filteredScoreDf %>% rename(Score = !!coverageInterval)
+ }
+ return (filteredScoreDf)
+}
+
+
+filterOverAllLocations = function(filteredScoreDf, scoreType) {
+ locationsIntersect = list()
+ filteredScoreDf = filteredScoreDf %>% filter(!is.na(Score))
+ # Create df with col for all locations across each unique date, ahead and forecaster combo
+ locationDf = filteredScoreDf %>% group_by(forecaster, target_end_date, ahead) %>%
+ summarize(location_list = paste(sort(unique(geo_value)),collapse=","))
+ locationDf = locationDf %>% filter(location_list != c('us'))
+ # Create a list containing each row's location list
+ locationList = sapply(locationDf$location_list, function(x) strsplit(x, ","))
+ locationList = lapply(locationList, function(x) x[x != 'us'])
+ # Get the intersection of all the locations in these lists
+ locationsIntersect = unique(Reduce(intersect, locationList))
+ filteredScoreDf = filteredScoreDf %>% filter(geo_value %in% locationsIntersect)
+ if (scoreType == "coverage") {
+ filteredScoreDf = filteredScoreDf %>%
+ group_by(forecaster, forecast_date, target_end_date, ahead) %>%
+ summarize(Score = sum(Score)/length(locationsIntersect), actual = sum(actual))
+ }
+ else {
+ filteredScoreDf = filteredScoreDf %>%
+ group_by(forecaster, forecast_date, target_end_date, ahead) %>%
+ summarize(Score = sum(Score), actual = sum(actual))
+ }
+ return (list(filteredScoreDf, locationsIntersect))
+}
+
+# Only use weekly aheads for hospitalizations
+# May change in the future
+filterHospitalizationsAheads = function(scoreDf) {
+ scoreDf['weekday'] = weekdays(as.Date(scoreDf$target_end_date))
+ scoreDf = scoreDf %>% filter(weekday == HOSPITALIZATIONS_TARGET_DAY)
+
+ oneAheadDf = scoreDf %>% filter(ahead >= HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 7 + HOSPITALIZATIONS_OFFSET) %>%
+ group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
+ mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[1])
+ twoAheadDf = scoreDf %>% filter(ahead >= 7 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 14 + HOSPITALIZATIONS_OFFSET) %>%
+ group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
+ mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[2])
+ threeAheadDf = scoreDf %>% filter(ahead >= 14 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 21 + HOSPITALIZATIONS_OFFSET) %>%
+ group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
+ mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[3])
+ fourAheadDf = scoreDf %>% filter(ahead >= 21 + HOSPITALIZATIONS_OFFSET) %>% filter(ahead < 28 + HOSPITALIZATIONS_OFFSET) %>%
+ group_by(target_end_date, forecaster) %>% filter(ahead == min(ahead)) %>%
+ mutate(ahead = HOSPITALIZATIONS_AHEAD_OPTIONS[4])
+
+ return(rbind(oneAheadDf, twoAheadDf, threeAheadDf, fourAheadDf))
+}
\ No newline at end of file
diff --git a/dashboard/export_scores.R b/dashboard/export_scores.R
new file mode 100644
index 0000000..9eb1954
--- /dev/null
+++ b/dashboard/export_scores.R
@@ -0,0 +1,60 @@
+source('./common.R')
+
+create_export_df = function(scoreDf, targetVariable, scoreType, forecasters, loc, coverageInterval) {
+ signalFilter = CASE_FILTER
+ if (targetVariable == "Deaths") {
+ signalFilter = DEATH_FILTER
+ } else if (targetVariable == "Hospitalizations") {
+ signalFilter = HOSPITALIZATIONS_FILTER
+ }
+ scoreDf = renameScoreCol(scoreDf, scoreType, coverageInterval)
+ scoreDf = scoreDf %>%
+ filter(signal == signalFilter) %>%
+ filter(forecaster %in% forecasters)
+ if (loc == TOTAL_LOCATIONS || scoreType == "coverage") {
+ if (signalFilter == HOSPITALIZATIONS_FILTER) {
+ scoreDf = filterHospitalizationsAheads(scoreDf)
+ }
+ scoreDf = filterOverAllLocations(scoreDf, scoreType)
+ return(scoreDf[[1]])
+ } else {
+ scoreDf = scoreDf %>% filter(geo_value == tolower(loc))
+ scoreDf = scoreDf[c("ahead", "geo_value", "forecaster", "forecast_date", "data_source", "target_end_date", "Score", "actual")]
+ return(scoreDf)
+ }
+}
+
+export_scores_ui = div(
+ downloadButton("exportScores", "Download CSV")
+)
+
+export_scores_server = function(input, output, df) {
+ output$exportScores <- downloadHandler(
+ filename = function() {
+ score = input$scoreType
+ if (input$scoreType == 'sharpness') {
+ score = 'spread'
+ }
+ filename = paste0("forecast-eval-", input$targetVariable, "-", score)
+ if (input$location != TOTAL_LOCATIONS) {
+ filename = paste0(filename, '-', input$location)
+ } else if (input$scoreType == 'coverage') {
+ filename = paste0(filename, '-', 'averaged-over-common-locations-Coverage-interval-', input$coverageInterval)
+ } else {
+ filename = paste0(filename, '-totaled-over-common-locations')
+ }
+ paste0(filename,'-', Sys.Date(), ".csv")
+ },
+ contentType = 'text/csv',
+ content = function(file) {
+ withProgress(message = 'Preparing export',
+ detail = 'This may take a while...', value = 0, max = 2, {
+ out_df = create_export_df(df, input$targetVariable, input$scoreType, input$forecasters,
+ input$location, input$coverageInterval)
+ incProgress(1)
+ write.csv(out_df, file, row.names=FALSE)
+ incProgress(2)
+ })
+ }
+ )
+}
diff --git a/dashboard/www/style.css b/dashboard/www/style.css
index 8a3413a..fa20980 100644
--- a/dashboard/www/style.css
+++ b/dashboard/www/style.css
@@ -85,16 +85,24 @@
margin-top:-20px;
font-size:12px;
}
+#horizon-disclaimer {
+ margin-top:-10px;
+ font-size:12px;
+}
#drag-to-zoom {
font-size:11px;
}
-#refreshColors {
+#refresh-colors {
height: 26px;
font-size: 12px;
}
#scale-score {
font-weight: bold;
}
+#loading-message {
+ font-style: italic;
+ font-size: 18px;
+}
@media (max-width: 1450px) {
#github-logo-container {