Question: how do WIS and coverage change with a constant learning rate, season by season, on the square-root scale and on the rate scale, and how do they compare with REF-op cold?

Setup:

  • Disease: flu. Base forecaster: the clean windowed_seasonal replay (all 53 locations, the 55 hub rounds). Horizons h0–h3.
  • Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Cold start, no burn-in season.
  • The tracker runs over both seasons, so 2025-26 starts from the offsets 2024-25 left behind; each season is scored on its own rounds. Scores: states only (US excluded).
  • Configs:
    • sqrt <lr>: the plain tracker with a constant learning rate on the square root of admissions per 100k; 11 rates from 0.001 to 0.32, a quarter decade apart, in sqrt(per-100k) units (not comparable number for number with the rate scale). The grid was set after a first look at results.
    • rate <lr>: the same tracker on admissions per 100k; 12 rates from 0.0032 to 1.8, a quarter decade apart. 0.1 is E11’s config, though E11 used the submitted ensemble.
    • REF-op cold: prod’s tracker without its warm start.
    • sqrt, adaptive: REF-paper on the sqrt scale with its adaptive learning rate (E05’s single, cold).
  • A sqrt rate is selected below as the highest mean gain with no loss in any season × horizon. The selection is in-sample.

WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage bias: lower is better.

source(here::here("scripts/calibration/calibration_ws_replay.R"))
wi <- ws_inputs("flu")
RATE_LRS <- signif(10^seq(-2.5, 0.25, by = 0.25), 2)
SQRT_LRS <- signif(10^seq(-3, -0.5, by = 0.25), 2)
all_cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE)
rate_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr))
sqrt_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr, transform = "sqrt"))
# Cached under cache/calibration/experiments/runs; E17, E18 and E07 reuse some of these runs.
run <- function(cfg) {
  do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers)))
}
rate_runs <- purrr::map(RATE_LRS, \(lr) run(rate_cfg(lr)))
sqrt_runs <- purrr::map(SQRT_LRS, \(lr) run(sqrt_cfg(lr)))
refop <- run(all_cfgs[["REF-op cold"]])
sqrt_adaptive <- run(ws_configs(wi$rate_scales, burn_in = TRUE)[["E05 single cold"]])
GEOS <- c(states = "states only (US excluded)")
SEASONS <- c(LIVE, "both")
SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45")
score <- function(cal) {
  fc <- cal$forecasts %>% mutate(season = season_of(reference_date))
  purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>%
    bind_rows() %>%
    select(geos, season, horizon, which, wis, cal_err) %>%
    tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>%
    mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base)
}
label <- function(df) {
  df %>% mutate(
    geos = factor(geos, levels = names(GEOS), labels = GEOS),
    season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")),
    h = factor(paste0("h", horizon), levels = paste0("h", 0:3))
  )
}
sweep_m <- bind_rows(
  purrr::map2(rate_runs, RATE_LRS, \(cal, lr) score(cal) %>% mutate(scale = "rate", lr = lr)),
  purrr::map2(sqrt_runs, SQRT_LRS, \(cal, lr) score(cal) %>% mutate(scale = "sqrt", lr = lr))
) %>% label()
ref_m <- bind_rows(
  score(refop) %>% mutate(ref = "REF-op cold"),
  score(sqrt_adaptive) %>% mutate(ref = "sqrt, adaptive")
) %>% label()
# Best sqrt rate: the highest mean WIS reduction among rates that improve WIS in both seasons at h0–h3 (states only).
best_tbl <- sweep_m %>%
  filter(scale == "sqrt", geos == GEOS[["states"]], season %in% LIVE) %>%
  group_by(lr) %>%
  summarize(worst = min(wis_pct), mean = mean(wis_pct), .groups = "drop")
BEST_SQRT <- best_tbl %>% filter(worst > 0) %>% slice_max(mean, n = 1) %>% pull(lr)

1. Coverage bias vs WIS, by season and scale

One curve per season and scale, tracing the learning rate from smallest (near the base, the open circle) to largest. Solid lines are the rate scale, dashed lines the sqrt scale. Labels mark a few rates on each curve. Triangles are REF-op cold, squares the adaptive sqrt tracker. The y axis is clipped at −10%. The sqrt rate selected by the rule below is 0.018.

LABELLED <- list(rate = c(0.01, 0.032, 0.1, 0.32), sqrt = c(0.0032, 0.01, 0.032, 0.1))
curve_plot <- function(g) {
  d <- sweep_m %>% filter(geos == g, season != "both seasons") %>% arrange(scale, lr)
  r <- ref_m %>% filter(geos == g, season != "both seasons")
  lab <- d %>% filter((scale == "rate" & lr %in% LABELLED$rate) | (scale == "sqrt" & lr %in% LABELLED$sqrt))
  base <- d %>% distinct(h, season, cal_err_base)
  ggplot(d, aes(cal_err_cal, wis_pct, colour = season)) +
    geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) +
    geom_path(aes(linetype = scale, group = interaction(season, scale)), linewidth = 0.7) +
    geom_point(aes(shape = scale), size = 1.1) +
    geom_text(data = lab, aes(label = paste0(ifelse(scale == "sqrt", "s", "r"), format(lr, drop0trailing = TRUE))),
      size = 2.3, vjust = -0.8, show.legend = FALSE) +
    geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) +
    geom_point(data = r, aes(shape = ref), size = 2.6) +
    scale_colour_manual(values = SEASON_COLS, name = NULL) +
    scale_linetype_manual(values = c(rate = "solid", sqrt = "22"), name = "scale") +
    scale_shape_manual(values = c(rate = 16, sqrt = 1, `REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) +
    facet_wrap(~h, nrow = 1) +
    coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) +
    labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g,
      caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate") +
    theme(legend.position = "bottom")
}
curve_plot(GEOS[["states"]])

Each metric against the learning rate

One panel set per scale; the two scales’ rates are in different units. Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 coverage bias (lower is better), each season’s base dotted. States only.

vs_lr_plot <- function(sc, breaks, xlab) {
  long <- sweep_m %>%
    filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>%
    transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>%
    tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>%
    mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)")))
  base_lines <- sweep_m %>%
    filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>%
    distinct(h, season, value = cal_err_base) %>%
    mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric)))
  zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0)
  ggplot(long, aes(lr, value, colour = season)) +
    geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) +
    geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") +
    geom_line() +
    geom_point(size = 1) +
    scale_x_log10(breaks = breaks, labels = format(breaks, drop0trailing = TRUE)) +
    scale_colour_manual(values = SEASON_COLS, name = NULL) +
    facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) +
    labs(x = xlab, y = NULL, title = paste(sc, "scale")) +
    theme(legend.position = "bottom")
}
vs_lr_plot("sqrt", c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), "constant learning rate (sqrt of per-100k units per step)")

vs_lr_plot("rate", c(0.003, 0.01, 0.03, 0.1, 0.3, 1), "constant learning rate (per 100k per step)")

Selecting a sqrt rate

For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons and h0–h3 (states only). The selected rate has the highest mean among those whose worst case is still a gain.

knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2))))
lr worst mean
0.0010 0.02 0.17
0.0018 0.03 0.29
0.0032 0.05 0.48
0.0056 0.08 0.74
0.0100 0.10 1.02
0.0180 0.06 1.15
0.0320 -0.26 0.81
0.0560 -1.59 -0.32
0.1000 -5.25 -3.15
0.1800 -13.13 -9.12
0.3200 -32.21 -20.41 
lr_lab <- function(lr) vapply(lr, \(x) format(x, drop0trailing = TRUE), character(1))
mv <- cal_month_view(c(
  setNames(sqrt_runs, paste("sqrt", lr_lab(SQRT_LRS))),
  setNames(rate_runs, paste("rate", lr_lab(RATE_LRS))),
  list(`REF-op cold` = refop, `sqrt, adaptive` = sqrt_adaptive)
), seasons = LIVE)
# The plots show the sqrt rates around the elbow, rate 0.032 and REF-op cold; the tables cover every rate on both scales.
MONTH_COLS <- c(`sqrt 0.01` = "#56B4E9", `sqrt 0.018` = "#CC79A7", `sqrt 0.032` = "#0072B2", `rate 0.032` = "#009E73", `REF-op cold` = "#D55E00")
mv_plot_cfgs <- names(MONTH_COLS)
mv_cols <- MONTH_COLS

Month view: WIS, coverage, median error

Scores by calendar month of the reference date, one figure per season, states only (US excluded). Rows are the three metrics, columns the horizons:

  • WIS reduction %: 100 × (WIS_base − WIS_cal) / WIS_base within the month, so positive = calibrated WIS lower than base. Grey bars are the month’s share of the season’s base WIS, so a large swing in a month with a small bar costs little overall.
  • L1 coverage bias reduction %: 100 × (bias_base − bias_cal) / bias_base, so positive = calibrated coverage bias lower than base. The bias is computed within the month: the mean over the 23 levels of |share of outcomes at or below the quantile − level|. A season-pooled bias lets under-prediction on the ramp cancel over-prediction after the peak; the monthly one does not. When the base’s bias is already small, a small absolute change shows as a large percentage; the month-average column of the coverage table below gives the absolute values.
  • Median AE reduction %: the same as WIS reduction % for the absolute error of the 0.5 quantile, so positive = calibrated median closer to the outcome.

A month holds one to five rounds of about 52 states, and outcomes in the same round move together, so a single month’s coverage bias is noisy and sits above zero even for a calibrated forecaster. That’s why the plot shows it relative to the base.

mv_plot_month <- mv$month
if (!is.null(get0("mv_plot_cfgs"))) {
  mv_plot_month <- mv_plot_month %>% filter(config %in% mv_plot_cfgs) %>% mutate(config = droplevels(config))
}
for (s in sort(unique(mv_plot_month$season))) {
  cat("\n\n", cal_h(2), " ", s, "\n\n", sep = "")
  print(cal_month_plot(mv_plot_month, s, colours = get0("mv_cols"), linetype = get0("mv_linetype"), colour_by = get0("mv_colour_by") %||% "config"))
}

2024-2025

2025-2026

cat("\n\n", cal_h(2), " By horizon\n\n", sep = "")

By horizon

cat("WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons.\n\n")

WIS reduction % (positive = calibrated WIS lower than base), states only; both pools the seasons.

print(knitr::kable(cal_month_wis_table(mv)))
config 2024-2025 h0 2024-2025 h1 2024-2025 h2 2024-2025 h3 2025-2026 h0 2025-2026 h1 2025-2026 h2 2025-2026 h3 both h0 both h1 both h2 both h3
sqrt 0.001 +0.1 +0.1 +0.0 +0.0 +0.5 +0.2 +0.2 +0.2 +0.3 +0.1 +0.1 +0.1
sqrt 0.0018 +0.2 +0.1 +0.1 +0.0 +0.9 +0.3 +0.3 +0.3 +0.5 +0.2 +0.2 +0.2
sqrt 0.0032 +0.4 +0.2 +0.1 +0.1 +1.5 +0.6 +0.5 +0.6 +0.8 +0.3 +0.3 +0.3
sqrt 0.0056 +0.7 +0.3 +0.2 +0.1 +2.2 +0.8 +0.8 +0.9 +1.3 +0.5 +0.4 +0.4
sqrt 0.01 +1.1 +0.4 +0.2 +0.1 +2.8 +1.1 +1.1 +1.3 +1.8 +0.7 +0.6 +0.6
sqrt 0.018 +1.7 +0.6 +0.3 +0.1 +3.0 +1.0 +1.1 +1.5 +2.2 +0.8 +0.6 +0.6
sqrt 0.032 +2.2 +0.6 +0.1 -0.3 +2.1 +0.1 +0.5 +1.1 +2.2 +0.4 +0.2 +0.2
sqrt 0.056 +2.4 +0.1 -0.8 -1.4 +0.1 -1.6 -1.0 -0.3 +1.5 -0.6 -0.9 -0.9
sqrt 0.1 +1.7 -2.1 -3.8 -4.7 -4.0 -5.3 -3.8 -3.2 -0.5 -3.4 -3.8 -4.1
sqrt 0.18 -1.9 -6.9 -11.3 -13.1 -12.0 -10.0 -8.4 -9.3 -5.9 -8.2 -10.1 -11.7
sqrt 0.32 -6.4 -16.7 -26.8 -32.2 -23.8 -18.2 -18.1 -21.2 -13.2 -17.3 -23.2 -28.0
rate 0.0032 +0.2 +0.1 +0.1 +0.1 +0.7 +0.4 +0.3 +0.3 +0.4 +0.2 +0.2 +0.2
rate 0.0056 +0.3 +0.1 +0.1 +0.1 +1.1 +0.6 +0.5 +0.6 +0.6 +0.3 +0.3 +0.3
rate 0.01 +0.4 +0.2 +0.2 +0.1 +1.6 +0.8 +0.8 +0.8 +0.9 +0.5 +0.4 +0.4
rate 0.018 +0.7 +0.4 +0.3 +0.2 +1.9 +1.0 +0.9 +1.0 +1.2 +0.6 +0.5 +0.5
rate 0.032 +1.0 +0.5 +0.3 +0.2 +1.9 +1.0 +0.8 +0.8 +1.3 +0.7 +0.5 +0.4
rate 0.056 +1.2 +0.5 +0.2 -0.0 +1.4 +0.4 +0.3 +0.2 +1.3 +0.5 +0.3 +0.0
rate 0.1 +1.1 +0.2 -0.5 -0.9 +0.2 -0.8 -1.0 -1.2 +0.7 -0.2 -0.7 -1.0
rate 0.18 +0.5 -1.0 -2.3 -3.2 -2.1 -3.2 -3.3 -3.7 -0.5 -1.9 -2.7 -3.4
rate 0.32 -0.9 -3.4 -6.0 -7.4 -6.5 -6.9 -7.2 -7.8 -3.1 -4.8 -6.5 -7.6
rate 0.56 -3.7 -7.8 -12.4 -14.9 -13.4 -12.0 -12.1 -14.2 -7.5 -9.6 -12.2 -14.7
rate 1 -8.6 -14.5 -22.9 -27.9 -25.3 -17.7 -18.1 -24.6 -15.2 -15.8 -20.9 -26.6
rate 1.8 -16.0 -24.5 -39.5 -50.7 -49.5 -32.4 -28.5 -38.6 -29.3 -27.7 -35.0 -46.1
REF-op cold +2.2 +0.9 -0.0 -0.8 +1.8 +0.1 +0.3 +0.9 +2.0 +0.6 +0.1 -0.2
sqrt, adaptive +1.8 +0.0 -1.4 -2.6 +2.2 -0.6 -0.7 -0.2 +1.9 -0.2 -1.1 -1.7 
print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION))
L1 coverage bias (lower is better), states only, as season-pooled / month-avg. Season-pooled scores all of a season’s rounds together. Month-avg is the mean over calendar months of the bias computed within each month, every month weighted equally (‘both’: every season-month weighted equally).
config 2024-2025 h0 2024-2025 h1 2024-2025 h2 2024-2025 h3 2025-2026 h0 2025-2026 h1 2025-2026 h2 2025-2026 h3 both h0 both h1 both h2 both h3
base 0.127 / 0.130 0.129 / 0.144 0.148 / 0.179 0.169 / 0.211 0.090 / 0.097 0.081 / 0.112 0.084 / 0.130 0.086 / 0.146 0.108 / 0.112 0.105 / 0.127 0.115 / 0.153 0.127 / 0.177
sqrt 0.001 0.125 / 0.129 0.128 / 0.144 0.146 / 0.178 0.167 / 0.211 0.085 / 0.094 0.077 / 0.109 0.079 / 0.128 0.082 / 0.144 0.105 / 0.110 0.102 / 0.125 0.112 / 0.151 0.124 / 0.175
sqrt 0.0018 0.124 / 0.128 0.127 / 0.143 0.145 / 0.178 0.166 / 0.210 0.080 / 0.090 0.073 / 0.108 0.076 / 0.126 0.078 / 0.142 0.102 / 0.108 0.099 / 0.124 0.110 / 0.150 0.121 / 0.174
sqrt 0.0032 0.122 / 0.127 0.125 / 0.142 0.142 / 0.177 0.164 / 0.210 0.073 / 0.085 0.068 / 0.106 0.070 / 0.124 0.072 / 0.139 0.097 / 0.104 0.096 / 0.123 0.106 / 0.149 0.117 / 0.172
sqrt 0.0056 0.118 / 0.124 0.120 / 0.140 0.138 / 0.175 0.159 / 0.208 0.061 / 0.078 0.059 / 0.100 0.061 / 0.121 0.062 / 0.134 0.089 / 0.100 0.089 / 0.119 0.099 / 0.146 0.110 / 0.169
sqrt 0.01 0.110 / 0.121 0.114 / 0.139 0.131 / 0.173 0.151 / 0.206 0.043 / 0.070 0.044 / 0.096 0.046 / 0.116 0.046 / 0.133 0.076 / 0.094 0.078 / 0.116 0.088 / 0.143 0.098 / 0.167
sqrt 0.018 0.099 / 0.115 0.104 / 0.136 0.120 / 0.170 0.137 / 0.204 0.021 / 0.075 0.023 / 0.098 0.024 / 0.119 0.022 / 0.138 0.059 / 0.094 0.062 / 0.116 0.071 / 0.143 0.078 / 0.169
sqrt 0.032 0.085 / 0.114 0.088 / 0.135 0.101 / 0.169 0.116 / 0.203 0.007 / 0.081 0.008 / 0.107 0.011 / 0.131 0.015 / 0.146 0.043 / 0.096 0.045 / 0.120 0.050 / 0.149 0.055 / 0.172
sqrt 0.056 0.067 / 0.114 0.068 / 0.136 0.077 / 0.171 0.087 / 0.209 0.007 / 0.078 0.012 / 0.112 0.018 / 0.136 0.025 / 0.150 0.031 / 0.095 0.030 / 0.123 0.031 / 0.152 0.033 / 0.177
sqrt 0.1 0.052 / 0.113 0.050 / 0.141 0.050 / 0.180 0.053 / 0.221 0.007 / 0.080 0.012 / 0.119 0.015 / 0.136 0.020 / 0.147 0.023 / 0.095 0.021 / 0.129 0.018 / 0.157 0.017 / 0.182
sqrt 0.18 0.040 / 0.103 0.038 / 0.143 0.030 / 0.185 0.040 / 0.230 0.006 / 0.088 0.011 / 0.123 0.016 / 0.141 0.021 / 0.148 0.018 / 0.095 0.015 / 0.133 0.014 / 0.162 0.012 / 0.186
sqrt 0.32 0.030 / 0.095 0.036 / 0.131 0.026 / 0.190 0.046 / 0.244 0.008 / 0.090 0.010 / 0.122 0.017 / 0.146 0.032 / 0.153 0.016 / 0.092 0.016 / 0.126 0.013 / 0.166 0.012 / 0.196
rate 0.0032 0.124 / 0.128 0.126 / 0.142 0.144 / 0.177 0.164 / 0.209 0.078 / 0.087 0.071 / 0.105 0.073 / 0.123 0.073 / 0.138 0.101 / 0.106 0.098 / 0.122 0.108 / 0.148 0.118 / 0.171
rate 0.0056 0.122 / 0.127 0.124 / 0.141 0.141 / 0.175 0.161 / 0.207 0.068 / 0.080 0.062 / 0.098 0.064 / 0.119 0.064 / 0.137 0.095 / 0.102 0.092 / 0.118 0.102 / 0.145 0.111 / 0.170
rate 0.01 0.118 / 0.124 0.120 / 0.138 0.136 / 0.173 0.154 / 0.205 0.053 / 0.077 0.050 / 0.094 0.050 / 0.118 0.046 / 0.140 0.085 / 0.099 0.084 / 0.115 0.092 / 0.144 0.099 / 0.170
rate 0.018 0.110 / 0.118 0.113 / 0.136 0.127 / 0.169 0.143 / 0.200 0.032 / 0.088 0.030 / 0.106 0.028 / 0.128 0.024 / 0.147 0.071 / 0.102 0.071 / 0.120 0.076 / 0.147 0.082 / 0.172
rate 0.032 0.098 / 0.112 0.101 / 0.131 0.111 / 0.164 0.123 / 0.196 0.013 / 0.104 0.010 / 0.123 0.010 / 0.147 0.014 / 0.161 0.053 / 0.108 0.053 / 0.127 0.056 / 0.155 0.061 / 0.177
rate 0.056 0.082 / 0.113 0.082 / 0.131 0.088 / 0.167 0.095 / 0.209 0.012 / 0.110 0.014 / 0.138 0.018 / 0.162 0.022 / 0.175 0.037 / 0.112 0.035 / 0.135 0.037 / 0.164 0.039 / 0.191
rate 0.1 0.063 / 0.122 0.059 / 0.141 0.059 / 0.187 0.062 / 0.232 0.013 / 0.111 0.020 / 0.144 0.022 / 0.163 0.021 / 0.177 0.025 / 0.116 0.020 / 0.143 0.020 / 0.174 0.024 / 0.203
rate 0.18 0.047 / 0.129 0.036 / 0.156 0.033 / 0.205 0.051 / 0.254 0.010 / 0.099 0.012 / 0.131 0.014 / 0.152 0.020 / 0.166 0.019 / 0.113 0.015 / 0.142 0.014 / 0.177 0.020 / 0.207
rate 0.32 0.037 / 0.132 0.027 / 0.161 0.036 / 0.213 0.063 / 0.265 0.007 / 0.096 0.011 / 0.130 0.021 / 0.148 0.031 / 0.157 0.018 / 0.113 0.014 / 0.145 0.014 / 0.178 0.019 / 0.207
rate 0.56 0.032 / 0.132 0.026 / 0.167 0.041 / 0.214 0.071 / 0.266 0.009 / 0.096 0.012 / 0.132 0.027 / 0.144 0.037 / 0.155 0.015 / 0.113 0.016 / 0.148 0.013 / 0.177 0.019 / 0.207
rate 1 0.027 / 0.120 0.027 / 0.163 0.041 / 0.207 0.071 / 0.257 0.012 / 0.100 0.013 / 0.123 0.023 / 0.142 0.033 / 0.155 0.017 / 0.109 0.018 / 0.142 0.017 / 0.172 0.019 / 0.203
rate 1.8 0.029 / 0.099 0.032 / 0.144 0.038 / 0.202 0.070 / 0.262 0.009 / 0.090 0.014 / 0.107 0.020 / 0.142 0.038 / 0.149 0.015 / 0.094 0.018 / 0.124 0.017 / 0.170 0.018 / 0.202
REF-op cold 0.106 / 0.121 0.107 / 0.140 0.117 / 0.175 0.124 / 0.213 0.057 / 0.078 0.050 / 0.100 0.049 / 0.121 0.047 / 0.127 0.081 / 0.098 0.077 / 0.119 0.082 / 0.146 0.085 / 0.167
sqrt, adaptive 0.085 / 0.115 0.077 / 0.138 0.078 / 0.174 0.075 / 0.213 0.008 / 0.078 0.010 / 0.105 0.014 / 0.130 0.016 / 0.143 0.045 / 0.095 0.039 / 0.120 0.037 / 0.150 0.032 / 0.176

Mean offset at the median over time

The calibrated minus base median, in admissions per 100k, averaged over states at each round, h0 and h2. The bottom row is the mean base median per 100k at h0, to place the season peak.

OFFSET_CFGS <- list(`sqrt 0.018` = sqrt_runs[[which(SQRT_LRS == 0.018)]], `rate 0.032` = rate_runs[[which(RATE_LRS == 0.032)]], `REF-op cold` = refop)
fc_pick <- purrr::imap(OFFSET_CFGS, \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>%
  bind_rows() %>%
  mutate(season = season_of(reference_date), config = factor(config, levels = names(OFFSET_CFGS))) %>%
  filter(season %in% LIVE, location != "US", !is.na(value_base))
pop <- wi$rate_scales %>% select(location, scale)
med <- fc_pick %>%
  filter(level == 0.5, horizon %in% c(0, 2), config %in% names(OFFSET_CFGS)) %>%
  left_join(pop, by = "location") %>%
  group_by(config, season, horizon, reference_date) %>%
  summarize(off = mean((value_cal - value_base) / scale), base = mean(value_base / scale), truth = mean(truth / scale, na.rm = TRUE), .groups = "drop")
off_long <- bind_rows(
  med %>% transmute(config, season, reference_date, row = paste0("offset, h", horizon), value = off),
  med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "base median", season, reference_date, row = "base median h0", value = base),
  med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "truth", season, reference_date, row = "base median h0", value = truth)
) %>%
  mutate(config = factor(config, levels = c(names(OFFSET_CFGS), "base median", "truth")), row = factor(row, levels = c("offset, h0", "offset, h2", "base median h0")))
ggplot(off_long, aes(reference_date, value, colour = config)) +
  geom_hline(yintercept = 0, colour = "grey60") +
  geom_line() +
  geom_point(size = 0.7) +
  facet_grid(row ~ season, scales = "free") +
  scale_colour_manual(values = c(MONTH_COLS[names(OFFSET_CFGS)], `base median` = "grey30", truth = "grey65"), name = NULL) +
  scale_x_date(date_breaks = "1 month", date_labels = "%b") +
  labs(x = "reference date", y = "admissions per 100k (mean over states)") +
  theme(legend.position = "bottom")