Question: what does each tracker do month by month? How large is its step, how large is the offset it plays, and does the WIS change in a month follow from those together with the base’s bias in that month? And how much of the second season’s result is the offset carried in from the first?

Setup:

  • Disease: flu. Base forecaster: the clean windowed_seasonal replay (53 locations, the hub rounds). Horizons h0–h3.
  • Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Each round’s offsets come from that round’s own re-run, so they are the state that played. The step size is taken from one pass on final truth per config, because in a round’s own re-run that round’s outcomes have not yet stepped the tracker.
  • Configs: the E20 grid ({adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}), prod’s REF-op warm and REF-op cold, and two season_policy = "reset" runs (each anchor, cold, no leak), which zero the offsets at the start of 2025-26.
  • Scores over the two live seasons, each on its own rounds; states only (US excluded).

Internals are unit-free: the learning rate and the median-level offset are divided by the base’s 90% interval width, the rate in the config’s working units (sqrt of counts, or of counts per 100k) and the offset in counts. Each value is the median over states for a (horizon, round), then averaged within the calendar month. “Share above base median” is the fraction of state outcomes above the base’s median, the base’s own bias that month.

source(here::here("scripts/calibration/calibration_ws_replay.R"))
wi <- ws_inputs(params$disease)
fc_burn_in <- bind_rows(ws_burn_in_forecasts(params$disease), wi$fc)
grid <- ws_grid_configs(wi$rate_scales)
cold_no_leak <- grid[c("adaptive 0.03 | cold | no leak", "constant 0.018 | cold | no leak")]
RESETS <- purrr::map(cold_no_leak, function(s) list(cfg = utils::modifyList(s$cfg, list(season_policy = "reset")), burn_in = FALSE))
names(RESETS) <- sub(" \\| no leak$", " | reset", names(RESETS))
SPECS <- c(grid, ws_prod_specs(wi$rate_scales), RESETS)
# Internals are not in the standard cache, so cached runs without them are re-run here.
cals <- purrr::map(SPECS, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers, internals = TRUE)
# An exact run's round-t rows come from the re-run at t, before round t's outcomes step the tracker,
# so their step size is zero. The step size comes from one pass on final truth per config instead.
cals_final <- purrr::map(SPECS, function(spec) {
  fc <- if (spec$burn_in) fc_burn_in else wi$fc
  do.call(cal_run, c(list(fc, wi$truth), spec$cfg, list(learn = "final")))
})
ANCHOR_COLS <- c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2", `REF-op` = "#000000")
anchor_of <- function(config) dplyr::case_when(
  grepl("^adaptive", config) ~ "adaptive 0.03",
  grepl("^constant", config) ~ "constant 0.018",
  TRUE ~ "REF-op"
)
internals <- purrr::imap(SPECS, function(spec, nm) {
  eta <- ws_internals(cals_final[[nm]], spec$cfg, wi$rate_scales) %>% select(horizon, reference_date, eta_rel)
  ws_internals(cals[[nm]], spec$cfg, wi$rate_scales) %>%
    select(-eta_rel) %>%
    left_join(eta, by = c("horizon", "reference_date")) %>%
    mutate(config = nm)
}) %>%
  bind_rows() %>%
  mutate(
    config = factor(config, levels = names(SPECS)),
    season = season_of(reference_date),
    month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)]),
    h = factor(paste0("h", horizon), levels = paste0("h", 0:3))
  ) %>%
  filter(season %in% LIVE)
mv <- cal_month_view(cals, seasons = LIVE)
by_month <- internals %>%
  group_by(config, season, month, h, horizon) %>%
  summarize(across(c(eta_rel, offset_rel, offset_abs_rel, above_base_median), \(x) mean(x, na.rm = TRUE)), .groups = "drop") %>%
  left_join(mv$month %>% select(config, season, month, horizon, wis_pct, share), by = c("config", "season", "month", "horizon")) %>%
  mutate(anchor = anchor_of(as.character(config)))
# Internals are only shown for the no-leak grid, prod and the resets; the leak runs' offsets are in the tables.
PLOT_CFGS <- c(names(grid)[c(1, 3, 5, 7)], "REF-op warm", "REF-op cold")

Per round

Median over states of the step size and the played offset, both relative to the base’s 90% width, per round. Warm configs start with an offset; cold ones at zero.

long <- internals %>%
  filter(config %in% PLOT_CFGS) %>%
  select(config, h, reference_date, `eta / width` = eta_rel, `median-level offset / width` = offset_rel, `mean |offset| / width` = offset_abs_rel) %>%
  tidyr::pivot_longer(-c(config, h, reference_date))
ggplot(long, aes(reference_date, value, colour = config)) +
  geom_hline(yintercept = 0, colour = "grey60") +
  geom_line() +
  facet_grid(name ~ h, scales = "free_y") +
  labs(x = NULL, y = "median over states", colour = NULL) +
  theme(legend.position = "bottom") +
  guides(colour = guide_legend(nrow = 2))

By month

One row per config, horizon and season-month: step size, offset, the base’s bias that month, and the month’s WIS change. The grey bar is the month’s share of the season’s base WIS.

pm <- by_month %>% filter(config %in% PLOT_CFGS)
for (s in LIVE) {
  d <- pm %>% filter(season == s)
  long <- d %>%
    select(config, h, month, `eta / width` = eta_rel, `offset / width` = offset_rel, `share above base median` = above_base_median, `WIS reduction %` = wis_pct) %>%
    tidyr::pivot_longer(-c(config, h, month)) %>%
    mutate(name = factor(name, levels = c("eta / width", "offset / width", "share above base median", "WIS reduction %")))
  bars <- d %>% distinct(h, month, share) %>% mutate(name = factor("WIS reduction %", levels = levels(long$name)))
  print(
    ggplot(long, aes(month, value, colour = config, group = config)) +
      geom_col(data = bars, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) +
      geom_hline(data = tibble(name = factor("share above base median", levels = levels(long$name)), y = 0.5), aes(yintercept = y), colour = "grey60") +
      geom_hline(data = tibble(name = factor(c("offset / width", "WIS reduction %"), levels = levels(long$name)), y = 0), aes(yintercept = y), colour = "grey60") +
      geom_line() +
      geom_point(size = 1) +
      facet_grid(name ~ h, scales = "free_y") +
      labs(title = s, x = NULL, y = NULL, colour = NULL) +
      theme(legend.position = "bottom") +
      guides(colour = guide_legend(nrow = 2))
  )
}

Tables

month_cell <- function(d, col, fmt) {
  d %>%
    transmute(config, h, col = paste(season, month), v = sprintf(fmt, .data[[col]])) %>%
    tidyr::pivot_wider(names_from = col, values_from = v) %>%
    arrange(h, config)
}
for (spec in list(
  c("eta_rel", "%.3f", "Step size / base 90% width (mean over rounds of the state median)"),
  c("offset_rel", "%+.3f", "Median-level offset / base 90% width"),
  c("wis_pct", "%+.1f", "WIS reduction % within the month (positive = calibrated WIS lower than base)")
)) {
  cat("\n\n", cal_h(3), " ", spec[3], "\n\n", sep = "")
  print(knitr::kable(month_cell(by_month, spec[1], spec[2])))
}

Step size / base 90% width (mean over rounds of the state median)

config h 2024-2025 Nov 2024-2025 Dec 2024-2025 Jan 2024-2025 Feb 2024-2025 Mar 2024-2025 Apr 2024-2025 May 2025-2026 Oct 2025-2026 Nov 2025-2026 Dec 2025-2026 Jan 2025-2026 Feb 2025-2026 Mar 2025-2026 Apr 2025-2026 May
adaptive 0.03 | cold | no leak h0 NaN 0.017 0.017 0.023 0.028 0.039 0.057 0.031 0.035 0.020 0.016 0.023 0.030 0.044 0.056
adaptive 0.03 | cold | leak 0.1 h0 NaN 0.017 0.017 0.023 0.028 0.039 0.057 0.031 0.035 0.020 0.016 0.023 0.030 0.044 0.056
adaptive 0.03 | warm | no leak h0 0.029 0.021 0.015 0.018 0.025 0.039 0.057 0.031 0.035 0.020 0.016 0.023 0.030 0.044 0.056
adaptive 0.03 | warm | leak 0.1 h0 0.029 0.021 0.015 0.018 0.025 0.039 0.057 0.031 0.035 0.020 0.016 0.023 0.030 0.044 0.056
constant 0.018 | cold | no leak h0 NaN 0.016 0.010 0.010 0.012 0.019 0.028 0.017 0.019 0.013 0.010 0.013 0.016 0.024 0.031
constant 0.018 | cold | leak 0.1 h0 NaN 0.016 0.010 0.010 0.012 0.019 0.028 0.017 0.019 0.013 0.010 0.013 0.016 0.024 0.031
constant 0.018 | warm | no leak h0 0.020 0.016 0.010 0.010 0.012 0.019 0.028 0.017 0.019 0.013 0.010 0.013 0.016 0.024 0.031
constant 0.018 | warm | leak 0.1 h0 0.020 0.016 0.010 0.010 0.012 0.019 0.028 0.017 0.019 0.013 0.010 0.013 0.016 0.024 0.031
REF-op warm h0 0.003 0.001 0.000 0.000 0.001 0.003 0.007 0.003 0.003 0.001 0.001 0.001 0.002 0.005 0.009
REF-op cold h0 NaN 0.001 0.000 0.001 0.001 0.003 0.007 0.003 0.003 0.001 0.001 0.001 0.002 0.005 0.009
adaptive 0.03 | cold | reset h0 NaN 0.017 0.017 0.023 0.028 0.039 0.057 0.031 0.035 0.020 0.016 0.023 0.030 0.044 0.056
constant 0.018 | cold | reset h0 NaN 0.016 0.010 0.010 0.012 0.019 0.028 0.017 0.019 0.013 0.010 0.013 0.016 0.024 0.031
adaptive 0.03 | cold | no leak h1 NaN 0.015 0.017 0.021 0.029 0.048 0.074 0.032 0.032 0.018 0.016 0.025 0.034 0.056 0.074
adaptive 0.03 | cold | leak 0.1 h1 NaN 0.015 0.017 0.021 0.029 0.048 0.074 0.032 0.032 0.018 0.016 0.025 0.034 0.056 0.074
adaptive 0.03 | warm | no leak h1 0.027 0.018 0.014 0.017 0.025 0.046 0.074 0.032 0.032 0.018 0.016 0.025 0.034 0.056 0.074
adaptive 0.03 | warm | leak 0.1 h1 0.027 0.018 0.014 0.017 0.025 0.046 0.074 0.032 0.032 0.018 0.016 0.025 0.034 0.056 0.074
constant 0.018 | cold | no leak h1 NaN 0.010 0.007 0.007 0.009 0.016 0.025 0.013 0.013 0.008 0.007 0.009 0.013 0.020 0.027
constant 0.018 | cold | leak 0.1 h1 NaN 0.010 0.007 0.007 0.009 0.016 0.025 0.013 0.013 0.008 0.007 0.009 0.013 0.020 0.027
constant 0.018 | warm | no leak h1 0.014 0.010 0.007 0.007 0.009 0.016 0.025 0.013 0.013 0.008 0.007 0.009 0.013 0.020 0.027
constant 0.018 | warm | leak 0.1 h1 0.014 0.010 0.007 0.007 0.009 0.016 0.025 0.013 0.013 0.008 0.007 0.009 0.013 0.020 0.027
REF-op warm h1 0.002 0.001 0.000 0.000 0.001 0.004 0.009 0.003 0.002 0.001 0.001 0.001 0.002 0.006 0.011
REF-op cold h1 NaN 0.001 0.000 0.001 0.001 0.004 0.009 0.003 0.002 0.001 0.001 0.001 0.002 0.006 0.011
adaptive 0.03 | cold | reset h1 NaN 0.015 0.017 0.021 0.029 0.048 0.074 0.032 0.032 0.018 0.016 0.025 0.034 0.056 0.074
constant 0.018 | cold | reset h1 NaN 0.010 0.007 0.007 0.009 0.016 0.025 0.013 0.013 0.008 0.007 0.009 0.013 0.020 0.027
adaptive 0.03 | cold | no leak h2 NaN 0.014 0.018 0.022 0.029 0.054 0.086 0.030 0.030 0.016 0.017 0.025 0.035 0.063 0.086
adaptive 0.03 | cold | leak 0.1 h2 NaN 0.014 0.018 0.022 0.029 0.054 0.086 0.030 0.030 0.016 0.017 0.025 0.035 0.063 0.086
adaptive 0.03 | warm | no leak h2 0.024 0.017 0.014 0.017 0.025 0.052 0.086 0.030 0.030 0.016 0.017 0.025 0.035 0.063 0.086
adaptive 0.03 | warm | leak 0.1 h2 0.024 0.017 0.014 0.017 0.025 0.052 0.086 0.030 0.030 0.016 0.017 0.025 0.035 0.063 0.086
constant 0.018 | cold | no leak h2 NaN 0.008 0.006 0.006 0.007 0.014 0.023 0.010 0.009 0.006 0.006 0.008 0.011 0.018 0.024
constant 0.018 | cold | leak 0.1 h2 NaN 0.008 0.006 0.006 0.007 0.014 0.023 0.010 0.009 0.006 0.006 0.008 0.011 0.018 0.024
constant 0.018 | warm | no leak h2 0.010 0.008 0.006 0.006 0.007 0.014 0.023 0.010 0.009 0.006 0.006 0.008 0.011 0.018 0.024
constant 0.018 | warm | leak 0.1 h2 0.010 0.008 0.006 0.006 0.007 0.014 0.023 0.010 0.009 0.006 0.006 0.008 0.011 0.018 0.024
REF-op warm h2 0.002 0.001 0.000 0.000 0.001 0.004 0.012 0.002 0.002 0.001 0.001 0.001 0.002 0.007 0.014
REF-op cold h2 NaN 0.001 0.001 0.001 0.001 0.005 0.012 0.002 0.002 0.001 0.001 0.001 0.002 0.007 0.014
adaptive 0.03 | cold | reset h2 NaN 0.014 0.018 0.022 0.029 0.054 0.086 0.030 0.030 0.016 0.017 0.025 0.035 0.063 0.086
constant 0.018 | cold | reset h2 NaN 0.008 0.006 0.006 0.007 0.014 0.023 0.010 0.009 0.006 0.006 0.008 0.011 0.018 0.024
adaptive 0.03 | cold | no leak h3 NaN 0.015 0.020 0.023 0.032 0.059 0.094 0.027 0.028 0.015 0.018 0.024 0.034 0.064 0.091
adaptive 0.03 | cold | leak 0.1 h3 NaN 0.015 0.020 0.023 0.032 0.059 0.094 0.027 0.028 0.015 0.018 0.024 0.034 0.064 0.091
adaptive 0.03 | warm | no leak h3 0.023 0.017 0.016 0.018 0.028 0.055 0.093 0.027 0.028 0.015 0.018 0.024 0.034 0.064 0.091
adaptive 0.03 | warm | leak 0.1 h3 0.023 0.017 0.016 0.018 0.028 0.055 0.093 0.027 0.028 0.015 0.018 0.024 0.034 0.064 0.091
constant 0.018 | cold | no leak h3 NaN 0.007 0.006 0.005 0.007 0.013 0.022 0.008 0.008 0.005 0.005 0.007 0.009 0.016 0.023
constant 0.018 | cold | leak 0.1 h3 NaN 0.007 0.006 0.005 0.007 0.013 0.022 0.008 0.008 0.005 0.005 0.007 0.009 0.016 0.023
constant 0.018 | warm | no leak h3 0.009 0.007 0.006 0.005 0.007 0.013 0.022 0.008 0.008 0.005 0.005 0.007 0.009 0.016 0.023
constant 0.018 | warm | leak 0.1 h3 0.009 0.007 0.006 0.005 0.007 0.013 0.022 0.008 0.008 0.005 0.005 0.007 0.009 0.016 0.023
REF-op warm h3 0.001 0.001 0.000 0.001 0.001 0.005 0.013 0.002 0.001 0.000 0.001 0.001 0.002 0.008 0.015
REF-op cold h3 NaN 0.001 0.001 0.001 0.001 0.005 0.013 0.002 0.001 0.000 0.001 0.001 0.002 0.008 0.015
adaptive 0.03 | cold | reset h3 NaN 0.015 0.020 0.023 0.032 0.059 0.094 0.027 0.028 0.015 0.018 0.024 0.034 0.064 0.091
constant 0.018 | cold | reset h3 NaN 0.007 0.006 0.005 0.007 0.013 0.022 0.008 0.008 0.005 0.005 0.007 0.009 0.016 0.023

Median-level offset / base 90% width

config h 2024-2025 Nov 2024-2025 Dec 2024-2025 Jan 2024-2025 Feb 2024-2025 Mar 2024-2025 Apr 2024-2025 May 2025-2026 Oct 2025-2026 Nov 2025-2026 Dec 2025-2026 Jan 2025-2026 Feb 2025-2026 Mar 2025-2026 Apr 2025-2026 May
adaptive 0.03 | cold | no leak h0 -0.000 -0.001 +0.010 +0.031 +0.063 +0.060 +0.061 +0.025 +0.052 +0.043 +0.059 +0.068 +0.098 +0.121 +0.111
adaptive 0.03 | cold | leak 0.1 h0 -0.000 -0.001 +0.011 +0.025 +0.040 +0.014 +0.011 +0.012 +0.038 +0.021 +0.029 +0.022 +0.032 +0.024 +0.027
adaptive 0.03 | warm | no leak h0 +0.039 +0.030 +0.034 +0.047 +0.076 +0.077 +0.078 +0.030 +0.062 +0.050 +0.062 +0.074 +0.102 +0.124 +0.121
adaptive 0.03 | warm | leak 0.1 h0 +0.039 +0.030 +0.034 +0.042 +0.058 +0.047 +0.050 +0.024 +0.055 +0.041 +0.046 +0.049 +0.061 +0.066 +0.073
constant 0.018 | cold | no leak h0 -0.000 -0.000 +0.006 +0.018 +0.035 +0.041 +0.049 +0.023 +0.050 +0.037 +0.046 +0.060 +0.078 +0.104 +0.114
constant 0.018 | cold | leak 0.1 h0 -0.000 -0.000 +0.007 +0.015 +0.021 +0.012 +0.008 +0.006 +0.024 +0.015 +0.020 +0.017 +0.023 +0.019 +0.019
constant 0.018 | warm | no leak h0 +0.043 +0.031 +0.030 +0.039 +0.057 +0.068 +0.083 +0.033 +0.064 +0.047 +0.053 +0.065 +0.085 +0.114 +0.126
constant 0.018 | warm | leak 0.1 h0 +0.043 +0.032 +0.030 +0.036 +0.044 +0.046 +0.057 +0.022 +0.046 +0.034 +0.038 +0.042 +0.052 +0.060 +0.073
REF-op warm h0 +0.039 +0.030 +0.036 +0.045 +0.061 +0.050 +0.051 +0.026 +0.059 +0.044 +0.051 +0.054 +0.070 +0.070 +0.078
REF-op cold h0 -0.000 -0.001 +0.012 +0.028 +0.045 +0.017 +0.014 +0.015 +0.044 +0.025 +0.035 +0.034 +0.046 +0.039 +0.046
adaptive 0.03 | cold | reset h0 -0.000 -0.001 +0.010 +0.031 +0.063 +0.060 +0.061 +0.000 +0.007 +0.017 +0.037 +0.046 +0.070 +0.073 +0.086
constant 0.018 | cold | reset h0 -0.000 -0.000 +0.006 +0.018 +0.035 +0.041 +0.049 +0.000 +0.004 +0.010 +0.024 +0.032 +0.047 +0.061 +0.080
adaptive 0.03 | cold | no leak h1 -0.000 -0.001 +0.012 +0.034 +0.074 +0.086 +0.057 +0.018 +0.045 +0.034 +0.050 +0.056 +0.085 +0.105 +0.109
adaptive 0.03 | cold | leak 0.1 h1 -0.000 -0.001 +0.012 +0.028 +0.045 +0.024 -0.012 +0.006 +0.038 +0.024 +0.029 +0.019 +0.035 +0.026 +0.004
adaptive 0.03 | warm | no leak h1 +0.049 +0.040 +0.042 +0.056 +0.094 +0.124 +0.096 +0.030 +0.053 +0.041 +0.055 +0.066 +0.092 +0.122 +0.108
adaptive 0.03 | warm | leak 0.1 h1 +0.049 +0.040 +0.041 +0.052 +0.073 +0.070 +0.057 +0.030 +0.055 +0.040 +0.051 +0.051 +0.077 +0.084 +0.075
constant 0.018 | cold | no leak h1 +0.000 -0.001 +0.007 +0.015 +0.029 +0.042 +0.047 +0.019 +0.033 +0.025 +0.032 +0.042 +0.057 +0.086 +0.112
constant 0.018 | cold | leak 0.1 h1 +0.000 -0.001 +0.007 +0.012 +0.016 +0.011 +0.003 +0.005 +0.016 +0.011 +0.015 +0.011 +0.016 +0.016 +0.012
constant 0.018 | warm | no leak h1 +0.050 +0.039 +0.033 +0.039 +0.060 +0.095 +0.123 +0.034 +0.059 +0.042 +0.047 +0.060 +0.080 +0.120 +0.152
constant 0.018 | warm | leak 0.1 h1 +0.050 +0.039 +0.033 +0.036 +0.048 +0.067 +0.085 +0.028 +0.046 +0.034 +0.036 +0.041 +0.056 +0.079 +0.099
REF-op warm h1 +0.049 +0.040 +0.043 +0.054 +0.079 +0.075 +0.058 +0.031 +0.055 +0.042 +0.055 +0.055 +0.085 +0.091 +0.078
REF-op cold h1 -0.000 -0.001 +0.013 +0.032 +0.052 +0.032 -0.007 +0.009 +0.044 +0.027 +0.036 +0.026 +0.046 +0.043 +0.019
adaptive 0.03 | cold | reset h1 -0.000 -0.001 +0.012 +0.034 +0.074 +0.086 +0.057 -0.000 +0.016 +0.014 +0.032 +0.040 +0.070 +0.087 +0.072
constant 0.018 | cold | reset h1 +0.000 -0.001 +0.007 +0.015 +0.029 +0.042 +0.047 +0.000 +0.007 +0.007 +0.016 +0.021 +0.034 +0.046 +0.060
adaptive 0.03 | cold | no leak h2 -0.000 +0.001 +0.016 +0.041 +0.089 +0.144 +0.100 +0.005 +0.028 +0.023 +0.040 +0.053 +0.089 +0.138 +0.128
adaptive 0.03 | cold | leak 0.1 h2 -0.000 +0.001 +0.014 +0.034 +0.054 +0.049 -0.024 +0.003 +0.029 +0.018 +0.028 +0.022 +0.043 +0.044 -0.011
adaptive 0.03 | warm | no leak h2 +0.059 +0.051 +0.053 +0.074 +0.121 +0.203 +0.162 +0.017 +0.028 +0.027 +0.043 +0.054 +0.093 +0.149 +0.118
adaptive 0.03 | warm | leak 0.1 h2 +0.059 +0.051 +0.051 +0.066 +0.092 +0.121 +0.068 +0.018 +0.048 +0.041 +0.055 +0.063 +0.097 +0.130 +0.097
constant 0.018 | cold | no leak h2 -0.000 +0.001 +0.007 +0.014 +0.027 +0.052 +0.061 +0.015 +0.031 +0.024 +0.028 +0.040 +0.059 +0.101 +0.133
constant 0.018 | cold | leak 0.1 h2 -0.000 +0.001 +0.006 +0.011 +0.015 +0.018 +0.005 +0.003 +0.012 +0.007 +0.011 +0.009 +0.015 +0.019 +0.012
constant 0.018 | warm | no leak h2 +0.066 +0.058 +0.048 +0.052 +0.076 +0.147 +0.202 +0.041 +0.058 +0.046 +0.047 +0.062 +0.092 +0.153 +0.206
constant 0.018 | warm | leak 0.1 h2 +0.066 +0.058 +0.048 +0.050 +0.065 +0.112 +0.151 +0.032 +0.048 +0.041 +0.042 +0.053 +0.076 +0.132 +0.165
REF-op warm h2 +0.059 +0.051 +0.053 +0.070 +0.098 +0.132 +0.070 +0.018 +0.053 +0.047 +0.059 +0.067 +0.102 +0.141 +0.100
REF-op cold h2 -0.000 +0.001 +0.016 +0.038 +0.062 +0.058 -0.020 +0.004 +0.031 +0.024 +0.036 +0.033 +0.057 +0.072 +0.010
adaptive 0.03 | cold | reset h2 -0.000 +0.001 +0.016 +0.041 +0.089 +0.144 +0.100 +0.000 +0.020 +0.020 +0.035 +0.047 +0.081 +0.124 +0.095
constant 0.018 | cold | reset h2 -0.000 +0.001 +0.007 +0.014 +0.027 +0.052 +0.061 -0.000 +0.008 +0.007 +0.015 +0.020 +0.033 +0.057 +0.073
adaptive 0.03 | cold | no leak h3 +0.000 -0.000 +0.012 +0.048 +0.105 +0.218 +0.186 +0.011 +0.018 +0.016 +0.032 +0.043 +0.085 +0.160 +0.150
adaptive 0.03 | cold | leak 0.1 h3 +0.000 -0.000 +0.012 +0.040 +0.064 +0.078 -0.012 -0.003 +0.021 +0.016 +0.025 +0.023 +0.042 +0.057 -0.011
adaptive 0.03 | warm | no leak h3 +0.069 +0.063 +0.066 +0.088 +0.151 +0.287 +0.278 +0.027 +0.027 +0.023 +0.040 +0.052 +0.088 +0.172 +0.143
adaptive 0.03 | warm | leak 0.1 h3 +0.069 +0.063 +0.065 +0.082 +0.116 +0.179 +0.132 +0.011 +0.039 +0.040 +0.055 +0.068 +0.106 +0.179 +0.131
constant 0.018 | cold | no leak h3 -0.000 +0.000 +0.005 +0.013 +0.026 +0.055 +0.072 +0.010 +0.027 +0.024 +0.026 +0.035 +0.054 +0.097 +0.143
constant 0.018 | cold | leak 0.1 h3 -0.000 +0.000 +0.005 +0.011 +0.015 +0.022 +0.008 +0.001 +0.010 +0.007 +0.009 +0.008 +0.014 +0.021 +0.006
constant 0.018 | warm | no leak h3 +0.078 +0.073 +0.062 +0.065 +0.090 +0.189 +0.292 +0.044 +0.065 +0.057 +0.053 +0.068 +0.099 +0.199 +0.287
constant 0.018 | warm | leak 0.1 h3 +0.078 +0.073 +0.062 +0.062 +0.078 +0.148 +0.219 +0.034 +0.053 +0.049 +0.049 +0.061 +0.086 +0.163 +0.241
REF-op warm h3 +0.069 +0.063 +0.066 +0.086 +0.125 +0.185 +0.141 +0.010 +0.039 +0.040 +0.058 +0.074 +0.114 +0.184 +0.144
REF-op cold h3 +0.000 -0.000 +0.013 +0.044 +0.075 +0.098 -0.002 -0.005 +0.017 +0.016 +0.031 +0.032 +0.061 +0.088 +0.013
adaptive 0.03 | cold | reset h3 +0.000 -0.000 +0.012 +0.048 +0.105 +0.218 +0.186 +0.000 -0.005 -0.000 +0.016 +0.021 +0.052 +0.116 +0.090
constant 0.018 | cold | reset h3 -0.000 +0.000 +0.005 +0.013 +0.026 +0.055 +0.072 -0.000 +0.006 +0.005 +0.012 +0.015 +0.027 +0.055 +0.067

WIS reduction % within the month (positive = calibrated WIS lower than base)

config h 2024-2025 Nov 2024-2025 Dec 2024-2025 Jan 2024-2025 Feb 2024-2025 Mar 2024-2025 Apr 2024-2025 May 2025-2026 Oct 2025-2026 Nov 2025-2026 Dec 2025-2026 Jan 2025-2026 Feb 2025-2026 Mar 2025-2026 Apr 2025-2026 May
adaptive 0.03 | cold | no leak h0 +0.0 +0.1 +1.7 +5.0 -6.8 -2.6 +17.8 -10.3 +4.0 +7.7 -2.4 +8.2 -21.7 -0.4 +9.8
adaptive 0.03 | cold | leak 0.1 h0 +0.0 +0.1 +1.6 +4.5 -2.5 +2.9 +8.5 +0.3 +5.6 +3.0 -0.9 +3.4 -2.7 +3.4 +4.6
adaptive 0.03 | warm | no leak h0 -0.1 +1.3 +2.0 +5.6 -3.1 -0.1 +18.3 -5.7 +6.0 +6.8 -1.7 +9.1 -20.8 +0.3 +12.1
adaptive 0.03 | warm | leak 0.1 h0 -0.1 +1.4 +1.9 +5.2 +0.6 +4.3 +11.8 +0.1 +8.8 +3.4 -1.0 +5.8 -3.8 +5.6 +8.3
constant 0.018 | cold | no leak h0 -0.0 +0.2 +1.6 +3.2 -2.2 +0.6 +14.3 +1.3 +8.0 +6.2 -0.5 +8.2 -14.4 +2.4 +13.0
constant 0.018 | cold | leak 0.1 h0 -0.0 +0.2 +1.4 +2.7 -0.6 +1.9 +5.2 +2.8 +4.8 +2.0 -0.2 +3.1 -1.7 +2.8 +4.7
constant 0.018 | warm | no leak h0 +0.8 +1.6 +2.6 +4.9 +1.0 +3.0 +19.5 +3.0 +11.1 +5.9 -0.1 +10.0 -16.0 +3.0 +15.2
constant 0.018 | warm | leak 0.1 h0 +0.8 +1.6 +2.5 +4.7 +3.7 +6.7 +13.3 +8.4 +12.6 +2.7 +0.1 +7.4 -2.4 +7.6 +14.6
REF-op warm h0 -0.1 +1.4 +2.0 +5.6 -0.1 +4.2 +12.9 -0.9 +8.5 +4.1 -1.2 +6.4 -6.1 +6.0 +10.0
REF-op cold h0 +0.0 +0.1 +1.8 +4.9 -3.3 +2.8 +10.4 -0.8 +5.5 +3.8 -1.1 +4.6 -5.0 +4.8 +7.5
adaptive 0.03 | cold | reset h0 +0.0 +0.1 +1.7 +5.0 -6.8 -2.6 +17.8 -0.0 +2.4 +1.8 -1.2 +4.7 -10.2 +3.3 +10.8
constant 0.018 | cold | reset h0 -0.0 +0.2 +1.6 +3.2 -2.2 +0.6 +14.3 +0.0 +1.9 +1.4 -0.3 +4.2 -6.5 +4.4 +12.0
adaptive 0.03 | cold | no leak h1 +0.0 +0.0 +2.5 +3.5 -16.6 -14.4 +17.6 -16.3 +0.4 +5.8 -6.0 +3.9 -37.5 -10.2 +8.3
adaptive 0.03 | cold | leak 0.1 h1 +0.0 +0.0 +2.3 +3.5 -8.0 +0.2 +6.9 -2.7 +3.8 +2.4 -3.0 +2.0 -8.9 +4.6 -1.7
adaptive 0.03 | warm | no leak h1 +1.7 +0.3 +3.5 +4.9 -13.2 -12.7 +19.8 -12.7 +2.9 +4.3 -5.2 +5.3 -35.9 -8.8 +11.7
adaptive 0.03 | warm | leak 0.1 h1 +1.7 +0.3 +3.3 +4.9 -6.1 +0.6 +10.9 -4.6 +6.9 +2.5 -4.3 +5.7 -14.7 +5.8 +3.4
constant 0.018 | cold | no leak h1 +0.0 +0.0 +1.5 +1.7 -4.4 -1.8 +12.6 -1.0 +4.9 +4.1 -3.0 +4.8 -19.6 +0.8 +17.4
constant 0.018 | cold | leak 0.1 h1 +0.0 +0.1 +1.3 +1.5 -2.1 +0.6 +3.5 +2.1 +2.5 +1.1 -1.1 +1.7 -4.0 +3.1 +2.5
constant 0.018 | warm | no leak h1 +3.7 -0.2 +4.1 +4.1 -4.0 -3.6 +20.9 -0.9 +11.7 +3.8 -3.8 +8.5 -25.7 -0.7 +20.2
constant 0.018 | warm | leak 0.1 h1 +3.7 -0.2 +4.0 +4.1 -0.4 +3.5 +15.0 +5.3 +11.5 +1.4 -2.7 +8.2 -12.8 +8.1 +17.2
REF-op warm h1 +1.7 +0.3 +3.5 +5.1 -7.7 -0.8 +12.3 -6.4 +6.2 +3.0 -5.0 +6.0 -18.6 +5.0 +5.2
REF-op cold h1 +0.0 +0.0 +2.6 +3.8 -9.9 -1.2 +8.1 -5.3 +4.1 +3.2 -3.9 +2.9 -13.6 +5.0 +1.6
adaptive 0.03 | cold | reset h1 +0.0 +0.0 +2.5 +3.5 -16.6 -14.4 +17.6 +0.0 +1.2 +1.1 -2.2 +2.4 -22.9 -1.2 +9.2
constant 0.018 | cold | reset h1 +0.0 +0.0 +1.5 +1.7 -4.4 -1.8 +12.6 +0.0 +0.8 +0.8 -0.8 +2.4 -10.4 +3.1 +12.8
adaptive 0.03 | cold | no leak h2 +0.0 +0.0 +2.7 +2.9 -31.4 -40.6 +18.0 -23.3 +5.2 +6.3 -4.2 -1.4 -60.2 -17.9 +3.4
adaptive 0.03 | cold | leak 0.1 h2 +0.0 +0.0 +2.5 +3.1 -16.4 -6.7 +3.5 -1.3 +4.4 +2.3 -1.8 +2.8 -16.3 +4.9 -4.8
adaptive 0.03 | warm | no leak h2 +4.9 +1.3 +4.3 +5.8 -30.8 -42.7 +21.2 -17.5 +5.8 +4.5 -3.1 +0.8 -59.0 -13.4 +10.1
adaptive 0.03 | warm | leak 0.1 h2 +4.9 +1.3 +4.2 +5.9 -17.8 -10.5 +12.1 -2.3 +6.6 +2.6 -3.3 +6.7 -30.2 +6.8 +4.0
constant 0.018 | cold | no leak h2 +0.0 +0.0 +1.2 +1.4 -7.2 -5.5 +14.7 -4.2 +6.2 +3.9 -1.9 +3.3 -27.9 +4.8 +17.3
constant 0.018 | cold | leak 0.1 h2 +0.0 +0.0 +1.1 +1.3 -3.8 -0.6 +3.0 +1.2 +2.3 +0.9 -0.3 +1.6 -6.2 +3.9 +2.4
constant 0.018 | warm | no leak h2 +8.6 +1.0 +4.4 +5.5 -14.0 -19.5 +26.1 -11.8 +13.0 +4.4 -3.7 +7.6 -43.4 +0.3 +16.0
constant 0.018 | warm | leak 0.1 h2 +8.6 +1.0 +4.3 +5.6 -9.0 -5.9 +21.4 -0.7 +10.9 +2.2 -2.7 +9.4 -29.0 +11.2 +20.4
REF-op warm h2 +4.9 +1.3 +4.4 +6.0 -20.8 -14.0 +13.4 -4.0 +7.3 +3.1 -3.9 +6.3 -35.9 +5.1 +6.2
REF-op cold h2 +0.0 +0.1 +2.8 +3.2 -19.8 -9.8 +6.3 -3.5 +5.0 +2.9 -2.5 +3.0 -24.3 +5.6 -0.0
adaptive 0.03 | cold | reset h2 +0.0 +0.0 +2.7 +2.9 -31.4 -40.6 +18.0 +0.0 +1.0 +0.0 -0.3 +1.9 -37.1 +1.1 +11.0
constant 0.018 | cold | reset h2 +0.0 +0.0 +1.2 +1.4 -7.2 -5.5 +14.7 -0.0 +1.0 +0.5 +0.1 +1.7 -14.8 +7.2 +14.8
adaptive 0.03 | cold | no leak h3 +0.0 +0.0 +2.0 +2.9 -48.4 -81.3 +9.4 -28.4 +8.3 +7.5 -1.0 -10.7 -80.6 -36.6 -11.2
adaptive 0.03 | cold | leak 0.1 h3 +0.0 +0.0 +1.9 +3.2 -26.8 -17.3 +3.8 +1.1 +3.1 +1.7 +1.3 +1.8 -22.0 +7.0 -5.0
adaptive 0.03 | warm | no leak h3 +6.0 +2.8 +4.6 +7.1 -52.5 -91.9 +12.1 -23.6 +7.4 +5.8 +0.3 -8.0 -75.7 -30.0 -5.5
adaptive 0.03 | warm | leak 0.1 h3 +6.0 +2.8 +4.5 +7.4 -34.0 -32.9 +15.6 -0.8 +4.2 +2.5 +0.8 +3.1 -47.5 +3.0 +1.8
constant 0.018 | cold | no leak h3 -0.0 +0.0 +0.8 +1.4 -9.0 -8.9 +15.7 -5.1 +5.7 +3.8 +0.5 +0.3 -34.5 +4.8 +11.0
constant 0.018 | cold | leak 0.1 h3 -0.0 +0.0 +0.7 +1.3 -5.1 -1.9 +3.6 +1.0 +1.7 +0.8 +0.7 +1.1 -7.7 +5.2 +3.0
constant 0.018 | warm | no leak h3 +9.7 +4.1 +5.6 +7.8 -26.9 -46.2 +22.7 -17.7 +11.3 +5.9 -1.1 +1.4 -64.5 -12.5 -10.7
constant 0.018 | warm | leak 0.1 h3 +9.7 +4.1 +5.5 +7.9 -20.9 -26.3 +26.5 -5.1 +8.2 +3.7 -0.1 +4.6 -51.0 +3.9 +0.5
REF-op warm h3 +6.0 +2.8 +4.7 +7.4 -38.5 -39.0 +16.7 -2.3 +4.6 +2.9 +0.9 +1.6 -55.3 -1.5 +4.1
REF-op cold h3 +0.0 +0.0 +2.1 +3.3 -32.2 -24.9 +7.0 -1.0 +3.9 +2.5 +1.4 +0.9 -33.5 +5.2 +0.1
adaptive 0.03 | cold | reset h3 +0.0 +0.0 +2.0 +2.9 -48.4 -81.3 +9.4 +0.0 -2.3 -1.4 +3.5 -0.0 -43.8 -3.5 +10.3
constant 0.018 | cold | reset h3 -0.0 +0.0 +0.8 +1.4 -9.0 -8.9 +15.7 +0.0 +0.6 +0.4 +1.4 +1.1 -16.9 +9.4 +15.1 
cat("\n\n", cal_h(3), " Share of state outcomes above the base median (the base's bias; same for every config)\n\n", sep = "")

Share of state outcomes above the base median (the base’s bias; same for every config)

print(knitr::kable(
  by_month %>%
    filter(config == names(SPECS)[1]) %>%
    transmute(h, col = paste(season, month), v = sprintf("%.2f", above_base_median)) %>%
    tidyr::pivot_wider(names_from = col, values_from = v)
))
h 2024-2025 Nov 2024-2025 Dec 2024-2025 Jan 2024-2025 Feb 2024-2025 Mar 2024-2025 Apr 2024-2025 May 2025-2026 Oct 2025-2026 Nov 2025-2026 Dec 2025-2026 Jan 2025-2026 Feb 2025-2026 Mar 2025-2026 Apr 2025-2026 May
h0 0.50 0.90 0.67 0.85 0.53 0.59 0.70 0.56 0.77 0.93 0.54 0.69 0.47 0.68 0.67
h1 0.61 0.98 0.71 0.80 0.44 0.55 0.77 0.60 0.83 0.99 0.44 0.68 0.44 0.66 0.69
h2 0.84 0.98 0.83 0.84 0.42 0.55 0.78 0.56 0.94 0.97 0.46 0.75 0.33 0.72 0.65
h3 0.97 1.00 0.94 0.86 0.43 0.54 0.81 0.62 1.00 0.88 0.58 0.71 0.27 0.71 0.62

Offset against WIS, month by month

Each point is a config in one season-month at one horizon: the median-level offset it played against the base’s bias that month, coloured by the month’s WIS change. If a positive offset only pays when the base under-predicts, the gains sit in the upper right and lower left.

ggplot(by_month %>% filter(config %in% names(grid)), aes(above_base_median, offset_rel, colour = wis_pct)) +
  geom_hline(yintercept = 0, colour = "grey60") +
  geom_vline(xintercept = 0.5, colour = "grey60") +
  geom_point(alpha = 0.8) +
  facet_grid(season ~ h) +
  scale_colour_gradient2(low = "#D55E00", mid = "grey80", high = "#009E73", midpoint = 0) +
  labs(x = "share of outcomes above the base median", y = "median-level offset / width", colour = "WIS reduction %")

Carry vs reset

The offset each cold, no-leak tracker brings into 2025-26, and what resetting it to zero at the season boundary does to the season’s WIS. E01 did this for the adaptive rate; the constant one is new.

pairs <- tibble::tibble(
  anchor = c("adaptive 0.03", "constant 0.018"),
  carry = names(cold_no_leak),
  reset = names(RESETS)
)
m <- ws_wis_by_season(cals[c(pairs$carry, pairs$reset)])
cr <- pairs %>%
  tidyr::pivot_longer(c(carry, reset), names_to = "policy", values_to = "config") %>%
  left_join(m %>% mutate(config = as.character(config)), by = "config")
knitr::kable(
  cr %>%
    transmute(anchor, policy, col = paste(season, h), v = sprintf("%+.1f", wis_pct)) %>%
    tidyr::pivot_wider(names_from = col, values_from = v),
  caption = "WIS reduction %, states only, by season and horizon"
)
WIS reduction %, states only, by season and horizon
anchor policy 2024-2025 h0 2024-2025 h1 2024-2025 h2 2024-2025 h3 2025-2026 h0 2025-2026 h1 2025-2026 h2 2025-2026 h3 both h0 both h1 both h2 both h3
adaptive 0.03 carry +1.8 +0.0 -1.4 -2.6 +2.2 -0.6 -0.7 -0.2 +1.9 -0.2 -1.1 -1.7
adaptive 0.03 reset +1.8 +0.0 -1.4 -2.6 +0.7 -0.9 -1.3 -1.6 +1.4 -0.3 -1.3 -2.2
constant 0.018 carry +1.7 +0.6 +0.3 +0.1 +3.0 +1.0 +1.1 +1.5 +2.2 +0.8 +0.6 +0.6
constant 0.018 reset +1.7 +0.6 +0.3 +0.1 +1.0 +0.1 +0.2 +0.3 +1.4 +0.4 +0.2 +0.2 
first_live <- internals %>%
  filter(season == LIVE[2], config %in% pairs$carry) %>%
  group_by(config, h) %>%
  slice_min(reference_date, n = 1, with_ties = FALSE) %>%
  ungroup()
knitr::kable(
  first_live %>%
    transmute(config, h, v = sprintf("%+.3f", offset_rel)) %>%
    tidyr::pivot_wider(names_from = h, values_from = v),
  caption = sprintf("Median-level offset / width at the first %s round, carried in from %s", LIVE[2], LIVE[1])
)
Median-level offset / width at the first 2025-2026 round, carried in from 2024-2025
config h0 h1 h2 h3
adaptive 0.03 | cold | no leak +0.025 +0.018 +0.005 +0.011
constant 0.018 | cold | no leak +0.023 +0.019 +0.015 +0.010 
cr_month <- by_month %>%
  filter(config %in% c(pairs$carry, pairs$reset), season == LIVE[2]) %>%
  mutate(policy = ifelse(config %in% pairs$reset, "reset", "carry"))
ggplot(cr_month, aes(month, wis_pct, colour = anchor, linetype = policy, group = config)) +
  geom_hline(yintercept = 0, colour = "grey60") +
  geom_line() +
  geom_point(size = 1) +
  facet_wrap(~h, nrow = 1) +
  scale_colour_manual(values = ANCHOR_COLS) +
  labs(title = paste(LIVE[2], "by month"), x = NULL, y = "WIS reduction %", colour = NULL, linetype = NULL) +
  theme(legend.position = "bottom")