cumulativeConfidence

Function for assessing impact and presence of chance between AB test variant measures. Outputs a cumulative time series of each variant mean and confidence interval. Can also assess non randomly assigned variants.

Languages and Tools

   

Packages

     

Arguments

Argument	Description
.data	A data frame, tibble, named matrix, or data.table that must be in machine readable long format.
date	Date column, which must be in the ISO 8601 format, or convertible to the format. Internally the function will attempt to cast the date to YYYY-MM-DD.
variants	The column/variable indicating whether the observation, or row, contains information for the A or B variant.
metric	The success metric that the means of will be used to estimate impact and presence of chance.
z_value	The Z value to determine which confidence level will be used. Defaulted to 1.39 to estimate at the 95% confidence level.

Function

rcumulativeConfidence <- function(.data, date, variants, metric, z_value = 1.39) {
  require(dplyr)
  require(rlang)
  require(extrafont)

data_table <- .data %>%
  mutate(date = as.Date( {{ date }} )  ) %>%
  arrange( {{ variants }}, date) %>%
  group_by( {{ variants }} ) %>%
  mutate(
    observations = 1,
    cumulative_mean = cummean( {{ metric }} ),
    cumulative_observations = cumsum(observations)
  ) %>%
  group_by( {{variants }}, date) %>%
  mutate(
    cumulative_mean = last(cumulative_mean),
    cumulative_observations = max(cumulative_observations),
    cumulative_squared_errors = cumsum(as.numeric( {{ metric }} - cumulative_mean)^2)
  ) %>%
  group_by( {{variants }}, date) %>%
  summarise(
    cumulative_ci_lower = max(cumulative_mean) - (z_value * (sqrt(max(cumulative_squared_errors) / max(cumulative_observations)) / sqrt(max(cumulative_observations))) ),
    cumulative_mean = max(cumulative_mean),
    cumulative_ci_upper = max(cumulative_mean) + (z_value * (sqrt(max(cumulative_squared_errors) / max(cumulative_observations)) / sqrt(max(cumulative_observations))) )
  )

print('Information for the latest date')
data_table %>% 
  group_by( {{ variants }} ) %>% 
  filter(date == max(date)) %>%
  rename(!!paste0(as_name(enquo(metric)), '_', names(.)[4]) := names(.)[4]) %>% 
  print()

data_table %>% 
  ggplot2::ggplot(
    aes(
      date,
      cumulative_mean,
      group = {{ variants }},
      col = {{ variants }}
    )
  ) +
  geom_errorbar(
    aes(
      ymin = cumulative_ci_lower,
      ymax = cumulative_ci_upper
    ),
    col = 'black',
    alpha = 0.5,
    width = 0.35
  ) +
  geom_line(alpha = 0.2) +
  geom_linerange(
    aes(
      ymin = cumulative_ci_lower,
      ymax = cumulative_ci_upper
    ),
    size = 5
  ) +
  xlab("Date") +
  ylab(paste0(as_label(enquo(metric)), ' mean')) +
  ggtitle(paste0('Cumulative Mean ', as_label(enquo(metric)), ': 83.4% Confidence Intervals')) +
  viridis::scale_color_viridis(
    option = "D",
    discrete = TRUE,
    begin = 0,
    end = 0.6
  ) +
  theme(
    panel.background = element_rect(fill = 'grey94'),
    text = element_text(family = 'Segoe UI')
  )

}

Output

#> [1] "Information for the latest date"
#> # A tibble: 2 x 5
#> # Groups:   Prototype [2]
#>   Prototype date       cumulative_ci_lower sessionLength_cumulative_mean cumulative_ci_upper
#>   <chr>     <date>                   <dbl>                         <dbl>               <dbl>
#> 1 proto_A   2018-07-29                17.3                          17.4                17.4
#> 2 proto_B   2018-07-29                12.5                          12.6                12.6