Documentation State Model
Hampus Broman & William Levén
2021-05
Looking at the data
We plot the data and can see that there is some difference between the debt versions
d.both_completed %>%
mutate_at("high_debt_version",
function(x) case_when(
x == "false" ~ "Low debt",
x == "true" ~ "High debt"
)) %>%
ggplot(aes(high_debt_version, fill = documentation)) +
geom_bar() +
scale_y_reverse() +
xlab("Debt level") +
scale_fill_manual("Documentation state", values = c("Correct" = "darkblue", "None" = "transparent", "Incorrect" = "lightblue"), guide = guide_legend(reverse = TRUE))
Initial model
The type of the outcome is categorical and is hence modeled as categorical.
We include high_debt_verison as a predictor in our model as this variable represent the very effect we want to measure. We also include a varying intercept for each individual to prevent the model from learning too much from single participants with extreme measurements.
Selecting priors
We iterate over the model until we have sane priors.
Base model with priors
documentation.with <- extendable_model(
base_name = "documentation",
base_formula = "documentation ~ 1 + high_debt_version + (1 | session)",
base_priors = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = d.both_completed,
)Default priors
prior_summary(documentation.with(only_priors= TRUE))Selected priors
prior_summary(documentation.with(sample_prior = "only"))Prior predictive check
pp_check(documentation.with(sample_prior = "only"), nsamples = 200, type = "bars")
Beta parameter influence
We choose a beta parameter priors allowing for the beta parameter to account for 100% of the effect but that is skeptical to such strong effects from the beta parameter.
sim.size <- 1000
sim.intercept <- rnorm(sim.size, 0, 1)
sim.beta <- rnorm(sim.size, 0, 1)
sim.beta.diff <- (plogis(sim.intercept + sim.beta) / plogis(sim.intercept) * 100) - 100
data.frame(x = sim.beta.diff) %>%
ggplot(aes(x)) +
geom_density() +
xlim(-80, 80) +
labs(
title = "Beta parameter prior influence",
x = "Estimate with beta as % of estimate without beta",
y = "Density"
)
Model fit
We check the posterior distribution and can see that the model seems to have been able to fit the data well. Sampling seems to also have worked well as Rhat values are close to 1 and the sampling plots look nice.
Posterior predictive check
pp_check(documentation.with(), nsamples = 200, type = "bars")
Summary
summary(documentation.with())## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + (1 | session)
## Data: as.data.frame(data) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 1.01 0.81 0.04 2.99 1.00 797
## sd(muNone_Intercept) 1.34 0.97 0.05 3.62 1.01 596
## Tail_ESS
## sd(muIncorrect_Intercept) 1783
## sd(muNone_Intercept) 1663
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## muIncorrect_Intercept 0.51 0.59 -0.73 1.64 1.00
## muNone_Intercept 0.76 0.61 -0.51 1.87 1.00
## muIncorrect_high_debt_versionfalse 0.04 0.65 -1.27 1.32 1.00
## muNone_high_debt_versionfalse 0.54 0.66 -0.71 1.85 1.00
## Bulk_ESS Tail_ESS
## muIncorrect_Intercept 3408 2395
## muNone_Intercept 2357 1890
## muIncorrect_high_debt_versionfalse 4751 2685
## muNone_high_debt_versionfalse 4644 2932
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Sampling plots
plot(documentation.with(), ask = FALSE)
Model predictor extenstions
# default prior for monotonic predictor
edlvl_prior <- c(
prior(dirichlet(2), class = "simo", coef = "moeducation_level1", dpar = "muIncorrect"),
prior(dirichlet(2), class = "simo", coef = "moeducation_level1", dpar = "muNone")
)We use loo to check some possible extensions on the model.
One variable
loo_result <- loo(
# Benchmark model(s)
documentation.with(),
# New model(s)
documentation.with("work_domain"),
documentation.with("work_experience_programming.s"),
documentation.with("work_experience_java.s"),
documentation.with("education_field"),
documentation.with("mo(education_level)", edlvl_prior),
documentation.with("workplace_peer_review"),
documentation.with("workplace_td_tracking"),
documentation.with("workplace_pair_programming"),
documentation.with("workplace_coding_standards"),
documentation.with("scenario"),
documentation.with("group")
)Comparison
loo_result[2]## $diffs
## elpd_diff se_diff
## documentation.with("group") 0.0 0.0
## documentation.with("work_domain") -0.2 1.4
## documentation.with("workplace_td_tracking") -0.4 1.0
## documentation.with("workplace_peer_review") -0.5 1.2
## documentation.with() -0.5 0.9
## documentation.with("workplace_pair_programming") -0.6 1.0
## documentation.with("education_field") -0.6 0.9
## documentation.with("workplace_coding_standards") -0.9 1.1
## documentation.with("mo(education_level)", edlvl_prior) -1.2 1.2
## documentation.with("work_experience_programming.s") -1.6 1.4
## documentation.with("scenario") -1.6 1.1
## documentation.with("work_experience_java.s") -2.1 1.4Diagnostics
loo_result[1]## $loos
## $loos$`documentation.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.6
## p_loo 14.1 1.2
## looic 88.0 7.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1265
## (0.5, 0.7] (ok) 12 27.3% 539
## (0.7, 1] (bad) 6 13.6% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.7 3.8
## p_loo 16.5 1.5
## looic 87.4 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 747
## (0.5, 0.7] (ok) 11 25.0% 685
## (0.7, 1] (bad) 4 9.1% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_experience_programming.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -45.1 4.1
## p_loo 16.6 1.8
## looic 90.2 8.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 805
## (0.5, 0.7] (ok) 16 36.4% 294
## (0.7, 1] (bad) 2 4.5% 479
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_experience_java.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -45.6 4.2
## p_loo 17.2 1.9
## looic 91.2 8.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 852
## (0.5, 0.7] (ok) 15 34.1% 413
## (0.7, 1] (bad) 6 13.6% 84
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 15.0 1.3
## looic 88.3 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 922
## (0.5, 0.7] (ok) 12 27.3% 658
## (0.7, 1] (bad) 5 11.4% 336
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.7 4.1
## p_loo 15.9 1.5
## looic 89.3 8.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 867
## (0.5, 0.7] (ok) 15 34.1% 529
## (0.7, 1] (bad) 3 6.8% 338
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.7
## p_loo 15.0 1.4
## looic 88.0 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 917
## (0.5, 0.7] (ok) 14 31.8% 711
## (0.7, 1] (bad) 3 6.8% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 3.8
## p_loo 14.4 1.4
## looic 87.9 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 909
## (0.5, 0.7] (ok) 12 27.3% 458
## (0.7, 1] (bad) 2 4.5% 334
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 14.8 1.3
## looic 88.2 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 680
## (0.5, 0.7] (ok) 12 27.3% 493
## (0.7, 1] (bad) 6 13.6% 355
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_coding_standards")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.4 3.8
## p_loo 15.1 1.4
## looic 88.8 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 922
## (0.5, 0.7] (ok) 16 36.4% 586
## (0.7, 1] (bad) 2 4.5% 337
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("scenario")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -45.1 3.8
## p_loo 16.2 1.5
## looic 90.2 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 859
## (0.5, 0.7] (ok) 10 22.7% 427
## (0.7, 1] (bad) 6 13.6% 136
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.9 1.4
## looic 87.0 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 980
## (0.5, 0.7] (ok) 16 36.4% 558
## (0.7, 1] (bad) 1 2.3% 807
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.Two variables
loo_result <- loo(
# Benchmark model(s)
documentation.with(),
documentation.with("group"),
documentation.with("work_domain"),
documentation.with("workplace_peer_review"),
documentation.with("workplace_td_tracking"),
documentation.with("workplace_pair_programming"),
documentation.with("education_field"),
# New model(s)
documentation.with(c("group", "work_domain")),
documentation.with(c("group", "workplace_peer_review")),
documentation.with(c("group", "workplace_td_tracking")),
documentation.with(c("group", "workplace_pair_programming")),
documentation.with(c("group", "education_field")),
documentation.with(c("work_domain", "workplace_peer_review")),
documentation.with(c("work_domain", "workplace_td_tracking")),
documentation.with(c("work_domain", "workplace_pair_programming")),
documentation.with(c("work_domain", "education_field")),
documentation.with(c("workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("workplace_peer_review", "workplace_pair_programming")),
documentation.with(c("workplace_peer_review", "education_field")),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("workplace_td_tracking", "education_field")),
documentation.with(c("workplace_pair_programming", "education_field"))
)Comparison
loo_result[2]## $diffs
## elpd_diff
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) 0.0
## documentation.with(c("group", "work_domain")) -0.1
## documentation.with("group") -0.2
## documentation.with(c("group", "workplace_peer_review")) -0.2
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) -0.3
## documentation.with("work_domain") -0.3
## documentation.with(c("group", "workplace_pair_programming")) -0.6
## documentation.with("workplace_td_tracking") -0.6
## documentation.with("workplace_peer_review") -0.6
## documentation.with() -0.6
## documentation.with(c("work_domain", "workplace_td_tracking")) -0.7
## documentation.with(c("workplace_pair_programming", "education_field")) -0.8
## documentation.with("workplace_pair_programming") -0.8
## documentation.with("education_field") -0.8
## documentation.with(c("workplace_peer_review", "education_field")) -0.8
## documentation.with(c("workplace_peer_review", "workplace_pair_programming")) -0.8
## documentation.with(c("workplace_td_tracking", "education_field")) -0.8
## documentation.with(c("group", "workplace_td_tracking")) -0.9
## documentation.with(c("work_domain", "workplace_peer_review")) -0.9
## documentation.with(c("work_domain", "workplace_pair_programming")) -1.0
## documentation.with(c("group", "education_field")) -1.0
## documentation.with(c("work_domain", "education_field")) -1.4
## se_diff
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) 0.0
## documentation.with(c("group", "work_domain")) 1.0
## documentation.with("group") 1.0
## documentation.with(c("group", "workplace_peer_review")) 1.5
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) 1.3
## documentation.with("work_domain") 1.2
## documentation.with(c("group", "workplace_pair_programming")) 1.1
## documentation.with("workplace_td_tracking") 0.7
## documentation.with("workplace_peer_review") 1.5
## documentation.with() 1.1
## documentation.with(c("work_domain", "workplace_td_tracking")) 1.1
## documentation.with(c("workplace_pair_programming", "education_field")) 1.1
## documentation.with("workplace_pair_programming") 0.9
## documentation.with("education_field") 1.2
## documentation.with(c("workplace_peer_review", "education_field")) 1.6
## documentation.with(c("workplace_peer_review", "workplace_pair_programming")) 1.5
## documentation.with(c("workplace_td_tracking", "education_field")) 0.9
## documentation.with(c("group", "workplace_td_tracking")) 0.8
## documentation.with(c("work_domain", "workplace_peer_review")) 1.5
## documentation.with(c("work_domain", "workplace_pair_programming")) 1.1
## documentation.with(c("group", "education_field")) 1.1
## documentation.with(c("work_domain", "education_field")) 1.3Diagnostics
loo_result[1]## $loos
## $loos$`documentation.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.6
## p_loo 14.1 1.2
## looic 88.0 7.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1265
## (0.5, 0.7] (ok) 12 27.3% 539
## (0.7, 1] (bad) 6 13.6% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.9 1.4
## looic 87.0 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 980
## (0.5, 0.7] (ok) 16 36.4% 558
## (0.7, 1] (bad) 1 2.3% 807
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.7 3.8
## p_loo 16.5 1.5
## looic 87.4 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 747
## (0.5, 0.7] (ok) 11 25.0% 685
## (0.7, 1] (bad) 4 9.1% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.7
## p_loo 15.0 1.4
## looic 88.0 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 917
## (0.5, 0.7] (ok) 14 31.8% 711
## (0.7, 1] (bad) 3 6.8% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 3.8
## p_loo 14.4 1.4
## looic 87.9 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 909
## (0.5, 0.7] (ok) 12 27.3% 458
## (0.7, 1] (bad) 2 4.5% 334
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 14.8 1.3
## looic 88.2 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 680
## (0.5, 0.7] (ok) 12 27.3% 493
## (0.7, 1] (bad) 6 13.6% 355
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 15.0 1.3
## looic 88.3 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 922
## (0.5, 0.7] (ok) 12 27.3% 658
## (0.7, 1] (bad) 5 11.4% 336
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 3.9
## p_loo 16.8 1.6
## looic 86.9 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 885
## (0.5, 0.7] (ok) 12 27.3% 486
## (0.7, 1] (bad) 2 4.5% 369
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.8 1.4
## looic 87.1 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 1033
## (0.5, 0.7] (ok) 19 43.2% 567
## (0.7, 1] (bad) 1 2.3% 965
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.2 4.0
## p_loo 15.9 1.5
## looic 88.4 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 571
## (0.5, 0.7] (ok) 18 40.9% 489
## (0.7, 1] (bad) 2 4.5% 609
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 4.0
## p_loo 15.5 1.5
## looic 87.8 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 720
## (0.5, 0.7] (ok) 14 31.8% 305
## (0.7, 1] (bad) 2 4.5% 381
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "education_field"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.4 4.1
## p_loo 16.5 1.6
## looic 88.8 8.1
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 930
## (0.5, 0.7] (ok) 17 38.6% 535
## (0.7, 1] (bad) 4 9.1% 212
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("work_domain", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.3 3.9
## p_loo 17.0 1.7
## looic 88.5 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 748
## (0.5, 0.7] (ok) 18 40.9% 549
## (0.7, 1] (bad) 2 4.5% 216
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("work_domain", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 4.1
## p_loo 16.9 1.7
## looic 88.1 8.1
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 1099
## (0.5, 0.7] (ok) 15 34.1% 284
## (0.7, 1] (bad) 2 4.5% 443
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("work_domain", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.4 4.0
## p_loo 17.6 1.7
## looic 88.7 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 22 50.0% 605
## (0.5, 0.7] (ok) 17 38.6% 531
## (0.7, 1] (bad) 5 11.4% 209
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("work_domain", "education_field"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.8 4.1
## p_loo 17.9 1.8
## looic 89.5 8.1
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 25 56.8% 805
## (0.5, 0.7] (ok) 16 36.4% 478
## (0.7, 1] (bad) 3 6.8% 260
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.6 3.9
## p_loo 14.6 1.4
## looic 87.2 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 1164
## (0.5, 0.7] (ok) 14 31.8% 351
## (0.7, 1] (bad) 3 6.8% 296
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.2 3.8
## p_loo 15.3 1.4
## looic 88.4 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 894
## (0.5, 0.7] (ok) 17 38.6% 495
## (0.7, 1] (bad) 4 9.1% 501
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "education_field"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.2 3.8
## p_loo 15.8 1.5
## looic 88.3 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 25 56.8% 1098
## (0.5, 0.7] (ok) 17 38.6% 514
## (0.7, 1] (bad) 2 4.5% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 14.5 1.4
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1023
## (0.5, 0.7] (ok) 12 27.3% 469
## (0.7, 1] (bad) 2 4.5% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "education_field"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.2 4.0
## p_loo 15.5 1.5
## looic 88.4 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 678
## (0.5, 0.7] (ok) 16 36.4% 549
## (0.7, 1] (bad) 1 2.3% 245
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_pair_programming", "education_field"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.9
## p_loo 15.4 1.5
## looic 88.2 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 494
## (0.5, 0.7] (ok) 13 29.5% 426
## (0.7, 1] (bad) 4 9.1% 285
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.Three variables
loo_result <- loo(
# Benchmark model(s)
documentation.with(),
documentation.with("group"),
documentation.with("work_domain"),
documentation.with("workplace_peer_review"),
documentation.with("workplace_td_tracking"),
documentation.with("workplace_pair_programming"),
documentation.with("education_field"),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("group", "work_domain")),
documentation.with(c("group", "workplace_peer_review")),
documentation.with(c("group", "workplace_pair_programming")),
documentation.with(c("workplace_peer_review", "workplace_td_tracking")),
# New model(s)
documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "group")),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "work_domain")),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")),
documentation.with(c("group", "work_domain", "workplace_td_tracking")),
documentation.with(c("group", "work_domain", "workplace_pair_programming")),
documentation.with(c("group", "work_domain", "workplace_peer_review")),
documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")),
documentation.with(c("group", "workplace_pair_programming", "work_domain")),
documentation.with(c("workplace_peer_review", "workplace_td_tracking", "work_domain"))
)Comparison
loo_result[2]## $diffs
## elpd_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) -0.5
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) -0.8
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) -1.0
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) -1.0
## documentation.with(c("group", "work_domain")) -1.0
## documentation.with("group") -1.1
## documentation.with(c("group", "workplace_peer_review")) -1.1
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) -1.2
## documentation.with("work_domain") -1.3
## documentation.with(c("workplace_peer_review", "workplace_td_tracking", "work_domain")) -1.5
## documentation.with(c("group", "workplace_pair_programming")) -1.5
## documentation.with("workplace_td_tracking") -1.6
## documentation.with("workplace_peer_review") -1.6
## documentation.with() -1.6
## documentation.with(c("group", "work_domain", "workplace_pair_programming")) -1.7
## documentation.with(c("group", "workplace_pair_programming", "work_domain")) -1.7
## documentation.with("workplace_pair_programming") -1.7
## documentation.with("education_field") -1.8
## documentation.with(c("group", "work_domain", "workplace_td_tracking")) -1.8
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "group")) -2.1
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "work_domain")) -2.4
## se_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) 1.2
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) 1.1
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) 1.3
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) 1.1
## documentation.with(c("group", "work_domain")) 0.8
## documentation.with("group") 1.0
## documentation.with(c("group", "workplace_peer_review")) 1.0
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) 1.3
## documentation.with("work_domain") 1.1
## documentation.with(c("workplace_peer_review", "workplace_td_tracking", "work_domain")) 1.1
## documentation.with(c("group", "workplace_pair_programming")) 1.1
## documentation.with("workplace_td_tracking") 1.3
## documentation.with("workplace_peer_review") 1.1
## documentation.with() 1.1
## documentation.with(c("group", "work_domain", "workplace_pair_programming")) 1.2
## documentation.with(c("group", "workplace_pair_programming", "work_domain")) 1.2
## documentation.with("workplace_pair_programming") 1.1
## documentation.with("education_field") 1.1
## documentation.with(c("group", "work_domain", "workplace_td_tracking")) 1.2
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "group")) 1.2
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "work_domain")) 1.3Diagnostics
loo_result[1]## $loos
## $loos$`documentation.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.6
## p_loo 14.1 1.2
## looic 88.0 7.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1265
## (0.5, 0.7] (ok) 12 27.3% 539
## (0.7, 1] (bad) 6 13.6% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.9 1.4
## looic 87.0 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 980
## (0.5, 0.7] (ok) 16 36.4% 558
## (0.7, 1] (bad) 1 2.3% 807
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.7 3.8
## p_loo 16.5 1.5
## looic 87.4 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 747
## (0.5, 0.7] (ok) 11 25.0% 685
## (0.7, 1] (bad) 4 9.1% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.7
## p_loo 15.0 1.4
## looic 88.0 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 917
## (0.5, 0.7] (ok) 14 31.8% 711
## (0.7, 1] (bad) 3 6.8% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 3.8
## p_loo 14.4 1.4
## looic 87.9 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 909
## (0.5, 0.7] (ok) 12 27.3% 458
## (0.7, 1] (bad) 2 4.5% 334
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 14.8 1.3
## looic 88.2 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 680
## (0.5, 0.7] (ok) 12 27.3% 493
## (0.7, 1] (bad) 6 13.6% 355
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 15.0 1.3
## looic 88.3 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 922
## (0.5, 0.7] (ok) 12 27.3% 658
## (0.7, 1] (bad) 5 11.4% 336
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 14.5 1.4
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1023
## (0.5, 0.7] (ok) 12 27.3% 469
## (0.7, 1] (bad) 2 4.5% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 3.9
## p_loo 16.8 1.6
## looic 86.9 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 885
## (0.5, 0.7] (ok) 12 27.3% 486
## (0.7, 1] (bad) 2 4.5% 369
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.8 1.4
## looic 87.1 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 1033
## (0.5, 0.7] (ok) 19 43.2% 567
## (0.7, 1] (bad) 1 2.3% 965
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 4.0
## p_loo 15.5 1.5
## looic 87.8 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 720
## (0.5, 0.7] (ok) 14 31.8% 305
## (0.7, 1] (bad) 2 4.5% 381
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.6 3.9
## p_loo 14.6 1.4
## looic 87.2 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 1164
## (0.5, 0.7] (ok) 14 31.8% 351
## (0.7, 1] (bad) 3 6.8% 296
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "group"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.5 4.2
## p_loo 16.6 1.7
## looic 89.0 8.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 716
## (0.5, 0.7] (ok) 11 25.0% 341
## (0.7, 1] (bad) 4 9.1% 256
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.8 4.2
## p_loo 17.9 1.9
## looic 89.6 8.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 785
## (0.5, 0.7] (ok) 13 29.5% 246
## (0.7, 1] (bad) 8 18.2% 199
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.9 3.8
## p_loo 13.7 1.3
## looic 85.8 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1223
## (0.5, 0.7] (ok) 14 31.8% 762
## (0.7, 1] (bad) 4 9.1% 496
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.2 4.2
## p_loo 17.9 1.9
## looic 88.5 8.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 923
## (0.5, 0.7] (ok) 9 20.5% 306
## (0.7, 1] (bad) 5 11.4% 321
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 4.2
## p_loo 17.9 1.9
## looic 88.1 8.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 922
## (0.5, 0.7] (ok) 13 29.5% 348
## (0.7, 1] (bad) 4 9.1% 359
## (1, Inf) (very bad) 1 2.3% 42
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.4 3.9
## p_loo 15.8 1.6
## looic 84.8 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 1090
## (0.5, 0.7] (ok) 11 25.0% 780
## (0.7, 1] (bad) 4 9.1% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 15.2 1.5
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 1108
## (0.5, 0.7] (ok) 19 43.2% 548
## (0.7, 1] (bad) 2 4.5% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.2 4.0
## p_loo 15.0 1.4
## looic 86.4 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 898
## (0.5, 0.7] (ok) 12 27.3% 503
## (0.7, 1] (bad) 4 9.1% 331
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_pair_programming", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 4.2
## p_loo 17.9 1.9
## looic 88.1 8.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 922
## (0.5, 0.7] (ok) 13 29.5% 348
## (0.7, 1] (bad) 4 9.1% 359
## (1, Inf) (very bad) 1 2.3% 42
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_td_tracking", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.8 4.1
## p_loo 16.7 1.7
## looic 87.7 8.1
## ------
## Monte Carlo SE of elpd_loo is 0.2.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 996
## (0.5, 0.7] (ok) 18 40.9% 431
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.Four variables
loo_result <- loo(
# Benchmark model(s)
documentation.with(),
documentation.with("group"),
documentation.with("work_domain"),
documentation.with("workplace_peer_review"),
documentation.with("workplace_td_tracking"),
documentation.with("workplace_pair_programming"),
documentation.with("education_field"),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("group", "work_domain")),
documentation.with(c("group", "workplace_peer_review")),
documentation.with(c("group", "workplace_pair_programming")),
documentation.with(c("workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("group", "work_domain", "workplace_peer_review")),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")),
documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")),
documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")),
# New model(s)
documentation.with(c("work_domain", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")),
documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking"))
)Comparison
loo_result[2]## $diffs
## elpd_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking")) -0.3
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) -0.5
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")) -0.5
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")) -0.6
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) -0.8
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) -1.0
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) -1.0
## documentation.with(c("group", "work_domain")) -1.0
## documentation.with(c("work_domain", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")) -1.1
## documentation.with("group") -1.1
## documentation.with(c("group", "workplace_peer_review")) -1.1
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) -1.2
## documentation.with("work_domain") -1.3
## documentation.with(c("group", "workplace_pair_programming")) -1.5
## documentation.with("workplace_td_tracking") -1.6
## documentation.with("workplace_peer_review") -1.6
## documentation.with() -1.6
## documentation.with("workplace_pair_programming") -1.7
## documentation.with("education_field") -1.8
## se_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking")) 0.6
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) 1.2
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")) 0.7
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")) 1.1
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) 1.1
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) 1.3
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) 1.1
## documentation.with(c("group", "work_domain")) 0.8
## documentation.with(c("work_domain", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming")) 1.0
## documentation.with("group") 1.0
## documentation.with(c("group", "workplace_peer_review")) 1.0
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) 1.3
## documentation.with("work_domain") 1.1
## documentation.with(c("group", "workplace_pair_programming")) 1.1
## documentation.with("workplace_td_tracking") 1.3
## documentation.with("workplace_peer_review") 1.1
## documentation.with() 1.1
## documentation.with("workplace_pair_programming") 1.1
## documentation.with("education_field") 1.1Diagnostics
loo_result[1]## $loos
## $loos$`documentation.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.6
## p_loo 14.1 1.2
## looic 88.0 7.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1265
## (0.5, 0.7] (ok) 12 27.3% 539
## (0.7, 1] (bad) 6 13.6% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.9 1.4
## looic 87.0 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 980
## (0.5, 0.7] (ok) 16 36.4% 558
## (0.7, 1] (bad) 1 2.3% 807
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.7 3.8
## p_loo 16.5 1.5
## looic 87.4 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 747
## (0.5, 0.7] (ok) 11 25.0% 685
## (0.7, 1] (bad) 4 9.1% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.7
## p_loo 15.0 1.4
## looic 88.0 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 917
## (0.5, 0.7] (ok) 14 31.8% 711
## (0.7, 1] (bad) 3 6.8% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 3.8
## p_loo 14.4 1.4
## looic 87.9 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 909
## (0.5, 0.7] (ok) 12 27.3% 458
## (0.7, 1] (bad) 2 4.5% 334
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 14.8 1.3
## looic 88.2 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 680
## (0.5, 0.7] (ok) 12 27.3% 493
## (0.7, 1] (bad) 6 13.6% 355
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 15.0 1.3
## looic 88.3 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 922
## (0.5, 0.7] (ok) 12 27.3% 658
## (0.7, 1] (bad) 5 11.4% 336
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 14.5 1.4
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1023
## (0.5, 0.7] (ok) 12 27.3% 469
## (0.7, 1] (bad) 2 4.5% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 3.9
## p_loo 16.8 1.6
## looic 86.9 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 885
## (0.5, 0.7] (ok) 12 27.3% 486
## (0.7, 1] (bad) 2 4.5% 369
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.8 1.4
## looic 87.1 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 1033
## (0.5, 0.7] (ok) 19 43.2% 567
## (0.7, 1] (bad) 1 2.3% 965
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 4.0
## p_loo 15.5 1.5
## looic 87.8 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 720
## (0.5, 0.7] (ok) 14 31.8% 305
## (0.7, 1] (bad) 2 4.5% 381
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.6 3.9
## p_loo 14.6 1.4
## looic 87.2 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 1164
## (0.5, 0.7] (ok) 14 31.8% 351
## (0.7, 1] (bad) 3 6.8% 296
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.4 3.9
## p_loo 15.8 1.6
## looic 84.8 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 1090
## (0.5, 0.7] (ok) 11 25.0% 780
## (0.7, 1] (bad) 4 9.1% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.9 3.8
## p_loo 13.7 1.3
## looic 85.8 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1223
## (0.5, 0.7] (ok) 14 31.8% 762
## (0.7, 1] (bad) 4 9.1% 496
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.2 4.0
## p_loo 15.0 1.4
## looic 86.4 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 898
## (0.5, 0.7] (ok) 12 27.3% 503
## (0.7, 1] (bad) 4 9.1% 331
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 15.2 1.5
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 1108
## (0.5, 0.7] (ok) 19 43.2% 548
## (0.7, 1] (bad) 2 4.5% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("work_domain", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 4.0
## p_loo 16.6 1.7
## looic 86.9 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 22 50.0% 1097
## (0.5, 0.7] (ok) 20 45.5% 477
## (0.7, 1] (bad) 2 4.5% 338
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.0 4.0
## p_loo 15.0 1.5
## looic 86.0 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 1063
## (0.5, 0.7] (ok) 14 31.8% 548
## (0.7, 1] (bad) 2 4.5% 1029
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.9 4.1
## p_loo 16.7 1.8
## looic 85.9 8.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 31 70.5% 819
## (0.5, 0.7] (ok) 8 18.2% 511
## (0.7, 1] (bad) 5 11.4% 264
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.7 4.0
## p_loo 16.1 1.7
## looic 85.4 8.0
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 996
## (0.5, 0.7] (ok) 16 36.4% 473
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.Five variables
loo_result <- loo(
# Benchmark model(s)
documentation.with(),
documentation.with("group"),
documentation.with("work_domain"),
documentation.with("workplace_peer_review"),
documentation.with("workplace_td_tracking"),
documentation.with("workplace_pair_programming"),
documentation.with("education_field"),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming")),
documentation.with(c("group", "work_domain")),
documentation.with(c("group", "workplace_peer_review")),
documentation.with(c("group", "workplace_pair_programming")),
documentation.with(c("workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("group", "work_domain", "workplace_peer_review")),
documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")),
documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")),
documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking")),
documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")),
# New model(s)
documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming", "workplace_td_tracking"))
)Comparison
loo_result[2]## $diffs
## elpd_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming", "workplace_td_tracking")) -0.3
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking")) -0.3
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) -0.5
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")) -0.5
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) -0.8
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) -1.0
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) -1.0
## documentation.with(c("group", "work_domain")) -1.0
## documentation.with("group") -1.1
## documentation.with(c("group", "workplace_peer_review")) -1.1
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) -1.2
## documentation.with("work_domain") -1.3
## documentation.with(c("group", "workplace_pair_programming")) -1.5
## documentation.with("workplace_td_tracking") -1.6
## documentation.with("workplace_peer_review") -1.6
## documentation.with() -1.6
## documentation.with("workplace_pair_programming") -1.7
## documentation.with("education_field") -1.8
## se_diff
## documentation.with(c("group", "work_domain", "workplace_peer_review")) 0.0
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming", "workplace_td_tracking")) 0.8
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking")) 0.6
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review")) 1.2
## documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming")) 0.7
## documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming")) 1.1
## documentation.with(c("workplace_td_tracking", "workplace_pair_programming")) 1.3
## documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking")) 1.1
## documentation.with(c("group", "work_domain")) 0.8
## documentation.with("group") 1.0
## documentation.with(c("group", "workplace_peer_review")) 1.0
## documentation.with(c("workplace_peer_review", "workplace_td_tracking")) 1.3
## documentation.with("work_domain") 1.1
## documentation.with(c("group", "workplace_pair_programming")) 1.1
## documentation.with("workplace_td_tracking") 1.3
## documentation.with("workplace_peer_review") 1.1
## documentation.with() 1.1
## documentation.with("workplace_pair_programming") 1.1
## documentation.with("education_field") 1.1Diagnostics
loo_result[1]## $loos
## $loos$`documentation.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.6
## p_loo 14.1 1.2
## looic 88.0 7.3
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1265
## (0.5, 0.7] (ok) 12 27.3% 539
## (0.7, 1] (bad) 6 13.6% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.9 1.4
## looic 87.0 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 980
## (0.5, 0.7] (ok) 16 36.4% 558
## (0.7, 1] (bad) 1 2.3% 807
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.7 3.8
## p_loo 16.5 1.5
## looic 87.4 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 747
## (0.5, 0.7] (ok) 11 25.0% 685
## (0.7, 1] (bad) 4 9.1% 437
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.0 3.7
## p_loo 15.0 1.4
## looic 88.0 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 917
## (0.5, 0.7] (ok) 14 31.8% 711
## (0.7, 1] (bad) 3 6.8% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 3.8
## p_loo 14.4 1.4
## looic 87.9 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 909
## (0.5, 0.7] (ok) 12 27.3% 458
## (0.7, 1] (bad) 2 4.5% 334
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 14.8 1.3
## looic 88.2 7.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 680
## (0.5, 0.7] (ok) 12 27.3% 493
## (0.7, 1] (bad) 6 13.6% 355
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -44.1 3.7
## p_loo 15.0 1.3
## looic 88.3 7.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 922
## (0.5, 0.7] (ok) 12 27.3% 658
## (0.7, 1] (bad) 5 11.4% 336
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 14.5 1.4
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 1023
## (0.5, 0.7] (ok) 12 27.3% 469
## (0.7, 1] (bad) 2 4.5% 383
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 3.9
## p_loo 16.8 1.6
## looic 86.9 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 885
## (0.5, 0.7] (ok) 12 27.3% 486
## (0.7, 1] (bad) 2 4.5% 369
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.5 3.8
## p_loo 14.8 1.4
## looic 87.1 7.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 24 54.5% 1033
## (0.5, 0.7] (ok) 19 43.2% 567
## (0.7, 1] (bad) 1 2.3% 965
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.9 4.0
## p_loo 15.5 1.5
## looic 87.8 8.0
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 720
## (0.5, 0.7] (ok) 14 31.8% 305
## (0.7, 1] (bad) 2 4.5% 381
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.6 3.9
## p_loo 14.6 1.4
## looic 87.2 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 27 61.4% 1164
## (0.5, 0.7] (ok) 14 31.8% 351
## (0.7, 1] (bad) 3 6.8% 296
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.4 3.9
## p_loo 15.8 1.6
## looic 84.8 7.8
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 29 65.9% 1090
## (0.5, 0.7] (ok) 11 25.0% 780
## (0.7, 1] (bad) 4 9.1% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("workplace_td_tracking", "workplace_pair_programming", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.9 3.8
## p_loo 13.7 1.3
## looic 85.8 7.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 26 59.1% 1223
## (0.5, 0.7] (ok) 14 31.8% 762
## (0.7, 1] (bad) 4 9.1% 496
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.2 4.0
## p_loo 15.0 1.4
## looic 86.4 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 898
## (0.5, 0.7] (ok) 12 27.3% 503
## (0.7, 1] (bad) 4 9.1% 331
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -43.4 4.0
## p_loo 15.2 1.5
## looic 86.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 23 52.3% 1108
## (0.5, 0.7] (ok) 19 43.2% 548
## (0.7, 1] (bad) 2 4.5% 267
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.7 4.0
## p_loo 16.1 1.7
## looic 85.4 8.0
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 28 63.6% 996
## (0.5, 0.7] (ok) 16 36.4% 473
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.9 4.1
## p_loo 16.7 1.8
## looic 85.9 8.2
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 31 70.5% 819
## (0.5, 0.7] (ok) 8 18.2% 511
## (0.7, 1] (bad) 5 11.4% 264
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`documentation.with(c("group", "work_domain", "workplace_peer_review", "workplace_pair_programming", "workplace_td_tracking"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -42.7 4.1
## p_loo 16.5 1.7
## looic 85.3 8.1
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 30 68.2% 953
## (0.5, 0.7] (ok) 9 20.5% 461
## (0.7, 1] (bad) 5 11.4% 280
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.Candidate models
We pick some of our top performing models as candidates and inspect them closer.
The candidate models are named and listed in order of complexity.
Documentation0
We select the simplest model as a baseline.
documentation0 <- brm(
"documentation ~ 1 + high_debt_version + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation0",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation0)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 1.01 0.81 0.04 2.99 1.00 797
## sd(muNone_Intercept) 1.34 0.97 0.05 3.62 1.01 596
## Tail_ESS
## sd(muIncorrect_Intercept) 1783
## sd(muNone_Intercept) 1663
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## muIncorrect_Intercept 0.51 0.59 -0.73 1.64 1.00
## muNone_Intercept 0.76 0.61 -0.51 1.87 1.00
## muIncorrect_high_debt_versionfalse 0.04 0.65 -1.27 1.32 1.00
## muNone_high_debt_versionfalse 0.54 0.66 -0.71 1.85 1.00
## Bulk_ESS Tail_ESS
## muIncorrect_Intercept 3408 2395
## muNone_Intercept 2357 1890
## muIncorrect_high_debt_versionfalse 4751 2685
## muNone_high_debt_versionfalse 4644 2932
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation0)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.5737990 1.1666252 -3.6257817 1.088651
## 6033d90a5af2c702367b3a96 0.3158600 0.9827580 -1.4174820 2.665932
## 6034fc165af2c702367b3a98 -0.4327795 1.1424426 -3.5186307 1.385535
## 603500725af2c702367b3a99 0.9493749 1.3628863 -0.5800635 4.545955
## 603f97625af2c702367b3a9d 0.9749455 1.3979445 -0.6243103 4.564458
## 603fd5d95af2c702367b3a9e -0.4563847 1.2106102 -3.6117975 1.445781
## 60409b7b5af2c702367b3a9f 0.9612890 1.3522290 -0.6180260 4.640545
## 604b82b5a7718fbed181b336 -0.4752766 1.2164002 -3.4140790 1.301414
## 6050c1bf856f36729d2e5218 0.9758416 1.4347923 -0.6359394 4.735828
## 6050e1e7856f36729d2e5219 0.1864220 0.9398340 -1.6493422 2.444286
## 6055fdc6856f36729d2e521b -0.4254398 1.1668660 -3.2592368 1.446640
## 60589862856f36729d2e521f 0.1771059 0.9115157 -1.5404795 2.318001
## 605afa3a856f36729d2e5222 -0.5610085 1.2058828 -3.4886668 1.200422
## 605c8bc6856f36729d2e5223 0.3214101 0.9718144 -1.3661377 2.676825
## 605f3f2d856f36729d2e5224 0.3221453 0.9539061 -1.4405652 2.641423
## 605f46c3856f36729d2e5225 -0.4454148 1.2092840 -3.4900442 1.471917
## 60605337856f36729d2e5226 0.1787544 0.9258103 -1.5802352 2.446948
## 60609ae6856f36729d2e5228 0.3118464 0.9780047 -1.3746567 2.706801
## 6061ce91856f36729d2e522e -0.5645222 1.2028423 -3.7873927 1.149801
## 6061f106856f36729d2e5231 -0.4482786 1.1732625 -3.4480425 1.395374
## 6068ea9f856f36729d2e523e -0.4278826 1.1010536 -3.1891605 1.373934
## 6075ab05856f36729d2e5247 -0.4519516 1.1911752 -3.5479702 1.330733
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.006507317 1.062590 -2.2740792 2.2540040
## 6033d90a5af2c702367b3a96 0.138723867 1.073052 -1.9716752 2.6135863
## 6034fc165af2c702367b3a98 1.096178382 1.529027 -0.8404242 5.0412483
## 603500725af2c702367b3a99 -0.901754303 1.462965 -4.6523647 1.1217253
## 603f97625af2c702367b3a9d -0.932456062 1.526069 -4.9169960 1.1212663
## 603fd5d95af2c702367b3a9e 1.105998334 1.573408 -0.8587135 5.3441000
## 60409b7b5af2c702367b3a9f -0.900769015 1.476327 -4.8135338 1.0593653
## 604b82b5a7718fbed181b336 1.084661092 1.541826 -0.8112732 5.1485220
## 6050c1bf856f36729d2e5218 -0.887398767 1.486081 -4.7704943 1.2544253
## 6050e1e7856f36729d2e5219 -1.022534549 1.502238 -4.7182448 0.8929634
## 6055fdc6856f36729d2e521b 1.131237889 1.581186 -0.8969573 5.2768453
## 60589862856f36729d2e521f -1.046842762 1.535243 -5.1387605 0.9084778
## 605afa3a856f36729d2e5222 -0.019666351 1.031074 -2.1767950 2.2454160
## 605c8bc6856f36729d2e5223 0.127285216 1.052688 -2.0006135 2.4268695
## 605f3f2d856f36729d2e5224 0.123993867 1.053939 -2.0184952 2.4661498
## 605f46c3856f36729d2e5225 1.122704336 1.552202 -0.8450688 5.0341688
## 60605337856f36729d2e5226 -1.017747281 1.489395 -5.0580122 0.9297134
## 60609ae6856f36729d2e5228 0.111090253 1.064487 -2.0841028 2.4787905
## 6061ce91856f36729d2e522e -0.035769736 1.040729 -2.1891035 2.1935713
## 6061f106856f36729d2e5231 1.107739334 1.563698 -0.8413139 5.2024412
## 6068ea9f856f36729d2e523e 1.087791296 1.523701 -0.8836985 4.9185203
## 6075ab05856f36729d2e5247 1.089195262 1.527408 -0.8296812 5.0528578Sampling plots
plot(documentation0, ask = FALSE)
Posterior predictive check
pp_check(documentation0, nsamples = 200, type = "bars")
Documentation1
We select the best performing model with one variable.
documentation1 <- brm(
"documentation ~ 1 + high_debt_version + group + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation1",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation1)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.97 0.77 0.04 2.86 1.00 932
## sd(muNone_Intercept) 1.35 0.98 0.04 3.75 1.01 689
## Tail_ESS
## sd(muIncorrect_Intercept) 1509
## sd(muNone_Intercept) 1448
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## muIncorrect_Intercept 0.43 0.69 -0.96 1.75 1.00
## muNone_Intercept 0.69 0.72 -0.79 2.05 1.00
## muIncorrect_high_debt_versionfalse 0.03 0.66 -1.27 1.33 1.00
## muIncorrect_groupconsultants -0.60 0.89 -2.35 1.15 1.00
## muIncorrect_groupfriends -0.17 0.79 -1.71 1.38 1.00
## muIncorrect_groupprofessionalMcontact -0.44 0.93 -2.23 1.39 1.00
## muIncorrect_groupstudents 0.59 0.76 -0.91 2.05 1.00
## muNone_high_debt_versionfalse 0.54 0.66 -0.73 1.88 1.00
## muNone_groupconsultants 0.96 0.85 -0.71 2.59 1.00
## muNone_groupfriends -0.10 0.80 -1.71 1.51 1.00
## muNone_groupprofessionalMcontact -0.10 0.94 -2.00 1.75 1.00
## muNone_groupstudents -0.04 0.79 -1.62 1.49 1.00
## Bulk_ESS Tail_ESS
## muIncorrect_Intercept 3060 2598
## muNone_Intercept 3136 1740
## muIncorrect_high_debt_versionfalse 4333 3245
## muIncorrect_groupconsultants 5029 3012
## muIncorrect_groupfriends 4473 2867
## muIncorrect_groupprofessionalMcontact 4802 2635
## muIncorrect_groupstudents 3966 2968
## muNone_high_debt_versionfalse 3926 2908
## muNone_groupconsultants 3718 2486
## muNone_groupfriends 3900 2704
## muNone_groupprofessionalMcontact 4746 2579
## muNone_groupstudents 3715 2595
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation1)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.6306495 1.1577765 -3.4844985 0.9388153
## 6033d90a5af2c702367b3a96 0.1921168 0.9287505 -1.5674312 2.5123360
## 6034fc165af2c702367b3a98 -0.5077010 1.1786269 -3.4048657 1.3277858
## 603500725af2c702367b3a99 0.8048654 1.2897482 -0.7889888 4.0963435
## 603f97625af2c702367b3a9d 0.7964224 1.2933628 -0.7491983 4.1176225
## 603fd5d95af2c702367b3a9e -0.4929869 1.1430599 -3.4919958 1.2328923
## 60409b7b5af2c702367b3a9f 0.7808253 1.3005092 -0.8164143 4.1959890
## 604b82b5a7718fbed181b336 -0.2336038 1.1112931 -3.0378915 1.8341830
## 6050c1bf856f36729d2e5218 0.9374294 1.3388951 -0.5767866 4.4519363
## 6050e1e7856f36729d2e5219 0.1971419 0.9596445 -1.6016715 2.4335223
## 6055fdc6856f36729d2e521b -0.4102454 1.1412424 -3.3948240 1.3488195
## 60589862856f36729d2e521f 0.1627261 0.9151046 -1.5837703 2.3830418
## 605afa3a856f36729d2e5222 -0.5008749 1.1687037 -3.4636487 1.2920895
## 605c8bc6856f36729d2e5223 0.3255948 0.9442163 -1.3000095 2.7076125
## 605f3f2d856f36729d2e5224 0.3145467 0.9887694 -1.4417235 2.6797683
## 605f46c3856f36729d2e5225 -0.4033429 1.0897531 -2.9733727 1.3568335
## 60605337856f36729d2e5226 0.1689644 0.8965671 -1.5581900 2.2454955
## 60609ae6856f36729d2e5228 0.3046351 0.9506905 -1.3852437 2.6894493
## 6061ce91856f36729d2e522e -0.5090801 1.1616754 -3.5081743 1.2249083
## 6061f106856f36729d2e5231 -0.3826653 1.0986590 -3.1205920 1.4682475
## 6068ea9f856f36729d2e523e -0.2708556 1.1597329 -3.1881682 1.7233010
## 6075ab05856f36729d2e5247 -0.2395347 1.1025047 -2.9395725 1.6348030
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.13747065 1.080145 -2.0393812 2.5362515
## 6033d90a5af2c702367b3a96 0.28090668 1.147366 -1.8696432 2.9346138
## 6034fc165af2c702367b3a98 1.24322759 1.611041 -0.7450943 5.2515163
## 603500725af2c702367b3a99 -0.80481298 1.529809 -4.5219347 1.3414593
## 603f97625af2c702367b3a9d -0.77279221 1.475958 -4.6239643 1.4417490
## 603fd5d95af2c702367b3a9e 1.23836436 1.611160 -0.7760658 5.3125558
## 60409b7b5af2c702367b3a9f -0.81828501 1.488723 -4.5958990 1.3535870
## 604b82b5a7718fbed181b336 0.80765357 1.518514 -1.2435117 4.8040460
## 6050c1bf856f36729d2e5218 -0.93656262 1.545261 -4.6051417 1.1725645
## 6050e1e7856f36729d2e5219 -1.02214089 1.509872 -4.7091335 0.9815069
## 6055fdc6856f36729d2e521b 1.13457646 1.587125 -0.8613241 5.2844875
## 60589862856f36729d2e521f -1.03810902 1.493961 -4.8889175 0.9482846
## 605afa3a856f36729d2e5222 -0.06069649 1.162165 -2.5213940 2.4905255
## 605c8bc6856f36729d2e5223 0.12770516 1.083060 -2.0898810 2.5477820
## 605f3f2d856f36729d2e5224 0.11646131 1.131229 -2.1600575 2.6348498
## 605f46c3856f36729d2e5225 1.12522468 1.580884 -0.7959631 5.2519745
## 60605337856f36729d2e5226 -1.05262888 1.553631 -4.9111313 0.9436859
## 60609ae6856f36729d2e5228 0.11109622 1.032156 -1.9857027 2.3695395
## 6061ce91856f36729d2e522e 0.01752046 1.066138 -2.2337413 2.3079593
## 6061f106856f36729d2e5231 1.10950111 1.602634 -0.8666377 5.1021900
## 6068ea9f856f36729d2e523e 0.86327907 1.565703 -1.2088132 4.8686805
## 6075ab05856f36729d2e5247 0.82573726 1.507768 -1.1825250 4.6364470Sampling plots
plot(documentation1, ask = FALSE)
Posterior predictive check
pp_check(documentation1, nsamples = 200, type = "bars")
Documentation2
We select the best performing model with two variables.
documentation2 <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation2",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation2)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.86 0.76 0.03 2.80 1.00 1072
## sd(muNone_Intercept) 1.40 0.98 0.06 3.66 1.01 946
## Tail_ESS
## sd(muIncorrect_Intercept) 2108
## sd(muNone_Intercept) 1935
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## muIncorrect_Intercept 0.41 0.79 -1.15 1.99 1.00
## muNone_Intercept 0.64 0.79 -0.89 2.17 1.00
## muIncorrect_high_debt_versionfalse 0.03 0.68 -1.32 1.32 1.00
## muIncorrect_groupconsultants -0.59 0.92 -2.39 1.17 1.00
## muIncorrect_groupfriends -0.28 0.81 -1.90 1.29 1.00
## muIncorrect_groupprofessionalMcontact -0.34 0.92 -2.14 1.40 1.00
## muIncorrect_groupstudents 0.64 0.78 -0.92 2.10 1.00
## muIncorrect_work_domainAutomotive -0.10 0.79 -1.63 1.41 1.00
## muIncorrect_work_domainDevops 0.72 0.89 -1.04 2.44 1.00
## muIncorrect_work_domainEmbedded 0.02 0.88 -1.68 1.77 1.00
## muIncorrect_work_domainFinance -0.85 0.90 -2.59 0.89 1.00
## muIncorrect_work_domainMixed 0.10 0.91 -1.70 1.87 1.00
## muIncorrect_work_domainMusic 0.24 0.95 -1.60 2.09 1.00
## muIncorrect_work_domainNone 0.72 0.80 -0.85 2.29 1.00
## muIncorrect_work_domainRetail -0.19 0.95 -2.04 1.68 1.00
## muIncorrect_work_domainTelecom -0.26 0.95 -2.13 1.56 1.00
## muIncorrect_work_domainWeb -0.23 0.84 -1.86 1.46 1.00
## muNone_high_debt_versionfalse 0.58 0.66 -0.65 1.91 1.00
## muNone_groupconsultants 0.93 0.87 -0.75 2.59 1.00
## muNone_groupfriends -0.05 0.82 -1.65 1.53 1.00
## muNone_groupprofessionalMcontact -0.10 0.90 -1.86 1.67 1.00
## muNone_groupstudents -0.10 0.78 -1.63 1.42 1.00
## muNone_work_domainAutomotive 0.32 0.79 -1.23 1.84 1.00
## muNone_work_domainDevops -0.41 0.94 -2.24 1.44 1.00
## muNone_work_domainEmbedded -0.53 0.95 -2.35 1.34 1.00
## muNone_work_domainFinance -0.12 0.89 -1.84 1.62 1.00
## muNone_work_domainMixed -0.54 0.96 -2.38 1.37 1.00
## muNone_work_domainMusic 0.03 0.93 -1.78 1.86 1.00
## muNone_work_domainNone 0.00 0.84 -1.64 1.67 1.00
## muNone_work_domainRetail 0.30 0.94 -1.62 2.17 1.00
## muNone_work_domainTelecom 0.45 0.95 -1.33 2.36 1.00
## muNone_work_domainWeb 0.21 0.83 -1.41 1.84 1.00
## Bulk_ESS Tail_ESS
## muIncorrect_Intercept 3818 2631
## muNone_Intercept 3561 2961
## muIncorrect_high_debt_versionfalse 4742 2651
## muIncorrect_groupconsultants 5557 2501
## muIncorrect_groupfriends 4486 2552
## muIncorrect_groupprofessionalMcontact 5601 3193
## muIncorrect_groupstudents 4913 3196
## muIncorrect_work_domainAutomotive 4497 2970
## muIncorrect_work_domainDevops 4996 3109
## muIncorrect_work_domainEmbedded 5297 2972
## muIncorrect_work_domainFinance 4922 3381
## muIncorrect_work_domainMixed 5439 2824
## muIncorrect_work_domainMusic 5087 2660
## muIncorrect_work_domainNone 4319 3109
## muIncorrect_work_domainRetail 5498 2873
## muIncorrect_work_domainTelecom 5089 2722
## muIncorrect_work_domainWeb 5658 2945
## muNone_high_debt_versionfalse 5524 2744
## muNone_groupconsultants 4101 2647
## muNone_groupfriends 4445 3164
## muNone_groupprofessionalMcontact 5492 2621
## muNone_groupstudents 3984 2894
## muNone_work_domainAutomotive 4364 3108
## muNone_work_domainDevops 5220 2931
## muNone_work_domainEmbedded 5131 3148
## muNone_work_domainFinance 4972 2799
## muNone_work_domainMixed 5111 2783
## muNone_work_domainMusic 5420 2998
## muNone_work_domainNone 4588 3057
## muNone_work_domainRetail 5942 3220
## muNone_work_domainTelecom 5544 2841
## muNone_work_domainWeb 4219 3057
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation2)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.4534212 1.0829016 -3.2691945 1.141082
## 6033d90a5af2c702367b3a96 0.1943614 0.9023927 -1.4070667 2.437164
## 6034fc165af2c702367b3a98 -0.3850312 1.0835746 -3.2039155 1.254079
## 603500725af2c702367b3a99 0.5680675 1.2277265 -0.9088084 3.814232
## 603f97625af2c702367b3a9d 0.7432528 1.2918492 -0.7168493 4.149034
## 603fd5d95af2c702367b3a9e -0.3805202 1.0805379 -3.0853310 1.238818
## 60409b7b5af2c702367b3a9f 0.5600197 1.2439574 -0.9583096 3.778486
## 604b82b5a7718fbed181b336 -0.1877565 1.1077771 -2.9695742 1.855451
## 6050c1bf856f36729d2e5218 0.6839102 1.2481341 -0.7521494 4.184574
## 6050e1e7856f36729d2e5219 0.1360999 0.8941909 -1.6869162 2.387112
## 6055fdc6856f36729d2e521b -0.2451385 1.1152257 -2.9141895 1.566186
## 60589862856f36729d2e521f 0.1064147 0.8425793 -1.5320383 2.087460
## 605afa3a856f36729d2e5222 -0.3403956 1.1377229 -3.2733290 1.428917
## 605c8bc6856f36729d2e5223 0.2490224 0.9536131 -1.4075382 2.555925
## 605f3f2d856f36729d2e5224 0.2995238 0.9057337 -1.1772752 2.570122
## 605f46c3856f36729d2e5225 -0.2926415 1.0375318 -3.0644132 1.423619
## 60605337856f36729d2e5226 0.1812055 0.8965576 -1.5183507 2.395896
## 60609ae6856f36729d2e5228 0.1527327 0.8877543 -1.5325027 2.454207
## 6061ce91856f36729d2e522e -0.3742271 1.0385842 -3.1719153 1.221211
## 6061f106856f36729d2e5231 -0.4040018 1.1031205 -3.1803425 1.249233
## 6068ea9f856f36729d2e523e -0.1978428 1.1000319 -2.9436030 1.698238
## 6075ab05856f36729d2e5247 -0.2041647 1.0238586 -2.6874982 1.652864
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.042665847 1.178389 -2.3956748 2.5755293
## 6033d90a5af2c702367b3a96 0.164635965 1.166899 -2.2210808 2.8762698
## 6034fc165af2c702367b3a98 1.222977147 1.620293 -0.8091479 5.4458228
## 603500725af2c702367b3a99 -0.748358788 1.525755 -4.5683090 1.5654520
## 603f97625af2c702367b3a9d -0.973034280 1.594516 -4.8699568 1.2263350
## 603fd5d95af2c702367b3a9e 1.240577435 1.662767 -0.9370886 5.3483670
## 60409b7b5af2c702367b3a9f -0.743264480 1.531772 -4.6668555 1.5453675
## 604b82b5a7718fbed181b336 0.829734868 1.632665 -1.5267792 5.0680540
## 6050c1bf856f36729d2e5218 -0.798161817 1.542274 -4.6767272 1.5169380
## 6050e1e7856f36729d2e5219 -0.971730513 1.635346 -5.1109132 1.3508853
## 6055fdc6856f36729d2e521b 1.048737175 1.619914 -1.1240443 5.2743293
## 60589862856f36729d2e521f -0.980485552 1.550714 -4.9627380 1.2048525
## 605afa3a856f36729d2e5222 -0.102048555 1.186491 -2.6893920 2.4496848
## 605c8bc6856f36729d2e5223 0.150708354 1.210744 -2.2521562 2.8135450
## 605f3f2d856f36729d2e5224 -0.003011347 1.132811 -2.4003835 2.4006725
## 605f46c3856f36729d2e5225 1.112843973 1.641824 -0.9679131 5.4774645
## 60605337856f36729d2e5226 -1.214732302 1.590013 -5.1261853 0.8335935
## 60609ae6856f36729d2e5228 0.274482194 1.174110 -1.9688735 3.0117148
## 6061ce91856f36729d2e522e -0.102063205 1.200972 -2.6849300 2.4152135
## 6061f106856f36729d2e5231 1.298494011 1.709837 -0.8278773 5.5831083
## 6068ea9f856f36729d2e523e 0.822125686 1.601245 -1.4582755 4.7572848
## 6075ab05856f36729d2e5247 0.860489905 1.542336 -1.2506522 4.7549928Sampling plots
plot(documentation2, ask = FALSE)
Posterior predictive check
pp_check(documentation2, nsamples = 200, type = "bars")
Documentation3
We select the best performing model with three variables.
documentation3 <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation3",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation3)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.83 0.75 0.03 2.67 1.01 1246
## sd(muNone_Intercept) 1.07 0.88 0.04 3.25 1.01 887
## Tail_ESS
## sd(muIncorrect_Intercept) 2296
## sd(muNone_Intercept) 1891
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI
## muIncorrect_Intercept 0.49 0.81 -1.07 2.02
## muNone_Intercept 0.36 0.82 -1.24 1.96
## muIncorrect_high_debt_versionfalse 0.02 0.70 -1.39 1.37
## muIncorrect_groupconsultants -0.56 0.91 -2.35 1.19
## muIncorrect_groupfriends -0.31 0.80 -1.95 1.26
## muIncorrect_groupprofessionalMcontact -0.36 0.95 -2.26 1.52
## muIncorrect_groupstudents 0.82 0.78 -0.69 2.29
## muIncorrect_work_domainAutomotive -0.09 0.78 -1.65 1.42
## muIncorrect_work_domainDevops 0.71 0.92 -1.10 2.46
## muIncorrect_work_domainEmbedded 0.02 0.90 -1.74 1.79
## muIncorrect_work_domainFinance -0.86 0.90 -2.64 0.94
## muIncorrect_work_domainMixed 0.08 0.91 -1.73 1.83
## muIncorrect_work_domainMusic 0.18 0.95 -1.72 2.03
## muIncorrect_work_domainNone 0.88 0.84 -0.76 2.53
## muIncorrect_work_domainRetail -0.11 0.96 -1.99 1.79
## muIncorrect_work_domainTelecom -0.28 0.97 -2.15 1.63
## muIncorrect_work_domainWeb -0.27 0.83 -1.91 1.36
## muIncorrect_workplace_peer_reviewfalse -0.36 0.79 -1.91 1.23
## muNone_high_debt_versionfalse 0.57 0.68 -0.75 1.94
## muNone_groupconsultants 0.88 0.87 -0.82 2.61
## muNone_groupfriends 0.03 0.81 -1.56 1.61
## muNone_groupprofessionalMcontact -0.03 0.92 -1.86 1.77
## muNone_groupstudents -0.41 0.81 -2.00 1.17
## muNone_work_domainAutomotive 0.35 0.77 -1.12 1.87
## muNone_work_domainDevops -0.43 0.94 -2.28 1.44
## muNone_work_domainEmbedded -0.51 0.91 -2.28 1.31
## muNone_work_domainFinance -0.18 0.84 -1.84 1.43
## muNone_work_domainMixed -0.55 0.93 -2.30 1.29
## muNone_work_domainMusic 0.11 0.92 -1.64 1.91
## muNone_work_domainNone -0.28 0.85 -1.97 1.44
## muNone_work_domainRetail 0.23 0.96 -1.62 2.11
## muNone_work_domainTelecom 0.54 0.93 -1.29 2.31
## muNone_work_domainWeb 0.21 0.81 -1.38 1.85
## muNone_workplace_peer_reviewfalse 1.00 0.79 -0.57 2.54
## Rhat Bulk_ESS Tail_ESS
## muIncorrect_Intercept 1.00 4427 2897
## muNone_Intercept 1.00 4543 2917
## muIncorrect_high_debt_versionfalse 1.00 4498 2707
## muIncorrect_groupconsultants 1.00 5946 3049
## muIncorrect_groupfriends 1.00 5347 2943
## muIncorrect_groupprofessionalMcontact 1.00 5566 2531
## muIncorrect_groupstudents 1.00 4575 2872
## muIncorrect_work_domainAutomotive 1.00 4948 2887
## muIncorrect_work_domainDevops 1.01 5535 2611
## muIncorrect_work_domainEmbedded 1.00 5255 3153
## muIncorrect_work_domainFinance 1.00 5220 3321
## muIncorrect_work_domainMixed 1.00 5603 3018
## muIncorrect_work_domainMusic 1.00 5703 3026
## muIncorrect_work_domainNone 1.00 4566 2680
## muIncorrect_work_domainRetail 1.00 5589 3096
## muIncorrect_work_domainTelecom 1.00 5668 2752
## muIncorrect_work_domainWeb 1.00 4755 2515
## muIncorrect_workplace_peer_reviewfalse 1.00 4489 2503
## muNone_high_debt_versionfalse 1.00 4647 2932
## muNone_groupconsultants 1.00 4530 2513
## muNone_groupfriends 1.00 4749 2852
## muNone_groupprofessionalMcontact 1.00 6501 2901
## muNone_groupstudents 1.00 4200 3018
## muNone_work_domainAutomotive 1.00 4564 3365
## muNone_work_domainDevops 1.00 5948 2921
## muNone_work_domainEmbedded 1.00 5696 3499
## muNone_work_domainFinance 1.00 4941 2861
## muNone_work_domainMixed 1.00 5913 2455
## muNone_work_domainMusic 1.00 4881 2890
## muNone_work_domainNone 1.00 5151 2929
## muNone_work_domainRetail 1.00 6808 3262
## muNone_work_domainTelecom 1.00 4779 2946
## muNone_work_domainWeb 1.00 5198 2800
## muNone_workplace_peer_reviewfalse 1.00 4168 2898
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation3)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.39708196 1.1061049 -3.4326737 1.221777
## 6033d90a5af2c702367b3a96 0.26507172 0.9462551 -1.3616410 2.632921
## 6034fc165af2c702367b3a98 -0.33934241 1.0442458 -3.0568600 1.311979
## 603500725af2c702367b3a99 0.56276801 1.1860076 -0.8731656 3.931903
## 603f97625af2c702367b3a9d 0.63948100 1.2045133 -0.7551168 3.973976
## 603fd5d95af2c702367b3a9e -0.35340958 1.0502143 -3.0852590 1.285347
## 60409b7b5af2c702367b3a9f 0.55234223 1.1574309 -0.8581536 3.852218
## 604b82b5a7718fbed181b336 -0.14646282 1.0438562 -2.7176965 1.821365
## 6050c1bf856f36729d2e5218 0.64816357 1.2251630 -0.8158113 3.972073
## 6050e1e7856f36729d2e5219 0.11780643 0.8476208 -1.5799020 2.218550
## 6055fdc6856f36729d2e521b -0.27065556 1.0230010 -2.9722515 1.404800
## 60589862856f36729d2e521f 0.06724852 0.8587797 -1.7205200 2.032314
## 605afa3a856f36729d2e5222 -0.35823576 1.0789389 -3.1439822 1.257902
## 605c8bc6856f36729d2e5223 0.22002331 0.9269878 -1.4248437 2.594944
## 605f3f2d856f36729d2e5224 0.23960297 0.8895824 -1.3489677 2.460739
## 605f46c3856f36729d2e5225 -0.33410443 1.0618253 -3.0877453 1.296548
## 60605337856f36729d2e5226 0.18592123 0.8454625 -1.4061927 2.266253
## 60609ae6856f36729d2e5228 0.20926114 0.8877203 -1.4081097 2.451157
## 6061ce91856f36729d2e522e -0.38592822 1.0699501 -3.2054700 1.135786
## 6061f106856f36729d2e5231 -0.36071827 1.0580383 -3.1640557 1.268992
## 6068ea9f856f36729d2e523e -0.16325942 1.0586185 -2.6662827 1.702343
## 6075ab05856f36729d2e5247 -0.23810238 1.0132598 -2.7685335 1.537150
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.06602263 0.9850132 -2.2718768 2.1093475
## 6033d90a5af2c702367b3a96 0.03454741 1.0070470 -1.9752765 2.3379255
## 6034fc165af2c702367b3a98 0.82676675 1.4018771 -0.8396074 4.6437900
## 603500725af2c702367b3a99 -0.52848640 1.3017088 -3.9043965 1.4743428
## 603f97625af2c702367b3a9d -0.55980427 1.3088170 -4.0550267 1.2888773
## 603fd5d95af2c702367b3a9e 0.80896073 1.4145747 -0.9488677 4.5379190
## 60409b7b5af2c702367b3a9f -0.53738045 1.2594012 -3.8211242 1.3407145
## 604b82b5a7718fbed181b336 0.46519681 1.2762613 -1.5044755 3.9338013
## 6050c1bf856f36729d2e5218 -0.54484006 1.2982153 -3.8711760 1.3998960
## 6050e1e7856f36729d2e5219 -0.67237022 1.3444842 -4.3530898 1.0995223
## 6055fdc6856f36729d2e521b 0.80212936 1.4547033 -0.9958702 4.6600005
## 60589862856f36729d2e521f -0.64240670 1.3075722 -4.0338025 1.1774598
## 605afa3a856f36729d2e5222 0.01506730 0.9945918 -2.2142828 2.1882155
## 605c8bc6856f36729d2e5223 0.18172069 1.0460513 -1.9283183 2.6834608
## 605f3f2d856f36729d2e5224 0.08512667 0.9757935 -1.8440365 2.3526025
## 605f46c3856f36729d2e5225 0.87449040 1.4369347 -0.8354989 4.5857623
## 60605337856f36729d2e5226 -0.77772258 1.2842776 -4.1642970 0.8371869
## 60609ae6856f36729d2e5228 0.10095417 0.9962745 -1.8364300 2.4090198
## 6061ce91856f36729d2e522e -0.04404011 0.9599969 -2.2318410 1.9868150
## 6061f106856f36729d2e5231 0.86343740 1.4598672 -0.8408385 4.6488940
## 6068ea9f856f36729d2e523e 0.47721448 1.3279067 -1.5211920 4.0237125
## 6075ab05856f36729d2e5247 0.72439494 1.3662365 -1.0838340 4.4568768Sampling plots
plot(documentation3, ask = FALSE)
Posterior predictive check
pp_check(documentation3, nsamples = 200, type = "bars")
Documentation4
We select the best performing model with four variables.
documentation4 <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + workplace_td_tracking + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation4",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation4)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + workplace_td_tracking + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.86 0.75 0.03 2.78 1.00 1219
## sd(muNone_Intercept) 1.01 0.85 0.03 3.14 1.00 939
## Tail_ESS
## sd(muIncorrect_Intercept) 2023
## sd(muNone_Intercept) 1680
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI
## muIncorrect_Intercept 0.30 0.93 -1.55 2.08
## muNone_Intercept 0.61 0.89 -1.14 2.43
## muIncorrect_high_debt_versionfalse 0.02 0.67 -1.31 1.30
## muIncorrect_groupconsultants -0.49 0.93 -2.28 1.29
## muIncorrect_groupfriends -0.37 0.79 -1.93 1.16
## muIncorrect_groupprofessionalMcontact -0.33 0.94 -2.17 1.54
## muIncorrect_groupstudents 0.75 0.82 -0.89 2.39
## muIncorrect_work_domainAutomotive -0.11 0.79 -1.65 1.46
## muIncorrect_work_domainDevops 0.68 0.94 -1.16 2.56
## muIncorrect_work_domainEmbedded 0.09 0.92 -1.70 1.90
## muIncorrect_work_domainFinance -0.82 0.90 -2.58 0.98
## muIncorrect_work_domainMixed 0.05 0.93 -1.78 1.82
## muIncorrect_work_domainMusic 0.24 0.91 -1.53 2.03
## muIncorrect_work_domainNone 0.82 0.85 -0.86 2.47
## muIncorrect_work_domainRetail -0.12 0.98 -2.03 1.87
## muIncorrect_work_domainTelecom -0.25 0.96 -2.16 1.65
## muIncorrect_work_domainWeb -0.32 0.82 -1.90 1.27
## muIncorrect_workplace_peer_reviewfalse -0.40 0.78 -1.89 1.18
## muIncorrect_workplace_td_trackingfalse 0.37 0.81 -1.18 1.95
## muNone_high_debt_versionfalse 0.57 0.66 -0.70 1.88
## muNone_groupconsultants 0.80 0.87 -0.95 2.53
## muNone_groupfriends 0.13 0.79 -1.41 1.68
## muNone_groupprofessionalMcontact -0.07 0.91 -1.84 1.76
## muNone_groupstudents -0.28 0.80 -1.83 1.29
## muNone_work_domainAutomotive 0.44 0.80 -1.08 1.98
## muNone_work_domainDevops -0.41 0.90 -2.21 1.37
## muNone_work_domainEmbedded -0.57 0.93 -2.38 1.26
## muNone_work_domainFinance -0.21 0.87 -1.95 1.44
## muNone_work_domainMixed -0.52 0.93 -2.33 1.32
## muNone_work_domainMusic 0.04 0.92 -1.76 1.78
## muNone_work_domainNone -0.20 0.85 -1.84 1.44
## muNone_work_domainRetail 0.19 0.95 -1.67 2.02
## muNone_work_domainTelecom 0.48 0.95 -1.35 2.35
## muNone_work_domainWeb 0.27 0.82 -1.37 1.85
## muNone_workplace_peer_reviewfalse 1.11 0.79 -0.45 2.67
## muNone_workplace_td_trackingfalse -0.56 0.78 -2.08 1.00
## Rhat Bulk_ESS Tail_ESS
## muIncorrect_Intercept 1.00 3980 2859
## muNone_Intercept 1.00 4447 2966
## muIncorrect_high_debt_versionfalse 1.00 4661 3071
## muIncorrect_groupconsultants 1.00 5373 3020
## muIncorrect_groupfriends 1.00 3942 3013
## muIncorrect_groupprofessionalMcontact 1.00 4581 3134
## muIncorrect_groupstudents 1.00 4136 3148
## muIncorrect_work_domainAutomotive 1.00 4288 2924
## muIncorrect_work_domainDevops 1.00 5134 2657
## muIncorrect_work_domainEmbedded 1.00 5137 3084
## muIncorrect_work_domainFinance 1.00 4243 2776
## muIncorrect_work_domainMixed 1.00 5305 3213
## muIncorrect_work_domainMusic 1.00 4895 3234
## muIncorrect_work_domainNone 1.00 4418 2977
## muIncorrect_work_domainRetail 1.00 5103 2922
## muIncorrect_work_domainTelecom 1.00 5574 2840
## muIncorrect_work_domainWeb 1.00 4709 3415
## muIncorrect_workplace_peer_reviewfalse 1.00 3803 2966
## muIncorrect_workplace_td_trackingfalse 1.00 4088 2858
## muNone_high_debt_versionfalse 1.00 4562 2925
## muNone_groupconsultants 1.00 4466 3058
## muNone_groupfriends 1.00 4823 2773
## muNone_groupprofessionalMcontact 1.00 4535 3281
## muNone_groupstudents 1.00 4474 2824
## muNone_work_domainAutomotive 1.00 4185 3189
## muNone_work_domainDevops 1.00 5100 3113
## muNone_work_domainEmbedded 1.00 4367 2896
## muNone_work_domainFinance 1.00 5035 2834
## muNone_work_domainMixed 1.00 4654 2848
## muNone_work_domainMusic 1.00 4835 3090
## muNone_work_domainNone 1.00 4312 2730
## muNone_work_domainRetail 1.00 4806 2570
## muNone_work_domainTelecom 1.00 5279 2551
## muNone_work_domainWeb 1.00 4521 3230
## muNone_workplace_peer_reviewfalse 1.00 3671 3356
## muNone_workplace_td_trackingfalse 1.00 4641 3057
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation4)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.45459555 1.0945679 -3.4443770 1.148222
## 6033d90a5af2c702367b3a96 0.24614102 0.9132162 -1.3245437 2.489270
## 6034fc165af2c702367b3a98 -0.37692357 1.0693403 -3.2382082 1.237913
## 603500725af2c702367b3a99 0.60339417 1.2436344 -0.9180882 3.886052
## 603f97625af2c702367b3a9d 0.64422814 1.1831608 -0.8173545 3.934954
## 603fd5d95af2c702367b3a9e -0.38259310 1.0407363 -3.1465842 1.197881
## 60409b7b5af2c702367b3a9f 0.58652178 1.2087131 -0.8799915 3.887938
## 604b82b5a7718fbed181b336 -0.14709605 1.0609918 -2.8226807 1.859086
## 6050c1bf856f36729d2e5218 0.66015479 1.2521519 -0.8210056 4.158218
## 6050e1e7856f36729d2e5219 0.08954383 0.8735408 -1.7014070 2.149240
## 6055fdc6856f36729d2e521b -0.26030082 1.0730859 -3.0363352 1.471033
## 60589862856f36729d2e521f 0.12971424 0.9023809 -1.7346360 2.270302
## 605afa3a856f36729d2e5222 -0.36611220 1.0775683 -3.1968007 1.244295
## 605c8bc6856f36729d2e5223 0.23516761 0.9358227 -1.4750237 2.556151
## 605f3f2d856f36729d2e5224 0.35288371 0.9454869 -1.1751800 2.789740
## 605f46c3856f36729d2e5225 -0.44437170 1.1456644 -3.2897155 1.250757
## 60605337856f36729d2e5226 0.10048333 0.8618785 -1.6200157 2.111144
## 60609ae6856f36729d2e5228 0.17019417 0.8950740 -1.4011060 2.408759
## 6061ce91856f36729d2e522e -0.40426916 1.1109643 -3.3397097 1.210615
## 6061f106856f36729d2e5231 -0.35883835 1.0437334 -2.8153545 1.283712
## 6068ea9f856f36729d2e523e -0.13742814 1.0775320 -2.7379265 1.923635
## 6075ab05856f36729d2e5247 -0.23801458 1.0280420 -2.7828342 1.616494
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.01701530 0.9802080 -2.2130690 2.1762808
## 6033d90a5af2c702367b3a96 0.03787395 0.9475342 -1.9999718 2.2560133
## 6034fc165af2c702367b3a98 0.73362599 1.3017724 -0.8783625 4.2739338
## 603500725af2c702367b3a99 -0.46896976 1.2810703 -3.7367415 1.4665093
## 603f97625af2c702367b3a9d -0.47529588 1.2464926 -3.6795160 1.3432618
## 603fd5d95af2c702367b3a9e 0.74217912 1.3302535 -0.9409412 4.4005418
## 60409b7b5af2c702367b3a9f -0.45918587 1.1886477 -3.6316238 1.3727195
## 604b82b5a7718fbed181b336 0.38585757 1.2310317 -1.4984743 3.6042928
## 6050c1bf856f36729d2e5218 -0.44797792 1.2373189 -3.7928557 1.4876753
## 6050e1e7856f36729d2e5219 -0.54994209 1.2103237 -3.8402582 1.1565843
## 6055fdc6856f36729d2e521b 0.66903313 1.3341724 -1.0774372 4.1251285
## 60589862856f36729d2e521f -0.64757281 1.3149641 -4.1175980 1.0619975
## 605afa3a856f36729d2e5222 -0.05153409 1.0186736 -2.3523410 2.1690940
## 605c8bc6856f36729d2e5223 0.11087886 0.9745474 -1.8046427 2.3609055
## 605f3f2d856f36729d2e5224 -0.02238921 0.9710114 -2.2080910 2.1816415
## 605f46c3856f36729d2e5225 0.85472267 1.3447751 -0.7229780 4.4307283
## 60605337856f36729d2e5226 -0.70635059 1.2642257 -4.1966920 0.8846255
## 60609ae6856f36729d2e5228 0.13096146 0.9989253 -1.8322915 2.5006245
## 6061ce91856f36729d2e522e 0.01719109 0.9199825 -1.8938495 2.1526118
## 6061f106856f36729d2e5231 0.80297166 1.3501398 -0.8269275 4.2818235
## 6068ea9f856f36729d2e523e 0.41497164 1.2625814 -1.5661140 3.7170620
## 6075ab05856f36729d2e5247 0.60274528 1.2793265 -1.1022975 3.9465653Sampling plots
plot(documentation4, ask = FALSE)
Posterior predictive check
pp_check(documentation4, nsamples = 200, type = "bars")
Documentation5
We select the best performing model with five variables.
documentation5 <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + workplace_td_tracking + workplace_pair_programming + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.both_completed),
file = "fits/documentation5",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation5)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + workplace_td_tracking + workplace_pair_programming + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.79 0.70 0.02 2.55 1.00 1025
## sd(muNone_Intercept) 1.02 0.86 0.03 3.10 1.01 912
## Tail_ESS
## sd(muIncorrect_Intercept) 2053
## sd(muNone_Intercept) 1846
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI
## muIncorrect_Intercept -0.19 1.12 -2.44
## muNone_Intercept 0.88 1.08 -1.22
## muIncorrect_high_debt_versionfalse 0.02 0.68 -1.30
## muIncorrect_groupconsultants -0.50 0.93 -2.32
## muIncorrect_groupfriends -0.36 0.83 -2.00
## muIncorrect_groupprofessionalMcontact -0.36 0.96 -2.20
## muIncorrect_groupstudents 0.70 0.80 -0.85
## muIncorrect_work_domainAutomotive -0.02 0.79 -1.51
## muIncorrect_work_domainDevops 0.63 0.92 -1.15
## muIncorrect_work_domainEmbedded 0.04 0.90 -1.71
## muIncorrect_work_domainFinance -0.85 0.87 -2.54
## muIncorrect_work_domainMixed 0.17 0.92 -1.64
## muIncorrect_work_domainMusic 0.20 0.93 -1.62
## muIncorrect_work_domainNone 0.79 0.86 -0.93
## muIncorrect_work_domainRetail -0.14 0.97 -2.04
## muIncorrect_work_domainTelecom -0.21 0.96 -2.10
## muIncorrect_work_domainWeb -0.38 0.84 -1.99
## muIncorrect_workplace_peer_reviewfalse -0.48 0.80 -2.06
## muIncorrect_workplace_td_trackingfalse 0.45 0.81 -1.10
## muIncorrect_workplace_pair_programmingfalse 0.62 0.83 -1.05
## muNone_high_debt_versionfalse 0.58 0.68 -0.73
## muNone_groupconsultants 0.79 0.90 -0.96
## muNone_groupfriends 0.10 0.81 -1.51
## muNone_groupprofessionalMcontact -0.09 0.93 -1.90
## muNone_groupstudents -0.29 0.80 -1.86
## muNone_work_domainAutomotive 0.35 0.79 -1.19
## muNone_work_domainDevops -0.34 0.96 -2.29
## muNone_work_domainEmbedded -0.56 0.94 -2.43
## muNone_work_domainFinance -0.16 0.88 -1.89
## muNone_work_domainMixed -0.56 0.96 -2.50
## muNone_work_domainMusic 0.05 0.92 -1.79
## muNone_work_domainNone -0.14 0.83 -1.76
## muNone_work_domainRetail 0.19 0.95 -1.64
## muNone_work_domainTelecom 0.42 0.94 -1.37
## muNone_work_domainWeb 0.30 0.81 -1.26
## muNone_workplace_peer_reviewfalse 1.16 0.80 -0.44
## muNone_workplace_td_trackingfalse -0.63 0.79 -2.22
## muNone_workplace_pair_programmingfalse -0.34 0.79 -1.84
## u-95% CI Rhat Bulk_ESS Tail_ESS
## muIncorrect_Intercept 1.93 1.00 4242 3065
## muNone_Intercept 3.01 1.00 3951 2901
## muIncorrect_high_debt_versionfalse 1.33 1.00 4094 2920
## muIncorrect_groupconsultants 1.34 1.00 4169 3079
## muIncorrect_groupfriends 1.25 1.00 4995 3133
## muIncorrect_groupprofessionalMcontact 1.50 1.00 5111 2885
## muIncorrect_groupstudents 2.28 1.00 3475 2836
## muIncorrect_work_domainAutomotive 1.55 1.00 4477 3370
## muIncorrect_work_domainDevops 2.46 1.00 4508 3085
## muIncorrect_work_domainEmbedded 1.81 1.00 5047 3454
## muIncorrect_work_domainFinance 0.89 1.00 5002 3443
## muIncorrect_work_domainMixed 2.00 1.00 4989 2642
## muIncorrect_work_domainMusic 2.05 1.00 5076 3088
## muIncorrect_work_domainNone 2.51 1.00 3931 3172
## muIncorrect_work_domainRetail 1.76 1.00 4294 3006
## muIncorrect_work_domainTelecom 1.62 1.00 4481 3103
## muIncorrect_work_domainWeb 1.32 1.00 4090 2942
## muIncorrect_workplace_peer_reviewfalse 1.10 1.00 3908 3133
## muIncorrect_workplace_td_trackingfalse 2.04 1.00 4241 3328
## muIncorrect_workplace_pair_programmingfalse 2.21 1.00 4396 3236
## muNone_high_debt_versionfalse 1.91 1.00 4479 2597
## muNone_groupconsultants 2.61 1.00 4370 2455
## muNone_groupfriends 1.69 1.00 4271 2862
## muNone_groupprofessionalMcontact 1.74 1.00 5014 2662
## muNone_groupstudents 1.29 1.00 3947 3273
## muNone_work_domainAutomotive 1.89 1.00 3937 3077
## muNone_work_domainDevops 1.53 1.00 4487 2857
## muNone_work_domainEmbedded 1.24 1.00 4425 2591
## muNone_work_domainFinance 1.56 1.00 4545 3074
## muNone_work_domainMixed 1.31 1.00 3700 2652
## muNone_work_domainMusic 1.81 1.00 5080 3015
## muNone_work_domainNone 1.48 1.00 4133 3256
## muNone_work_domainRetail 2.01 1.00 5627 3219
## muNone_work_domainTelecom 2.23 1.00 5134 3059
## muNone_work_domainWeb 1.92 1.00 4207 2826
## muNone_workplace_peer_reviewfalse 2.70 1.00 3013 2722
## muNone_workplace_td_trackingfalse 0.90 1.00 4432 3238
## muNone_workplace_pair_programmingfalse 1.20 1.00 4303 3190
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation5)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.40858347 1.0843811 -3.2259567 1.131172
## 6033d90a5af2c702367b3a96 0.18420564 0.8485479 -1.4259585 2.384032
## 6034fc165af2c702367b3a98 -0.34047065 0.9847740 -3.0313420 1.181284
## 603500725af2c702367b3a99 0.48827992 1.0499185 -0.8356684 3.403754
## 603f97625af2c702367b3a9d 0.55011653 1.0992946 -0.8327003 3.602522
## 603fd5d95af2c702367b3a9e -0.26433151 1.0034062 -2.8386742 1.393835
## 60409b7b5af2c702367b3a9f 0.48516158 1.0740207 -0.8592317 3.440226
## 604b82b5a7718fbed181b336 -0.06441518 0.9620942 -2.2716805 1.956163
## 6050c1bf856f36729d2e5218 0.55282653 1.1191550 -0.8533992 3.723779
## 6050e1e7856f36729d2e5219 0.15271554 0.8513114 -1.4569860 2.300560
## 6055fdc6856f36729d2e521b -0.13842202 0.9541326 -2.3411638 1.630875
## 60589862856f36729d2e521f 0.09487108 0.8324820 -1.5729533 2.111632
## 605afa3a856f36729d2e5222 -0.29625964 1.0021585 -2.8853010 1.245209
## 605c8bc6856f36729d2e5223 0.21877143 0.9131045 -1.3429940 2.607244
## 605f3f2d856f36729d2e5224 0.26818138 0.9033592 -1.2479377 2.598723
## 605f46c3856f36729d2e5225 -0.30297371 0.9886001 -3.0203815 1.314907
## 60605337856f36729d2e5226 0.18637713 0.8502602 -1.3681750 2.215234
## 60609ae6856f36729d2e5228 0.12464665 0.8148810 -1.5172295 2.102443
## 6061ce91856f36729d2e522e -0.38746129 1.0518772 -3.1339002 1.052015
## 6061f106856f36729d2e5231 -0.35108761 0.9945136 -2.9886930 1.162314
## 6068ea9f856f36729d2e523e -0.10668062 0.9645467 -2.4643705 1.827181
## 6075ab05856f36729d2e5247 -0.23999858 1.0124046 -2.8544155 1.515558
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.01344447 0.9695053 -2.1293567 2.1592085
## 6033d90a5af2c702367b3a96 0.08523177 0.9688481 -1.8598762 2.3479613
## 6034fc165af2c702367b3a98 0.78541690 1.3698472 -0.8585884 4.3387583
## 603500725af2c702367b3a99 -0.49074240 1.2778814 -3.9486590 1.3558128
## 603f97625af2c702367b3a9d -0.46452799 1.2678886 -3.6774052 1.3988540
## 603fd5d95af2c702367b3a9e 0.66474574 1.3092774 -1.0398092 4.0152150
## 60409b7b5af2c702367b3a9f -0.46694399 1.2868252 -3.9961310 1.4808670
## 604b82b5a7718fbed181b336 0.34897821 1.2120489 -1.6253632 3.5691415
## 6050c1bf856f36729d2e5218 -0.45874246 1.2401273 -3.7193345 1.3600990
## 6050e1e7856f36729d2e5219 -0.62211095 1.3072679 -4.0215437 1.1795875
## 6055fdc6856f36729d2e521b 0.59853572 1.2948234 -1.1961992 4.1445710
## 60589862856f36729d2e521f -0.63341584 1.2911511 -4.0802737 1.0773713
## 605afa3a856f36729d2e5222 -0.02166087 0.9733929 -2.1731395 2.1364245
## 605c8bc6856f36729d2e5223 0.12656266 0.9849885 -1.8965610 2.4741465
## 605f3f2d856f36729d2e5224 0.02663598 0.9717173 -2.1434050 2.2136930
## 605f46c3856f36729d2e5225 0.79593248 1.3271703 -0.8121544 4.2543080
## 60605337856f36729d2e5226 -0.76625246 1.3423321 -4.1760237 0.8603179
## 60609ae6856f36729d2e5228 0.17790932 0.9426243 -1.6849312 2.4306525
## 6061ce91856f36729d2e522e 0.10196004 1.0107577 -1.9845980 2.3822257
## 6061f106856f36729d2e5231 0.81711310 1.4191507 -0.9447214 4.5875085
## 6068ea9f856f36729d2e523e 0.37569496 1.2135918 -1.4707255 3.5198080
## 6075ab05856f36729d2e5247 0.63969607 1.3108798 -1.0711980 4.1068360Sampling plots
plot(documentation5, ask = FALSE)
Posterior predictive check
pp_check(documentation5, nsamples = 200, type = "bars")
Final model
All candidate models look nice, candidate 3 is significantly better than the other candidates, we will proceed with: documentation3
Variations
We will try a few different variations of the selected candidate model.
All data points
Some participants did only complete one scenario. Those has been excluded from the initial dataset to improve sampling of the models. We do however want to use all data we can and will therefore try to fit the model with the complete dataset.
documentation3.all <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.completed),
file = "fits/documentation3.all",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation3.all)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + (1 | session)
## Data: as.data.frame(d.completed) (Number of observations: 51)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 29)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.83 0.72 0.03 2.58 1.00 1281
## sd(muNone_Intercept) 1.11 0.92 0.03 3.35 1.00 834
## Tail_ESS
## sd(muIncorrect_Intercept) 2075
## sd(muNone_Intercept) 1536
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI
## muIncorrect_Intercept 0.38 0.78 -1.13 1.88
## muNone_Intercept -0.00 0.79 -1.53 1.56
## muIncorrect_high_debt_versionfalse -0.45 0.65 -1.73 0.83
## muIncorrect_groupconsultants -0.13 0.89 -1.96 1.57
## muIncorrect_groupfriends -0.22 0.77 -1.74 1.29
## muIncorrect_groupopen -0.28 0.92 -2.10 1.52
## muIncorrect_groupproductMcompany -0.21 1.00 -2.18 1.75
## muIncorrect_groupprofessionalMcontact -0.34 0.93 -2.21 1.47
## muIncorrect_groupstudents 0.77 0.77 -0.71 2.27
## muIncorrect_work_domainApp -0.25 0.94 -2.17 1.68
## muIncorrect_work_domainAutomotive 0.03 0.79 -1.56 1.60
## muIncorrect_work_domainDevops 0.76 0.90 -0.98 2.49
## muIncorrect_work_domainEMCommerce 0.49 0.95 -1.36 2.37
## muIncorrect_work_domainEmbedded 0.09 0.88 -1.67 1.87
## muIncorrect_work_domainFinance -0.80 0.90 -2.53 1.00
## muIncorrect_work_domainMixed 0.14 0.91 -1.64 1.99
## muIncorrect_work_domainMusic 0.21 0.91 -1.54 1.99
## muIncorrect_work_domainNone 0.96 0.81 -0.65 2.55
## muIncorrect_work_domainRetail -0.17 0.96 -2.02 1.74
## muIncorrect_work_domainTelecom -0.47 0.92 -2.30 1.36
## muIncorrect_work_domainWeb -0.42 0.79 -1.99 1.15
## muIncorrect_workplace_peer_reviewfalse -0.11 0.76 -1.60 1.37
## muNone_high_debt_versionfalse 0.41 0.64 -0.84 1.68
## muNone_groupconsultants 0.78 0.85 -0.85 2.46
## muNone_groupfriends 0.38 0.79 -1.20 1.89
## muNone_groupopen 0.44 0.97 -1.40 2.31
## muNone_groupproductMcompany -0.39 0.94 -2.22 1.39
## muNone_groupprofessionalMcontact 0.04 0.92 -1.77 1.82
## muNone_groupstudents -0.52 0.76 -2.03 0.98
## muNone_work_domainApp 0.48 0.96 -1.43 2.36
## muNone_work_domainAutomotive 0.57 0.77 -0.98 2.01
## muNone_work_domainDevops -0.41 0.94 -2.28 1.41
## muNone_work_domainEMCommerce -0.28 0.95 -2.14 1.58
## muNone_work_domainEmbedded -0.47 0.93 -2.26 1.35
## muNone_work_domainFinance -0.05 0.86 -1.76 1.62
## muNone_work_domainMixed -0.53 0.97 -2.42 1.37
## muNone_work_domainMusic 0.20 0.90 -1.62 1.91
## muNone_work_domainNone -0.05 0.80 -1.60 1.55
## muNone_work_domainRetail 0.31 0.94 -1.52 2.14
## muNone_work_domainTelecom 0.22 0.89 -1.54 1.94
## muNone_work_domainWeb 0.27 0.77 -1.23 1.78
## muNone_workplace_peer_reviewfalse 1.19 0.76 -0.39 2.62
## Rhat Bulk_ESS Tail_ESS
## muIncorrect_Intercept 1.00 4771 3195
## muNone_Intercept 1.00 4550 2644
## muIncorrect_high_debt_versionfalse 1.00 4914 2969
## muIncorrect_groupconsultants 1.00 5547 2809
## muIncorrect_groupfriends 1.00 5746 2679
## muIncorrect_groupopen 1.00 6450 2806
## muIncorrect_groupproductMcompany 1.00 7302 3016
## muIncorrect_groupprofessionalMcontact 1.00 6958 3107
## muIncorrect_groupstudents 1.00 6201 3077
## muIncorrect_work_domainApp 1.00 6546 2790
## muIncorrect_work_domainAutomotive 1.00 5836 3270
## muIncorrect_work_domainDevops 1.00 5293 3449
## muIncorrect_work_domainEMCommerce 1.00 7262 3057
## muIncorrect_work_domainEmbedded 1.00 6471 3070
## muIncorrect_work_domainFinance 1.00 5210 2956
## muIncorrect_work_domainMixed 1.00 6627 2936
## muIncorrect_work_domainMusic 1.00 6602 3194
## muIncorrect_work_domainNone 1.00 5095 3258
## muIncorrect_work_domainRetail 1.00 7451 3358
## muIncorrect_work_domainTelecom 1.00 5774 2837
## muIncorrect_work_domainWeb 1.00 5391 2704
## muIncorrect_workplace_peer_reviewfalse 1.00 5015 2876
## muNone_high_debt_versionfalse 1.00 5630 3073
## muNone_groupconsultants 1.00 4700 2592
## muNone_groupfriends 1.00 5623 2979
## muNone_groupopen 1.00 5483 3009
## muNone_groupproductMcompany 1.00 6387 3167
## muNone_groupprofessionalMcontact 1.00 5664 3022
## muNone_groupstudents 1.00 5157 3033
## muNone_work_domainApp 1.00 6288 2908
## muNone_work_domainAutomotive 1.00 5204 3203
## muNone_work_domainDevops 1.00 5818 2864
## muNone_work_domainEMCommerce 1.00 7031 2520
## muNone_work_domainEmbedded 1.00 6148 3027
## muNone_work_domainFinance 1.00 5995 3224
## muNone_work_domainMixed 1.00 5889 2976
## muNone_work_domainMusic 1.00 5530 3232
## muNone_work_domainNone 1.00 6073 3291
## muNone_work_domainRetail 1.00 7240 2810
## muNone_work_domainTelecom 1.00 5675 2598
## muNone_work_domainWeb 1.00 5082 3091
## muNone_workplace_peer_reviewfalse 1.00 4671 2871
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation3.all)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 0.31251805 1.0992337 -1.3999530 3.229259
## 6033d69a5af2c702367b3a95 -0.41623517 1.0444304 -3.2093065 1.148748
## 6033d90a5af2c702367b3a96 0.25415802 0.9365164 -1.4391935 2.643902
## 6034fc165af2c702367b3a98 -0.33039985 1.0610878 -3.1378898 1.350503
## 603500725af2c702367b3a99 0.55989071 1.1142353 -0.8897120 3.495869
## 603f84f15af2c702367b3a9b -0.38332965 1.0145975 -3.1247672 1.147858
## 603f97625af2c702367b3a9d 0.69334404 1.2055884 -0.6604976 4.133973
## 603fd5d95af2c702367b3a9e -0.34489170 1.0310598 -3.0141050 1.314672
## 60409b7b5af2c702367b3a9f 0.52560658 1.1127958 -0.9232042 3.412855
## 604b82b5a7718fbed181b336 -0.16211937 1.0137456 -2.7518010 1.744156
## 604f1239a7718fbed181b33f 0.45524981 1.1420123 -1.1333257 3.519329
## 6050c1bf856f36729d2e5218 0.68276541 1.1805476 -0.7041875 3.895359
## 6050e1e7856f36729d2e5219 0.16195674 0.8897539 -1.5298822 2.325791
## 6055fdc6856f36729d2e521b -0.25718137 0.9906795 -2.6626508 1.410862
## 60579f2a856f36729d2e521e -0.13356407 1.0107575 -2.5330992 1.828127
## 60589862856f36729d2e521f 0.08263065 0.8493684 -1.6925650 2.137735
## 605a30a7856f36729d2e5221 -0.21596851 1.0043653 -2.7860335 1.593972
## 605afa3a856f36729d2e5222 -0.30502638 1.0034896 -2.9348890 1.345881
## 605c8bc6856f36729d2e5223 0.23640265 0.8975680 -1.3743950 2.485852
## 605f3f2d856f36729d2e5224 0.29696072 0.8880001 -1.2015275 2.668881
## 605f46c3856f36729d2e5225 -0.30627158 1.0111130 -3.0427317 1.263982
## 60605337856f36729d2e5226 0.19245986 0.8557660 -1.3962992 2.311771
## 60609ae6856f36729d2e5228 0.20244177 0.8730207 -1.4457107 2.274103
## 6061ce91856f36729d2e522e -0.30881825 1.0739736 -3.0099105 1.319146
## 6061f106856f36729d2e5231 -0.30861574 0.9940178 -2.8636172 1.381595
## 60672faa856f36729d2e523c -0.27152724 0.9588690 -2.7341610 1.414948
## 6068ea9f856f36729d2e523e -0.15816233 1.0471595 -2.7920395 1.853779
## 606db69d856f36729d2e5243 -0.23020913 1.0217729 -2.8339200 1.603413
## 6075ab05856f36729d2e5247 -0.22225896 1.0634869 -2.9897282 1.685337
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 -0.339557438 1.3198310 -3.8462762 1.8812483
## 6033d69a5af2c702367b3a95 0.001288553 1.0480481 -2.0936865 2.3097038
## 6033d90a5af2c702367b3a96 0.078241341 1.0262336 -2.0025762 2.4034970
## 6034fc165af2c702367b3a98 0.868851362 1.4893388 -0.9076599 4.8157828
## 603500725af2c702367b3a99 -0.540035495 1.3130759 -3.9501270 1.4082805
## 603f84f15af2c702367b3a9b -0.420802353 1.3405285 -3.8862697 1.7978453
## 603f97625af2c702367b3a9d -0.558929305 1.3293585 -4.0294158 1.3512218
## 603fd5d95af2c702367b3a9e 0.848938173 1.4374147 -0.9123348 4.7342940
## 60409b7b5af2c702367b3a9f -0.524651442 1.3116059 -3.9729690 1.4643620
## 604b82b5a7718fbed181b336 0.552007500 1.3491260 -1.4316960 4.1367100
## 604f1239a7718fbed181b33f -0.444787440 1.3662184 -3.9048552 1.8369768
## 6050c1bf856f36729d2e5218 -0.536135223 1.2821552 -3.9605202 1.4752140
## 6050e1e7856f36729d2e5219 -0.663351604 1.3279509 -4.2721672 1.1202390
## 6055fdc6856f36729d2e521b 0.874917276 1.4839163 -1.0005275 4.8420368
## 60579f2a856f36729d2e521e 0.513894938 1.3478059 -1.4302435 4.1305503
## 60589862856f36729d2e521f -0.593102225 1.4105460 -4.4024495 1.3618308
## 605a30a7856f36729d2e5221 -0.621916004 1.3656367 -4.1606167 1.3114643
## 605afa3a856f36729d2e5222 0.042581624 1.0659868 -2.2301195 2.3748903
## 605c8bc6856f36729d2e5223 0.239802484 1.0482742 -1.7339435 2.8279185
## 605f3f2d856f36729d2e5224 0.122232884 1.0057334 -1.8820782 2.5231573
## 605f46c3856f36729d2e5225 0.920288011 1.4434798 -0.8435772 4.6280133
## 60605337856f36729d2e5226 -0.818395097 1.3615752 -4.4780742 0.8562092
## 60609ae6856f36729d2e5228 0.127427485 1.0305606 -1.8861500 2.5064683
## 6061ce91856f36729d2e522e -0.040537427 0.9901817 -2.1101865 2.1194365
## 6061f106856f36729d2e5231 0.830658996 1.5001193 -1.0490500 4.8699783
## 60672faa856f36729d2e523c 0.658618616 1.4198841 -1.3408257 4.3498465
## 6068ea9f856f36729d2e523e 0.584098092 1.4165348 -1.4092493 4.4203138
## 606db69d856f36729d2e5243 -0.537210449 1.3203322 -3.9311135 1.4808260
## 6075ab05856f36729d2e5247 0.768253385 1.4107931 -1.0844925 4.6295123Sampling plots
plot(documentation3.all, ask = FALSE)
Posterior predictive check
pp_check(documentation3.all, nsamples = 200, type = "bars")
With experience predictor
As including all data points didn’t harm the model we will create this variant with all data points as well.
This variation includes work_experience_programming.s predictors as it can give further insight into how experience play a factor in the effect we try to measure. This is especially important as our sampling shewed towards containing less experienced developer than the population at large.
documentation3.all.exp <- brm(
"documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + work_experience_programming.s + (1 | session)",
prior = c(
prior(normal(0, 1), class = "b"),
prior(normal(0, 1), class = "Intercept"),
prior(exponential(1), class = "sd", dpar = "muIncorrect"),
prior(exponential(1), class = "sd", dpar = "muNone")
),
family = categorical(),
data = as.data.frame(d.completed),
file = "fits/documentation3.all.exp",
file_refit = "on_change",
seed = 20210421
)Summary
summary(documentation3.all.exp)## Family: categorical
## Links: muIncorrect = logit; muNone = logit
## Formula: documentation ~ 1 + high_debt_version + group + work_domain + workplace_peer_review + work_experience_programming.s + (1 | session)
## Data: as.data.frame(d.completed) (Number of observations: 51)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 29)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## sd(muIncorrect_Intercept) 0.87 0.76 0.03 2.74 1.00 1176
## sd(muNone_Intercept) 1.29 0.98 0.03 3.62 1.00 821
## Tail_ESS
## sd(muIncorrect_Intercept) 2012
## sd(muNone_Intercept) 1316
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI
## muIncorrect_Intercept 0.38 0.83 -1.28 1.97
## muNone_Intercept 0.12 0.82 -1.54 1.73
## muIncorrect_high_debt_versionfalse -0.45 0.65 -1.73 0.81
## muIncorrect_groupconsultants -0.11 0.88 -1.80 1.56
## muIncorrect_groupfriends -0.19 0.84 -1.85 1.50
## muIncorrect_groupopen -0.25 0.95 -2.07 1.59
## muIncorrect_groupproductMcompany -0.27 0.95 -2.17 1.55
## muIncorrect_groupprofessionalMcontact -0.33 0.95 -2.13 1.53
## muIncorrect_groupstudents 0.78 0.80 -0.78 2.32
## muIncorrect_work_domainApp -0.25 0.96 -2.13 1.63
## muIncorrect_work_domainAutomotive 0.01 0.81 -1.55 1.63
## muIncorrect_work_domainDevops 0.75 0.90 -1.01 2.50
## muIncorrect_work_domainEMCommerce 0.46 0.96 -1.50 2.34
## muIncorrect_work_domainEmbedded 0.04 0.92 -1.79 1.84
## muIncorrect_work_domainFinance -0.83 0.94 -2.65 0.99
## muIncorrect_work_domainMixed 0.14 0.90 -1.64 1.92
## muIncorrect_work_domainMusic 0.22 0.92 -1.57 2.07
## muIncorrect_work_domainNone 0.93 0.83 -0.70 2.58
## muIncorrect_work_domainRetail -0.19 0.98 -2.09 1.69
## muIncorrect_work_domainTelecom -0.48 0.91 -2.27 1.30
## muIncorrect_work_domainWeb -0.45 0.80 -2.07 1.12
## muIncorrect_workplace_peer_reviewfalse -0.06 0.79 -1.58 1.49
## muIncorrect_work_experience_programming.s -0.01 0.48 -0.97 0.91
## muNone_high_debt_versionfalse 0.46 0.66 -0.81 1.77
## muNone_groupconsultants 0.69 0.85 -0.98 2.36
## muNone_groupfriends 0.26 0.80 -1.31 1.80
## muNone_groupopen 0.38 0.96 -1.51 2.27
## muNone_groupproductMcompany -0.32 0.97 -2.22 1.57
## muNone_groupprofessionalMcontact 0.10 0.92 -1.73 1.89
## muNone_groupstudents -0.65 0.80 -2.18 0.97
## muNone_work_domainApp 0.43 0.92 -1.31 2.20
## muNone_work_domainAutomotive 0.57 0.82 -1.06 2.15
## muNone_work_domainDevops -0.35 0.97 -2.26 1.51
## muNone_work_domainEMCommerce -0.33 0.99 -2.27 1.57
## muNone_work_domainEmbedded -0.30 0.94 -2.15 1.49
## muNone_work_domainFinance 0.03 0.86 -1.68 1.69
## muNone_work_domainMixed -0.51 0.92 -2.32 1.32
## muNone_work_domainMusic 0.17 0.91 -1.64 1.94
## muNone_work_domainNone -0.11 0.83 -1.73 1.54
## muNone_work_domainRetail 0.30 0.95 -1.61 2.20
## muNone_work_domainTelecom 0.21 0.92 -1.57 2.05
## muNone_work_domainWeb 0.17 0.79 -1.42 1.71
## muNone_workplace_peer_reviewfalse 1.04 0.80 -0.54 2.60
## muNone_work_experience_programming.s -0.44 0.51 -1.50 0.55
## Rhat Bulk_ESS Tail_ESS
## muIncorrect_Intercept 1.00 3585 2640
## muNone_Intercept 1.00 3495 2379
## muIncorrect_high_debt_versionfalse 1.00 4362 2954
## muIncorrect_groupconsultants 1.00 4927 3141
## muIncorrect_groupfriends 1.00 3864 2579
## muIncorrect_groupopen 1.00 4486 3037
## muIncorrect_groupproductMcompany 1.00 6680 2756
## muIncorrect_groupprofessionalMcontact 1.00 4857 2633
## muIncorrect_groupstudents 1.00 4538 3116
## muIncorrect_work_domainApp 1.00 5234 3066
## muIncorrect_work_domainAutomotive 1.00 4042 2504
## muIncorrect_work_domainDevops 1.00 4505 3040
## muIncorrect_work_domainEMCommerce 1.00 4867 3175
## muIncorrect_work_domainEmbedded 1.00 4707 2875
## muIncorrect_work_domainFinance 1.00 4208 2920
## muIncorrect_work_domainMixed 1.00 4732 3137
## muIncorrect_work_domainMusic 1.00 4593 2870
## muIncorrect_work_domainNone 1.00 4648 3005
## muIncorrect_work_domainRetail 1.00 6056 3159
## muIncorrect_work_domainTelecom 1.00 5260 2980
## muIncorrect_work_domainWeb 1.00 4476 2891
## muIncorrect_workplace_peer_reviewfalse 1.00 3988 2780
## muIncorrect_work_experience_programming.s 1.00 3792 2716
## muNone_high_debt_versionfalse 1.00 4686 3289
## muNone_groupconsultants 1.00 4036 2770
## muNone_groupfriends 1.00 3978 2991
## muNone_groupopen 1.00 5731 3166
## muNone_groupproductMcompany 1.00 5099 3083
## muNone_groupprofessionalMcontact 1.00 4869 2273
## muNone_groupstudents 1.00 3594 2669
## muNone_work_domainApp 1.00 4503 2599
## muNone_work_domainAutomotive 1.00 4404 2631
## muNone_work_domainDevops 1.00 4649 2655
## muNone_work_domainEMCommerce 1.01 4957 2120
## muNone_work_domainEmbedded 1.00 4606 2841
## muNone_work_domainFinance 1.00 4241 2836
## muNone_work_domainMixed 1.00 4054 2465
## muNone_work_domainMusic 1.00 4420 2969
## muNone_work_domainNone 1.00 4126 3089
## muNone_work_domainRetail 1.00 5310 3051
## muNone_work_domainTelecom 1.00 4793 3009
## muNone_work_domainWeb 1.00 4264 2608
## muNone_workplace_peer_reviewfalse 1.00 3760 2739
## muNone_work_experience_programming.s 1.00 3069 2289
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).Random effects
ranef(documentation3.all.exp)## $session
## , , muIncorrect_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 0.35215390 1.0835251 -1.2720685 3.281790
## 6033d69a5af2c702367b3a95 -0.43561757 1.1263735 -3.4445645 1.224746
## 6033d90a5af2c702367b3a96 0.24156892 0.9353124 -1.3891317 2.612974
## 6034fc165af2c702367b3a98 -0.33075330 1.0285055 -2.9808523 1.414331
## 603500725af2c702367b3a99 0.55148516 1.1491438 -0.9191708 3.677129
## 603f84f15af2c702367b3a9b -0.39538162 1.0482339 -3.3175530 1.227430
## 603f97625af2c702367b3a9d 0.73209361 1.2266347 -0.6640618 4.069779
## 603fd5d95af2c702367b3a9e -0.37154350 1.0522945 -3.2215080 1.314815
## 60409b7b5af2c702367b3a9f 0.56638611 1.1565224 -0.9117672 3.815698
## 604b82b5a7718fbed181b336 -0.17935264 1.0451424 -2.7512758 1.803570
## 604f1239a7718fbed181b33f 0.53447096 1.1847965 -1.0010298 3.654788
## 6050c1bf856f36729d2e5218 0.74160895 1.3208277 -0.7071550 4.264883
## 6050e1e7856f36729d2e5219 0.15684035 0.8838772 -1.5655212 2.286051
## 6055fdc6856f36729d2e521b -0.25927427 1.0604283 -3.0708818 1.563680
## 60579f2a856f36729d2e521e -0.15739867 1.0795794 -3.0287840 1.900561
## 60589862856f36729d2e521f 0.07744758 0.9137508 -1.7814982 2.159088
## 605a30a7856f36729d2e5221 -0.26701334 1.0576854 -3.0317007 1.433679
## 605afa3a856f36729d2e5222 -0.35107584 1.1078604 -3.3853897 1.257651
## 605c8bc6856f36729d2e5223 0.22409680 0.9461695 -1.5222905 2.542913
## 605f3f2d856f36729d2e5224 0.22281990 0.9998923 -1.6854625 2.683510
## 605f46c3856f36729d2e5225 -0.31262937 1.1033293 -3.1636317 1.460914
## 60605337856f36729d2e5226 0.19280649 0.9028720 -1.4939667 2.435949
## 60609ae6856f36729d2e5228 0.23971050 0.9409840 -1.4196755 2.744929
## 6061ce91856f36729d2e522e -0.33126056 1.0569761 -3.0013500 1.375219
## 6061f106856f36729d2e5231 -0.32879441 1.1498438 -3.2380597 1.552472
## 60672faa856f36729d2e523c -0.24587406 1.1192609 -3.1639960 1.738808
## 6068ea9f856f36729d2e523e -0.20712573 1.0385930 -2.8681940 1.707668
## 606db69d856f36729d2e5243 -0.25072741 1.0875796 -2.9572958 1.619125
## 6075ab05856f36729d2e5247 -0.24751321 1.0647372 -3.0519630 1.618389
##
## , , muNone_Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 -0.42647882 1.520614 -4.3129678 2.1956208
## 6033d69a5af2c702367b3a95 -0.08362818 1.133242 -2.5254800 2.4428723
## 6033d90a5af2c702367b3a96 0.05741262 1.137033 -2.2148512 2.7185120
## 6034fc165af2c702367b3a98 1.06288040 1.566210 -0.7984319 5.1082460
## 603500725af2c702367b3a99 -0.67139383 1.487560 -4.5275185 1.3690868
## 603f84f15af2c702367b3a9b -0.54026918 1.468193 -4.3440175 1.6737628
## 603f97625af2c702367b3a9d -0.73974016 1.551183 -4.8643175 1.4962128
## 603fd5d95af2c702367b3a9e 1.02555832 1.560676 -1.0336660 5.1107053
## 60409b7b5af2c702367b3a9f -0.64294382 1.474245 -4.3687188 1.5455868
## 604b82b5a7718fbed181b336 0.71397514 1.569034 -1.5271695 4.9410785
## 604f1239a7718fbed181b33f -0.54071836 1.470321 -4.1133670 1.8291110
## 6050c1bf856f36729d2e5218 -0.67734865 1.462037 -4.5721932 1.5920803
## 6050e1e7856f36729d2e5219 -0.86216737 1.482463 -4.7927935 1.2540868
## 6055fdc6856f36729d2e521b 0.98806355 1.524456 -1.0644357 4.9061420
## 60579f2a856f36729d2e521e 0.62323621 1.544044 -1.5771407 4.5573240
## 60589862856f36729d2e521f -0.57742440 1.443344 -4.1114985 1.7692588
## 605a30a7856f36729d2e5221 -0.76885720 1.531704 -4.8535828 1.3615910
## 605afa3a856f36729d2e5222 0.25118737 1.222459 -2.2070710 3.1049965
## 605c8bc6856f36729d2e5223 0.30838754 1.163071 -1.9311173 3.1032765
## 605f3f2d856f36729d2e5224 0.60797863 1.391179 -1.6173983 4.0709578
## 605f46c3856f36729d2e5225 1.04986759 1.609727 -0.9360428 5.2276928
## 60605337856f36729d2e5226 -1.06768130 1.547569 -5.1500085 0.9587315
## 60609ae6856f36729d2e5228 0.08153189 1.121600 -2.2196452 2.6199338
## 6061ce91856f36729d2e522e -0.12148428 1.097464 -2.5825582 2.1760213
## 6061f106856f36729d2e5231 0.97575188 1.522577 -1.1027430 4.9884745
## 60672faa856f36729d2e523c 0.76549321 1.544689 -1.5899847 4.8397988
## 6068ea9f856f36729d2e523e 0.71412286 1.560641 -1.4606168 4.7469630
## 606db69d856f36729d2e5243 -0.62570292 1.462999 -4.4105200 1.6686938
## 6075ab05856f36729d2e5247 0.87853310 1.555323 -1.1618170 4.8460635Loo comparison
loo(
documentation3.all,
documentation3.all.exp
)## Output of model 'documentation3.all':
##
## Computed from 4000 by 51 log-likelihood matrix
##
## Estimate SE
## elpd_loo -51.9 4.0
## p_loo 19.7 1.7
## looic 103.7 7.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 31 60.8% 1089
## (0.5, 0.7] (ok) 17 33.3% 556
## (0.7, 1] (bad) 3 5.9% 294
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## Output of model 'documentation3.all.exp':
##
## Computed from 4000 by 51 log-likelihood matrix
##
## Estimate SE
## elpd_loo -52.8 4.4
## p_loo 21.5 2.0
## looic 105.6 8.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 36 70.6% 702
## (0.5, 0.7] (ok) 12 23.5% 238
## (0.7, 1] (bad) 3 5.9% 470
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## Model comparisons:
## elpd_diff se_diff
## documentation3.all 0.0 0.0
## documentation3.all.exp -0.9 1.3Sampling plots
plot(documentation3.all.exp, ask = FALSE)
Posterior predictive check
pp_check(documentation3.all.exp, nsamples = 200, type = "bars")
Final model
- Fitting the model to all data point did not significantly damage the model and will be used as is a more fair representation of reality.
- Adding the experience predictors did not significantly damage the model and will be used as it provides useful insight.
This means that our final model, with all data points and experience predictors, is documentation3.all.exp
Interpreting the model
To begin interpreting the model we look at how it’s parameters were estimated. As our research is focused on how the outcome of the model is effected we will mainly analyze the \(\beta\) parameters.
\(\beta\) parameters
No documentation
mcmc_areas(documentation3.all.exp, pars = c(
"b_muNone_high_debt_versionfalse",
"b_muNone_work_experience_programming.s",
"b_muNone_workplace_peer_reviewfalse",
"b_muNone_groupconsultants",
"b_muNone_groupfriends",
"b_muNone_groupopen",
"b_muNone_groupproductMcompany",
"b_muNone_groupprofessionalMcontact",
"b_muNone_groupstudents",
"b_muNone_work_domainApp",
"b_muNone_work_domainAutomotive",
"b_muNone_work_domainDevops",
"b_muNone_work_domainEMCommerce",
"b_muNone_work_domainEmbedded",
"b_muNone_work_domainFinance",
"b_muNone_work_domainMixed",
"b_muNone_work_domainMusic",
"b_muNone_work_domainNone",
"b_muNone_work_domainRetail",
"b_muNone_work_domainTelecom",
"b_muNone_work_domainWeb"
), prob = 0.95) + scale_y_discrete() +
scale_y_discrete(labels=c(
"High debt version: false",
"Professional programming experience",
"Group: consultants",
"Group: frinds",
"Group: open",
"Group: product-company",
"Group: professional-contacts",
"Group: students",
"Domain: App",
"Domain: Automotive",
"Domain: Dev-ops",
"Domain: E-Commerce",
"Domain: students",
"Domain: Embedded",
"Domain: Finance",
"Domain: Mixed",
"Domain: Music",
"Domain: None",
"Domain: Retail",
"Domain: Telecom",
"Domain: Web"
)) +
ggtitle("Beta parameters densities for no documentation", subtitle = "Shaded region marks 95% of the density. Line marks the median")
Incorrect documentation
mcmc_areas(documentation3.all.exp, pars = c(
"b_muIncorrect_high_debt_versionfalse",
"b_muIncorrect_work_experience_programming.s",
"b_muIncorrect_workplace_peer_reviewfalse",
"b_muIncorrect_groupconsultants",
"b_muIncorrect_groupfriends",
"b_muIncorrect_groupopen",
"b_muIncorrect_groupproductMcompany",
"b_muIncorrect_groupprofessionalMcontact",
"b_muIncorrect_groupstudents",
"b_muIncorrect_work_domainApp",
"b_muIncorrect_work_domainAutomotive",
"b_muIncorrect_work_domainDevops",
"b_muIncorrect_work_domainEMCommerce",
"b_muIncorrect_work_domainEmbedded",
"b_muIncorrect_work_domainFinance",
"b_muIncorrect_work_domainMixed",
"b_muIncorrect_work_domainMusic",
"b_muIncorrect_work_domainNone",
"b_muIncorrect_work_domainRetail",
"b_muIncorrect_work_domainTelecom",
"b_muIncorrect_work_domainWeb"
), prob = 0.95) + scale_y_discrete() +
scale_y_discrete(labels=c(
"High debt version: false",
"Professional programming experience",
"Group: consultants",
"Group: frinds",
"Group: open",
"Group: product-company",
"Group: professional-contacts",
"Group: students",
"Domain: App",
"Domain: Automotive",
"Domain: Dev-ops",
"Domain: E-Commerce",
"Domain: students",
"Domain: Embedded",
"Domain: Finance",
"Domain: Mixed",
"Domain: Music",
"Domain: None",
"Domain: Retail",
"Domain: Telecom",
"Domain: Web"
)) +
ggtitle("Beta parameters densities for incorrect documentation", subtitle = "Shaded region marks 95% of the density. Line marks the median")
Effects sizes
scale_programming_experience <- function(x) {
(x - mean(d.completed$work_experience_programming))/ sd(d.completed$work_experience_programming)
}
unscale_programming_experience <- function(x) {
x * sd(d.completed$work_experience_programming) + mean(d.completed$work_experience_programming)
}
post_settings <- expand.grid(
high_debt_version = c("false", "true"),
group = NA,
work_domain = NA,
session = NA,
workplace_peer_review = NA,
work_experience_programming.s = sapply(c(0, 3, 10, 25, 40), scale_programming_experience)
)
post <- posterior_predict(documentation3.all.exp, newdata = post_settings) %>%
melt(value.name = "estimate", varnames = c("sample_number", "settings_id")) %>%
left_join(
rowid_to_column(post_settings, var= "settings_id"),
by = "settings_id"
) %>%
mutate(work_experience_programming = unscale_programming_experience(work_experience_programming.s)) %>%
select(
estimate,
high_debt_version,
work_experience_programming
)
post %>%
mutate_at("high_debt_version",
function(x) case_when(
x == "false" ~ "Low debt",
x == "true" ~ "High debt"
)) %>%
mutate_at("estimate",
function(x) case_when(
x == 1 ~ "Correct",
x == 2 ~ "Incorrect",
x == 3 ~ "Missing"
)) %>%
ggplot(aes(high_debt_version)) +
geom_bar(aes(fill = estimate), position = "fill") +
facet_grid(rows = vars(work_experience_programming)) +
scale_y_reverse() +
scale_fill_manual("Legend", values = c("darkblue", "#7070FF", "lightblue"), guide = guide_legend(reverse = TRUE)) +
labs(title = "Documentation state") +
xlab("Debt version") +
ylab("Ratio of documentation state")
We can see that task completion ratios are similar for both high and low debt but that there is some difference in how often they leave incorrect documentation and will therefore proceed to calculate some probabilities of leaving incorrect documentation.
d <- post %>% filter(estimate == 2)
d.high <- d %>% filter(high_debt_version == "true") %>% pull(estimate)
d.low <- d %>% filter(high_debt_version == "false") %>% pull(estimate)
x <- length(d.high) / length(d.low)
x## [1] 1.316179Given all the simulated cases we find that developers are 32% more likely to leave incorrect documentation in the high debt version of the scenarios.
d <- post %>% filter(estimate == 2, work_experience_programming == 10)
d.high <- d %>% filter(high_debt_version == "true") %>% pull(estimate)
d.low <- d %>% filter(high_debt_version == "false") %>% pull(estimate)
x <- length(d.high) / length(d.low)
x## [1] 1.31405Given developers with 10 years of professional programming experience we find that they are 31% more likely to leave incorrect documentation in the high debt version of the scenarios.
d <- post %>% filter(estimate == 2, work_experience_programming == 25)
d.high <- d %>% filter(high_debt_version == "true") %>% pull(estimate)
d.low <- d %>% filter(high_debt_version == "false") %>% pull(estimate)
x <- length(d.high) / length(d.low)
x## [1] 1.317723Given developers with 25 years of professional programming experience we find that they are 32% more likely to leave incorrect documentation in the high debt version of the scenarios.
LS0tCnRpdGxlOiAiRG9jdW1lbnRhdGlvbiBTdGF0ZSBNb2RlbCIKYXV0aG9yOiBIYW1wdXMgQnJvbWFuICYgV2lsbGlhbSBMZXbDqW4KZGF0ZTogMjAyMS0wNQpvdXRwdXQ6IAogIGh0bWxfZG9jdW1lbnQ6IAogICAgcGFuZG9jX2FyZ3M6IFsgIi1vIiwgImRvY3MvZG9jdW1lbnRhdGlvbl9zdGF0ZS5odG1sIiBdCi0tLQoKYGBge3IgaW5jbHVkZS1zZXR1cCwgaW5jbHVkZT1GQUxTRX0KIyBMb2FkIHNldHVwIGZpbGUKc291cmNlKGtuaXRyOjpwdXJsKCdzZXR1cC5SbWQnLCBvdXRwdXQgPSB0ZW1wZmlsZSgpKSkKYGBgCgoKCiMjIExvb2tpbmcgYXQgdGhlIGRhdGEKV2UgcGxvdCB0aGUgZGF0YSBhbmQgY2FuIHNlZSB0aGF0IHRoZXJlIGlzIHNvbWUgZGlmZmVyZW5jZSBiZXR3ZWVuIHRoZSBkZWJ0IHZlcnNpb25zCgpgYGB7cn0KZC5ib3RoX2NvbXBsZXRlZCAlPiUKICBtdXRhdGVfYXQoImhpZ2hfZGVidF92ZXJzaW9uIiwgCiAgICAgICAgICAgIGZ1bmN0aW9uKHgpIGNhc2Vfd2hlbigKICAgICAgICAgICAgICB4ID09ICJmYWxzZSIgfiAiTG93IGRlYnQiLAogICAgICAgICAgICAgIHggPT0gInRydWUiIH4gIkhpZ2ggZGVidCIKICAgICAgICAgICAgKSkgJT4lCiAgZ2dwbG90KGFlcyhoaWdoX2RlYnRfdmVyc2lvbiwgZmlsbCA9IGRvY3VtZW50YXRpb24pKSArCiAgZ2VvbV9iYXIoKSArCiAgc2NhbGVfeV9yZXZlcnNlKCkgKwogIHhsYWIoIkRlYnQgbGV2ZWwiKSArCiAgc2NhbGVfZmlsbF9tYW51YWwoIkRvY3VtZW50YXRpb24gc3RhdGUiLCB2YWx1ZXMgPSBjKCJDb3JyZWN0IiA9ICJkYXJrYmx1ZSIsICJOb25lIiA9ICJ0cmFuc3BhcmVudCIsICJJbmNvcnJlY3QiID0gImxpZ2h0Ymx1ZSIpLCBndWlkZSA9IGd1aWRlX2xlZ2VuZChyZXZlcnNlID0gVFJVRSkpCmBgYAoKIyMgSW5pdGlhbCBtb2RlbApUaGUgdHlwZSBvZiB0aGUgb3V0Y29tZSBpcyBjYXRlZ29yaWNhbCBhbmQgaXMgaGVuY2UgbW9kZWxlZCBhcyBjYXRlZ29yaWNhbC4gCgpXZSBpbmNsdWRlIGBoaWdoX2RlYnRfdmVyaXNvbmAgYXMgYSBwcmVkaWN0b3IgaW4gb3VyIG1vZGVsIGFzIHRoaXMgdmFyaWFibGUgcmVwcmVzZW50IHRoZSB2ZXJ5IGVmZmVjdCB3ZSB3YW50IHRvIG1lYXN1cmUuCldlIGFsc28gaW5jbHVkZSBhIHZhcnlpbmcgaW50ZXJjZXB0IGZvciBlYWNoIGluZGl2aWR1YWwgdG8gcHJldmVudCB0aGUgbW9kZWwgZnJvbSBsZWFybmluZyB0b28gbXVjaCBmcm9tIHNpbmdsZSBwYXJ0aWNpcGFudHMgd2l0aCBleHRyZW1lIG1lYXN1cmVtZW50cy4KCiMjIyBTZWxlY3RpbmcgcHJpb3JzIHsudGFic2V0fQoKV2UgaXRlcmF0ZSBvdmVyIHRoZSBtb2RlbCB1bnRpbCB3ZSBoYXZlIHNhbmUgcHJpb3JzLgoKIyMjIyBCYXNlIG1vZGVsIHdpdGggcHJpb3JzCmBgYHtyIGluaXRpYWwtbW9kZWwtZGVmaW5pdGlvbiwgY2xhc3Muc291cmNlID0gJ2ZvbGQtc2hvdyd9CmRvY3VtZW50YXRpb24ud2l0aCA8LSBleHRlbmRhYmxlX21vZGVsKAogIGJhc2VfbmFtZSA9ICJkb2N1bWVudGF0aW9uIiwKICBiYXNlX2Zvcm11bGEgPSAiZG9jdW1lbnRhdGlvbiB+IDEgKyBoaWdoX2RlYnRfdmVyc2lvbiArICgxIHwgc2Vzc2lvbikiLAogIGJhc2VfcHJpb3JzID0gYygKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiYiIpLAogICAgcHJpb3Iobm9ybWFsKDAsIDEpLCBjbGFzcyA9ICJJbnRlcmNlcHQiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVJbmNvcnJlY3QiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVOb25lIikKICApLAogIGZhbWlseSA9IGNhdGVnb3JpY2FsKCksCiAgZGF0YSA9IGQuYm90aF9jb21wbGV0ZWQsCikKYGBgCgoKIyMjIyBEZWZhdWx0IHByaW9ycwoKYGBge3IgZGVmYXVsdC1wcmlvcnN9CnByaW9yX3N1bW1hcnkoZG9jdW1lbnRhdGlvbi53aXRoKG9ubHlfcHJpb3JzPSBUUlVFKSkKYGBgCgojIyMjIFNlbGVjdGVkIHByaW9ycwoKYGBge3Igc2VsZWN0ZWQtcHJpb3JzLCB3YXJuaW5nPUZBTFNFfQpwcmlvcl9zdW1tYXJ5KGRvY3VtZW50YXRpb24ud2l0aChzYW1wbGVfcHJpb3IgPSAib25seSIpKQpgYGAKCiMjIyMgUHJpb3IgcHJlZGljdGl2ZSBjaGVjawoKYGBge3IgcHJpb3JzLWNoZWNrLCB3YXJuaW5nPUZBTFNFfQpwcF9jaGVjayhkb2N1bWVudGF0aW9uLndpdGgoc2FtcGxlX3ByaW9yID0gIm9ubHkiKSwgbnNhbXBsZXMgPSAyMDAsIHR5cGUgPSAiYmFycyIpCmBgYAoKIyMjIyBCZXRhIHBhcmFtZXRlciBpbmZsdWVuY2UKCldlIGNob29zZSBhIGJldGEgcGFyYW1ldGVyIHByaW9ycyBhbGxvd2luZyBmb3IgdGhlIGJldGEgcGFyYW1ldGVyIHRvIGFjY291bnQgZm9yIDEwMCUgb2YgdGhlIGVmZmVjdCBidXQgdGhhdCBpcyBza2VwdGljYWwgdG8gc3VjaCBzdHJvbmcgZWZmZWN0cyBmcm9tIHRoZSBiZXRhIHBhcmFtZXRlci4KCmBgYHtyIHByaW9ycy1iZXRhLCB3YXJuaW5nPUZBTFNFfQpzaW0uc2l6ZSA8LSAxMDAwCnNpbS5pbnRlcmNlcHQgPC0gcm5vcm0oc2ltLnNpemUsIDAsIDEpCnNpbS5iZXRhIDwtIHJub3JtKHNpbS5zaXplLCAwLCAxKQpzaW0uYmV0YS5kaWZmIDwtIChwbG9naXMoc2ltLmludGVyY2VwdCArIHNpbS5iZXRhKSAvIHBsb2dpcyhzaW0uaW50ZXJjZXB0KSAqIDEwMCkgLSAxMDAKCmRhdGEuZnJhbWUoeCA9IHNpbS5iZXRhLmRpZmYpICU+JQogIGdncGxvdChhZXMoeCkpICsKICBnZW9tX2RlbnNpdHkoKSArCiAgeGxpbSgtODAsIDgwKSArCiAgbGFicygKICAgIHRpdGxlID0gIkJldGEgcGFyYW1ldGVyIHByaW9yIGluZmx1ZW5jZSIsCiAgICB4ID0gIkVzdGltYXRlIHdpdGggYmV0YSBhcyAlIG9mIGVzdGltYXRlIHdpdGhvdXQgYmV0YSIsCiAgICB5ID0gIkRlbnNpdHkiCiAgKQoKYGBgCgojIyMgTW9kZWwgZml0IHsudGFic2V0fQoKV2UgY2hlY2sgdGhlIHBvc3RlcmlvciBkaXN0cmlidXRpb24gYW5kIGNhbiBzZWUgdGhhdCB0aGUgbW9kZWwgc2VlbXMgdG8gaGF2ZSBiZWVuIGFibGUgdG8gZml0IHRoZSBkYXRhIHdlbGwuIApTYW1wbGluZyBzZWVtcyB0byBhbHNvIGhhdmUgd29ya2VkIHdlbGwgYXMgUmhhdCB2YWx1ZXMgYXJlIGNsb3NlIHRvIDEgYW5kIHRoZSBzYW1wbGluZyBwbG90cyBsb29rIG5pY2UuCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBiYXNlLXBwLWNoZWNrLCB3YXJuaW5nPUZBTFNFfQpwcF9jaGVjayhkb2N1bWVudGF0aW9uLndpdGgoKSwgbnNhbXBsZXMgPSAyMDAsIHR5cGUgPSAiYmFycyIpCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBiYXNlLXN1bW1hcnksIHdhcm5pbmc9RkFMU0V9CnN1bW1hcnkoZG9jdW1lbnRhdGlvbi53aXRoKCkpCmBgYAoKIyMjIyBTYW1wbGluZyBwbG90cwoKYGBge3IgYmFzZS1wbG90LCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpwbG90KGRvY3VtZW50YXRpb24ud2l0aCgpLCBhc2sgPSBGQUxTRSkKYGBgCgojIyBNb2RlbCBwcmVkaWN0b3IgZXh0ZW5zdGlvbnMgey50YWJzZXR9CgpgYGB7ciBtby1wcmlvcnN9CiMgZGVmYXVsdCBwcmlvciBmb3IgbW9ub3RvbmljIHByZWRpY3RvcgplZGx2bF9wcmlvciA8LSBjKAogIHByaW9yKGRpcmljaGxldCgyKSwgY2xhc3MgPSAic2ltbyIsIGNvZWYgPSAibW9lZHVjYXRpb25fbGV2ZWwxIiwgZHBhciA9ICJtdUluY29ycmVjdCIpLAogIHByaW9yKGRpcmljaGxldCgyKSwgY2xhc3MgPSAic2ltbyIsIGNvZWYgPSAibW9lZHVjYXRpb25fbGV2ZWwxIiwgZHBhciA9ICJtdU5vbmUiKQopCmBgYAoKV2UgdXNlIGBsb29gIHRvIGNoZWNrIHNvbWUgcG9zc2libGUgZXh0ZW5zaW9ucyBvbiB0aGUgbW9kZWwuCgojIyMgT25lIHZhcmlhYmxlIHsudGFic2V0fQoKYGBge3IgbW9kZWwtZXh0ZW5zaW9uLTEsIHdhcm5pbmc9RkFMU0UsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnfQpsb29fcmVzdWx0IDwtIGxvbygKICAjIEJlbmNobWFyayBtb2RlbChzKQogIGRvY3VtZW50YXRpb24ud2l0aCgpLAogIAogICMgTmV3IG1vZGVsKHMpCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrX2RvbWFpbiIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnMiKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtfZXhwZXJpZW5jZV9qYXZhLnMiKSwKICBkb2N1bWVudGF0aW9uLndpdGgoImVkdWNhdGlvbl9maWVsZCIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgibW8oZWR1Y2F0aW9uX2xldmVsKSIsIGVkbHZsX3ByaW9yKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtwbGFjZV9wZWVyX3JldmlldyIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX3RkX3RyYWNraW5nIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX2NvZGluZ19zdGFuZGFyZHMiKSwKICBkb2N1bWVudGF0aW9uLndpdGgoInNjZW5hcmlvIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJncm91cCIpCikKYGBgCgojIyMjIENvbXBhcmlzb24KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi0xLXN1bSwgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsyXQpgYGAKCiMjIyMgRGlhZ25vc3RpY3MKCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi0xLWRpZywgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsxXQpgYGAKCiMjIyBUd28gdmFyaWFibGVzIHsudGFic2V0fQoKYGBge3IgbW9kZWwtZXh0ZW5zaW9uLTIsIHdhcm5pbmc9RkFMU0UsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnfQpsb29fcmVzdWx0IDwtIGxvbygKICAjIEJlbmNobWFyayBtb2RlbChzKQogIGRvY3VtZW50YXRpb24ud2l0aCgpLAogIAogIGRvY3VtZW50YXRpb24ud2l0aCgiZ3JvdXAiKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtfZG9tYWluIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtwbGFjZV90ZF90cmFja2luZyIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciKSwKICBkb2N1bWVudGF0aW9uLndpdGgoImVkdWNhdGlvbl9maWVsZCIpLAogIAogICMgTmV3IG1vZGVsKHMpCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtfZG9tYWluIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgImVkdWNhdGlvbl9maWVsZCIpKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrX2RvbWFpbiIsICJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoIndvcmtfZG9tYWluIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrX2RvbWFpbiIsICJlZHVjYXRpb25fZmllbGQiKSksCiAgCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgImVkdWNhdGlvbl9maWVsZCIpKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciLCAiZWR1Y2F0aW9uX2ZpZWxkIikpLAogIAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIsICJlZHVjYXRpb25fZmllbGQiKSkKKQpgYGAKCiMjIyMgQ29tcGFyaXNvbgoKYGBge3IgbW9kZWwtZXh0ZW5zaW9uLTItc3VtLCB3YXJuaW5nPUZBTFNFfQpsb29fcmVzdWx0WzJdCmBgYAoKIyMjIyBEaWFnbm9zdGljcwoKYGBge3IgbW9kZWwtZXh0ZW5zaW9uLTItZGlnLCB3YXJuaW5nPUZBTFNFfQpsb29fcmVzdWx0WzFdCmBgYAoKIyMjIFRocmVlIHZhcmlhYmxlcyB7LnRhYnNldH0KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi0zLCB3YXJuaW5nPUZBTFNFLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93J30KbG9vX3Jlc3VsdCA8LSBsb28oCiAgIyBCZW5jaG1hcmsgbW9kZWwocykKICBkb2N1bWVudGF0aW9uLndpdGgoKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoImdyb3VwIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrX2RvbWFpbiIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX3BlZXJfcmV2aWV3IiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJlZHVjYXRpb25fZmllbGQiKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIpKSwKICAKICAjIE5ldyBtb2RlbChzKQogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciLCAiZ3JvdXAiKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoIndvcmtwbGFjZV90ZF90cmFja2luZyIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIsICJ3b3JrX2RvbWFpbiIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIiwgIndvcmtwbGFjZV9wZWVyX3JldmlldyIpKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya19kb21haW4iLCAid29ya3BsYWNlX3RkX3RyYWNraW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3RkX3RyYWNraW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciKSksCiAgCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIiwgIndvcmtfZG9tYWluIikpLAogIAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtfZG9tYWluIikpCikKYGBgCgojIyMjIENvbXBhcmlzb24KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi0zLXN1bSwgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsyXQpgYGAKCiMjIyMgRGlhZ25vc3RpY3MKCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi0zLWRpZywgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsxXQpgYGAKCiMjIyBGb3VyIHZhcmlhYmxlcyB7LnRhYnNldH0KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi00LCB3YXJuaW5nPUZBTFNFLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93J30KbG9vX3Jlc3VsdCA8LSBsb28oCiAgIyBCZW5jaG1hcmsgbW9kZWwocykKICBkb2N1bWVudGF0aW9uLndpdGgoKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoImdyb3VwIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrX2RvbWFpbiIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX3BlZXJfcmV2aWV3IiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJlZHVjYXRpb25fZmllbGQiKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIpKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSksCiAgCiAgIyBOZXcgbW9kZWwocykKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3RkX3RyYWNraW5nIikpCikKYGBgCgojIyMjIENvbXBhcmlzb24KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi00LXN1bSwgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsyXQpgYGAKCiMjIyMgRGlhZ25vc3RpY3MKCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi00LWRpZywgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsxXQpgYGAKCiMjIyBGaXZlIHZhcmlhYmxlcyB7LnRhYnNldH0KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi01LCB3YXJuaW5nPUZBTFNFLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93J30KbG9vX3Jlc3VsdCA8LSBsb28oCiAgIyBCZW5jaG1hcmsgbW9kZWwocykKICBkb2N1bWVudGF0aW9uLndpdGgoKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoImdyb3VwIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrX2RvbWFpbiIpLAogIGRvY3VtZW50YXRpb24ud2l0aCgid29ya3BsYWNlX3BlZXJfcmV2aWV3IiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSwKICBkb2N1bWVudGF0aW9uLndpdGgoIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIiksCiAgZG9jdW1lbnRhdGlvbi53aXRoKCJlZHVjYXRpb25fZmllbGQiKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3RkX3RyYWNraW5nIiwgIndvcmtwbGFjZV9wYWlyX3Byb2dyYW1taW5nIikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICBkb2N1bWVudGF0aW9uLndpdGgoYygid29ya3BsYWNlX3BlZXJfcmV2aWV3IiwgIndvcmtwbGFjZV90ZF90cmFja2luZyIpKSwKICAKICBkb2N1bWVudGF0aW9uLndpdGgoYygiZ3JvdXAiLCAid29ya19kb21haW4iLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciLCAid29ya3BsYWNlX3BlZXJfcmV2aWV3IikpLAogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSksCiAgCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtfZG9tYWluIiwgIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfdGRfdHJhY2tpbmciKSksCiAgZG9jdW1lbnRhdGlvbi53aXRoKGMoImdyb3VwIiwgIndvcmtfZG9tYWluIiwgIndvcmtwbGFjZV9wZWVyX3JldmlldyIsICJ3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZyIpKSwKICAKICAjIE5ldyBtb2RlbChzKQogIGRvY3VtZW50YXRpb24ud2l0aChjKCJncm91cCIsICJ3b3JrX2RvbWFpbiIsICJ3b3JrcGxhY2VfcGVlcl9yZXZpZXciLCAid29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmciLCAid29ya3BsYWNlX3RkX3RyYWNraW5nIikpCikKYGBgCgojIyMjIENvbXBhcmlzb24KCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi01LXN1bSwgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsyXQpgYGAKCiMjIyMgRGlhZ25vc3RpY3MKCmBgYHtyIG1vZGVsLWV4dGVuc2lvbi01LWRpZywgd2FybmluZz1GQUxTRX0KbG9vX3Jlc3VsdFsxXQpgYGAKCgojIyBDYW5kaWRhdGUgbW9kZWxzICB7LnRhYnNldH0KV2UgcGljayBzb21lIG9mIG91ciB0b3AgcGVyZm9ybWluZyBtb2RlbHMgYXMgY2FuZGlkYXRlcyBhbmQgaW5zcGVjdCB0aGVtIGNsb3Nlci4KClRoZSBjYW5kaWRhdGUgbW9kZWxzIGFyZSBuYW1lZCBhbmQgbGlzdGVkIGluIG9yZGVyIG9mIGNvbXBsZXhpdHkuCgojIyMgRG9jdW1lbnRhdGlvbjAgIHsudGFic2V0fQoKV2Ugc2VsZWN0IHRoZSBzaW1wbGVzdCBtb2RlbCBhcyBhIGJhc2VsaW5lLgoKYGBge3IgZG9jdW1lbnRhdGlvbjAsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQpkb2N1bWVudGF0aW9uMCA8LSBicm0oCiAgImRvY3VtZW50YXRpb24gfiAxICsgaGlnaF9kZWJ0X3ZlcnNpb24gKyAoMSB8IHNlc3Npb24pIiwKICBwcmlvciA9IGMoCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gImIiKSwKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11SW5jb3JyZWN0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11Tm9uZSIpCiAgKSwKICBmYW1pbHkgPSBjYXRlZ29yaWNhbCgpLAogIGRhdGEgPSBhcy5kYXRhLmZyYW1lKGQuYm90aF9jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uMCIsCiAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogIHNlZWQgPSAyMDIxMDQyMQopCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBkb2N1bWVudGF0aW9uMC1zdW19CnN1bW1hcnkoZG9jdW1lbnRhdGlvbjApCmBgYAoKIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgZG9jdW1lbnRhdGlvbjAtcmFuZWZmfQpyYW5lZihkb2N1bWVudGF0aW9uMCkKYGBgCgojIyMjIFNhbXBsaW5nIHBsb3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uMC1wbG90fQpwbG90KGRvY3VtZW50YXRpb24wLCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBkb2N1bWVudGF0aW9uMC1wcH0KcHBfY2hlY2soZG9jdW1lbnRhdGlvbjAsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIyBEb2N1bWVudGF0aW9uMSAgey50YWJzZXR9CgpXZSBzZWxlY3QgdGhlIGJlc3QgcGVyZm9ybWluZyBtb2RlbCB3aXRoIG9uZSB2YXJpYWJsZS4KCmBgYHtyIGRvY3VtZW50YXRpb24xLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93Jywgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KZG9jdW1lbnRhdGlvbjEgPC0gYnJtKAogICJkb2N1bWVudGF0aW9uIH4gMSArIGhpZ2hfZGVidF92ZXJzaW9uICsgZ3JvdXAgKyAoMSB8IHNlc3Npb24pIiwKICBwcmlvciA9IGMoCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gImIiKSwKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11SW5jb3JyZWN0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11Tm9uZSIpCiAgKSwKICBmYW1pbHkgPSBjYXRlZ29yaWNhbCgpLAogIGRhdGEgPSBhcy5kYXRhLmZyYW1lKGQuYm90aF9jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uMSIsCiAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogIHNlZWQgPSAyMDIxMDQyMQopCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBkb2N1bWVudGF0aW9uMS1zdW19CnN1bW1hcnkoZG9jdW1lbnRhdGlvbjEpCmBgYAoKIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgZG9jdW1lbnRhdGlvbjEtcmFuZWZmfQpyYW5lZihkb2N1bWVudGF0aW9uMSkKYGBgCgojIyMjIFNhbXBsaW5nIHBsb3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uMS1wbG90fQpwbG90KGRvY3VtZW50YXRpb24xLCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBkb2N1bWVudGF0aW9uMS1wcH0KcHBfY2hlY2soZG9jdW1lbnRhdGlvbjEsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIyBEb2N1bWVudGF0aW9uMiAgey50YWJzZXR9CgpXZSBzZWxlY3QgdGhlIGJlc3QgcGVyZm9ybWluZyBtb2RlbCB3aXRoIHR3byB2YXJpYWJsZXMuCgpgYGB7ciBkb2N1bWVudGF0aW9uMiwgY2xhc3Muc291cmNlID0gJ2ZvbGQtc2hvdycsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0V9CmRvY3VtZW50YXRpb24yIDwtIGJybSgKICAiZG9jdW1lbnRhdGlvbiB+IDEgKyBoaWdoX2RlYnRfdmVyc2lvbiArIGdyb3VwICsgd29ya19kb21haW4gKyAoMSB8IHNlc3Npb24pIiwKICBwcmlvciA9IGMoCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gImIiKSwKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11SW5jb3JyZWN0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11Tm9uZSIpCiAgKSwKICBmYW1pbHkgPSBjYXRlZ29yaWNhbCgpLAogIGRhdGEgPSBhcy5kYXRhLmZyYW1lKGQuYm90aF9jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uMiIsCiAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogIHNlZWQgPSAyMDIxMDQyMQopCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBkb2N1bWVudGF0aW9uMi1zdW19CnN1bW1hcnkoZG9jdW1lbnRhdGlvbjIpCmBgYAoKIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgZG9jdW1lbnRhdGlvbjItcmFuZWZmfQpyYW5lZihkb2N1bWVudGF0aW9uMikKYGBgCgojIyMjIFNhbXBsaW5nIHBsb3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uMi1wbG90fQpwbG90KGRvY3VtZW50YXRpb24yLCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBkb2N1bWVudGF0aW9uMi1wcH0KcHBfY2hlY2soZG9jdW1lbnRhdGlvbjIsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIyBEb2N1bWVudGF0aW9uMyAgey50YWJzZXR9CgpXZSBzZWxlY3QgdGhlIGJlc3QgcGVyZm9ybWluZyBtb2RlbCB3aXRoIHRocmVlIHZhcmlhYmxlcy4KCmBgYHtyIGRvY3VtZW50YXRpb24zLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93Jywgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KZG9jdW1lbnRhdGlvbjMgPC0gYnJtKAogICJkb2N1bWVudGF0aW9uIH4gMSArIGhpZ2hfZGVidF92ZXJzaW9uICsgZ3JvdXAgKyB3b3JrX2RvbWFpbiArIHdvcmtwbGFjZV9wZWVyX3JldmlldyArICgxIHwgc2Vzc2lvbikiLAogIHByaW9yID0gYygKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiYiIpLAogICAgcHJpb3Iobm9ybWFsKDAsIDEpLCBjbGFzcyA9ICJJbnRlcmNlcHQiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVJbmNvcnJlY3QiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVOb25lIikKICApLAogIGZhbWlseSA9IGNhdGVnb3JpY2FsKCksCiAgZGF0YSA9IGFzLmRhdGEuZnJhbWUoZC5ib3RoX2NvbXBsZXRlZCksCiAgZmlsZSA9ICJmaXRzL2RvY3VtZW50YXRpb24zIiwKICBmaWxlX3JlZml0ID0gIm9uX2NoYW5nZSIsCiAgc2VlZCA9IDIwMjEwNDIxCikKYGBgCgojIyMjIFN1bW1hcnkKCmBgYHtyIGRvY3VtZW50YXRpb24zLXN1bX0Kc3VtbWFyeShkb2N1bWVudGF0aW9uMykKYGBgCgojIyMjIFJhbmRvbSBlZmZlY3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uMy1yYW5lZmZ9CnJhbmVmKGRvY3VtZW50YXRpb24zKQpgYGAKCiMjIyMgU2FtcGxpbmcgcGxvdHMKCmBgYHtyIGRvY3VtZW50YXRpb24zLXBsb3R9CnBsb3QoZG9jdW1lbnRhdGlvbjMsIGFzayA9IEZBTFNFKQpgYGAKCiMjIyMgUG9zdGVyaW9yIHByZWRpY3RpdmUgY2hlY2sKCmBgYHtyIGRvY3VtZW50YXRpb24zLXBwfQpwcF9jaGVjayhkb2N1bWVudGF0aW9uMywgbnNhbXBsZXMgPSAyMDAsIHR5cGUgPSAiYmFycyIpCmBgYAoKIyMjIERvY3VtZW50YXRpb240ICB7LnRhYnNldH0KCldlIHNlbGVjdCB0aGUgYmVzdCBwZXJmb3JtaW5nIG1vZGVsIHdpdGggZm91ciB2YXJpYWJsZXMuCgpgYGB7ciBkb2N1bWVudGF0aW9uNCwgY2xhc3Muc291cmNlID0gJ2ZvbGQtc2hvdycsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0V9CmRvY3VtZW50YXRpb240IDwtIGJybSgKICAiZG9jdW1lbnRhdGlvbiB+IDEgKyBoaWdoX2RlYnRfdmVyc2lvbiArIGdyb3VwICsgd29ya19kb21haW4gKyB3b3JrcGxhY2VfcGVlcl9yZXZpZXcgKyB3b3JrcGxhY2VfdGRfdHJhY2tpbmcgKyAoMSB8IHNlc3Npb24pIiwKICBwcmlvciA9IGMoCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gImIiKSwKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11SW5jb3JyZWN0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11Tm9uZSIpCiAgKSwKICBmYW1pbHkgPSBjYXRlZ29yaWNhbCgpLAogIGRhdGEgPSBhcy5kYXRhLmZyYW1lKGQuYm90aF9jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uNCIsCiAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogIHNlZWQgPSAyMDIxMDQyMQopCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBkb2N1bWVudGF0aW9uNC1zdW19CnN1bW1hcnkoZG9jdW1lbnRhdGlvbjQpCmBgYAoKIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgZG9jdW1lbnRhdGlvbjQtcmFuZWZmfQpyYW5lZihkb2N1bWVudGF0aW9uNCkKYGBgCgojIyMjIFNhbXBsaW5nIHBsb3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uNC1wbG90fQpwbG90KGRvY3VtZW50YXRpb240LCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBkb2N1bWVudGF0aW9uNC1wcH0KcHBfY2hlY2soZG9jdW1lbnRhdGlvbjQsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIyBEb2N1bWVudGF0aW9uNSAgey50YWJzZXR9CgpXZSBzZWxlY3QgdGhlIGJlc3QgcGVyZm9ybWluZyBtb2RlbCB3aXRoIGZpdmUgdmFyaWFibGVzLgoKYGBge3IgZG9jdW1lbnRhdGlvbjUsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnLCB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQpkb2N1bWVudGF0aW9uNSA8LSBicm0oCiAgImRvY3VtZW50YXRpb24gfiAxICsgaGlnaF9kZWJ0X3ZlcnNpb24gKyBncm91cCArIHdvcmtfZG9tYWluICsgd29ya3BsYWNlX3BlZXJfcmV2aWV3ICsgd29ya3BsYWNlX3RkX3RyYWNraW5nICsgd29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmcgKyAoMSB8IHNlc3Npb24pIiwKICBwcmlvciA9IGMoCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gImIiKSwKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11SW5jb3JyZWN0IiksCiAgICBwcmlvcihleHBvbmVudGlhbCgxKSwgY2xhc3MgPSAic2QiLCBkcGFyID0gIm11Tm9uZSIpCiAgKSwKICBmYW1pbHkgPSBjYXRlZ29yaWNhbCgpLAogIGRhdGEgPSBhcy5kYXRhLmZyYW1lKGQuYm90aF9jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uNSIsCiAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogIHNlZWQgPSAyMDIxMDQyMQopCmBgYAoKIyMjIyBTdW1tYXJ5CgpgYGB7ciBkb2N1bWVudGF0aW9uNS1zdW19CnN1bW1hcnkoZG9jdW1lbnRhdGlvbjUpCmBgYAoKIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgZG9jdW1lbnRhdGlvbjUtcmFuZWZmfQpyYW5lZihkb2N1bWVudGF0aW9uNSkKYGBgCgojIyMjIFNhbXBsaW5nIHBsb3RzCgpgYGB7ciBkb2N1bWVudGF0aW9uNS1wbG90fQpwbG90KGRvY3VtZW50YXRpb241LCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIFBvc3RlcmlvciBwcmVkaWN0aXZlIGNoZWNrCgpgYGB7ciBkb2N1bWVudGF0aW9uNS1wcH0KcHBfY2hlY2soZG9jdW1lbnRhdGlvbjUsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIEZpbmFsIG1vZGVsIApBbGwgY2FuZGlkYXRlIG1vZGVscyBsb29rIG5pY2UsIGNhbmRpZGF0ZSAzIGlzIHNpZ25pZmljYW50bHkgYmV0dGVyIHRoYW4gdGhlIG90aGVyIGNhbmRpZGF0ZXMsIHdlIHdpbGwgcHJvY2VlZCB3aXRoOiBgZG9jdW1lbnRhdGlvbjNgCgojIyMgVmFyaWF0aW9ucyB7LnRhYnNldH0KV2Ugd2lsbCB0cnkgYSBmZXcgZGlmZmVyZW50IHZhcmlhdGlvbnMgb2YgdGhlIHNlbGVjdGVkIGNhbmRpZGF0ZSBtb2RlbC4KCiMjIyMgQWxsIGRhdGEgcG9pbnRzIHsudGFic2V0fQoKU29tZSBwYXJ0aWNpcGFudHMgZGlkIG9ubHkgY29tcGxldGUgb25lIHNjZW5hcmlvLiBUaG9zZSBoYXMgYmVlbiBleGNsdWRlZCBmcm9tIHRoZSBpbml0aWFsIGRhdGFzZXQgdG8gaW1wcm92ZSBzYW1wbGluZyBvZiB0aGUgbW9kZWxzLiBXZSBkbyBob3dldmVyIHdhbnQgdG8gdXNlIGFsbCBkYXRhIHdlIGNhbiBhbmQgd2lsbCB0aGVyZWZvcmUgdHJ5IHRvIGZpdCB0aGUgbW9kZWwgd2l0aCB0aGUgY29tcGxldGUgZGF0YXNldC4KCmBgYHtyIHZhcmlhdGlvbi5hbGwsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0UsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnfQpkb2N1bWVudGF0aW9uMy5hbGwgPC0gYnJtKAogICJkb2N1bWVudGF0aW9uIH4gMSArIGhpZ2hfZGVidF92ZXJzaW9uICsgZ3JvdXAgKyB3b3JrX2RvbWFpbiArIHdvcmtwbGFjZV9wZWVyX3JldmlldyArICgxIHwgc2Vzc2lvbikiLAogIHByaW9yID0gYygKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiYiIpLAogICAgcHJpb3Iobm9ybWFsKDAsIDEpLCBjbGFzcyA9ICJJbnRlcmNlcHQiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVJbmNvcnJlY3QiKSwKICAgIHByaW9yKGV4cG9uZW50aWFsKDEpLCBjbGFzcyA9ICJzZCIsIGRwYXIgPSAibXVOb25lIikKICApLAogIGZhbWlseSA9IGNhdGVnb3JpY2FsKCksCiAgZGF0YSA9IGFzLmRhdGEuZnJhbWUoZC5jb21wbGV0ZWQpLAogIGZpbGUgPSAiZml0cy9kb2N1bWVudGF0aW9uMy5hbGwiLAogIGZpbGVfcmVmaXQgPSAib25fY2hhbmdlIiwKICBzZWVkID0gMjAyMTA0MjEKKQpgYGAKCiMjIyMjIFN1bW1hcnkKCmBgYHtyIHZhcmlhdGlvbi5hbGwtc3VtfQpzdW1tYXJ5KGRvY3VtZW50YXRpb24zLmFsbCkKYGBgCgojIyMjIyBSYW5kb20gZWZmZWN0cwoKYGBge3IgdmFyaWF0aW9uLmFsbC1yYW5lZmZ9CnJhbmVmKGRvY3VtZW50YXRpb24zLmFsbCkKYGBgCgojIyMjIyBTYW1wbGluZyBwbG90cwoKYGBge3IgdmFyaWF0aW9uLmFsbC1wbG90fQpwbG90KGRvY3VtZW50YXRpb24zLmFsbCwgYXNrID0gRkFMU0UpCmBgYAoKIyMjIyMgUG9zdGVyaW9yIHByZWRpY3RpdmUgY2hlY2sKCmBgYHtyIHZhcmlhdGlvbi5hbGwtcHB9CnBwX2NoZWNrKGRvY3VtZW50YXRpb24zLmFsbCwgbnNhbXBsZXMgPSAyMDAsIHR5cGUgPSAiYmFycyIpCmBgYAoKIyMjIyBXaXRoIGV4cGVyaWVuY2UgcHJlZGljdG9yIHsudGFic2V0fQoKQXMgaW5jbHVkaW5nIGFsbCBkYXRhIHBvaW50cyBkaWRuJ3QgaGFybSB0aGUgbW9kZWwgd2Ugd2lsbCBjcmVhdGUgdGhpcyB2YXJpYW50IHdpdGggYWxsIGRhdGEgcG9pbnRzIGFzIHdlbGwuCgpUaGlzIHZhcmlhdGlvbiBpbmNsdWRlcyBgd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnNgIHByZWRpY3RvcnMgYXMgaXQgY2FuIGdpdmUgZnVydGhlciBpbnNpZ2h0IGludG8gaG93IGV4cGVyaWVuY2UgcGxheSBhIGZhY3RvciBpbiB0aGUgZWZmZWN0IHdlIHRyeSB0byBtZWFzdXJlLiBUaGlzIGlzIGVzcGVjaWFsbHkgaW1wb3J0YW50IGFzIG91ciBzYW1wbGluZyBzaGV3ZWQgdG93YXJkcyBjb250YWluaW5nIGxlc3MgZXhwZXJpZW5jZWQgZGV2ZWxvcGVyIHRoYW4gdGhlIHBvcHVsYXRpb24gYXQgbGFyZ2UuCgpgYGB7ciB2YXJpYXRpb24uYWxsLmV4cCwgY2xhc3Muc291cmNlID0gJ2ZvbGQtc2hvdycsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CmRvY3VtZW50YXRpb24zLmFsbC5leHAgPC0gYnJtKAogICJkb2N1bWVudGF0aW9uIH4gMSArIGhpZ2hfZGVidF92ZXJzaW9uICsgZ3JvdXAgKyB3b3JrX2RvbWFpbiArIHdvcmtwbGFjZV9wZWVyX3JldmlldyArIHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZy5zICsgKDEgfCBzZXNzaW9uKSIsCiAgcHJpb3IgPSBjKAogICAgcHJpb3Iobm9ybWFsKDAsIDEpLCBjbGFzcyA9ICJiIiksCiAgICBwcmlvcihub3JtYWwoMCwgMSksIGNsYXNzID0gIkludGVyY2VwdCIpLAogICAgcHJpb3IoZXhwb25lbnRpYWwoMSksIGNsYXNzID0gInNkIiwgZHBhciA9ICJtdUluY29ycmVjdCIpLAogICAgcHJpb3IoZXhwb25lbnRpYWwoMSksIGNsYXNzID0gInNkIiwgZHBhciA9ICJtdU5vbmUiKQogICksCiAgZmFtaWx5ID0gY2F0ZWdvcmljYWwoKSwKICBkYXRhID0gYXMuZGF0YS5mcmFtZShkLmNvbXBsZXRlZCksCiAgZmlsZSA9ICJmaXRzL2RvY3VtZW50YXRpb24zLmFsbC5leHAiLAogIGZpbGVfcmVmaXQgPSAib25fY2hhbmdlIiwKICBzZWVkID0gMjAyMTA0MjEKKQpgYGAKCiMjIyMjIFN1bW1hcnkKCmBgYHtyIHZhcmlhdGlvbi5hbGwuZXhwLXN1bX0Kc3VtbWFyeShkb2N1bWVudGF0aW9uMy5hbGwuZXhwKQpgYGAKCiMjIyMjIFJhbmRvbSBlZmZlY3RzCgpgYGB7ciB2YXJpYXRpb24uYWxsLmV4cC1yYW5lZmZ9CnJhbmVmKGRvY3VtZW50YXRpb24zLmFsbC5leHApCmBgYAoKIyMjIyMgTG9vIGNvbXBhcmlzb24KCmBgYHtyIHZhcmlhdGlvbi5hbGwuZXhwLWxvbywgd2FybmluZz1GQUxTRX0KbG9vKAogIGRvY3VtZW50YXRpb24zLmFsbCwKICBkb2N1bWVudGF0aW9uMy5hbGwuZXhwCikKYGBgCgojIyMjIyBTYW1wbGluZyBwbG90cwoKYGBge3IgdmFyaWF0aW9uLmFsbC5leHAtcGxvdH0KcGxvdChkb2N1bWVudGF0aW9uMy5hbGwuZXhwLCBhc2sgPSBGQUxTRSkKYGBgCgojIyMjIyBQb3N0ZXJpb3IgcHJlZGljdGl2ZSBjaGVjawoKYGBge3IgdmFyaWF0aW9uLmFsbC5leHAtcHB9CnBwX2NoZWNrKGRvY3VtZW50YXRpb24zLmFsbC5leHAsIG5zYW1wbGVzID0gMjAwLCB0eXBlID0gImJhcnMiKQpgYGAKCiMjIyBGaW5hbCBtb2RlbAoqIEZpdHRpbmcgdGhlIG1vZGVsIHRvIGFsbCBkYXRhIHBvaW50IGRpZCBub3Qgc2lnbmlmaWNhbnRseSBkYW1hZ2UgdGhlIG1vZGVsIGFuZCB3aWxsIGJlIHVzZWQgYXMgaXMgYSBtb3JlIGZhaXIgcmVwcmVzZW50YXRpb24gb2YgcmVhbGl0eS4KKiBBZGRpbmcgdGhlIGV4cGVyaWVuY2UgcHJlZGljdG9ycyBkaWQgbm90IHNpZ25pZmljYW50bHkgZGFtYWdlIHRoZSBtb2RlbCBhbmQgd2lsbCBiZSB1c2VkIGFzIGl0IHByb3ZpZGVzIHVzZWZ1bCBpbnNpZ2h0LgoKVGhpcyBtZWFucyB0aGF0IG91ciBmaW5hbCBtb2RlbCwgd2l0aCBhbGwgZGF0YSBwb2ludHMgYW5kIGV4cGVyaWVuY2UgcHJlZGljdG9ycywgaXMgYGRvY3VtZW50YXRpb24zLmFsbC5leHBgCgojIyBJbnRlcnByZXRpbmcgdGhlIG1vZGVsClRvIGJlZ2luIGludGVycHJldGluZyB0aGUgbW9kZWwgd2UgbG9vayBhdCBob3cgaXQncyBwYXJhbWV0ZXJzIHdlcmUgZXN0aW1hdGVkLiBBcyBvdXIgcmVzZWFyY2ggaXMgZm9jdXNlZCBvbiBob3cgdGhlIG91dGNvbWUgb2YgdGhlIG1vZGVsIGlzIGVmZmVjdGVkIHdlIHdpbGwgbWFpbmx5IGFuYWx5emUgdGhlICRcYmV0YSQgcGFyYW1ldGVycy4KCiMjIyAkXGJldGEkIHBhcmFtZXRlcnMKCiMjIyMgTm8gZG9jdW1lbnRhdGlvbgoKYGBge3IgaW50ZXJwcmV0LWJldGEtcGxvdCwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KbWNtY19hcmVhcyhkb2N1bWVudGF0aW9uMy5hbGwuZXhwLCBwYXJzID0gYygKICAgICJiX211Tm9uZV9oaWdoX2RlYnRfdmVyc2lvbmZhbHNlIiwgCiAgICAiYl9tdU5vbmVfd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnMiLAogICAgImJfbXVOb25lX3dvcmtwbGFjZV9wZWVyX3Jldmlld2ZhbHNlIiwKICAgICJiX211Tm9uZV9ncm91cGNvbnN1bHRhbnRzIiwKICAgICJiX211Tm9uZV9ncm91cGZyaWVuZHMiLAogICAgImJfbXVOb25lX2dyb3Vwb3BlbiIsCiAgICAiYl9tdU5vbmVfZ3JvdXBwcm9kdWN0TWNvbXBhbnkiLAogICAgImJfbXVOb25lX2dyb3VwcHJvZmVzc2lvbmFsTWNvbnRhY3QiLAogICAgImJfbXVOb25lX2dyb3Vwc3R1ZGVudHMiLAogICAgImJfbXVOb25lX3dvcmtfZG9tYWluQXBwIiwKICAgICJiX211Tm9uZV93b3JrX2RvbWFpbkF1dG9tb3RpdmUiLAogICAgImJfbXVOb25lX3dvcmtfZG9tYWluRGV2b3BzIiwKICAgICJiX211Tm9uZV93b3JrX2RvbWFpbkVNQ29tbWVyY2UiLAogICAgImJfbXVOb25lX3dvcmtfZG9tYWluRW1iZWRkZWQiLAogICAgImJfbXVOb25lX3dvcmtfZG9tYWluRmluYW5jZSIsCiAgICAiYl9tdU5vbmVfd29ya19kb21haW5NaXhlZCIsCiAgICAiYl9tdU5vbmVfd29ya19kb21haW5NdXNpYyIsCiAgICAiYl9tdU5vbmVfd29ya19kb21haW5Ob25lIiwKICAgICJiX211Tm9uZV93b3JrX2RvbWFpblJldGFpbCIsCiAgICAiYl9tdU5vbmVfd29ya19kb21haW5UZWxlY29tIiwKICAgICJiX211Tm9uZV93b3JrX2RvbWFpbldlYiIKICApLCBwcm9iID0gMC45NSkgKyBzY2FsZV95X2Rpc2NyZXRlKCkgKwogIHNjYWxlX3lfZGlzY3JldGUobGFiZWxzPWMoCiAgICAiSGlnaCBkZWJ0IHZlcnNpb246IGZhbHNlIiwgCiAgICAiUHJvZmVzc2lvbmFsIHByb2dyYW1taW5nIGV4cGVyaWVuY2UiLAogICAgIkdyb3VwOiBjb25zdWx0YW50cyIsCiAgICAiR3JvdXA6IGZyaW5kcyIsCiAgICAiR3JvdXA6IG9wZW4iLAogICAgIkdyb3VwOiBwcm9kdWN0LWNvbXBhbnkiLAogICAgIkdyb3VwOiBwcm9mZXNzaW9uYWwtY29udGFjdHMiLAogICAgIkdyb3VwOiBzdHVkZW50cyIsCiAgICAiRG9tYWluOiBBcHAiLAogICAgIkRvbWFpbjogQXV0b21vdGl2ZSIsCiAgICAiRG9tYWluOiBEZXYtb3BzIiwKICAgICJEb21haW46IEUtQ29tbWVyY2UiLAogICAgIkRvbWFpbjogc3R1ZGVudHMiLAogICAgIkRvbWFpbjogRW1iZWRkZWQiLAogICAgIkRvbWFpbjogRmluYW5jZSIsCiAgICAiRG9tYWluOiBNaXhlZCIsCiAgICAiRG9tYWluOiBNdXNpYyIsCiAgICAiRG9tYWluOiBOb25lIiwKICAgICJEb21haW46IFJldGFpbCIsCiAgICAiRG9tYWluOiBUZWxlY29tIiwKICAgICJEb21haW46IFdlYiIKICApKSArCiAgZ2d0aXRsZSgiQmV0YSBwYXJhbWV0ZXJzIGRlbnNpdGllcyBmb3Igbm8gZG9jdW1lbnRhdGlvbiIsIHN1YnRpdGxlID0gIlNoYWRlZCByZWdpb24gbWFya3MgOTUlIG9mIHRoZSBkZW5zaXR5LiBMaW5lIG1hcmtzIHRoZSBtZWRpYW4iKQpgYGAKCiMjIyMgSW5jb3JyZWN0IGRvY3VtZW50YXRpb24KCmBgYHtyIGludGVycHJldC1iZXRhLXBsb3QtMiwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KbWNtY19hcmVhcyhkb2N1bWVudGF0aW9uMy5hbGwuZXhwLCBwYXJzID0gYygKICAgICJiX211SW5jb3JyZWN0X2hpZ2hfZGVidF92ZXJzaW9uZmFsc2UiLCAKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZy5zIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtwbGFjZV9wZWVyX3Jldmlld2ZhbHNlIiwKICAgICJiX211SW5jb3JyZWN0X2dyb3VwY29uc3VsdGFudHMiLAogICAgImJfbXVJbmNvcnJlY3RfZ3JvdXBmcmllbmRzIiwKICAgICJiX211SW5jb3JyZWN0X2dyb3Vwb3BlbiIsCiAgICAiYl9tdUluY29ycmVjdF9ncm91cHByb2R1Y3RNY29tcGFueSIsCiAgICAiYl9tdUluY29ycmVjdF9ncm91cHByb2Zlc3Npb25hbE1jb250YWN0IiwKICAgICJiX211SW5jb3JyZWN0X2dyb3Vwc3R1ZGVudHMiLAogICAgImJfbXVJbmNvcnJlY3Rfd29ya19kb21haW5BcHAiLAogICAgImJfbXVJbmNvcnJlY3Rfd29ya19kb21haW5BdXRvbW90aXZlIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluRGV2b3BzIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluRU1Db21tZXJjZSIsCiAgICAiYl9tdUluY29ycmVjdF93b3JrX2RvbWFpbkVtYmVkZGVkIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluRmluYW5jZSIsCiAgICAiYl9tdUluY29ycmVjdF93b3JrX2RvbWFpbk1peGVkIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluTXVzaWMiLAogICAgImJfbXVJbmNvcnJlY3Rfd29ya19kb21haW5Ob25lIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluUmV0YWlsIiwKICAgICJiX211SW5jb3JyZWN0X3dvcmtfZG9tYWluVGVsZWNvbSIsCiAgICAiYl9tdUluY29ycmVjdF93b3JrX2RvbWFpbldlYiIKICApLCBwcm9iID0gMC45NSkgKyBzY2FsZV95X2Rpc2NyZXRlKCkgKwogIHNjYWxlX3lfZGlzY3JldGUobGFiZWxzPWMoCiAgICAiSGlnaCBkZWJ0IHZlcnNpb246IGZhbHNlIiwgCiAgICAiUHJvZmVzc2lvbmFsIHByb2dyYW1taW5nIGV4cGVyaWVuY2UiLAogICAgIkdyb3VwOiBjb25zdWx0YW50cyIsCiAgICAiR3JvdXA6IGZyaW5kcyIsCiAgICAiR3JvdXA6IG9wZW4iLAogICAgIkdyb3VwOiBwcm9kdWN0LWNvbXBhbnkiLAogICAgIkdyb3VwOiBwcm9mZXNzaW9uYWwtY29udGFjdHMiLAogICAgIkdyb3VwOiBzdHVkZW50cyIsCiAgICAiRG9tYWluOiBBcHAiLAogICAgIkRvbWFpbjogQXV0b21vdGl2ZSIsCiAgICAiRG9tYWluOiBEZXYtb3BzIiwKICAgICJEb21haW46IEUtQ29tbWVyY2UiLAogICAgIkRvbWFpbjogc3R1ZGVudHMiLAogICAgIkRvbWFpbjogRW1iZWRkZWQiLAogICAgIkRvbWFpbjogRmluYW5jZSIsCiAgICAiRG9tYWluOiBNaXhlZCIsCiAgICAiRG9tYWluOiBNdXNpYyIsCiAgICAiRG9tYWluOiBOb25lIiwKICAgICJEb21haW46IFJldGFpbCIsCiAgICAiRG9tYWluOiBUZWxlY29tIiwKICAgICJEb21haW46IFdlYiIKICApKSArCiAgZ2d0aXRsZSgiQmV0YSBwYXJhbWV0ZXJzIGRlbnNpdGllcyBmb3IgaW5jb3JyZWN0IGRvY3VtZW50YXRpb24iLCBzdWJ0aXRsZSA9ICJTaGFkZWQgcmVnaW9uIG1hcmtzIDk1JSBvZiB0aGUgZGVuc2l0eS4gTGluZSBtYXJrcyB0aGUgbWVkaWFuIikKYGBgCgojIyMgRWZmZWN0cyBzaXplcwoKYGBge3IgZWZmZWN0LXNpemUsIGZpZy53aWR0aD04fQoKc2NhbGVfcHJvZ3JhbW1pbmdfZXhwZXJpZW5jZSA8LSBmdW5jdGlvbih4KSB7CiAgKHggLSBtZWFuKGQuY29tcGxldGVkJHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZykpLyBzZChkLmNvbXBsZXRlZCR3b3JrX2V4cGVyaWVuY2VfcHJvZ3JhbW1pbmcpCn0KdW5zY2FsZV9wcm9ncmFtbWluZ19leHBlcmllbmNlIDwtIGZ1bmN0aW9uKHgpIHsKICB4ICogc2QoZC5jb21wbGV0ZWQkd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nKSArIG1lYW4oZC5jb21wbGV0ZWQkd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nKQp9Cgpwb3N0X3NldHRpbmdzIDwtIGV4cGFuZC5ncmlkKAogIGhpZ2hfZGVidF92ZXJzaW9uID0gYygiZmFsc2UiLCAidHJ1ZSIpLAogIGdyb3VwID0gTkEsCiAgd29ya19kb21haW4gPSBOQSwKICBzZXNzaW9uID0gTkEsCiAgd29ya3BsYWNlX3BlZXJfcmV2aWV3ID0gTkEsCiAgd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnMgPSBzYXBwbHkoYygwLCAzLCAxMCwgMjUsIDQwKSwgc2NhbGVfcHJvZ3JhbW1pbmdfZXhwZXJpZW5jZSkKKQoKcG9zdCA8LSBwb3N0ZXJpb3JfcHJlZGljdChkb2N1bWVudGF0aW9uMy5hbGwuZXhwLCBuZXdkYXRhID0gcG9zdF9zZXR0aW5ncykgJT4lCiAgbWVsdCh2YWx1ZS5uYW1lID0gImVzdGltYXRlIiwgdmFybmFtZXMgPSBjKCJzYW1wbGVfbnVtYmVyIiwgInNldHRpbmdzX2lkIikpICU+JQogIGxlZnRfam9pbigKICAgIHJvd2lkX3RvX2NvbHVtbihwb3N0X3NldHRpbmdzLCB2YXI9ICJzZXR0aW5nc19pZCIpLAogICAgYnkgPSAic2V0dGluZ3NfaWQiCiAgKSAlPiUKICBtdXRhdGUod29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nID0gdW5zY2FsZV9wcm9ncmFtbWluZ19leHBlcmllbmNlKHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZy5zKSkgJT4lCiAgc2VsZWN0KAogICAgZXN0aW1hdGUsCiAgICBoaWdoX2RlYnRfdmVyc2lvbiwKICAgIHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZwogICkKCnBvc3QgJT4lCiAgbXV0YXRlX2F0KCJoaWdoX2RlYnRfdmVyc2lvbiIsIAogICAgICAgICAgICBmdW5jdGlvbih4KSBjYXNlX3doZW4oCiAgICAgICAgICAgICAgeCA9PSAiZmFsc2UiIH4gIkxvdyBkZWJ0IiwKICAgICAgICAgICAgICB4ID09ICJ0cnVlIiB+ICJIaWdoIGRlYnQiCiAgICAgICAgICAgICkpICU+JQogIG11dGF0ZV9hdCgiZXN0aW1hdGUiLCAKICAgICAgICAgICAgZnVuY3Rpb24oeCkgY2FzZV93aGVuKAogICAgICAgICAgICAgIHggPT0gMSB+ICJDb3JyZWN0IiwKICAgICAgICAgICAgICB4ID09IDIgfiAiSW5jb3JyZWN0IiwKICAgICAgICAgICAgICB4ID09IDMgfiAiTWlzc2luZyIKICAgICAgICAgICAgKSkgJT4lCiAgZ2dwbG90KGFlcyhoaWdoX2RlYnRfdmVyc2lvbikpICsKICBnZW9tX2JhcihhZXMoZmlsbCA9IGVzdGltYXRlKSwgcG9zaXRpb24gPSAiZmlsbCIpICsgCiAgZmFjZXRfZ3JpZChyb3dzID0gdmFycyh3b3JrX2V4cGVyaWVuY2VfcHJvZ3JhbW1pbmcpKSArCiAgc2NhbGVfeV9yZXZlcnNlKCkgKwogIHNjYWxlX2ZpbGxfbWFudWFsKCJMZWdlbmQiLCB2YWx1ZXMgPSBjKCJkYXJrYmx1ZSIsICIjNzA3MEZGIiwgImxpZ2h0Ymx1ZSIpLCBndWlkZSA9IGd1aWRlX2xlZ2VuZChyZXZlcnNlID0gVFJVRSkpICsKICBsYWJzKHRpdGxlID0gIkRvY3VtZW50YXRpb24gc3RhdGUiKSArCiAgeGxhYigiRGVidCB2ZXJzaW9uIikgKwogIHlsYWIoIlJhdGlvIG9mIGRvY3VtZW50YXRpb24gc3RhdGUiKQoKYGBgCgpXZSBjYW4gc2VlIHRoYXQgdGFzayBjb21wbGV0aW9uIHJhdGlvcyBhcmUgc2ltaWxhciBmb3IgYm90aCBoaWdoIGFuZCBsb3cgZGVidCBidXQgdGhhdCB0aGVyZSBpcyBzb21lIGRpZmZlcmVuY2UgaW4gaG93IG9mdGVuIHRoZXkgbGVhdmUgaW5jb3JyZWN0IGRvY3VtZW50YXRpb24gYW5kIHdpbGwgdGhlcmVmb3JlIHByb2NlZWQgdG8gY2FsY3VsYXRlIHNvbWUgcHJvYmFiaWxpdGllcyBvZiBsZWF2aW5nIGluY29ycmVjdCBkb2N1bWVudGF0aW9uLgoKYGBge3IsIGNsYXNzLnNvdXJjZSA9ICdmb2xkLXNob3cnfQpkIDwtIHBvc3QgJT4lIGZpbHRlcihlc3RpbWF0ZSA9PSAyKQpkLmhpZ2ggPC0gZCAlPiUgZmlsdGVyKGhpZ2hfZGVidF92ZXJzaW9uID09ICJ0cnVlIikgJT4lIHB1bGwoZXN0aW1hdGUpCmQubG93IDwtIGQgJT4lIGZpbHRlcihoaWdoX2RlYnRfdmVyc2lvbiA9PSAiZmFsc2UiKSAlPiUgcHVsbChlc3RpbWF0ZSkKeCA8LSBsZW5ndGgoZC5oaWdoKSAvIGxlbmd0aChkLmxvdykKeApgYGAKR2l2ZW4gYWxsIHRoZSBzaW11bGF0ZWQgY2FzZXMgd2UgZmluZCB0aGF0IGRldmVsb3BlcnMgYXJlIGByIHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKHggLSAxKWAgbW9yZSBsaWtlbHkgdG8gbGVhdmUgaW5jb3JyZWN0IGRvY3VtZW50YXRpb24gaW4gdGhlIGhpZ2ggZGVidCB2ZXJzaW9uIG9mIHRoZSBzY2VuYXJpb3MuCgpgYGB7ciwgY2xhc3Muc291cmNlID0gJ2ZvbGQtc2hvdyd9CmQgPC0gcG9zdCAlPiUgZmlsdGVyKGVzdGltYXRlID09IDIsIHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZyA9PSAxMCkKZC5oaWdoIDwtIGQgJT4lIGZpbHRlcihoaWdoX2RlYnRfdmVyc2lvbiA9PSAidHJ1ZSIpICU+JSBwdWxsKGVzdGltYXRlKQpkLmxvdyA8LSBkICU+JSBmaWx0ZXIoaGlnaF9kZWJ0X3ZlcnNpb24gPT0gImZhbHNlIikgJT4lIHB1bGwoZXN0aW1hdGUpCnggPC0gbGVuZ3RoKGQuaGlnaCkgLyBsZW5ndGgoZC5sb3cpCngKYGBgCkdpdmVuIGRldmVsb3BlcnMgd2l0aCAxMCB5ZWFycyBvZiBwcm9mZXNzaW9uYWwgcHJvZ3JhbW1pbmcgZXhwZXJpZW5jZSB3ZSBmaW5kIHRoYXQgdGhleSBhcmUgYHIgc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoeCAtIDEpYCBtb3JlIGxpa2VseSB0byBsZWF2ZSBpbmNvcnJlY3QgZG9jdW1lbnRhdGlvbiBpbiB0aGUgaGlnaCBkZWJ0IHZlcnNpb24gb2YgdGhlIHNjZW5hcmlvcy4KCmBgYHtyLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93J30KZCA8LSBwb3N0ICU+JSBmaWx0ZXIoZXN0aW1hdGUgPT0gMiwgd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nID09IDI1KQpkLmhpZ2ggPC0gZCAlPiUgZmlsdGVyKGhpZ2hfZGVidF92ZXJzaW9uID09ICJ0cnVlIikgJT4lIHB1bGwoZXN0aW1hdGUpCmQubG93IDwtIGQgJT4lIGZpbHRlcihoaWdoX2RlYnRfdmVyc2lvbiA9PSAiZmFsc2UiKSAlPiUgcHVsbChlc3RpbWF0ZSkKeCA8LSBsZW5ndGgoZC5oaWdoKSAvIGxlbmd0aChkLmxvdykKeApgYGAKR2l2ZW4gZGV2ZWxvcGVycyB3aXRoIDI1IHllYXJzIG9mIHByb2Zlc3Npb25hbCBwcm9ncmFtbWluZyBleHBlcmllbmNlIHdlIGZpbmQgdGhhdCB0aGV5IGFyZSBgciBzY2FsZXM6OmxhYmVsX3BlcmNlbnQoKSh4IC0gMSlgIG1vcmUgbGlrZWx5IHRvIGxlYXZlIGluY29ycmVjdCBkb2N1bWVudGF0aW9uIGluIHRoZSBoaWdoIGRlYnQgdmVyc2lvbiBvZiB0aGUgc2NlbmFyaW9zLgo=