Simulation example with mirtCAT

Example simulation for computerized adaptive testing with the mirtCAT package. Use of SimDesign for this package demonstrates a different type of parallelization, whereby the work is distributed within the analysis step rather than across all replications. This showcases how within-function parallelization can be utilized for code which has already been optimized for simulations, but still needs an ‘outer-shell’ to collect the simulation results. Currently requires mirtCAT version 1.1.4 or higher.

This study explored two major design conditions: size of the item bank, and the magnitude of the discrimination parameter. Just for didactic purposes the discrimination parameters were all set equal, however real studies will call for more intimate control of the properties of the item bank.

library(SimDesign)
library(mirtCAT)

Loading required package: mirt

Loading required package: stats4

Loading required package: lattice

Loading required package: shiny

library(parallel)

Design <- createDesign(nitems = c(100, 300, 500),
                       discrimination = c(.5, 1.5))
cl <- makeCluster(detectCores())
fo <- list(cl=cl, 
           design=list(min_SEM = .25, min_items = 10, max_items = 50))

#-------------------------------------------------------------------

Generate <- function(condition, fixed_objects = NULL) {
    Attach(condition)
    Theta <- matrix(rnorm(1000))
    a <- matrix(rep(discrimination, nitems))
    d <- rnorm(nitems)
    pars <- data.frame(a1 = a, d = d, g = 0.2)
    mirt_object <- generate.mirt_object(pars, '3PL')
    responses <- generate_pattern(mirt_object, Theta = Theta)
    list(mirt_object=mirt_object, responses=responses)
}

Analyse <- function(condition, dat, fixed_objects = NULL) {
    ret <- with(fixed_objects, 
                mirtCAT(mo = dat$mirt_object, local_pattern = dat$responses,
                        start_item = 'MI', method = 'EAP',
                        criteria = 'MI', design = design, cl = cl))
    ret
}

#-------------------------------------------------------------------

# only 1 replication (note: no Summarise() function)
res <- runSimulation(design=Design, replications=1, seed = 1:nrow(Design),
                     generate=Generate, analyse=Analyse, fixed_objects=fo, debug='none')

save, stop_on_fatal, and print_RAM flags disabled for testing purposes

Design: 1/6;   Replications: 1   Total Time: 0.00s 
 Conditions: nitems=100, discrimination=0.5


Design: 2/6;   Replications: 1   Total Time: 20.51s 
 Conditions: nitems=300, discrimination=0.5


Design: 3/6;   Replications: 1   Total Time: 38.43s 
 Conditions: nitems=500, discrimination=0.5


Design: 4/6;   Replications: 1   Total Time: 01m 0.45s 
 Conditions: nitems=100, discrimination=1.5


Design: 5/6;   Replications: 1   Total Time: 01m 13.56s 
 Conditions: nitems=300, discrimination=1.5


Design: 6/6;   Replications: 1   Total Time: 01m 30.89s 
 Conditions: nitems=500, discrimination=1.5

dplyr::glimpse(res[1:2])

List of 2
 $ nitems=100; discrimination=0.5:List of 1
  ..$ :List of 1000
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. .. [list output truncated]
 $ nitems=300; discrimination=0.5:List of 1
  ..$ :List of 1000
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. ..$ :List of 15
  .. .. ..- attr(*, "class")= chr "mirtCAT"
  .. .. [list output truncated]

# person 100 from the first condition
plot(res$`nitems=100; discrimination=0.5`[[1]][[100]], SE = 1.96)

# person 50 from the last condition
plot(res$`nitems=500; discrimination=1.5`[[1]][[50]], SE = 1.96)

diff <- lapply(res, function(x){
    sapply(x[[1]], function(y){
        so <- summary(y)
        so$final_estimates[1,] - so$true_thetas
    })
})

nitems_answered <- lapply(res, function(x){
    sapply(x[[1]], function(y){
        so <- summary(y)
        length(so$items_answered)
    })
})

meta_stats <- data.frame(Design, bias=sapply(diff, bias), RMSE=sapply(diff, RMSE), 
                         nitems_answered=sapply(nitems_answered, mean), row.names = NULL)
meta_stats

  nitems discrimination      bias  RMSE nitems_answered
1    100            0.5 -0.000277 0.599            50.0
2    300            0.5 -0.024392 0.567            50.0
3    500            0.5 -0.009807 0.590            50.0
4    100            1.5 -0.002053 0.264            46.8
5    300            1.5 -0.000141 0.264            45.2
6    500            1.5  0.007965 0.263            44.9

From this simple summary we can see that not only did the conditions with larger discrimination parameters provide better accuracy, but the CATs were actually terminated before the maximum 50 item cutoff.