The FDP for different \(g_1\) figure in cashr

Introduction

library(ggplot2)

Warning: package 'ggplot2' was built under R version 3.4.4

boxplot.quantile <- function(x) {
  r <- quantile(x, probs = c(0.05, 0.25, 0.5, 0.75, 0.95))
  names(r) <- c("ymin", "lower", "middle", "upper", "ymax")
  return(r)
}

boxplot.quantile.sq <- function (x) {
  r <- sqrt(quantile(x^2, probs = c(0.05, 0.25, 0.5, 0.75, 0.95)))
  names(r) <- c("ymin", "lower", "middle", "upper", "ymax")
  return(r)
}

mean.sq <- function (x) {
  r <- sqrt(mean(x^2))
  return(r)
}

mysqrt_trans <- function() {
  scales::trans_new("mysqrt", 
                    transform = base::sqrt,
                    inverse = function(x) ifelse(x<0, 0, x^2),
                    domain = c(0, Inf))
}

q <- 0.1
method.name.FDR <- c("cashr", "BH", "qvalue", "ashr", "locfdr")
method.col.FDR <- scales::hue_pal()(length(method.name.FDR))[c(5, 1, 2, 4, 3)]

density.ggdata.normal <- readRDS("../output/paper/simulation/density.ggdata.normal.rds")
density.ggdata.nearnormal <- readRDS("../output/paper/simulation/density.ggdata.nearnormal.rds")
density.ggdata.spiky <- readRDS("../output/paper/simulation/density.ggdata.spiky.rds")
density.ggdata.flattop <- readRDS("../output/paper/simulation/density.ggdata.flattop.rds")
density.ggdata.skew <- readRDS("../output/paper/simulation/density.ggdata.skew.rds")
density.ggdata.bimodal <- readRDS("../output/paper/simulation/density.ggdata.bimodal.rds")

g1.names <- c(
  'Gaussian',
  'Near Gaussian',
  'Spiky',
  'Flat Top',
  'Skew',
  'Bimodal'
)

density.g.ggdata <- rbind.data.frame(
  density.ggdata.normal,
  density.ggdata.nearnormal,
  density.ggdata.spiky,
  density.ggdata.flattop,
  density.ggdata.skew,
  density.ggdata.bimodal
)

density.g.ggdata$g <- plyr::mapvalues(density.g.ggdata$g, from = levels(density.g.ggdata$g), to = g1.names)

density.g.ggdata$g <- factor(density.g.ggdata$g, levels = g1.names[c(1, 2, 3, 5, 4, 6)])

density.g.plot <- ggplot(data = density.g.ggdata, aes(x = plotx, y = ploty)) +
  geom_line() +
  facet_wrap(~g, nrow = 1) +
  labs(x = expression(theta), y = expression(g[1](theta))) +
  theme(# axis.title.x = element_text(size = 15),
        axis.title.x = element_blank(),
        axis.text.x = element_text(size = 10),
        axis.title.y = element_text(size = 15),
        axis.text.y = element_text(size = 10),
        strip.text = element_text(size = 15),
        legend.position = "none",
        legend.background = element_rect(color = "grey"),
        legend.text = element_text(size = 12))

FDP.q.all.ggdata.normal <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.normal.rds")
FDP.q.all.ggdata.nearnormal <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.nearnormal.rds")
FDP.q.all.ggdata.spiky <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.spiky.rds")
FDP.q.all.ggdata.flattop <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.flattop.rds")
FDP.q.all.ggdata.skew <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.skew.rds")
FDP.q.all.ggdata.bimodal <- readRDS("../output/paper/simulation/FDP.q.all.ggdata.bimodal.rds")

FDP.q.g.ggdata <- rbind.data.frame(
  FDP.q.all.ggdata.normal,
  FDP.q.all.ggdata.nearnormal,
  FDP.q.all.ggdata.spiky,
  FDP.q.all.ggdata.flattop,
  FDP.q.all.ggdata.skew,
  FDP.q.all.ggdata.bimodal
)

FDP.q.g.ggdata$Method <- plyr::mapvalues(FDP.q.g.ggdata$Method, from = c("CASH", "BHq", "ASH"), to = c("cashr", "BH", "ashr"))

FDP.q.g.ggdata$Method <- factor(FDP.q.g.ggdata$Method, levels = c("cashr", "BH", "qvalue", "ashr", "locfdr"))

FDP.q.g.ggdata$pi0 <- as.numeric(levels(FDP.q.g.ggdata$pi0))[FDP.q.g.ggdata$pi0]

FDP.q.g.ggdata$g <- plyr::mapvalues(FDP.q.g.ggdata$g, from = levels(FDP.q.g.ggdata$g), to = g1.names)

FDP.q.g.ggdata$g <- factor(FDP.q.g.ggdata$g, levels = g1.names[c(1, 2, 3, 5, 4, 6)])

FDP.q.g.plot <- ggplot(data = FDP.q.g.ggdata, aes(x = Method, y = FDP, fill = Method, color = Method)) +
  stat_summary(fun.data = boxplot.quantile.sq, geom = "boxplot", position = "dodge", aes(width = 0.75), show.legend = FALSE) +
  stat_summary(fun.y = mean.sq, geom = "point", position = position_dodge(width = 0.9), show.legend = FALSE, shape = 13, size = 3) +
  scale_color_manual(labels = method.name.FDR, values = method.col.FDR) +
  scale_fill_manual(labels = method.name.FDR, values = alpha(method.col.FDR, 0.35)) +
  scale_y_continuous(trans = "mysqrt", breaks = c(0, 0.1, 0.2, 0.4, 0.6, 0.8)) +
  facet_grid(pi0 ~ g, labeller = label_bquote(rows = pi[0] == .(pi0))) +
  scale_x_discrete(limits = rev(levels(FDP.q.g.ggdata$Method))) +
  coord_flip() +
  geom_hline(yintercept = q, col = "black", linetype = "dashed", size = 1) +
  labs(y = "FDP", title = bquote(paste("Nominal FDR = ", .(q)))) +
  theme(plot.title = element_text(size = 12, hjust = 0),
        axis.title.y = element_blank(),
        axis.text.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        axis.text.x = element_text(size = 10),
        strip.text = element_text(size = 15),
        legend.position = "bottom",
        legend.background = element_rect(color = "grey"),
        legend.text = element_text(size = 12))

Warning: Ignoring unknown aesthetics: width

FDP.q.g.plot.save <- gridExtra::arrangeGrob(
  density.g.plot + 
    theme(plot.margin = grid::unit(c(5.5, 30, 5.5, 10), "points")) +
    theme(plot.title = element_blank()),
  FDP.q.g.plot + theme(strip.text.x = element_blank(), plot.title = element_blank()),
  heights = c(1, 2),
  top = grid::textGrob(bquote(paste("Nominal FDR = ", .(q))), gp = grid::gpar(fontsize = 15), hjust = 2.35)
)

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 5 rows containing non-finite values (stat_summary).

ggsave("../output/paper/FDPqg.pdf", FDP.q.g.plot.save, height = 6, width = 10)

mean.table <- aggregate(FDP ~ Method + pi0 + g, data = FDP.q.g.ggdata, FUN = "mean")
mse.sqrt.table <- aggregate(FDP ~ Method + pi0 + g, data = FDP.q.g.ggdata, FUN = function (x) {sqrt(mean((x - 0.1)^2))})
mad.table <- aggregate(FDP ~ Method + pi0 + g, data = FDP.q.g.ggdata, FUN = function (x) {mean(abs(x - 0.1))})

mse.sqrt.table$pi0 <- factor(mse.sqrt.table$pi0)
FDP.sqrt.mse.plot <- ggplot(data = mse.sqrt.table,
                            aes(x = pi0,
                                y = FDP,
                                group = Method,
                                col = Method,
                                linetype = Method,
                                shape = Method)) +
  geom_line(size = 1.25) +
  geom_point(size = 3) + 
  scale_color_manual(values = method.col.FDR) +
  facet_wrap(~ g, nrow = 1) +
  labs(y = 'Root MSE', x = expression(pi[0])) +
  theme(
    axis.title.y = element_text(size = 15),
    axis.text.y = element_text(size = 12),
    axis.title.x = element_text(size = 15),
    axis.text.x = element_text(size = 12),
    strip.text = element_text(size = 15),
    legend.text = element_text(size = 15),
    legend.title = element_blank()
  )

FDPqgplot.save <- gridExtra::arrangeGrob(
  density.g.plot + 
    labs(title = expression(paste('Distribution ', g[1], ' of non-null effects'))) +
    theme(plot.margin = grid::unit(c(5.5, 92, 25, 10), "points")) +
    theme(plot.title = element_text(size = 15)),
  FDP.q.g.plot +
    labs(title = bquote(paste('Distribution of empirical FDP at nominal FDR = ', .(q)))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 67.5, 20, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),
  FDP.sqrt.mse.plot +
    labs(title = 'Root MSE of empirical FDP from nominal value') +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15)),
  heights = c(1.5, 2, 1.5)
  # top = grid::textGrob(bquote(paste("Nominal FDR = ", .(q))), gp = grid::gpar(fontsize = 15), hjust = 4)
)

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 5 rows containing non-finite values (stat_summary).

ggsave("../output/paper/FDP_sqrtMSE.pdf", FDPqgplot.save, height = 10, width = 15)

TDP.q <- function (g1, M) {cbind.data.frame(g1 = g1, M[M$Noise == "All", 2 : 4])}
TDP.q.gaussian <- TDP.q('Gaussian', readRDS("../output/paper/simulation/TDP.q.normal.rds"))
TDP.q.neargaussian <- TDP.q('Near Gaussian', readRDS("../output/paper/simulation/TDP.q.nearnormal.rds"))
TDP.q.spiky <- TDP.q('Spiky', readRDS("../output/paper/simulation/TDP.q.spiky.rds"))
TDP.q.skew <- TDP.q('Skew', readRDS("../output/paper/simulation/TDP.q.skew.rds"))
TDP.q.flattop <- TDP.q('Flat Top', readRDS("../output/paper/simulation/TDP.q.flattop.rds"))
TDP.q.bimodal <- TDP.q('Bimodal', readRDS("../output/paper/simulation/TDP.q.bimodal.rds"))
TDP.q <- rbind.data.frame(
  TDP.q.gaussian,
  TDP.q.neargaussian,
  TDP.q.spiky,
  TDP.q.skew,
  TDP.q.flattop,
  TDP.q.bimodal
)

TDP.q$Method <- plyr::mapvalues(TDP.q$Method, from = levels(TDP.q$Method), to = c('BH', 'qvalue', 'locfdr', 'ashr', 'cashr', 'BH'))
TDP.q$Method <- factor(TDP.q$Method, levels = c("cashr", "BH", "qvalue", "ashr", "locfdr"))
TDP.q$pi0 <- as.numeric(levels(TDP.q$pi0))[TDP.q$pi0]

TDP.q.g.plot <- ggplot(data = TDP.q, aes(x = Method, y = TDP, fill = Method, color = Method)) +
  stat_summary(fun.data = boxplot.quantile, geom = "boxplot", position = "dodge", aes(width = 0.75), show.legend = FALSE) +
  stat_summary(fun.y = mean, geom = "point", position = position_dodge(width = 0.9), show.legend = FALSE, shape = 13, size = 3) +
  scale_color_manual(labels = method.name.FDR, values = method.col.FDR) +
  scale_fill_manual(labels = method.name.FDR, values = alpha(method.col.FDR, 0.35)) +
# scale_y_continuous(trans = "mysqrt", breaks = c(0, 0.1, 0.2, 0.4, 0.6, 0.8)) +
  facet_grid(pi0 ~ g1, labeller = label_bquote(rows = pi[0] == .(pi0))) +
  scale_x_discrete(limits = rev(levels(TDP.q$Method))) +
  coord_flip() +
# geom_hline(yintercept = q, col = "black", linetype = "dashed", size = 1) +
  labs(y = "TDP", title = bquote(paste("Nominal FDR = ", .(q)))) +
  theme(plot.title = element_text(size = 12, hjust = 0),
        axis.title.y = element_blank(),
        axis.text.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        axis.text.x = element_text(size = 10),
        strip.text = element_text(size = 15),
        legend.position = "bottom",
        legend.background = element_rect(color = "grey"),
        legend.text = element_text(size = 12))

Warning: Ignoring unknown aesthetics: width

FDP.TDP.qg.save <- gridExtra::arrangeGrob(
  density.g.plot + 
    labs(title = expression(paste('Distribution ', g[1], ' of non-null effects'))) +
    theme(plot.margin = grid::unit(c(5.5, 92, 25, 10), "points")) +
    theme(plot.title = element_text(size = 15)),
  FDP.q.g.plot +
    labs(title = bquote(paste('Distribution of empirical FDP at nominal FDR = ', .(q)))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 67.5, 20, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),
  FDP.sqrt.mse.plot +
    labs(title = 'Root MSE of empirical FDP from nominal value') +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 5.5, 20, 5.5), 'points')
    ),
  TDP.q.g.plot +
    labs(title = bquote(paste('Distribution of empirical TDP at nominal FDR = ', .(q)))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 67.5, 5.5, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),    
  heights = c(1.5, 2, 1.5, 2)
  # top = grid::textGrob(bquote(paste("Nominal FDR = ", .(q))), gp = grid::gpar(fontsize = 15), hjust = 4)
)

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 8 rows containing non-finite values (stat_summary).

Warning: Removed 8 rows containing non-finite values (stat_summary).

ggsave("../output/paper/FDP_sqrtMSE_TDP.pdf", FDP.TDP.qg.save, height = 15, width = 15)

mean.TDP <- aggregate(TDP ~ Method + pi0 + g1, data = TDP.q, FUN = "mean")

mean.TDP$pi0 <- factor(mean.TDP$pi0)
mean.TDP.plot <- ggplot(data = mean.TDP,
                            aes(x = pi0,
                                y = TDP,
                                group = Method,
                                col = Method,
                                linetype = Method,
                                shape = Method)) +
  geom_line(size = 1.25) +
  geom_point(size = 3) + 
  scale_color_manual(values = method.col.FDR) +
  facet_wrap(~ g1, nrow = 1) +
  labs(y = 'Mean TDP', x = expression(pi[0])) +
  theme(
    axis.title.y = element_text(size = 15),
    axis.text.y = element_text(size = 12),
    axis.title.x = element_text(size = 15),
    axis.text.x = element_text(size = 12),
    strip.text = element_text(size = 15),
    legend.text = element_text(size = 15),
    legend.title = element_blank()
  )

FDP.TDP.power.qg.save <- gridExtra::arrangeGrob(
  density.g.plot + 
    labs(title = expression(paste('Distribution ', g[1], ' of non-null effects'))) +
    theme(plot.margin = grid::unit(c(5.5, 92, 5.5, 10), "points")) +
    theme(plot.title = element_text(size = 15)),
  FDP.q.g.plot +
    labs(title = bquote(paste('Distribution of empirical FDP at nominal FDR = ', .(q)))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(20, 67.5, 5.5, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),
  FDP.sqrt.mse.plot +
    labs(title = 'Root MSE of empirical FDP from nominal value') +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(20, 5.5, 5.5, 5.5), 'points')
    ),
  TDP.q.g.plot +
    labs(title = bquote(paste('Distribution of empirical TDP at nominal FDR = ', .(q)))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(20, 67.5, 5.5, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),
  mean.TDP.plot +
    labs(title = bquote(paste('Mean of empirical TDP at nominal FDR = ', .(q)))) +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(20, 5.5, 5.5, 5.5), 'points')
    ),    
  heights = c(1.5, 2, 1.5, 2, 1.5)
  # top = grid::textGrob(bquote(paste("Nominal FDR = ", .(q))), gp = grid::gpar(fontsize = 15), hjust = 4)
)

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 8 rows containing non-finite values (stat_summary).

Warning: Removed 8 rows containing non-finite values (stat_summary).

ggsave("../output/paper/FDP_sqrtMSE_TDP.pdf", FDP.TDP.power.qg.save, height = 17, width = 15)

scaleFUN <- function(x) sprintf("%.2f", x)

FDP.power.qg.save <- gridExtra::arrangeGrob(
  density.g.plot + 
    labs(title = expression(paste('(a): Distribution ', g[1], ' of non-null effects'))) +
    theme(plot.margin = grid::unit(c(20, 92, 20, 10), "points")) +
    theme(plot.title = element_text(size = 15)),
  FDP.q.g.plot +
    labs(title = bquote(paste('(b): Distribution of FDP'))) +
    theme(strip.text.x = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 67.5, 5.5, 5.5), 'points'),
          axis.text.x = element_text(size = 12)
    ),
  FDP.sqrt.mse.plot +
    labs(title = '(c): Root MSE of FDP from nominal FDR') +
    expand_limits(y = 0) +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 5.5, 5.5, 15), 'points')
    ),
  mean.TDP.plot +
    labs(title = bquote(paste('(d): Mean of TDP'))) +
#    scale_y_continuous(labels = scaleFUN) +
    expand_limits(y = 0) +
    theme(strip.text = element_blank(),
          plot.title = element_text(size = 15),
          plot.margin = grid::unit(c(5.5, 5.5, 5.5, 15), 'points')
    ),    
  heights = c(1.7, 2, 1.5, 1.5),
  top = grid::textGrob(bquote(paste("Nominal FDR = ", .(q))), gp = grid::gpar(fontsize = 17), hjust = 3.6)
)

Warning: Removed 5 rows containing non-finite values (stat_summary).

Warning: Removed 5 rows containing non-finite values (stat_summary).

ggsave("../output/paper/FDP_sqrtMSE_Power.pdf", FDP.power.qg.save, height = 13.5, width = 15)

---

Not used

# pi0.plot <- 0.9
# 
# FDP.q.g.pi0.plot <- ggplot(data = FDP.q.g.ggdata[FDP.q.g.ggdata$pi0 == pi0.plot, ], aes(x = Method, y = FDP, fill = Method, color = Method)) +
#   stat_summary(fun.data = boxplot.quantile, geom = "boxplot", position = "dodge", aes(width = 0.5), show.legend = FALSE) +
#   stat_summary(fun.y = mean, geom = "point", position = position_dodge(width = 0.9), show.legend = FALSE) +
#   scale_color_manual(labels = method.name.FDR, values = method.col.FDR) +
#   scale_fill_manual(labels = method.name.FDR, values = alpha(method.col.FDR, 0.35)) +
#   scale_y_continuous(trans = "mysqrt", breaks = c(0, 0.1, 0.2, 0.4, 0.6, 0.8)) +
#   facet_wrap(~ g, nrow = 1) +
#   scale_x_discrete(limits = rev(levels(FDP.q.g.ggdata$Method))) +
#   coord_flip() +
#   geom_hline(yintercept = q, col = "black", linetype = "dashed", size = 1) +
#   labs(y = "FDP", title = bquote(paste("Nominal FDR = ", .(q), "(", pi[0] == .(pi0.plot), ")"))) +
#   theme(plot.title = element_text(size = 12, hjust = 0),
#         axis.title.y = element_blank(),
#         axis.text.y = element_text(size = 15),
#         axis.title.x = element_text(size = 15),
#         axis.text.x = element_text(size = 10),
#         strip.text = element_text(size = 15),
#         legend.position = "bottom",
#         legend.background = element_rect(color = "grey"),
#         legend.text = element_text(size = 12))

# FDP.q.g.pi0.plot.save <- gridExtra::arrangeGrob(
#   density.g.plot,
#   FDP.q.g.pi0.plot,
#   heights = c(1, 1.1)
# )
# 
# ggsave("../output/paper/FDP.q.g.pi0.pdf", FDP.q.g.pi0.plot.save, height = 4, width = 12)

# FDP.q.g.plot <- ggplot(data = FDP.q.g.ggdata, aes(x = Method, y = FDP, fill = Method, color = Method)) +
#   stat_summary(fun.data = boxplot.quantile, geom = "boxplot", position = "dodge", aes(width = 0.75), show.legend = FALSE) +
#   stat_summary(fun.y = mean, geom = "point", position = position_dodge(width = 0.9), show.legend = FALSE, shape = 13, size = 3) +
#   scale_color_manual(labels = method.name.FDR, values = method.col.FDR) +
#   scale_fill_manual(labels = method.name.FDR, values = alpha(method.col.FDR, 0.35)) +
#   facet_grid(pi0 ~ g, labeller = label_bquote(rows = pi[0] == .(pi0))) +
#   scale_x_discrete(limits = rev(levels(FDP.q.g.ggdata$Method))) +
#   coord_flip() +
#   geom_hline(yintercept = q, col = "black", linetype = "dashed", size = 1) +
#   labs(y = "FDP", title = bquote(paste("Nominal FDR = ", .(q)))) +
#   theme(plot.title = element_text(size = 12, hjust = 0),
#         axis.title.y = element_blank(),
#         axis.text.y = element_text(size = 15),
#         axis.title.x = element_text(size = 15),
#         axis.text.x = element_text(size = 10),
#         strip.text = element_text(size = 15),
#         legend.position = "bottom",
#         legend.background = element_rect(color = "grey"),
#         legend.text = element_text(size = 12))

Session information

sessionInfo()

R version 3.4.3 (2017-11-30)
Platform: x86_64-apple-darwin15.6.0 (64-bit)
Running under: macOS  10.14.1

Matrix products: default
BLAS: /Library/Frameworks/R.framework/Versions/3.4/Resources/lib/libRblas.0.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/3.4/Resources/lib/libRlapack.dylib

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
[1] ggplot2_3.1.0

loaded via a namespace (and not attached):
 [1] Rcpp_0.12.16      compiler_3.4.3    pillar_1.2.2     
 [4] git2r_0.23.0      plyr_1.8.4        workflowr_1.1.1  
 [7] bindr_0.1.1       R.methodsS3_1.7.1 R.utils_2.6.0    
[10] tools_3.4.3       digest_0.6.15     evaluate_0.10.1  
[13] tibble_1.4.2      gtable_0.2.0      pkgconfig_2.0.1  
[16] rlang_0.3.0.1     yaml_2.1.19       bindrcpp_0.2.2   
[19] gridExtra_2.3     withr_2.1.2       stringr_1.3.1    
[22] dplyr_0.7.4       knitr_1.20        rprojroot_1.3-2  
[25] grid_3.4.3        glue_1.2.0        R6_2.2.2         
[28] rmarkdown_1.9     reshape2_1.4.3    magrittr_1.5     
[31] whisker_0.3-2     backports_1.1.2   scales_0.5.0     
[34] htmltools_0.3.6   assertthat_0.2.0  colorspace_1.3-2 
[37] labeling_0.3      stringi_1.2.2     lazyeval_0.2.1   
[40] munsell_0.4.3     R.oo_1.22.0