Purpose: How do the TMB calculations change with adjustments to the VAF cutoff filter? More specifically, if we increase VAF filter cutoff, do the callers’ TMB stats relate to each other more?

Summary of Findings:

VarDict is the least related to the other callers, but increasing the VAF filter cutoff does somewhat recover how its TMB stats relate to the other callers’ TMB stats. Regardless, VarDict’s data should still be dropped as it is too aberrant from the other callers as well as overly sensitive.

Usage

To run this from the command line, use:

Rscript -e "rmarkdown::render('analyses/snv-callers/vaf_cutoff_experiment.Rmd', 
                              clean = TRUE)"

This assumes you are in the top directory of the repository.

Setup

Packages and functions

Read in set up script.

if (!("GGally" %in% installed.packages())) {
  install.packages("GGally")
}
if (!("ggupset" %in% installed.packages())) {
  install.packages("ggupset")
}
# Magrittr pipe
`%>%` <- dplyr::`%>%`

Set up output directories.

base_results_dir <- "results"
base_plots_dir <- "plots"

Make new directories for the comparison analysis.

vaf_cutoff_results_dir <- file.path(base_results_dir, "vaf_cutoff_data")

# Make caller specific plots folder
if (!dir.exists(vaf_cutoff_results_dir)) {
  dir.create(vaf_cutoff_results_dir)
}

Function for creating a TMB correlation matrix amongst the callers for a particular cutoff.

cor_matrix_plot <- function(cutoff) {
  # For a given VAF cutoff, calculate TMB correlations amongst the callers and plot it using
  # GGally::ggpairs
  #
  # Args:
  #   cutoff: The VAF cutoff set to analyze
  #
  # Returns:
  #   A correlation matrix plot amongst the callers

  # Isolate the TMB stats for this cutoff
  cutoff_df <- tmb_filter_long %>%
    dplyr::filter(vaf_cutoff == cutoff) %>%
    dplyr::select("lancet", "mutect2", "strelka2", "vardict")

  # Make the plot
  cor_plot <- GGally::ggpairs(cutoff_df, mapping = ggplot2::aes(alpha = 0.05)) +
    ggplot2::theme_classic() +
    ggplot2::ggtitle(paste0("VAF cutoff: ", cutoff))

  # Return the plot
  return(cor_plot)
}

Function for correlating TMB statistics for each VAF cutoff set.

cor_by_vaf <- function(cutoff = 0.1) {
  # For a given VAF cutoff, isolate the TMB statistics and correlate them.
  #
  # Args:
  #   cutoff: the VAF cutoff to calculate the correlation by.
  #
  # Returns:
  #   R values for Spearman's correlation amongst the callers

  cutoff_df <- tmb_filter_long %>%
    dplyr::filter(vaf_cutoff == cutoff) %>%
    dplyr::select(-Tumor_Sample_Barcode, -vaf_cutoff)

  # Do correlations between callers
  cor_res <- cor(cutoff_df, method = "spearman", use = "na.or.complete")

  # This is to remove redundancy as upper correlation matrix == lower
  cor_res[upper.tri(cor_res, diag = TRUE)] <- NA

  # Format into long format data.frame
  cor_df <- reshape2::melt(cor_res, na.rm = TRUE, value.name = "cor") %>%
    dplyr::mutate(
      compare = paste0(Var1, "-", Var2),
      cutoff
    )

  # Return a data.frame ready for being added to the bigger data.frame for plotting.
  return(cor_df)
}

Run the data with different VAF filter cutoffs

We need to re-calculate TMB statistics with various VAF filter cutoffs. We will run this script to do it: system("bash scripts/run_caller_evals_vaf_filter_exp.sh")

Import data

Get the lists of each type of file.

# Make a list of the vaf filter experiment tmb files
filter_tmb_files <- list.files(vaf_cutoff_results_dir,
  pattern = "_tmb.rds$", recursive = TRUE,
  full.names = TRUE
)

Take a look at the TMBs from different VAF filter cutoffs.

filter_tmb_files
 [1] "results/vaf_cutoff_data/cutoff_0.10/lancet_tmb.rds"   "results/vaf_cutoff_data/cutoff_0.10/mutect2_tmb.rds" 
 [3] "results/vaf_cutoff_data/cutoff_0.10/strelka2_tmb.rds" "results/vaf_cutoff_data/cutoff_0.10/vardict_tmb.rds" 
 [5] "results/vaf_cutoff_data/cutoff_0.20/lancet_tmb.rds"   "results/vaf_cutoff_data/cutoff_0.20/mutect2_tmb.rds" 
 [7] "results/vaf_cutoff_data/cutoff_0.20/strelka2_tmb.rds" "results/vaf_cutoff_data/cutoff_0.20/vardict_tmb.rds" 
 [9] "results/vaf_cutoff_data/cutoff_0.30/lancet_tmb.rds"   "results/vaf_cutoff_data/cutoff_0.30/mutect2_tmb.rds" 
[11] "results/vaf_cutoff_data/cutoff_0.30/strelka2_tmb.rds" "results/vaf_cutoff_data/cutoff_0.30/vardict_tmb.rds" 
[13] "results/vaf_cutoff_data/cutoff_0.40/lancet_tmb.rds"   "results/vaf_cutoff_data/cutoff_0.40/mutect2_tmb.rds" 
[15] "results/vaf_cutoff_data/cutoff_0.40/strelka2_tmb.rds" "results/vaf_cutoff_data/cutoff_0.40/vardict_tmb.rds"

Convert the VAF cutoffs into their own vector.

vaf_filter <- stringr::word(filter_tmb_files, sep = "/", 3)
vaf_filter <- as.numeric(gsub("cutoff_", "", vaf_filter))
vaf_filter
 [1] 0.1 0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.3 0.3 0.3 0.3 0.4 0.4 0.4 0.4

Get caller information as a vector.

caller_filter <- stringr::word(filter_tmb_files, sep = "/", 4)
caller_filter <- gsub("_tmb.rds", "", caller_filter)
caller_filter
 [1] "lancet"   "mutect2"  "strelka2" "vardict"  "lancet"   "mutect2"  "strelka2" "vardict"  "lancet"   "mutect2" 
[11] "strelka2" "vardict"  "lancet"   "mutect2"  "strelka2" "vardict"

Read in the data. Name each with their caller and VAF filter cutoff.

Turn this into a data frame.

tmb_filter_df <- dplyr::bind_rows(filter_tmb_list, .id = "NAME") %>%
  dplyr::mutate(
    caller = stringr::word(NAME, sep = "_", 1),
    vaf_cutoff = as.numeric(stringr::word(NAME, sep = "_", 2))
  )

Make this into a long form data.frame for plotting purposes.

tmb_filter_long <- tmb_filter_df %>%
  dplyr::distinct(Tumor_Sample_Barcode, caller, tmb, vaf_cutoff) %>%
  tidyr::spread(caller, tmb)

TMB correlations across callers

First we’ll make a series of corrletion matrix plots.

lapply(unique(vaf_filter), cor_matrix_plot)
Registered S3 method overwritten by 'GGally':
  method from   
  +.gg   ggplot2
[[1]]

[[2]]

[[3]]

[[4]]

Let’s make a summary plot of the Spearman’s correlations. Get correlations by vaf cutoff.

# Run this on each set
cor_by_vaf_df <- do.call(
  "rbind.data.frame",
  lapply(unique(vaf_filter), cor_by_vaf)
)

Plot the correlations and the vaf_cutoff per each caller combination.

ggplot2::ggplot(cor_by_vaf_df, ggplot2::aes(x = reorder(compare, -cor), fill = as.factor(cutoff), y = cor)) +
  ggplot2::geom_point(shape = 21, size = 2.5, stroke = .75, color = "black") +
  ggplot2::theme_classic() +
  ggupset::scale_x_mergelist(sep = "-") +
  ggupset::axis_combmatrix(sep = "-") +
  ggplot2::xlab("") +
  ggplot2::ylab("TMB Spearman Correlation") +
  ggplot2::scale_fill_brewer("VAF Filter Cutoff", palette = "YlGnBu")

Session Info

sessionInfo()
R version 3.6.0 (2019-04-26)
Platform: x86_64-pc-linux-gnu (64-bit)
Running under: Debian GNU/Linux 9 (stretch)

Matrix products: default
BLAS/LAPACK: /usr/lib/libopenblasp-r0.2.19.so

locale:
 [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C               LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8    
 [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=C              LC_PAPER=en_US.UTF-8       LC_NAME=C                 
 [9] LC_ADDRESS=C               LC_TELEPHONE=C             LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C       

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

loaded via a namespace (and not attached):
 [1] Rcpp_1.0.1         rstudioapi_0.10    GGally_1.4.0       knitr_1.23         magrittr_1.5      
 [6] progress_1.2.2     hms_0.4.2          ggupset_0.1.0      munsell_0.5.0      tidyselect_0.2.5  
[11] colorspace_1.4-1   R6_2.4.0           rlang_0.4.0        plyr_1.8.4         stringr_1.4.0     
[16] dplyr_0.8.3        tools_3.6.0        grid_3.6.0         gtable_0.3.0       xfun_0.8          
[21] htmltools_0.3.6    lazyeval_0.2.2     assertthat_0.2.1   digest_0.6.20      tibble_2.1.3      
[26] crayon_1.3.4       reshape2_1.4.3     purrr_0.3.2        readr_1.3.1        RColorBrewer_1.1-2
[31] ggplot2_3.2.0      glue_1.3.1         labeling_0.3       stringi_1.4.3      compiler_3.6.0    
[36] pillar_1.4.2       prettyunits_1.0.2  scales_1.0.0       reshape_0.8.8      pkgconfig_2.0.2
LS0tCnRpdGxlOiAiVkFGIEN1dG9mZnMgYW5kIFRNQiBDYWxjdWxhdGlvbiIKb3V0cHV0OiAKICBodG1sX25vdGVib29rOgogICAgdG9jOiBUUlVFCiAgICB0b2NfZmxvYXQ6IFRSVUUKYXV0aG9yOiBDLiBTYXZvbmVuIGZvciBBTFNGIENDREwKZGF0ZTogMjAxOQotLS0KCioqUHVycG9zZToqKiBIb3cgZG8gdGhlIFRNQiBjYWxjdWxhdGlvbnMgY2hhbmdlIHdpdGggYWRqdXN0bWVudHMgdG8gdGhlIFZBRiBjdXRvZmYKZmlsdGVyPyBNb3JlIHNwZWNpZmljYWxseSwgaWYgd2UgaW5jcmVhc2UgVkFGIGZpbHRlciBjdXRvZmYsIGRvIHRoZSBjYWxsZXJzJyBUTUIgc3RhdHMgCnJlbGF0ZSB0byBlYWNoIG90aGVyIG1vcmU/IAoKIyMjIFN1bW1hcnkgb2YgRmluZGluZ3M6CgpWYXJEaWN0IGlzIHRoZSBsZWFzdCByZWxhdGVkIHRvIHRoZSBvdGhlciBjYWxsZXJzLCBidXQgaW5jcmVhc2luZyB0aGUgVkFGIGZpbHRlciBjdXRvZmYgZG9lcyBzb21ld2hhdApyZWNvdmVyIGhvdyBpdHMgVE1CIHN0YXRzIHJlbGF0ZSB0byB0aGUgb3RoZXIgY2FsbGVycycgVE1CIHN0YXRzLiAKUmVnYXJkbGVzcywgVmFyRGljdCdzIGRhdGEgc2hvdWxkIHN0aWxsIGJlIGRyb3BwZWQgYXMgaXQgaXMgdG9vIGFiZXJyYW50IGZyb20gdGhlIG90aGVyIGNhbGxlcnMgYXMgCndlbGwgYXMgb3Zlcmx5IHNlbnNpdGl2ZS4gCgojIyMjIFVzYWdlCgpUbyBydW4gdGhpcyBmcm9tIHRoZSBjb21tYW5kIGxpbmUsIHVzZToKYGBgClJzY3JpcHQgLWUgInJtYXJrZG93bjo6cmVuZGVyKCdhbmFseXNlcy9zbnYtY2FsbGVycy92YWZfY3V0b2ZmX2V4cGVyaW1lbnQuUm1kJywgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNsZWFuID0gVFJVRSkiCmBgYAoKX1RoaXMgYXNzdW1lcyB5b3UgYXJlIGluIHRoZSB0b3AgZGlyZWN0b3J5IG9mIHRoZSByZXBvc2l0b3J5Ll8KCiMjIFNldHVwCgojIyMjIFBhY2thZ2VzIGFuZCBmdW5jdGlvbnMKClJlYWQgaW4gc2V0IHVwIHNjcmlwdC4KCmBgYHtyfQppZiAoISgiR0dhbGx5IiAlaW4lIGluc3RhbGxlZC5wYWNrYWdlcygpKSkgewogIGluc3RhbGwucGFja2FnZXMoIkdHYWxseSIpCn0KaWYgKCEoImdndXBzZXQiICVpbiUgaW5zdGFsbGVkLnBhY2thZ2VzKCkpKSB7CiAgaW5zdGFsbC5wYWNrYWdlcygiZ2d1cHNldCIpCn0KIyBNYWdyaXR0ciBwaXBlCmAlPiVgIDwtIGRwbHlyOjpgJT4lYApgYGAKClNldCB1cCBvdXRwdXQgZGlyZWN0b3JpZXMuIAoKYGBge3J9CmJhc2VfcmVzdWx0c19kaXIgPC0gInJlc3VsdHMiCmJhc2VfcGxvdHNfZGlyIDwtICJwbG90cyIKYGBgCgpNYWtlIG5ldyBkaXJlY3RvcmllcyBmb3IgdGhlIGNvbXBhcmlzb24gYW5hbHlzaXMuCgpgYGB7cn0KdmFmX2N1dG9mZl9yZXN1bHRzX2RpciA8LSBmaWxlLnBhdGgoYmFzZV9yZXN1bHRzX2RpciwgInZhZl9jdXRvZmZfZGF0YSIpCgojIE1ha2UgY2FsbGVyIHNwZWNpZmljIHBsb3RzIGZvbGRlcgppZiAoIWRpci5leGlzdHModmFmX2N1dG9mZl9yZXN1bHRzX2RpcikpIHsKICBkaXIuY3JlYXRlKHZhZl9jdXRvZmZfcmVzdWx0c19kaXIpCn0KYGBgCgpGdW5jdGlvbiBmb3IgY3JlYXRpbmcgYSBUTUIgY29ycmVsYXRpb24gbWF0cml4IGFtb25nc3QgdGhlIGNhbGxlcnMgZm9yIGEgcGFydGljdWxhciBjdXRvZmYuCgpgYGB7cn0KY29yX21hdHJpeF9wbG90IDwtIGZ1bmN0aW9uKGN1dG9mZikgewogICMgRm9yIGEgZ2l2ZW4gVkFGIGN1dG9mZiwgY2FsY3VsYXRlIFRNQiBjb3JyZWxhdGlvbnMgYW1vbmdzdCB0aGUgY2FsbGVycyBhbmQgcGxvdCBpdCB1c2luZwogICMgR0dhbGx5OjpnZ3BhaXJzCiAgIwogICMgQXJnczoKICAjICAgY3V0b2ZmOiBUaGUgVkFGIGN1dG9mZiBzZXQgdG8gYW5hbHl6ZQogICMKICAjIFJldHVybnM6CiAgIyAgIEEgY29ycmVsYXRpb24gbWF0cml4IHBsb3QgYW1vbmdzdCB0aGUgY2FsbGVycwoKICAjIElzb2xhdGUgdGhlIFRNQiBzdGF0cyBmb3IgdGhpcyBjdXRvZmYKICBjdXRvZmZfZGYgPC0gdG1iX2ZpbHRlcl9sb25nICU+JQogICAgZHBseXI6OmZpbHRlcih2YWZfY3V0b2ZmID09IGN1dG9mZikgJT4lCiAgICBkcGx5cjo6c2VsZWN0KCJsYW5jZXQiLCAibXV0ZWN0MiIsICJzdHJlbGthMiIsICJ2YXJkaWN0IikKCiAgIyBNYWtlIHRoZSBwbG90CiAgY29yX3Bsb3QgPC0gR0dhbGx5OjpnZ3BhaXJzKGN1dG9mZl9kZiwgbWFwcGluZyA9IGdncGxvdDI6OmFlcyhhbHBoYSA9IDAuMDUpKSArCiAgICBnZ3Bsb3QyOjp0aGVtZV9jbGFzc2ljKCkgKwogICAgZ2dwbG90Mjo6Z2d0aXRsZShwYXN0ZTAoIlZBRiBjdXRvZmY6ICIsIGN1dG9mZikpCgogICMgUmV0dXJuIHRoZSBwbG90CiAgcmV0dXJuKGNvcl9wbG90KQp9CmBgYAoKRnVuY3Rpb24gZm9yIGNvcnJlbGF0aW5nIFRNQiBzdGF0aXN0aWNzIGZvciBlYWNoIFZBRiBjdXRvZmYgc2V0LiAKCmBgYHtyfQpjb3JfYnlfdmFmIDwtIGZ1bmN0aW9uKGN1dG9mZiA9IDAuMSkgewogICMgRm9yIGEgZ2l2ZW4gVkFGIGN1dG9mZiwgaXNvbGF0ZSB0aGUgVE1CIHN0YXRpc3RpY3MgYW5kIGNvcnJlbGF0ZSB0aGVtLgogICMKICAjIEFyZ3M6CiAgIyAgIGN1dG9mZjogdGhlIFZBRiBjdXRvZmYgdG8gY2FsY3VsYXRlIHRoZSBjb3JyZWxhdGlvbiBieS4KICAjCiAgIyBSZXR1cm5zOgogICMgICBSIHZhbHVlcyBmb3IgU3BlYXJtYW4ncyBjb3JyZWxhdGlvbiBhbW9uZ3N0IHRoZSBjYWxsZXJzCgogIGN1dG9mZl9kZiA8LSB0bWJfZmlsdGVyX2xvbmcgJT4lCiAgICBkcGx5cjo6ZmlsdGVyKHZhZl9jdXRvZmYgPT0gY3V0b2ZmKSAlPiUKICAgIGRwbHlyOjpzZWxlY3QoLVR1bW9yX1NhbXBsZV9CYXJjb2RlLCAtdmFmX2N1dG9mZikKCiAgIyBEbyBjb3JyZWxhdGlvbnMgYmV0d2VlbiBjYWxsZXJzCiAgY29yX3JlcyA8LSBjb3IoY3V0b2ZmX2RmLCBtZXRob2QgPSAic3BlYXJtYW4iLCB1c2UgPSAibmEub3IuY29tcGxldGUiKQoKICAjIFRoaXMgaXMgdG8gcmVtb3ZlIHJlZHVuZGFuY3kgYXMgdXBwZXIgY29ycmVsYXRpb24gbWF0cml4ID09IGxvd2VyCiAgY29yX3Jlc1t1cHBlci50cmkoY29yX3JlcywgZGlhZyA9IFRSVUUpXSA8LSBOQQoKICAjIEZvcm1hdCBpbnRvIGxvbmcgZm9ybWF0IGRhdGEuZnJhbWUKICBjb3JfZGYgPC0gcmVzaGFwZTI6Om1lbHQoY29yX3JlcywgbmEucm0gPSBUUlVFLCB2YWx1ZS5uYW1lID0gImNvciIpICU+JQogICAgZHBseXI6Om11dGF0ZSgKICAgICAgY29tcGFyZSA9IHBhc3RlMChWYXIxLCAiLSIsIFZhcjIpLAogICAgICBjdXRvZmYKICAgICkKCiAgIyBSZXR1cm4gYSBkYXRhLmZyYW1lIHJlYWR5IGZvciBiZWluZyBhZGRlZCB0byB0aGUgYmlnZ2VyIGRhdGEuZnJhbWUgZm9yIHBsb3R0aW5nLgogIHJldHVybihjb3JfZGYpCn0KYGBgCgojIyBSdW4gdGhlIGRhdGEgd2l0aCBkaWZmZXJlbnQgVkFGIGZpbHRlciBjdXRvZmZzCgpXZSBuZWVkIHRvIHJlLWNhbGN1bGF0ZSBUTUIgc3RhdGlzdGljcyB3aXRoIHZhcmlvdXMgVkFGIGZpbHRlciBjdXRvZmZzLiAKV2Ugd2lsbCBydW4gdGhpcyBzY3JpcHQgdG8gZG8gaXQ6IGBzeXN0ZW0oImJhc2ggc2NyaXB0cy9ydW5fY2FsbGVyX2V2YWxzX3ZhZl9maWx0ZXJfZXhwLnNoIilgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQpzeXN0ZW0oImJhc2ggc2NyaXB0cy9ydW5fY2FsbGVyX2V2YWxzX3ZhZl9maWx0ZXJfZXhwLnNoIikKYGBgCgojIyBJbXBvcnQgZGF0YSAKCkdldCB0aGUgbGlzdHMgb2YgZWFjaCB0eXBlIG9mIGZpbGUuIAoKYGBge3J9CiMgTWFrZSBhIGxpc3Qgb2YgdGhlIHZhZiBmaWx0ZXIgZXhwZXJpbWVudCB0bWIgZmlsZXMKZmlsdGVyX3RtYl9maWxlcyA8LSBsaXN0LmZpbGVzKHZhZl9jdXRvZmZfcmVzdWx0c19kaXIsCiAgcGF0dGVybiA9ICJfdG1iLnJkcyQiLCByZWN1cnNpdmUgPSBUUlVFLAogIGZ1bGwubmFtZXMgPSBUUlVFCikKYGBgCgpUYWtlIGEgbG9vayBhdCB0aGUgVE1CcyBmcm9tIGRpZmZlcmVudCBWQUYgZmlsdGVyIGN1dG9mZnMuIAoKYGBge3J9CmZpbHRlcl90bWJfZmlsZXMKYGBgCgpDb252ZXJ0IHRoZSBWQUYgY3V0b2ZmcyBpbnRvIHRoZWlyIG93biB2ZWN0b3IuIAoKYGBge3J9CnZhZl9maWx0ZXIgPC0gc3RyaW5ncjo6d29yZChmaWx0ZXJfdG1iX2ZpbGVzLCBzZXAgPSAiLyIsIDMpCnZhZl9maWx0ZXIgPC0gYXMubnVtZXJpYyhnc3ViKCJjdXRvZmZfIiwgIiIsIHZhZl9maWx0ZXIpKQp2YWZfZmlsdGVyCmBgYAoKR2V0IGNhbGxlciBpbmZvcm1hdGlvbiBhcyBhIHZlY3Rvci4gCgpgYGB7cn0KY2FsbGVyX2ZpbHRlciA8LSBzdHJpbmdyOjp3b3JkKGZpbHRlcl90bWJfZmlsZXMsIHNlcCA9ICIvIiwgNCkKY2FsbGVyX2ZpbHRlciA8LSBnc3ViKCJfdG1iLnJkcyIsICIiLCBjYWxsZXJfZmlsdGVyKQpjYWxsZXJfZmlsdGVyCmBgYAoKUmVhZCBpbiB0aGUgZGF0YS4gCk5hbWUgZWFjaCB3aXRoIHRoZWlyIGNhbGxlciBhbmQgVkFGIGZpbHRlciBjdXRvZmYuIAoKYGBge3IgaW5jbHVkZT1GQUxTRX0KZmlsdGVyX3RtYl9saXN0IDwtIGxhcHBseShmaWx0ZXJfdG1iX2ZpbGVzLCBmdW5jdGlvbihmaWxlX25hbWUpIHsKICByZWFkcjo6cmVhZF9yZHMoZmlsZV9uYW1lKSAlPiUgZHBseXI6OnVuZ3JvdXAoKQp9KQpuYW1lcyhmaWx0ZXJfdG1iX2xpc3QpIDwtIHBhc3RlMChjYWxsZXJfZmlsdGVyLCAiXyIsIHZhZl9maWx0ZXIpCmBgYAoKVHVybiB0aGlzIGludG8gYSBkYXRhIGZyYW1lLgoKYGBge3J9CnRtYl9maWx0ZXJfZGYgPC0gZHBseXI6OmJpbmRfcm93cyhmaWx0ZXJfdG1iX2xpc3QsIC5pZCA9ICJOQU1FIikgJT4lCiAgZHBseXI6Om11dGF0ZSgKICAgIGNhbGxlciA9IHN0cmluZ3I6OndvcmQoTkFNRSwgc2VwID0gIl8iLCAxKSwKICAgIHZhZl9jdXRvZmYgPSBhcy5udW1lcmljKHN0cmluZ3I6OndvcmQoTkFNRSwgc2VwID0gIl8iLCAyKSkKICApCmBgYAoKTWFrZSB0aGlzIGludG8gYSBsb25nIGZvcm0gZGF0YS5mcmFtZSBmb3IgcGxvdHRpbmcgcHVycG9zZXMuIAoKYGBge3J9CnRtYl9maWx0ZXJfbG9uZyA8LSB0bWJfZmlsdGVyX2RmICU+JQogIGRwbHlyOjpkaXN0aW5jdChUdW1vcl9TYW1wbGVfQmFyY29kZSwgY2FsbGVyLCB0bWIsIHZhZl9jdXRvZmYpICU+JQogIHRpZHlyOjpzcHJlYWQoY2FsbGVyLCB0bWIpCmBgYAoKIyMgVE1CIGNvcnJlbGF0aW9ucyBhY3Jvc3MgY2FsbGVycwoKRmlyc3Qgd2UnbGwgbWFrZSBhIHNlcmllcyBvZiBjb3JybGV0aW9uIG1hdHJpeCBwbG90cy4gCgpgYGB7cn0KbGFwcGx5KHVuaXF1ZSh2YWZfZmlsdGVyKSwgY29yX21hdHJpeF9wbG90KQpgYGAKCkxldCdzIG1ha2UgYSBzdW1tYXJ5IHBsb3Qgb2YgdGhlIFNwZWFybWFuJ3MgY29ycmVsYXRpb25zLiAKR2V0IGNvcnJlbGF0aW9ucyBieSB2YWYgY3V0b2ZmLiAKCmBgYHtyfQojIFJ1biB0aGlzIG9uIGVhY2ggc2V0CmNvcl9ieV92YWZfZGYgPC0gZG8uY2FsbCgKICAicmJpbmQuZGF0YS5mcmFtZSIsCiAgbGFwcGx5KHVuaXF1ZSh2YWZfZmlsdGVyKSwgY29yX2J5X3ZhZikKKQpgYGAKClBsb3QgdGhlIGNvcnJlbGF0aW9ucyBhbmQgdGhlIHZhZl9jdXRvZmYgcGVyIGVhY2ggY2FsbGVyIGNvbWJpbmF0aW9uLiAKCmBgYHtyfQpnZ3Bsb3QyOjpnZ3Bsb3QoY29yX2J5X3ZhZl9kZiwgZ2dwbG90Mjo6YWVzKHggPSByZW9yZGVyKGNvbXBhcmUsIC1jb3IpLCBmaWxsID0gYXMuZmFjdG9yKGN1dG9mZiksIHkgPSBjb3IpKSArCiAgZ2dwbG90Mjo6Z2VvbV9wb2ludChzaGFwZSA9IDIxLCBzaXplID0gMi41LCBzdHJva2UgPSAuNzUsIGNvbG9yID0gImJsYWNrIikgKwogIGdncGxvdDI6OnRoZW1lX2NsYXNzaWMoKSArCiAgZ2d1cHNldDo6c2NhbGVfeF9tZXJnZWxpc3Qoc2VwID0gIi0iKSArCiAgZ2d1cHNldDo6YXhpc19jb21ibWF0cml4KHNlcCA9ICItIikgKwogIGdncGxvdDI6OnhsYWIoIiIpICsKICBnZ3Bsb3QyOjp5bGFiKCJUTUIgU3BlYXJtYW4gQ29ycmVsYXRpb24iKSArCiAgZ2dwbG90Mjo6c2NhbGVfZmlsbF9icmV3ZXIoIlZBRiBGaWx0ZXIgQ3V0b2ZmIiwgcGFsZXR0ZSA9ICJZbEduQnUiKQpgYGAKCiMjIFNlc3Npb24gSW5mbwoKYGBge3J9CnNlc3Npb25JbmZvKCkKYGBgCg==