From 9a40c5b260ee1ae949a933638879d5f6239844e9 Mon Sep 17 00:00:00 2001 From: James Duncan Date: Thu, 15 Dec 2022 10:34:26 -0800 Subject: [PATCH 1/3] Ignore renv --- .Rbuildignore | 2 ++ .gitignore | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.Rbuildignore b/.Rbuildignore index 3a85bf9c..174f540b 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,3 +1,5 @@ +^renv$ +^renv\.lock$ ^_pkgdown\.yml$ ^docs$ ^pkgdown$ diff --git a/.gitignore b/.gitignore index c3b74f49..66ed97ea 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,12 @@ results/ ## ---- R +# renv + +renv/ +.Rprofile +renv.lock + # R project files *.Rproj From c54799b01e1d5b59cde0d2f2f5d825d6b47141c4 Mon Sep 17 00:00:00 2001 From: James Duncan Date: Thu, 15 Dec 2022 11:29:00 -0800 Subject: [PATCH 2/3] Update GHA workflows --- .github/workflows/check-standard.yaml | 11 +++++------ .github/workflows/pkgdown.yaml | 12 ++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml index 7cd40d7c..c6a61b0d 100644 --- a/.github/workflows/check-standard.yaml +++ b/.github/workflows/check-standard.yaml @@ -1,14 +1,13 @@ -# Workflow derived from https://github.com/r-lib/actions/blob/v2/examples/check-standard.yaml +# Workflow derived from https://github.com/r-lib/actions/blob/v2.3.1/examples/check-standard.yaml on: push: - branches: main + branches: [main] pull_request: - branches: main + branches: [main] name: R-CMD-check jobs: - # Check R build R-CMD-check: runs-on: ${{ matrix.config.os }} @@ -18,7 +17,7 @@ jobs: fail-fast: false matrix: config: - - {os: macOS-latest, r: 'release'} + - {os: macos-latest, r: 'release'} - {os: windows-latest, r: 'release'} - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'} - {os: ubuntu-latest, r: 'release'} @@ -29,7 +28,7 @@ jobs: R_KEEP_PKG_SOURCE: yes steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: r-lib/actions/setup-pandoc@v2 diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index d4463b60..0ef8c0fc 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -1,9 +1,9 @@ -# Workflow derived from https://github.com/r-lib/actions/blob/v2/examples/pkgdown.yaml +# Workflow derived from https://github.com/r-lib/actions/blob/v2.3.1/examples/pkgdown.yaml on: push: - branches: main + branches: [main] pull_request: - branches: main + branches: [main] release: types: [published] workflow_dispatch: @@ -11,7 +11,7 @@ on: name: pkgdown jobs: - # Build website + # Build docs website pkgdown: runs-on: ubuntu-latest # Only restrict concurrency for non-PR jobs @@ -20,7 +20,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: r-lib/actions/setup-pandoc@v2 @@ -44,7 +44,7 @@ jobs: - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' - uses: JamesIves/github-pages-deploy-action@4.1.4 + uses: JamesIves/github-pages-deploy-action@v4.4.1 with: clean: false branch: gh-pages From befc6a49292cf3ec108749e7fa507b1a9d36f225 Mon Sep 17 00:00:00 2001 From: James Duncan Date: Thu, 15 Dec 2022 11:42:58 -0800 Subject: [PATCH 3/3] Reduce time to render and re-add example .md files for simChef.Rmd --- .gitignore | 2 +- vignettes/docs/dgps/Linear Gaussian DGP.md | 19 ++ .../Rejection Prob. (alpha = 0.1).md | 5 + vignettes/docs/methods/OLS.md | 19 ++ vignettes/docs/objectives.md | 5 + vignettes/docs/visualizers/Power.md | 5 + .../Rejection Prob. (alpha = 0.1) Plot.md | 6 + vignettes/simChef.Rmd | 166 ++++++++++-------- 8 files changed, 148 insertions(+), 79 deletions(-) create mode 100644 vignettes/docs/dgps/Linear Gaussian DGP.md create mode 100644 vignettes/docs/evaluators/Rejection Prob. (alpha = 0.1).md create mode 100644 vignettes/docs/methods/OLS.md create mode 100644 vignettes/docs/objectives.md create mode 100644 vignettes/docs/visualizers/Power.md create mode 100644 vignettes/docs/visualizers/Rejection Prob. (alpha = 0.1) Plot.md diff --git a/.gitignore b/.gitignore index 66ed97ea..92e7dc2c 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ vignettes/*.pdf .Renviron # pkgdown site -docs/ +/docs/ # translation temp files po/*~ diff --git a/vignettes/docs/dgps/Linear Gaussian DGP.md b/vignettes/docs/dgps/Linear Gaussian DGP.md new file mode 100644 index 00000000..c69d86d0 --- /dev/null +++ b/vignettes/docs/dgps/Linear Gaussian DGP.md @@ -0,0 +1,19 @@ +In the Linear Gaussian DGP, we simulate the feature/design matrix $\mathbf{X} \in \mathbb{R}^{n \times p}$ from a normal distribution and the response vector $\mathbf{y} \in \mathbb{R}^n$ from a linear model. Specifically, + +\begin{gather*} +\mathbf{X} \sim N\left(\mathbf{0}, \begin{pmatrix} 1 & \rho \\ \rho & 1 \end{pmatrix}\right), \\ +\mathbf{y} = \mathbf{X} \boldsymbol{\beta} + \boldsymbol{\epsilon},\\ +\boldsymbol{\epsilon} \sim N(\mathbf{0}, \sigma^2 \mathbf{I}_n) +\end{gather*} + +**Default Parameters in DGP** + +- Number of samples: $n = 200$ +- Number of features: $p = 2$ +- Correlation among features: $\rho = 0$ +- Amount of noise: $\sigma = 1$ +- Coefficients: $\boldsymbol{\beta} = (1, 0)^\top$ + + + [In practice, documentation of DGPs should answer the questions “what” and “why”. That is, “what” is the DGP, and “why” are we using/studying it? As this simulation experiment is a contrived example, we omit the “why” here.] + diff --git a/vignettes/docs/evaluators/Rejection Prob. (alpha = 0.1).md b/vignettes/docs/evaluators/Rejection Prob. (alpha = 0.1).md new file mode 100644 index 00000000..d14124c3 --- /dev/null +++ b/vignettes/docs/evaluators/Rejection Prob. (alpha = 0.1).md @@ -0,0 +1,5 @@ +We define the rejection probability as the proportion of repetitions in the simulation experiment that result in a p-value $\leq \alpha$. Here, we choose to set the significance level $\alpha = 0.1$. + + + [In practice, documentation of evaluation metrics should answer the questions “what” and “why”. That is, “what” is the metric, and “why” are we using/studying it? As this simulation experiment is a contrived example, we omit the “why” here.] + diff --git a/vignettes/docs/methods/OLS.md b/vignettes/docs/methods/OLS.md new file mode 100644 index 00000000..736d6c5f --- /dev/null +++ b/vignettes/docs/methods/OLS.md @@ -0,0 +1,19 @@ +Given some data $\mathbf{X}$ and $\mathbf{y}$, we fit ordinary least squares (OLS) and examine the p-values for each coefficient in the model. The p-values are computed using a two-sided t-test (see `summary.lm()`). + +Elaborating further on the testing, we are interested in testing: + +\begin{align*} +H_0: \beta_i = 0 \quad vs. \quad H_1: \beta_i \neq 0. +\end{align*} + +To test this, we compute the observed T-statistic, defined as + +\begin{align*} +T = \frac{\hat{\beta}_i}{\hat{SE}(\hat{\beta_i})}, +\end{align*} + +and then compute the two-sided p-value under the t distribution with $n - p - 1$ degrees of freedom. If the p-value is lower than some significance value $\alpha$, then there is sufficient evidence to reject the null hypothesis $H_0$. Otherwise, there is not sufficient evidence, and we fail to reject the null hypothesis. + + + [In practice, documentation of methods should answer the questions “what” and “why”. That is, “what” is the method, and “why” are we using/studying it? As this simulation experiment is a contrived example, we omit the “why” here.] + diff --git a/vignettes/docs/objectives.md b/vignettes/docs/objectives.md new file mode 100644 index 00000000..c0df7601 --- /dev/null +++ b/vignettes/docs/objectives.md @@ -0,0 +1,5 @@ +The objective of this simulation experiment is to provide a toy example on how to use `simChef` and showcase the automated R Markdown-generated documentation. For the sake of illustration, this toy simulation experiment studies the performance of linear regression at the surface-level with the sole purpose of facilitating an easy-to-understand walkthrough. + + + [Typically, the objective of the simulation experiment (and this blurb) will be more scientific than instructive and will warrant additional context/background and domain knowledge.] + diff --git a/vignettes/docs/visualizers/Power.md b/vignettes/docs/visualizers/Power.md new file mode 100644 index 00000000..0e446b1c --- /dev/null +++ b/vignettes/docs/visualizers/Power.md @@ -0,0 +1,5 @@ +To examine the power of the test, we plot the rejection probability as a function of $\alpha$, that is, $\mathbb{P}(\text{p-value} \leq \alpha)$ vs. $\alpha$. If the coefficient is non-zero in the underlying DGP, then a larger AUC would indicate better performance in terms of the power. We will primarily focus on plotting the power of the first coefficient $\beta_1$. + + + [In practice, documentation of the plotters should answer the questions “what” and “why”. That is, “what” is the plot, and “why” are we using/studying it? As this simulation experiment is a contrived example, we omit the “why” here.] + diff --git a/vignettes/docs/visualizers/Rejection Prob. (alpha = 0.1) Plot.md b/vignettes/docs/visualizers/Rejection Prob. (alpha = 0.1) Plot.md new file mode 100644 index 00000000..7bf5c96a --- /dev/null +++ b/vignettes/docs/visualizers/Rejection Prob. (alpha = 0.1) Plot.md @@ -0,0 +1,6 @@ +We plot the rejection probability for $\beta_1$ across varying parameters of the DGP to understand how characteristics of the DGP affect the test. +We define the rejection probability as the proportion of repetitions in the simulation experiment that result in a p-value $\leq \alpha$. Here, we choose to set the significance level $\alpha = 0.1$. + + + [In practice, documentation of the plotters should answer the questions “what” and “why”. That is, “what” is the plot, and “why” are we using/studying it? As this simulation experiment is a contrived example, we omit the “why” here.] + diff --git a/vignettes/simChef.Rmd b/vignettes/simChef.Rmd index f2e7aa9d..6a0367c5 100644 --- a/vignettes/simChef.Rmd +++ b/vignettes/simChef.Rmd @@ -39,7 +39,7 @@ The goal of `simChef` is to seamlessly and efficiently run simulation experiment 1. Create individual parts of the simulation experiment recipe a. Create DGP(s) (data-generating processes) via `create_dgp()` - ```{r} + ```{r overview1} dgp <- create_dgp( function(n = 100, p = 10, noise_max = 5) { noise_level <- runif(1, max = noise_max) @@ -52,7 +52,7 @@ The goal of `simChef` is to seamlessly and efficiently run simulation experiment dgp ``` b. Create Method(s) via `create_method()` - ```{r} + ```{r overview2} method <- create_method( # method function args include the names in the dgp output list (`X` and `y` here) function(X, y, ...) { @@ -65,7 +65,7 @@ The goal of `simChef` is to seamlessly and efficiently run simulation experiment method ``` c. Create Evaluator(s) via `create_evaluator()` - ```{r} + ```{r overview3} evaluator <- create_evaluator( # the main computational loop of the simulation will return a `tibble::tibble` # which is passed as 'fit_results' to our evaluation functions @@ -80,7 +80,7 @@ The goal of `simChef` is to seamlessly and efficiently run simulation experiment evaluator ``` d. Create Visualizer(s) via `create_visualizer()` - ```{r} + ```{r overview4} visualizer <- create_visualizer( # you can use 'fit_results' as well here by adding it as an argument function(eval_results) { @@ -94,7 +94,7 @@ The goal of `simChef` is to seamlessly and efficiently run simulation experiment visualizer ``` 2. Assemble these recipe parts into a complete simulation experiment, e.g.: - ```{r} + ```{r overview5} experiment <- create_experiment(name = "Experiment") %>% add_dgp(dgp, name = "DGP1") %>% add_method(method, name = "Method1") %>% @@ -145,7 +145,7 @@ As a toy DGP example, let us create a function to simulate a random Gaussian dat \boldsymbol{\epsilon} \sim N(\mathbf{0}, \sigma^2 \mathbf{I}_n) \end{gather*} -```{r} +```{r basics1} dgp_fun <- function(n, beta, rho, sigma) { cov_mat <- matrix(c(1, rho, rho, 1), byrow = T, nrow = 2, ncol = 2) X <- MASS::mvrnorm(n = n, mu = rep(0, 2), Sigma = cov_mat) @@ -156,7 +156,7 @@ dgp_fun <- function(n, beta, rho, sigma) { We can create an object of class `DGP()` from this function via -```{r} +```{r basics2} dgp <- create_dgp(.dgp_fun = dgp_fun, .name = "Linear Gaussian DGP", # additional named parameters to pass to dgp_fun() n = 200, beta = c(1, 0), rho = 0, sigma = 1) @@ -168,7 +168,7 @@ Note that the additional arguments in `create_dgp()` must be named arguments tha Given the above DGP, suppose we want to investigate the performance of linear regression, specifically the p-values that are outputted for the non-intercept coefficients from `summary.lm()`. -```{r} +```{r basics3} lm_fun <- function(X, y, cols = c("X1", "X2")) { lm_fit <- lm(y ~ X) pvals <- summary(lm_fit)$coefficients[cols, "Pr(>|t|)"] %>% @@ -179,7 +179,7 @@ lm_fun <- function(X, y, cols = c("X1", "X2")) { We can create an object of class `Method()` from this function via -```{r} +```{r basics4} lm_method <- create_method(.method_fun = lm_fun) ``` @@ -192,7 +192,7 @@ A couple notes here: To evaluate the performance of linear regression, one metric (or statistic) of interest could be the rejection probability at some level $\alpha$, which we compute in the following function. -```{r} +```{r basics5} reject_prob_fun <- function(fit_results, alpha = 0.05) { group_vars <- c(".dgp_name", ".method_name") eval_out <- fit_results %>% @@ -207,7 +207,7 @@ reject_prob_fun <- function(fit_results, alpha = 0.05) { We can create an object of class `Evaluator()` from this function via -```{r} +```{r basics6} reject_prob_eval <- create_evaluator(.eval_fun = reject_prob_fun, alpha = 0.1) ``` @@ -222,7 +222,7 @@ As before, additional arguments can be passed into `create_evaluator()`, and in Lastly, we may want to plot the results from the `Method` fits (stored in `fit_results`) and/or the outputs from our `Evaluators` (stored in `eval_results`). For example, we could plot the power of the hypothesis test. -```{r} +```{r basics7} power_plot_fun <- function(fit_results, col = "X1") { plt <- ggplot2::ggplot(fit_results) + ggplot2::aes(x = .data[[paste(col, "p-value")]], @@ -239,7 +239,7 @@ power_plot_fun <- function(fit_results, col = "X1") { We can create an object of class `Visualizer()` from this function via -```{r} +```{r basics8} power_plot <- create_visualizer(.viz_fun = power_plot_fun) ``` @@ -251,7 +251,7 @@ As with the other `create_*` functions, additional arguments that need to pass i At this point, we have created a `DGP` (`dgp`), `Method` (`lm_method`), `Evaluator` (`reject_prob_eval`), and `Visualizer` (`power_plot`). The next step is to create the simulation experiment recipe and `add` each component to the recipe via -```{r} +```{r basics9} experiment <- create_experiment(name = "Linear Regression Experiment") %>% add_dgp(dgp, name = "Linear Gaussian DGP") %>% add_method(lm_method, name = "OLS") %>% @@ -263,7 +263,7 @@ Any number of DGPs, Methods, Evaluators, and Visualizers can be added to the sim We can easily see the individual parts of the simulation experiment recipe by printing the experiment. -```{r} +```{r basics10} print(experiment) ``` @@ -277,21 +277,28 @@ A crucial component when running veridical simulations is **documentation**. We init_docs(experiment) ``` -This creates a series of blank .md files for the user to fill out with descriptions of the simulation experiment and its recipe components. These blank .md files can be found in the experiment's root results directory under docs/. To find the experiment's root results directory, use `experiment$get_save_dir()`. +```{r cp_docs, echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} +# copy pre-written .md files in vignettes/docs to the experiment's docs dir +file.copy(from = here::here("vignettes/docs"), + to = file.path(experiment$get_save_dir()), + recursive = TRUE) +``` + +This creates a series of blank .md files for the user to fill out with descriptions of the simulation experiment and its recipe components. These blank .md files can be found in the experiment's root results directory under docs/. To find the experiment's root results directory, use `experiment$get_save_dir()`. You can find example .md files corresponding to the current experiment [here](https://github.com/Yu-Group/simChef/tree/main/vignettes/docs). ## Run the simulation experiment -Thus far, we have created and documented the simulation experiment recipe, but we have not generated any results from the experiment. That is, we have only given the simulation experiment instructions on what to do. To run the experiment, say over 100 replicates, we can do so via +Thus far, we have created and documented the simulation experiment recipe, but we have not generated any results from the experiment. That is, we have only given the simulation experiment instructions on what to do. To run the experiment, say over 10 replicates, we can do so via -```{r} -results <- experiment$run(n_reps = 100, save = TRUE) +```{r basics11} +results <- experiment$run(n_reps = 10, save = TRUE) ``` or alternatively, ```{r eval = FALSE} results <- experiment %>% - run_experiment(n_reps = 100, save = TRUE) + run_experiment(n_reps = 10, save = TRUE) ``` The output of `experiment$run()` is a list of length three: @@ -300,7 +307,7 @@ The output of `experiment$run()` is a list of length three: - `eval_results`: output of `Evaluators` (see `experiment$evaluate()`) - `viz_results`: output of `Visualizers` (see `experiment$visualize()`) -```{r} +```{r basics12} str(results, max.level = 2) results$fit_results results$eval_results @@ -315,12 +322,13 @@ The experiment can also be run in parallel. For a more detailed walkthrough on h Finally, we can easily visualize all results from the simulation experiment in an html file (generated using R Markdown) or browser. -```{r warning = FALSE} +```{r render_docs1, eval = FALSE} render_docs(experiment, open = FALSE) ``` -```{r echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} - +```{r cp_html1, echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} +# create pkgdown/assets directory, which will be copied to the gh-pages branch +# during the pkgdown workflow (see .github/workflows/pkgdown.yaml) assets_dir <- here::here("pkgdown/assets") dir.create(assets_dir, recursive = TRUE) @@ -340,12 +348,12 @@ Note that if the documentation template has not yet been created for `experiment Now, going slightly beyond the most basic usage of `simChef`, it is often helpful to understand how a method's performance is affected as we vary a single parameter in the DGP across different values. For instance, what happens to the power as the amount of noise in the linear model increases? Using the simple grammar of `simChef`, we can investigate this question by adding a "vary across" component to the simulation experiment. -```{r} +```{r vary1} experiment <- experiment %>% add_vary_across(.dgp = "Linear Gaussian DGP", sigma = c(1, 2, 4, 8)) ``` -```{r} +```{r vary2} print(experiment) ``` @@ -353,15 +361,15 @@ In `add_vary_across()`, the `dgp` argument is the name of the DGP to vary or the However, when we run the experiment, the results are not quite what we expect. The results are summarized/aggregated across all values of `sigma`. -```{r} -vary_results <- experiment$run(n_reps = 100, save = TRUE) +```{r vary3} +vary_results <- experiment$run(n_reps = 10, save = TRUE) vary_results$eval_results vary_results$viz_results ``` To see how different values of `sigma` affect the experiment results, we need to modify our `Evaluator` and `Visualizer` functions. Specifically, in `reject_prob_eval`, we want to group `fit_results` by `sigma` in addition to `.dgp_name` and `.method_name`. To do this, we need to add a `vary_params` argument to our custom `Evaluator` function. When we run the experiment, `vary_params` will be auto-filled by a vector of the parameter names that are varied (i.e., those that have been added via `add_vary_across()`). In this case, `vary_params` will be auto-filled by `c("sigma")`. -```{r} +```{r vary4} reject_prob_fun <- function(fit_results, vary_params = NULL, alpha = 0.05) { group_vars <- c(".dgp_name", ".method_name", vary_params) eval_out <- fit_results %>% @@ -377,7 +385,7 @@ reject_prob_eval <- create_evaluator(.eval_fun = reject_prob_fun, alpha = 0.1) Similarly in `power_plot_fun`, we need to add a `vary_params` argument to plot the results across different values of `sigma`. -```{r} +```{r vary5} power_plot_fun <- function(fit_results, vary_params = NULL, col = "X1") { if (!is.null(vary_params)) { @@ -410,11 +418,11 @@ power_plot <- create_visualizer(.viz_fun = power_plot_fun) Here, we have also added a pre-processing step to deal with the potential case when we vary across a *list* of parameter values. This pre-processing step uses a helper function, `list_col_to_chr()`, to convert a list-type tibble column to a character-type tibble column that is amenable for plotting (unlike the list-type column). Now, we are ready to update our experiment and run the experiment via -```{r} +```{r vary6} vary_results <- experiment %>% update_evaluator(reject_prob_eval, name = "Rejection Prob. (alpha = 0.1)") %>% update_visualizer(power_plot, name = "Power") %>% - run_experiment(n_reps = 100, save = TRUE) + run_experiment(n_reps = 10, save = TRUE) vary_results$eval_results vary_results$viz_results ``` @@ -423,7 +431,7 @@ Note here we need to use `update_*` instead of `add_*` since an `Evaluator` name For fun, let's add another plot (in fact, an interactive plot using `plotly::ggplotly`) to our `Experiment` and run the `Experiment` across various values of the coefficient $\boldsymbol{\beta}_2$ and the correlation $\rho$ between features in $\mathbf{X}$. (Note: the visualizer function below (`reject_prob_plot_fun`) takes in the `Evaluator` results, stored as `eval_results`, and plots the rejection probability for the $\boldsymbol{\beta}_1$ at $\alpha = 0.1$.) -```{r} +```{r vary7} # create rejection probability plot reject_prob_plot_fun <- function(eval_results, vary_params = NULL, alpha = 0.05) { @@ -466,28 +474,22 @@ vary_beta2_results <- experiment %>% c(1, 1), c(1, 1.5), c(1, 2))) %>% - run_experiment(n_reps = 100, save = TRUE) + run_experiment(n_reps = 10, save = TRUE) # run experiment across values of rho (correlation) vary_cor_results <- experiment %>% remove_vary_across(dgp = "Linear Gaussian DGP") %>% add_vary_across(.dgp = "Linear Gaussian DGP", rho = c(0, 0.2, 0.5, 0.9)) %>% - run_experiment(n_reps = 100, save = TRUE) + run_experiment(n_reps = 10, save = TRUE) ``` Now when generating the R Markdown report summary for an `Experiment`, the R Markdown will compile results (both evaluation and visualization results) from all saved `Experiments` under the root results directory `experiment$get_save_dir()`. Since the results from the many `vary_across` runs above are all saved under the original `experiment`'s results directory, then the following will include all of the above results in a single document. -```{r warning = FALSE} +```{r render_docs2, warning = FALSE} render_docs(experiment, open = FALSE) ``` -Equivalently, we can create the same R Markdown report summary by directly specifying the experiment's root results directory. - -```{r eval = FALSE} -render_docs(save_dir = experiment$get_save_dir(), open = FALSE) -``` - -```{r echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} +```{r cp_html2, echo = FALSE, warning = FALSE, message = FALSE, results = "hide"} # copy html output to pkgdown/assets directory for website file.copy(from = file.path(experiment$get_save_dir(), paste0(experiment$name, ".html")), @@ -495,6 +497,13 @@ file.copy(from = file.path(experiment$get_save_dir(), overwrite = TRUE) ``` +Equivalently, we can create the same R Markdown report summary by directly specifying the experiment's root results directory. + +```{r eval = FALSE} +render_docs(save_dir = experiment$get_save_dir(), open = FALSE) +``` + + The results can be found [here](../linear_regression_output.html). In addition to showing all results under the root results directory, `render_docs()` will automatically generate a blank documentation template for every `DGP()`, `Method()`, `Evaluator()`, and `Visualizer()` found in any one of the `Experiments` under the root results directory. If one would like to generate the documentation template but not create the R Markdown report, see `init_docs()`. @@ -512,13 +521,13 @@ Currently, we have implemented the following templates: These functions will print out code to the console that can be easily copied and/or run. For example, -```{r} +```{r templates1} use_prediction_template(type = "regression") ``` For more guidance, we can also include concrete examples of a DGP and Method via: -```{r} +```{r templates2} use_prediction_template(type = "regression", include_dgp_example = TRUE, include_method_example = TRUE) @@ -530,7 +539,7 @@ use_prediction_template(type = "regression", It is important to note that the `Experiment()` class is an `R6`. With this, we need to be careful about clones versus pointers. In the following, it may look like the `vary_experiment` object has a `vary_across` component while the `experiment` object does not have a `vary_across` component. However, when `experiment` is piped into `add_vary_across()`, this is in itself modifying `experiment`, and `vary_experiment` is simply pointing to this modified `experiment`. -```{r} +```{r notes1} experiment <- experiment %>% remove_vary_across(dgp = "Linear Gaussian DGP") experiment @@ -545,19 +554,19 @@ experiment To modify `vary_experiment` without making changes to `experiment`, we need to create `vary_experiment` as a new experiment by explicitly *cloning* from the old experiment `experiment`. -```{r} +```{r notes2} vary_experiment <- create_experiment(name = "I am a clone", clone_from = experiment) data.table::address(experiment) == data.table::address(vary_experiment) ``` -```{r} +```{r notes3} vary_experiment ``` When creating an `Experiment` from a clone, we are making a deep clone of the parent experiment's `DGPs`, `Methods`, `Evaluators`, and `Visualizers`, but not the `vary_across` component. We thus need to add a `vary_across` component to `vary_experiment` using `add_vary_across()`. -```{r} +```{r notes4} # this works without an error vary_experiment <- vary_experiment %>% add_vary_across(.dgp = "Linear Gaussian DGP", sigma = c(1, 2, 4, 8)) @@ -565,7 +574,7 @@ vary_experiment <- vary_experiment %>% We can also add/update DGPs, Methods, Evaluators, and Visualizers to the cloned experiment without modifying the parent `experiment`. -```{r} +```{r notes5} # add DGP dgp_new <- create_dgp(.dgp_fun = dgp_fun, .name = "Linear Gaussian DGP (large n)", n = 500, beta = c(1, 0), rho = 0, sigma = 1) @@ -581,52 +590,52 @@ To reduce computation, one may want to avoid re-running previously computed and As an example, let us run the following experiment and save its results. -```{r} -orig_results <- experiment$run(n_reps = 100, save = TRUE) +```{r caching1} +orig_results <- experiment$run(n_reps = 10, save = TRUE) ``` When setting `use_cached = TRUE`, the following line of code will not generate any new results but will instead simply read in the saved or cached results from disk. -```{r} -cached_results <- experiment$run(n_reps = 100, use_cached = TRUE) +```{r caching2} +cached_results <- experiment$run(n_reps = 10, use_cached = TRUE) all.equal(orig_results, cached_results) ``` We can also choose to read in a smaller number of the cached replicates -```{r} -smaller_results <- experiment$run(n_reps = 10, use_cached = TRUE) +```{r caching3} +smaller_results <- experiment$run(n_reps = 5, use_cached = TRUE) max(as.numeric(smaller_results$fit_results$.rep)) ``` or increase the number of replicates without re-doing computation for the previously cached replicates -```{r} -larger_results <- experiment$run(n_reps = 150, use_cached = TRUE) +```{r caching4} +larger_results <- experiment$run(n_reps = 15, use_cached = TRUE) all.equal(orig_results$fit_results, - larger_results$fit_results %>% dplyr::filter(as.numeric(.rep) <= 100)) + larger_results$fit_results %>% dplyr::filter(as.numeric(.rep) <= 10)) ``` In addition, caching works intuitively when adding or modifying components of an Experiment. In the following, we add a new DGP to the Experiment, so when we run the Experiment with `use_cached = TRUE`, only the replicates involving the new DGP are run while the replicates involving the old DGP (i.e., the `Linear Gaussian DGP`) are loaded in from the cache. -```{r} +```{r caching5} experiment <- experiment %>% add_dgp(dgp = dgp_new, name = "New DGP") -new_results <- experiment$run(n_reps = 100, use_cached = TRUE) +new_results <- experiment$run(n_reps = 10, use_cached = TRUE) all.equal(new_results$fit_results %>% dplyr::filter(.dgp_name == "Linear Gaussian DGP"), orig_results$fit_results) ``` Note that since we did not save the Experiment results above, then the following will re-run the replicates corresponding to the new DGP as above. Please set `save = TRUE` in order to cache the results for future use. -```{r} -new_results2 <- experiment$run(n_reps = 100, use_cached = TRUE) +```{r caching6} +new_results2 <- experiment$run(n_reps = 10, use_cached = TRUE) identical(new_results$fit_results %>% dplyr::filter(.dgp_name == "New DGP"), new_results2$fit_results %>% dplyr::filter(.dgp_name == "New DGP")) ``` Some helpful functions regarding caching include: -```{r} +```{r caching7} # to load in the cached fit results for an experiment fit_results <- get_cached_results(experiment, "fit") # to load in the cached evaluation results for an experiment @@ -692,10 +701,11 @@ experiment$fit(n_reps = 100, checkpoint_n_reps = 25) #> Fit results saved | time taken: 0.124632 seconds #> ============================== ``` + ## Additional R Markdown notes and options There are several customizable options regarding the aesthetics of Evaluators and Visualizers displayed in the Rmd report. These can be modified using the `doc_options` argument in `create_evaluator()` and `create_visualizer()`. For example, we can customize the height and width of the Power plot via -```{r} +```{r rmd_notes1} power_plot <- create_visualizer(.viz_fun = power_plot_fun, .doc_options = list(height = 10, width = 8)) experiment <- experiment %>% @@ -703,7 +713,7 @@ experiment <- experiment %>% ``` and the number of digits in the evaluation table outputs via -```{r} +```{r rmd_notes2} reject_prob_eval <- create_evaluator(.eval_fun = reject_prob_fun, alpha = 0.1, .doc_options = list(digits = 3)) experiment <- experiment %>% @@ -711,7 +721,7 @@ experiment <- experiment %>% ``` Alternatively, `set_doc_options()` can be used to update the R Markdown option for an existing object, rather than having to recreate the `Evaluator()` or `Visualizer()` from scratch, e.g., -```{r} +```{r rmd_notes3} experiment <- experiment %>% set_doc_options(field_name = "visualizer", name = "Power", show = TRUE, height = 10, width = 8) @@ -789,8 +799,8 @@ fs::dir_tree(get_save_dir(experiment)) 2. `experiment$evaluate()`: evaluate the experiment through the Evaluator(s) and return the evaluation results 3. `experiment$visualize()`: create visualizations of the fit/evaluation results from the experiment using the Visualizer(s) and return the visualization results -```{r} -fit_results <- experiment$fit(n_reps = 100) +```{r handling1} +fit_results <- experiment$fit(n_reps = 10) eval_results <- experiment$evaluate(fit_results) viz_results <- experiment$visualize(fit_results, eval_results) ``` @@ -798,7 +808,7 @@ viz_results <- experiment$visualize(fit_results, eval_results) or alternatively, ```{r eval = FALSE} -fit_results <- fit_experiment(experiment, n_reps = 100) +fit_results <- fit_experiment(experiment, n_reps = 10) eval_results <- evaluate_experiment(experiment, fit_results) viz_results <- visualize_experiment(experiment, fit_results, eval_results) ``` @@ -807,7 +817,7 @@ viz_results <- visualize_experiment(experiment, fit_results, eval_results) Thus far, we have neither stored nor returned any data from the `DGPs` since these can be large objects that require high memory loads when `n_reps` is large. However, one can generate small samples of data from the `DGPs` in the experiment via -```{r} +```{r handling2} data_list <- experiment$generate_data(n_reps = 1) str(data_list) ``` @@ -821,7 +831,7 @@ data_list <- generate_data(experiment, n_reps = 1) This might be helpful for exploratory data analysis or further diagnosis of the experiment results. Other helpful methods for handling the experiment include the `get_*` family of methods, i.e., -```{r} +```{r handling3} get_dgps(experiment) # or `experiment$get_dgps()` get_methods(experiment) # or `experiment$get_methods()` get_evaluators(experiment) # or `experiment$get_evaluators()` @@ -829,7 +839,7 @@ get_visualizers(experiment) # or `experiment$get_visualizers()` get_vary_across(experiment) # or `experiment$get_vary_across()` ``` -```{r results = "hide"} +```{r handling4, results = "hide"} get_save_dir(experiment) # or `experiment$get_save_dir()` #> [1] "./results/Linear Regression Experiment" ``` @@ -838,7 +848,7 @@ get_save_dir(experiment) # or `experiment$get_save_dir()` In the event of an error, `simChef` makes it possible to both retrieve results from completed replicates (before the error occurred) and to gracefully debug errors in user-defined functions. For the sake of demonstration, let us create an artificial example. -```{r error = TRUE} +```{r debug1, error = TRUE} # create experiment dgp_fun <- function() return("data") dgp_fun_err <- function() { stop("Uh oh!") } @@ -858,24 +868,24 @@ results <- run_experiment(experiment, n_reps = 2) If working interactively, the error can be inspected using the usual call to `rlang::last_error()`. Results that were run and completed before the error can be recovered via `rlang::last_error()$partial_results` and you can find errors that occurred within the simulation loop via `rlang::last_error()$errors`. Alternatively, we can wrap the call that ran the error (in this case, `run_experiment()`) in `tryCatch()` as follows: -```{r error = TRUE} +```{r debug2, error = TRUE} results <- tryCatch(run_experiment(experiment, n_reps = 2), error = identity) results ``` From this, we can view the results that were completed before we ran into the error via -```{r} +```{r debug3} results$partial_results ``` and extract the error object(s) via -```{r} +```{r debug4} results$errors ``` Note that the error tibble contains the `DGP` and `Method` (if the `DGP` was not the error cause) that were being processed when the error occurred in the `.dgp` and `.method` columns, respectively. The corresponding input parameters are in the `.dgp_params` and `.method_params` columns (both `NA` here since no parameters were varied in the experiment). Finally, the captured error itself is in the `.err` column. Using this, we can easily investigate and reproduce the error and if desired, work directly with the user-specified function that raised the error, e.g., -```{r error = TRUE} +```{r debug5, error = TRUE} # get dgp that ran the error err_fun <- results$errors$.dgp[[1]]$dgp_fun # reproduce error via a direct call to the DGP function that raised the error