Fall 2019 release: v1.3.5 (#253)

* remove old reindex definition * disable tests for travis cache * restyle + update site; re-enable tests * update DOI + site * minor readme/pkgdown update * fix broken link * three screeners and tests * oops found & fixed error * line 5 of corP and corRank * `sl3_task` bug fix; complexity & efficiency improvement (#246) * initial density and pooled hazards code * port tests * survival fixes + update documentation * re-style * implement semiparametric variance estimator * add test sketch for pooled hazards densities * fix merge conflicts * finish pooled hazards density test * re-style * adapt to changes in try-error data structure * fixed arguments not passed bug * Revert "fixed arguments not passed bug" This reverts commit b1358f1. * added learner caret * fixed formatting * added Learner GAM * fixed formatting * added gam to binomial learners test * fixed argument passing bug * fixed argument passing bug for svm * update contributors, mostly rebuild docs site * remove old reindex definition * improved complexity and efficiency of data treatment * debugged changes to sl3_task * modified DESCRIPTION * bug fix (flag_name) * bug fix (empty task) * bug fix (columns treatment and mapping) * fixed case where node list is given * fixed case where outcome is multivariate * complexity reduction * change task creation in Lrnr_density_discretize + re-style code * test refinement * fix for a special case * revert changes to survival utils * Subset covariates correctly for sublearners (#250) * fix covariate subsetting * only delay subset if task is delayed * don't count NA as a value when guessing variable types from data (#251) * Overhaul data preprocessing (#252) * don't count NA as a value when guessing variable types from data * overhaul data preprocessing * Convex combination with `Lrnr_nnls` (#249) convex combo with test * bump version; update docs * update site * add NEWS * use --ignore-installed in travis pip call * minor version bump; fix use of here::here; re-style; update docs and site * update DOI for v1.3.5 and update docs/site
tlverse · Nov 30, 2019 · 84f6423 · 84f6423
1 parent e09eb63
commit 84f6423
Show file tree

Hide file tree

Showing 229 changed files with 9,457 additions and 9,940 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -16,6 +16,7 @@
 xgboost.model
 deploy.sh
 ^docs$
+^revdep$
 ^README\.Rmd$
 ^README-.*\.Rmd$
 ^README\.html$
@@ -27,5 +28,3 @@ deploy.sh
 ^LICENSE$
 man-roxygen
 ^_pkgdown\.yml$
-^doc$
-^Meta$
diff --git a/.travis.yml b/.travis.yml
@@ -21,11 +21,9 @@ r:
   - devel
 
 before_install:
-  - sudo apt-get -y install python3-pip python-dev
   - sudo apt-get -y install libx11-dev mesa-common-dev libglu1-mesa-dev
-
-  - sudo apt-get remove python-enum34
-  - sudo pip install numpy tensorflow keras
+  - sudo apt-get -y install python3-pip python-dev
+  - sudo -H pip install --ignore-installed numpy tensorflow keras
   #- Rscript -e 'update.packages(ask = FALSE)'
 
 r_packages:
@@ -37,14 +35,14 @@ r_packages:
 r_github_packages:
   - r-lib/covr
   - r-lib/sessioninfo
-  - tpospisi/cdetools/r
-  - tpospisi/RFCDE/r
   - osofr/simcausal
   - osofr/condensier
-  - tlverse/delayed
   - tlverse/origami
+  - tlverse/delayed
   - tlverse/hal9001
   - nhejazi/haldensify
+  - tpospisi/cdetools/r
+  - tpospisi/RFCDE/r
 
 after_success:
   - travis_wait 80 Rscript -e 'covr::codecov()'

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: sl3
 Title: Pipelines for Machine Learning and Super Learning
-Version: 1.3.0
+Version: 1.3.5
 Authors@R: c(
     person("Jeremy", "Coyle", email = "[email protected]",
            role = c("aut", "cre", "cph"),
@@ -40,7 +40,8 @@ Imports:
     methods,
     ggplot2,
     stringr,
-    digest
+    digest,
+    imputeMissings
 Suggests:
     testthat,
     rmarkdown,
@@ -102,5 +103,5 @@ LazyLoad: yes
 VignetteBuilder:
     knitr,
     R.rsp
-RoxygenNote: 6.1.1
-Roxygen: list(markdown = TRUE)
+RoxygenNote: 7.0.1
+Roxygen: list(markdown = TRUE, r6 = FALSE)
diff --git a/Makefile b/Makefile
@@ -11,6 +11,9 @@ check:
 checkfast:
 	Rscript -e "devtools::check(build_args = '--no-build-vignettes')"
 
+revdep:
+	Rscript -e "revdepcheck::revdep_check()"
+
 test:
 	Rscript -e "devtools::test()"
 
@@ -30,4 +33,4 @@ style:
 	Rscript -e "styler::style_pkg()"
 
 pr: style doc check site
-	echo "If all checks have passed, you are ready to submit PR"
+	echo "If all checks have passed, you are ready to submit PR"
diff --git a/NAMESPACE b/NAMESPACE
@@ -49,6 +49,9 @@ export(Lrnr_revere_task)
 export(Lrnr_rfcde)
 export(Lrnr_rpart)
 export(Lrnr_rugarch)
+export(Lrnr_screener_corP)
+export(Lrnr_screener_corRank)
+export(Lrnr_screener_randomForest)
 export(Lrnr_sl)
 export(Lrnr_solnp)
 export(Lrnr_solnp_density)
@@ -70,12 +73,14 @@ export(debugonce_train)
 export(define_h2o_X)
 export(delayed_learner_fit_chain)
 export(delayed_learner_fit_predict)
+export(delayed_learner_subset_covariates)
 export(delayed_learner_train)
 export(delayed_make_learner)
 export(dt_expand_factors)
 export(factor_to_indicators)
 export(learner_fit_chain)
 export(learner_fit_predict)
+export(learner_subset_covariates)
 export(learner_train)
 export(loss_loglik_binomial)
 export(loss_loglik_multinomial)
@@ -119,12 +124,16 @@ importFrom(R6,R6Class)
 importFrom(assertthat,assert_that)
 importFrom(assertthat,is.count)
 importFrom(assertthat,is.flag)
+importFrom(data.table,set)
+importFrom(data.table,setnames)
 importFrom(digest,digest)
 importFrom(ggplot2,cut_interval)
 importFrom(ggplot2,cut_number)
+importFrom(imputeMissings,impute)
 importFrom(methods,is)
 importFrom(origami,cross_validate)
 importFrom(origami,fold_index)
+importFrom(origami,id_folds_to_folds)
 importFrom(origami,make_folds)
 importFrom(origami,training)
 importFrom(origami,validation)

diff --git a/NEWS.md b/NEWS.md
@@ -0,0 +1,32 @@
+# sl3 1.3.1
+* Overhaul of data preprocessing.
+* New screening methods and convex combination in `Lrnr_nnls`.
+* Bug fixes including covariate subsetting and better handling of NAs.
+* Package and documentation cleanup.
+
+# sl3 1.3.0
+* Fixes incorrect handling of missingness in the automatic imputation procedure.
+* Adds new standard learners, including from the `gam` and `caret` packages.
+* Adds custom learners for conditional density estimation, including
+  semiparametric methods based on conditional mean and conditional mean/variance
+  estimation as well as generalized functionality for density estimation via a
+  pooled hazards approach.
+
+# sl3 1.2.0
+* Default metalearners based on task outcome types.
+* Handling of imputation internally in task objects.
+* Addition of several new learners, including from the `gbm`, `earth`,
+  `polspline` packages.
+* Fixing errors in existing learners (e.g., subtle parallelization in `xgboost`
+  and `ranger`).
+* Support for multivariate outcomes
+* Sets default cross-validation to be revere-style.
+* Support for cross-validated super learner and variable importance.
+
+# sl3 1.1.0
+* A full-featured and stable release of the project.
+* Numerous learners are included and many bugs have been fixed relative to
+  earlier versions (esp v1.0.0) of the software.
+
+# sl3 1.0.0
+* An initial stable release.
diff --git a/R/Lrnr_arima.R b/R/Lrnr_arima.R
@@ -41,8 +41,8 @@ Lrnr_arima <- R6Class(
   class = TRUE,
   public = list(
     initialize = function(order = NULL,
-                              seasonal = list(order = c(0L, 0L, 0L), period = NA),
-                              n.ahead = NULL, ...) {
+                          seasonal = list(order = c(0L, 0L, 0L), period = NA),
+                          n.ahead = NULL, ...) {
       super$initialize(params = args_to_list(), ...)
     }
   ),

diff --git a/R/Lrnr_bartMachine.R b/R/Lrnr_bartMachine.R
@@ -62,9 +62,9 @@ Lrnr_bartMachine <- R6Class(
   inherit = Lrnr_base, portable = TRUE, class = TRUE,
   public = list(
     initialize = function(num_trees = 50, num_burn_in = 250, verbose = F,
-                              alpha = 0.95, beta = 2, k = 2, q = 0.9, nu = 3,
-                              num_iterations_after_burn_in = 1000,
-                              prob_rule_class = 0.5, ...) {
+                          alpha = 0.95, beta = 2, k = 2, q = 0.9, nu = 3,
+                          num_iterations_after_burn_in = 1000,
+                          prob_rule_class = 0.5, ...) {
       super$initialize(params = args_to_list(), ...)
     }
   ),

diff --git a/R/Lrnr_base.R b/R/Lrnr_base.R
@@ -81,7 +81,7 @@ Lrnr_base <- R6Class(
         fit_object <- private$.train(subsetted_task)
       }
       new_object <- self$clone() # copy parameters, and whatever else
-      new_object$set_train(fit_object, task)
+      new_object$set_train(fit_object, subsetted_task)
       return(new_object)
     },
 
@@ -146,7 +146,10 @@ Lrnr_base <- R6Class(
     },
 
     train_sublearners = function(task) {
-      return(private$.train_sublearners(task))
+      # TODO: add error handling
+      subsetted_task <- delayed_learner_subset_covariates(self, task)
+
+      return(private$.train_sublearners(subsetted_task))
     },
 
     train = function(task) {

diff --git a/R/Lrnr_bilstm.R b/R/Lrnr_bilstm.R
@@ -31,16 +31,16 @@ Lrnr_bilstm <- R6Class(
   classname = "Lrnr_bilstm", inherit = Lrnr_base, portable = TRUE, class = TRUE,
   public = list(
     initialize = function(units = 4,
-                              loss = "mean_squared_error",
-                              optimizer = "adam",
-                              batch_size = 1,
-                              epochs = 500,
-                              window = 5,
-                              n.ahead = 1,
-                              activation = "linear",
-                              dense = 1,
-                              dropout = 0,
-                              ...) {
+                          loss = "mean_squared_error",
+                          optimizer = "adam",
+                          batch_size = 1,
+                          epochs = 500,
+                          window = 5,
+                          n.ahead = 1,
+                          activation = "linear",
+                          dense = 1,
+                          dropout = 0,
+                          ...) {
       params <- list(
         units = units, loss = loss, optimizer = optimizer,
         batch_size = batch_size, epochs = epochs, window = window,

diff --git a/R/Lrnr_caret.R b/R/Lrnr_caret.R
@@ -51,10 +51,10 @@ Lrnr_caret <- R6Class(
   portable = TRUE, class = TRUE,
   public = list(
     initialize = function(algorithm,
-                              method = "CV",
-                              metric = NULL,
-                              trControl = caret::trainControl(method = method),
-                              ...) {
+                          method = "CV",
+                          metric = NULL,
+                          trControl = caret::trainControl(method = method),
+                          ...) {
       params <- list(
         method = algorithm,
         ...

diff --git a/R/Lrnr_condensier.R b/R/Lrnr_condensier.R
@@ -53,17 +53,17 @@ Lrnr_condensier <- R6Class(
   class = TRUE,
   public = list(
     initialize = function(bin_method = c("equal.mass", "equal.len", "dhist"),
-                              nbins = 5,
-                              max_n_cat = 20,
-                              pool = FALSE,
-                              max_n_bin = NA_integer_,
-                              parfit = FALSE,
-                              bin_estimator = make_learner(
-                                Lrnr_glm_fast,
-                                family = binomial()
-                              ),
-                              intrvls = NULL,
-                              ...) {
+                          nbins = 5,
+                          max_n_cat = 20,
+                          pool = FALSE,
+                          max_n_bin = NA_integer_,
+                          parfit = FALSE,
+                          bin_estimator = make_learner(
+                            Lrnr_glm_fast,
+                            family = binomial()
+                          ),
+                          intrvls = NULL,
+                          ...) {
       params <- args_to_list()
       assert_that(is(bin_estimator, "Lrnr_base") || is(
         bin_estimator,

diff --git a/R/Lrnr_dbarts.R b/R/Lrnr_dbarts.R
@@ -85,7 +85,7 @@ Lrnr_dbarts <- R6Class(
   inherit = Lrnr_base, portable = TRUE, class = TRUE,
   public = list(
     initialize = function(ndpost = 500, nskip = 100,
-                              ntree = 200L, verbose = FALSE, keeptrees = TRUE, ...) {
+                          ntree = 200L, verbose = FALSE, keeptrees = TRUE, ...) {
       super$initialize(params = args_to_list(), ...)
     }
   ),

diff --git a/R/Lrnr_density_discretize.R b/R/Lrnr_density_discretize.R
@@ -31,7 +31,7 @@ Lrnr_density_discretize <- R6Class(
   class = TRUE,
   public = list(
     initialize = function(categorical_learner = NULL, type = "equal_mass",
-                              n_bins = 20, ...) {
+                          n_bins = 20, ...) {
       if (is.null(categorical_learner)) {
         categorical_learner <- make_learner(Lrnr_glmnet)
       }
@@ -63,7 +63,6 @@ Lrnr_density_discretize <- R6Class(
         outcome = "discrete_Y",
         column_names = new_columns
       )
-
       # fit categorical learner to discretized task
       categorical_fit <- self$params$categorical_learner$train(discrete_task)
 

diff --git a/R/Lrnr_earth.R b/R/Lrnr_earth.R
@@ -60,8 +60,8 @@ Lrnr_earth <- R6Class(
   portable = TRUE, class = TRUE,
   public = list(
     initialize = function(degree = 2, penalty = 3, pmethod = "backward",
-                              nfold = 0, ncross = 1, minspan = 0, endspan = 0,
-                              ...) {
+                          nfold = 0, ncross = 1, minspan = 0, endspan = 0,
+                          ...) {
       params <- args_to_list()
       super$initialize(params = params, ...)
     }

diff --git a/R/Lrnr_expSmooth.R b/R/Lrnr_expSmooth.R
@@ -72,14 +72,14 @@ Lrnr_expSmooth <- R6Class(
   class = TRUE,
   public = list(
     initialize = function(model = "ZZZ", damped = NULL, alpha = NULL,
-                              beta = NULL, gamma = NULL, phi = NULL, lambda = NULL,
-                              additive.only = FALSE, biasadj = FALSE,
-                              lower = c(rep(1e-04, 3), 0.8),
-                              upper = c(rep(0.9999, 3), 0.98),
-                              opt.crit = "lik", nmse = 3, bounds = "both",
-                              ic = "aic", restrict = TRUE,
-                              allow.multiplicative.trend = FALSE,
-                              use.initial.values = FALSE, freq = 1, ...) {
+                          beta = NULL, gamma = NULL, phi = NULL, lambda = NULL,
+                          additive.only = FALSE, biasadj = FALSE,
+                          lower = c(rep(1e-04, 3), 0.8),
+                          upper = c(rep(0.9999, 3), 0.98),
+                          opt.crit = "lik", nmse = 3, bounds = "both",
+                          ic = "aic", restrict = TRUE,
+                          allow.multiplicative.trend = FALSE,
+                          use.initial.values = FALSE, freq = 1, ...) {
       params <- args_to_list()
       super$initialize(params = params, ...)
     }

diff --git a/R/Lrnr_gam.R b/R/Lrnr_gam.R
@@ -49,9 +49,9 @@ Lrnr_gam <- R6Class(
   portable = TRUE, class = TRUE,
   public = list(
     initialize = function(formula = NULL,
-                              family = NULL,
-                              method = "GCV.Cp",
-                              ...) {
+                          family = NULL,
+                          method = "GCV.Cp",
+                          ...) {
       params <- args_to_list()
       super$initialize(params = params, ...)
     }

diff --git a/R/Lrnr_gbm.R b/R/Lrnr_gbm.R
@@ -46,7 +46,7 @@ Lrnr_gbm <- R6Class(
   portable = TRUE, class = TRUE,
   public = list(
     initialize = function(n.trees = 10000, interaction.depth = 2,
-                              shrinkage = 0.001, ...) {
+                          shrinkage = 0.001, ...) {
       params <- args_to_list()
       super$initialize(params = params, ...)
     }

diff --git a/R/Lrnr_glm.R b/R/Lrnr_glm.R
@@ -67,9 +67,12 @@ Lrnr_glm <- R6Class(
       }
 
       args$ctrl <- glm.control(trace = FALSE)
-      SuppressGivenWarnings({
-        fit_object <- call_with_args(stats::glm.fit, args)
-      }, GetWarningsToSuppress())
+      SuppressGivenWarnings(
+        {
+          fit_object <- call_with_args(stats::glm.fit, args)
+        },
+        GetWarningsToSuppress()
+      )
 
       fit_object$linear.predictors <- NULL
       fit_object$weights <- NULL