More verbose logging when fetching data from Genesys

R/api-client.R

@@ -79,7 +79,7 @@ authorization <- function(authorization) {
 #' Ensure that environment has OAuth token
 check_auth <- function() {
   if (is.null(.genesysEnv$Authorization)) {
-    stop("You must first authorize with Genesys with user_login or client_login.");
+    warning("You must first authorize with Genesys with user_login() or client_login(...).");
   }
 }
 

R/genesys.R

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#' Default page size
-FETCH_PAGESIZE <- 1000
-MAX_ALLOWED_PAGES <- 500
+#' Max pages to retrieve
+.MAX_ALLOWED_PAGES <- 500
 
 #' Who am i?
 #'
@@ -39,13 +38,15 @@ me <- function() {
 #'   accessions <- fetch_accessions(mcpd_filter(ORIGCTY = c("DEU", "SVN")))
 #'
 #' @return Paged data structure
-fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGESIZE, selector = NULL) {
-  resp <- post(path = "/acn/filter", query=list(p = page, s = size), body = filters)
+fetch_accessions_page <- function(filters = list(), page = 0, size = 1000, selector = NULL) {
+  start_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
+  resp <- post(path = "/acn/filter", query=list(p = page, l = size), body = filters)
   if (httr::status_code(resp) != 200) {
     stop("Genesys responded with HTTP status code ", httr::status_code(resp), ". Expected 200.")
   }
+  end_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
   paged <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE)
-  message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows"))
+  message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows in", end_time - start_time, "ms."))
   
   # Apply selector
   if (is.function(selector)) {
@@ -60,6 +61,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
 #' @param size number of records to load per page (page size)
 #' @param page the page index (0-based)
 #' @param selector NULL or a function to "select" variables of interest
+#' @param at.least stop fetching when at.least records are received from Genesys
 #'
 #' @examples
 #'   # Retrieve all accession data by country of origin
@@ -75,7 +77,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
 #'
 #' @export
 #' @return Paged data structure
-fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZE, selector = NULL) {
+fetch_accessions <- function(filters = list(), page = NULL, size = 1000, selector = NULL, at.least = NULL) {
   if (! is.null(page)) {
     # Fetch page
     return(fetch_accessions_page(filters, page, size, selector));
@@ -86,9 +88,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
   pages <- paged$totalPages
   
   for (page in 1:pages) {
-    if (page > MAX_ALLOWED_PAGES) {
+    if (page > .MAX_ALLOWED_PAGES) {
       # Break if over max pages
-      return(page)
+      message(paste("Not requesting data after page", .MAX_ALLOWED_PAGES, "Stopping."))
+      break
     }
     p <- fetch_accessions_page(filters, page, size, selector)
     
@@ -100,6 +103,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
       # print("Got last page")
       break
     }
+    if (! is.null(maxRecords) && maxRecords <= paged$numberOfElements) {
+      message(paste("Receved", paged$numberOfElements, "of", maxRecords, "requested. Stopping."))
+      break
+    }
   }
   
   paged