Commit 37211d91 authored by Matija Obreza's avatar Matija Obreza
Browse files

More verbose logging when fetching data from Genesys

parent 75131536
...@@ -79,7 +79,7 @@ authorization <- function(authorization) { ...@@ -79,7 +79,7 @@ authorization <- function(authorization) {
#' Ensure that environment has OAuth token #' Ensure that environment has OAuth token
check_auth <- function() { check_auth <- function() {
if (is.null(.genesysEnv$Authorization)) { if (is.null(.genesysEnv$Authorization)) {
stop("You must first authorize with Genesys with user_login or client_login."); warning("You must first authorize with Genesys with user_login() or client_login(...).");
} }
} }
......
...@@ -12,9 +12,8 @@ ...@@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#' Default page size #' Max pages to retrieve
FETCH_PAGESIZE <- 1000 .MAX_ALLOWED_PAGES <- 500
MAX_ALLOWED_PAGES <- 500
#' Who am i? #' Who am i?
#' #'
...@@ -39,13 +38,15 @@ me <- function() { ...@@ -39,13 +38,15 @@ me <- function() {
#' accessions <- fetch_accessions(mcpd_filter(ORIGCTY = c("DEU", "SVN"))) #' accessions <- fetch_accessions(mcpd_filter(ORIGCTY = c("DEU", "SVN")))
#' #'
#' @return Paged data structure #' @return Paged data structure
fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGESIZE, selector = NULL) { fetch_accessions_page <- function(filters = list(), page = 0, size = 1000, selector = NULL) {
resp <- post(path = "/acn/filter", query=list(p = page, s = size), body = filters) start_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
resp <- post(path = "/acn/filter", query=list(p = page, l = size), body = filters)
if (httr::status_code(resp) != 200) { if (httr::status_code(resp) != 200) {
stop("Genesys responded with HTTP status code ", httr::status_code(resp), ". Expected 200.") stop("Genesys responded with HTTP status code ", httr::status_code(resp), ". Expected 200.")
} }
end_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
paged <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE) paged <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE)
message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows")) message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows in", end_time - start_time, "ms."))
# Apply selector # Apply selector
if (is.function(selector)) { if (is.function(selector)) {
...@@ -60,6 +61,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES ...@@ -60,6 +61,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
#' @param size number of records to load per page (page size) #' @param size number of records to load per page (page size)
#' @param page the page index (0-based) #' @param page the page index (0-based)
#' @param selector NULL or a function to "select" variables of interest #' @param selector NULL or a function to "select" variables of interest
#' @param at.least stop fetching when at.least records are received from Genesys
#' #'
#' @examples #' @examples
#' # Retrieve all accession data by country of origin #' # Retrieve all accession data by country of origin
...@@ -75,7 +77,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES ...@@ -75,7 +77,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
#' #'
#' @export #' @export
#' @return Paged data structure #' @return Paged data structure
fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZE, selector = NULL) { fetch_accessions <- function(filters = list(), page = NULL, size = 1000, selector = NULL, at.least = NULL) {
if (! is.null(page)) { if (! is.null(page)) {
# Fetch page # Fetch page
return(fetch_accessions_page(filters, page, size, selector)); return(fetch_accessions_page(filters, page, size, selector));
...@@ -86,9 +88,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ ...@@ -86,9 +88,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
pages <- paged$totalPages pages <- paged$totalPages
for (page in 1:pages) { for (page in 1:pages) {
if (page > MAX_ALLOWED_PAGES) { if (page > .MAX_ALLOWED_PAGES) {
# Break if over max pages # Break if over max pages
return(page) message(paste("Not requesting data after page", .MAX_ALLOWED_PAGES, "Stopping."))
break
} }
p <- fetch_accessions_page(filters, page, size, selector) p <- fetch_accessions_page(filters, page, size, selector)
...@@ -100,6 +103,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ ...@@ -100,6 +103,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
# print("Got last page") # print("Got last page")
break break
} }
if (! is.null(maxRecords) && maxRecords <= paged$numberOfElements) {
message(paste("Receved", paged$numberOfElements, "of", maxRecords, "requested. Stopping."))
break
}
} }
paged paged
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment