Commit 37211d91 authored by Matija Obreza's avatar Matija Obreza

More verbose logging when fetching data from Genesys

parent 75131536
......@@ -79,7 +79,7 @@ authorization <- function(authorization) {
#' Ensure that environment has OAuth token
check_auth <- function() {
if (is.null(.genesysEnv$Authorization)) {
stop("You must first authorize with Genesys with user_login or client_login.");
warning("You must first authorize with Genesys with user_login() or client_login(...).");
}
}
......
......@@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#' Default page size
FETCH_PAGESIZE <- 1000
MAX_ALLOWED_PAGES <- 500
#' Max pages to retrieve
.MAX_ALLOWED_PAGES <- 500
#' Who am i?
#'
......@@ -39,13 +38,15 @@ me <- function() {
#' accessions <- fetch_accessions(mcpd_filter(ORIGCTY = c("DEU", "SVN")))
#'
#' @return Paged data structure
fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGESIZE, selector = NULL) {
resp <- post(path = "/acn/filter", query=list(p = page, s = size), body = filters)
fetch_accessions_page <- function(filters = list(), page = 0, size = 1000, selector = NULL) {
start_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
resp <- post(path = "/acn/filter", query=list(p = page, l = size), body = filters)
if (httr::status_code(resp) != 200) {
stop("Genesys responded with HTTP status code ", httr::status_code(resp), ". Expected 200.")
}
end_time <- as.numeric(as.numeric(Sys.time())*1000, digits=15)
paged <- jsonlite::fromJSON(httr::content(resp, "text"), simplifyVector = FALSE)
message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows"))
message(paste("Retrieved page", page + 1, "of", paged$totalPages, "with", paged$numberOfElements, "rows in", end_time - start_time, "ms."))
# Apply selector
if (is.function(selector)) {
......@@ -60,6 +61,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
#' @param size number of records to load per page (page size)
#' @param page the page index (0-based)
#' @param selector NULL or a function to "select" variables of interest
#' @param at.least stop fetching when at.least records are received from Genesys
#'
#' @examples
#' # Retrieve all accession data by country of origin
......@@ -75,7 +77,7 @@ fetch_accessions_page <- function(filters = list(), page = 0, size = FETCH_PAGES
#'
#' @export
#' @return Paged data structure
fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZE, selector = NULL) {
fetch_accessions <- function(filters = list(), page = NULL, size = 1000, selector = NULL, at.least = NULL) {
if (! is.null(page)) {
# Fetch page
return(fetch_accessions_page(filters, page, size, selector));
......@@ -86,9 +88,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
pages <- paged$totalPages
for (page in 1:pages) {
if (page > MAX_ALLOWED_PAGES) {
if (page > .MAX_ALLOWED_PAGES) {
# Break if over max pages
return(page)
message(paste("Not requesting data after page", .MAX_ALLOWED_PAGES, "Stopping."))
break
}
p <- fetch_accessions_page(filters, page, size, selector)
......@@ -100,6 +103,10 @@ fetch_accessions <- function(filters = list(), page = NULL, size = FETCH_PAGESIZ
# print("Got last page")
break
}
if (! is.null(maxRecords) && maxRecords <= paged$numberOfElements) {
message(paste("Receved", paged$numberOfElements, "of", maxRecords, "requested. Stopping."))
break
}
}
paged
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment