Commit d7250b7b authored by Matija Obreza's avatar Matija Obreza

Merge branch '412-scheduled-glis-update' into 'master'

Resolve "Scheduled GLIS update"

Closes #412

See merge request genesys-pgr/genesys-server!350
parents 463fd3c4 89364650
......@@ -18,6 +18,7 @@ package org.genesys2.server.api.v1;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import java.util.List;
import io.swagger.annotations.Api;
......@@ -36,15 +37,18 @@ import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.FaoInstituteRepository;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.worker.ScheduledGLISUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
......@@ -83,6 +87,9 @@ public class AdminController {
@Autowired
private DatasetService datasetService;
@Autowired
private ScheduledGLISUpdater scheduledGLISUpdater;
@PostMapping(value = "/ensure-inst-folders")
public Boolean ensureInstituteFolders() throws Exception {
LOG.info("Ensure institute folders");
......@@ -143,6 +150,13 @@ public class AdminController {
aclService.cleanupAcl();
return true;
}
@PostMapping(value = "/update-glis")
public Boolean updateGLIS(@RequestParam(value="from", required = true) @DateTimeFormat(pattern="yyyy-MM-dd") final Date from) {
LOG.info("Update GLIS with accessions with DOI");
scheduledGLISUpdater.notifyGLIS(from);
return true;
}
@PostMapping(value = "/acl", params = { "institutes" })
......@@ -185,6 +199,6 @@ public class AdminController {
aclService.createOrUpdatePermissions(gallery);
});
}
}
......@@ -21,6 +21,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
......@@ -73,6 +74,7 @@ import org.genesys2.server.service.worker.AccessionProcessor;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
import org.genesys2.server.service.worker.ScheduledGLISUpdater;
import org.genesys2.util.PDCICalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -80,6 +82,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Transactional;
......@@ -182,6 +185,9 @@ public class AdminController {
@Autowired
private CustomAclService aclService;
@Autowired
private ScheduledGLISUpdater scheduledGLISUpdater;
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -597,7 +603,14 @@ public class AdminController {
return "redirect:/admin/";
}
@PostMapping(value = "/update-glis")
public String updateGLIS(@RequestParam(value="from", required = true) @DateTimeFormat(pattern="yyyy-MM-dd") final Date from) {
LOG.info("Update GLIS with accessions with DOI");
scheduledGLISUpdater.notifyGLIS(from);
return "redirect:/admin/";
}
@Autowired
private AccessionIdRepository accessionIdRepository;
......
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.worker;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import javax.persistence.EntityManager;
import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.core.types.dsl.PathBuilderFactory;
import com.querydsl.jpa.JPQLQuery;
import net.javacrumbs.shedlock.core.SchedulerLock;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.glis.v1.TermConstants;
import org.genesys.glis.v1.api.GenesysApi;
import org.genesys.glis.v1.model.UpdateTargets;
import org.genesys.glis.v1.model.UpdatedTarget;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.jpa.repository.support.Querydsl;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
/**
* Updating GLIS with new accessions with DOI that were modified in the last 8 hours.
*
* Task runs on 1 cluster node every 6 hours.
*
* @author Matija Obreza
* @author Maxym Borodenko
*/
@Component
public class ScheduledGLISUpdater {
// Handle data in batches of 100
private static final int BATCH_SIZE = 100;
/** The Constant LOG. */
public static final Logger LOG = LoggerFactory.getLogger(ScheduledGLISUpdater.class);
/** The GLIS Genesys API. */
@Autowired(required = false)
private GenesysApi glisGenesys;
/** The entity manager. */
@Autowired
private EntityManager em;
@Scheduled(cron = "0 0 1/3 ? * *") // every three hours
@SchedulerLock(name = "org.genesys2.server.service.worker.ScheduledGLISUpdater")
public void runGLISUpdater() {
if (glisGenesys == null) {
LOG.warn("GLIS Client not available. Not initializing GLIS Updater.");
return;
}
Calendar cal = Calendar.getInstance();
cal.add(Calendar.HOUR, -4);
LOG.warn("Starting scheduled GLIS update with updates since {}", cal);
StopWatch stopWatch = StopWatch.createStarted();
notifyGLIS(cal.getTime());
stopWatch.stop();
LOG.warn("Scheduled GLIS update took {}ms", stopWatch.getTime());
}
/**
* Notify GLIS.
*
* @param date the date
* @return the stop watch
*/
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void notifyGLIS(Date date) {
LOG.info("Starting GLIS update with updates since {}", date);
PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
Querydsl querydsl = new Querydsl(em, builder);
JPQLQuery<String> query = querydsl.createQuery(QAccession.accession)
// select DOI only
.select(QAccession.accession.doi)
// order by id
.orderBy(QAccession.accession.id.asc())
.where(QAccession.accession.lastModifiedDate.after(date).and(QAccession.accession.doi.isNotNull()))
.limit(BATCH_SIZE);
int startPosition = 0;
List<String> results;
StopWatch stopWatch = StopWatch.createStarted();
do {
query.offset(startPosition);
results = query.fetch();
// Update GLIS using loaded DOIs
processBatch(results);
// Start position is updated above.
startPosition += results.size();
// Clear anything cached in the entity manager
em.clear();
} while (results.size() > 0);
stopWatch.stop();
LOG.warn("Scheduled GLIS update took {}ms", stopWatch.getTime());
}
private void processBatch(List<String> accessionDOIs) {
if (CollectionUtils.isEmpty(accessionDOIs)) {
return;
}
UpdateTargets targets = new UpdateTargets();
targets.addKwsItem(TermConstants.PASSPORT_DATA);
targets.setDois(accessionDOIs);
for (int i = 5; i >= 0; i--) {
try {
LOG.info("Updating GLIS for {} accessions with DOIs", targets.getDois().size());
List<UpdatedTarget> glisResponse = glisGenesys.registerGenesysAsTarget(targets);
glisResponse.forEach(updatedTarget -> {
if (StringUtils.equals("OK", updatedTarget.getResult())) {
LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult());
} else if (StringUtils.equals("KO", updatedTarget.getResult())) {
LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult());
}
});
break;
} catch (Throwable e) {
if (i == 0) {
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e);
} else {
LOG.warn("Problem updating GLIS targets with DOIs: {}. {}. Retrying.", targets.getDois(), e.getMessage());
}
}
}
}
}
......@@ -17,28 +17,15 @@
package org.genesys2.spring.config;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.glis.v1.TermConstants;
import org.genesys.glis.v1.api.GenesysApi;
import org.genesys.glis.v1.model.UpdateTargets;
import org.genesys.glis.v1.model.UpdatedTarget;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.worker.AccessionModifiedAspect.AccessionModifiedListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.google.common.collect.Lists;
/**
* The AccessionListenersConfig configures hard-wired listeners to accession
* modifications in Genesys.
* modifications in Genesys (no listeners)
*
* @author Matija Obreza
*/
......@@ -46,60 +33,7 @@ import com.google.common.collect.Lists;
public class AccessionListenersConfig {
@Bean
public List<AccessionModifiedListener> accessionModifiedListeners(GenesysApi glisGenesysApi) {
ArrayList<AccessionModifiedListener> listeners = Lists.newArrayList(glisUpdater(glisGenesysApi));
System.err.println("Declared AccessionModifiedListeners " + listeners.size());
List<AccessionModifiedListener> active = listeners.stream().filter(listener -> listener != null).collect(Collectors.toList());
System.err.println("Active AccessionModifiedListeners " + active.size());
return active;
}
/**
* An in-line implementation of the Genesys updater to GLIS.
*
* @param glisGenesysApi the glis genesys api
* @return the listener
*/
private AccessionModifiedListener glisUpdater(GenesysApi glisGenesysApi) {
if (glisGenesysApi == null) {
System.err.println("GLIS Client not available. Not initializing GLIS Updater.");
return null;
}
return new AccessionModifiedListener() {
private final Logger LOG = LoggerFactory.getLogger(AccessionListenersConfig.class + ".GlisUpdater");
@Override
public void onUpdate(Collection<Accession> accessions) {
UpdateTargets targets = new UpdateTargets();
targets.addKwsItem(TermConstants.PASSPORT_DATA);
accessions.stream().map(a -> a.getDoi()).filter(doi -> doi != null).forEach(doi -> targets.addDoisItem(doi));
if (! CollectionUtils.isEmpty(targets.getDois())) {
try {
LOG.info("Updating GLIS for {} accessions with DOIs", targets.getDois().size());
List<UpdatedTarget> glisResponse = glisGenesysApi.registerGenesysAsTarget(targets);
glisResponse.forEach(updatedTarget -> {
if (StringUtils.equals("OK", updatedTarget.getResult())) {
LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult());
} else if (StringUtils.equals("KO", updatedTarget.getResult())) {
LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult());
}
});
} catch (Throwable e) {
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e);
}
} else {
LOG.info("Updated {} accessions do not have DOIs assigned", accessions.size());
}
}
@Override
public void onRemove(Collection<Accession> accessions) {
// No operation
}
};
public List<AccessionModifiedListener> accessionModifiedListeners() {
return new ArrayList<>();
}
}
......@@ -208,5 +208,15 @@
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>GLIS</h3>
<form method="post" action="<c:url value="/admin/update-glis" />">
<input type="date" name="from" pattern="yyyy-MM-dd" required />
<button type="submit" class="btn btn-default">Update GLIS with accessions with DOI</button>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
</body>
</html>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment