From a025c927961f01220c1fe96d900ad22583b337f2 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Tue, 9 Oct 2018 06:13:48 -0400 Subject: [PATCH] AccessionProcessor for processing Lists of Accessions - Admin: recalculate Accession#tileIndex - Needed after database upgrade - Longer delay before updating Institute numbers and PDCI --- .../server/mvc/admin/AdminController.java | 67 ++++++++++++++----- .../server/service/AccessionService.java | 20 ++++++ .../service/impl/AccessionServiceImpl.java | 13 ++++ .../impl/GenesysFilterServiceImpl.java | 13 ++-- .../service/worker/AccessionCounter.java | 4 +- .../service/worker/AccessionProcessor.java | 5 +- .../service/worker/AccessionRefAspect.java | 4 +- src/main/webapp/WEB-INF/jsp/admin/index.jsp | 9 +++ 8 files changed, 105 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/genesys2/server/mvc/admin/AdminController.java b/src/main/java/org/genesys2/server/mvc/admin/AdminController.java index ac80cba7b..bc50eedd6 100644 --- a/src/main/java/org/genesys2/server/mvc/admin/AdminController.java +++ b/src/main/java/org/genesys2/server/mvc/admin/AdminController.java @@ -29,6 +29,8 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang.time.StopWatch; import org.genesys.catalog.service.DatasetService; +import org.genesys.worldclim.WorldClimUtil; +import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.genesys.AccessionId; import org.genesys2.server.model.genesys.PDCI; import org.genesys2.server.model.impl.FaoInstitute; @@ -243,30 +245,61 @@ public class AdminController { AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class); LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter); - accessionProcessor.apply(filter, (accession) -> { + accessionProcessor.apply(filter, (accessions) -> { // Everything here is executed within a @Transaction(readOnly = false) context - - AccessionId accessionId = accession.getAccessionId(); - PDCI pdci = accessionId.getPdci(); - // create new PDCI if missing - PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession); - // if PDCI was missing, link it with accession - if (pdci == null) { - LOG.trace("Assigning new PDCI for {}", accession); - resultingPdci.setAccession(accessionId); - accessionId.setPdci(resultingPdci); - // also updates accession.accessionId.pdci - return accessionRepository.save(accession); - } else { - LOG.trace("Updating PDCI for {}", accession); - accessionId.setPdci(pdciRepository.save(resultingPdci)); - return accession; + if (accessions == null) { + return accessions; } + + accessions.forEach(accession -> { + AccessionId accessionId = accession.getAccessionId(); + PDCI pdci = accessionId.getPdci(); + // create new PDCI if missing + PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession); + // if PDCI was missing, link it with accession + if (pdci == null) { + LOG.trace("Assigning new PDCI for {}", accession); + resultingPdci.setAccession(accessionId); + accessionId.setPdci(resultingPdci); + } else { + LOG.trace("Updating PDCI for {}", accession); + accessionId.setPdci(pdciRepository.save(resultingPdci)); + } + }); + + LOG.debug("Updated {} PDCI entries", accessions.size()); + return accessionRepository.save(accessions); }); return "redirect:/admin/"; } + + + @RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "action=tile-index") + public String updateTileIndex(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException { + + AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class); + LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter); + + accessionProcessor.apply(filter, (accessions) -> { + if (accessions == null) { + return accessions; + } + + accessions.forEach(accession -> { + AccessionGeo geo = accession.getAccessionId().getGeo(); + if (geo != null) { + accession.setTileIndex(WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude())); + } + }); + + LOG.debug("Regenerated {} tileIndexes", accessions.size()); + return accessionRepository.save(accessions); + }); + + return "redirect:/admin/"; + } @RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion") public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException { diff --git a/src/main/java/org/genesys2/server/service/AccessionService.java b/src/main/java/org/genesys2/server/service/AccessionService.java index ea753006d..ce54b7219 100644 --- a/src/main/java/org/genesys2/server/service/AccessionService.java +++ b/src/main/java/org/genesys2/server/service/AccessionService.java @@ -97,6 +97,16 @@ public interface AccessionService { */ List processAccessions(List accessionIds, IAccessionAction action); + /** + * Executes the action on a list of accession in a + * Spring transaction. Spring security context not supported. + * + * @param accessionIds List of accession IDs + * @param action the action to execute on each accession + * @return the list of processed accessions + */ + List processAccessions(List accessionIds, IAccessionBatchAction action); + public static interface IAccessionAction { /** * Run action on Accession @@ -106,4 +116,14 @@ public interface AccessionService { */ Accession apply(Accession a) throws Exception; } + + public static interface IAccessionBatchAction { + /** + * Run action on batch of Accessions + * + * @param a the accession + * @return must return the resulting {@link Accession} + */ + List apply(List a) throws Exception; + } } diff --git a/src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java index 7650bc32b..4532b4fdd 100644 --- a/src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java @@ -209,4 +209,17 @@ public class AccessionServiceImpl implements AccessionService { return accessions; } + @Override + @Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW) + public List processAccessions(List accessionIds, IAccessionBatchAction action) { + List accessions = accessionRepository.findAll(accessionIds); + accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute)); + LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size()); + try { + return action.apply(accessions); + } catch (Exception e) { + LOG.info("Error processing accession: {}", e.getMessage()); + return accessions; + } + } } diff --git a/src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java index cfdaab890..16104208e 100644 --- a/src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import javax.persistence.EntityManager; import javax.sql.DataSource; import org.genesys2.server.exception.MaxPageLimitException; @@ -74,8 +73,8 @@ import org.springframework.transaction.annotation.Transactional; import com.querydsl.core.BooleanBuilder; import com.querydsl.core.Tuple; -import com.querydsl.jpa.JPQLQuery; import com.querydsl.jpa.impl.JPAQuery; +import com.querydsl.jpa.impl.JPAQueryFactory; @Service @Transactional(readOnly = true) @@ -98,9 +97,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { @Autowired private GeoService geoService; - @Autowired - private EntityManager entityManager; - private JdbcTemplate jdbcTemplate; @Autowired @@ -117,6 +113,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { @Autowired private GeoRegionService geoRegionService; + + @Autowired + private JPAQueryFactory jpaQueryFactory; @Autowired public void setDataSource(final DataSource dataSource) { @@ -449,10 +448,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { QAccession accession = QAccession.accession; QAccessionGeo accessionGeo = accession.accessionId.geo; - JPQLQuery query = new JPAQuery<>(entityManager); - query.select(accessionGeo.longitude, accessionGeo.latitude).distinct().from(accession); + JPAQuery query = jpaQueryFactory.selectFrom(accession).select(accessionGeo.longitude, accessionGeo.latitude).distinct(); BooleanBuilder filt = new BooleanBuilder(); + // filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull()); filt.and(accession.tileIndex.isNotNull()); if (filter != null) { filt.and(filter.buildQuery()); diff --git a/src/main/java/org/genesys2/server/service/worker/AccessionCounter.java b/src/main/java/org/genesys2/server/service/worker/AccessionCounter.java index fa3f2df6f..dc562fccf 100644 --- a/src/main/java/org/genesys2/server/service/worker/AccessionCounter.java +++ b/src/main/java/org/genesys2/server/service/worker/AccessionCounter.java @@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean { private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class); // Interval time (in ms) for scanning the queue for expired updates - private static final int DELAY_BETWEEN_QUEUESCANS = 2000; + private static final int DELAY_BETWEEN_QUEUESCANS = 5000; // If a request comes before this delay (in ms), reschedule - private static final long MINTIME_BETWEEN_UPDATES = 10000; + private static final long MINTIME_BETWEEN_UPDATES = 30000; private DelayQueue> instituteQueue; diff --git a/src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java b/src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java index be2da0dca..d0293aeaa 100644 --- a/src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java +++ b/src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java @@ -26,6 +26,7 @@ import org.genesys2.server.model.genesys.QAccession; import org.genesys2.server.persistence.AccessionRepository; import org.genesys2.server.service.AccessionService; import org.genesys2.server.service.AccessionService.IAccessionAction; +import org.genesys2.server.service.AccessionService.IAccessionBatchAction; import org.genesys2.server.service.filter.AccessionFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,7 +122,7 @@ public class AccessionProcessor { * @param action the action */ @Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW) - public void apply(AccessionFilter filter, IAccessionAction action) { + public void apply(AccessionFilter filter, IAccessionBatchAction action) { Predicate predicate = filter.buildQuery(); long count = accessionRepository.count(predicate); @@ -163,7 +164,7 @@ public class AccessionProcessor { LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime()); } - private void asyncUpdate(List accessionIds, IAccessionAction action) { + private void asyncUpdate(List accessionIds, IAccessionBatchAction action) { if (accessionIds.size() == 0) { return; } diff --git a/src/main/java/org/genesys2/server/service/worker/AccessionRefAspect.java b/src/main/java/org/genesys2/server/service/worker/AccessionRefAspect.java index 52dd5bee5..bb3a0c605 100644 --- a/src/main/java/org/genesys2/server/service/worker/AccessionRefAspect.java +++ b/src/main/java/org/genesys2/server/service/worker/AccessionRefAspect.java @@ -58,7 +58,7 @@ public class AccessionRefAspect { @Before(value = "execution(* org.genesys2.server.persistence.AccessionRepository.delete(*)) && args(arg)") public void beforeDeleteAccession(JoinPoint joinPoint, Object arg) throws Throwable { - LOG.debug("Before delete accession" + arg); + LOG.trace("Before delete accession" + arg); if (arg == null) { return; } @@ -121,7 +121,7 @@ public class AccessionRefAspect { } private void updateDatasets(Accession accession) { - LOG.debug("After returning saved accession" + accession); + LOG.trace("After returning saved accession" + accession); List datasets = (List) datasetRepository.findAll(QDataset.dataset.accessionRefs.any().acceNumb.eq(accession.getAccessionNumber()).and( QDataset.dataset.accessionRefs.any().genus.eq(accession.getTaxonomy().getGenus())).and(QDataset.dataset.accessionRefs.any().instCode.eq(accession.getInstitute() .getCode()))); diff --git a/src/main/webapp/WEB-INF/jsp/admin/index.jsp b/src/main/webapp/WEB-INF/jsp/admin/index.jsp index aac2f9610..5699e0e32 100644 --- a/src/main/webapp/WEB-INF/jsp/admin/index.jsp +++ b/src/main/webapp/WEB-INF/jsp/admin/index.jsp @@ -20,6 +20,15 @@ + +
"> + + + + +
+ +

PDCI

"> -- GitLab