Commit a025c927 authored by Matija Obreza's avatar Matija Obreza

AccessionProcessor for processing Lists of Accessions

- Admin: recalculate Accession#tileIndex
- Needed after database upgrade
- Longer delay before updating Institute numbers and PDCI
parent d0e2612d
...@@ -29,6 +29,8 @@ import javax.xml.parsers.ParserConfigurationException; ...@@ -29,6 +29,8 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch; import org.apache.commons.lang.time.StopWatch;
import org.genesys.catalog.service.DatasetService; import org.genesys.catalog.service.DatasetService;
import org.genesys.worldclim.WorldClimUtil;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionId; import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.PDCI; import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.impl.FaoInstitute; import org.genesys2.server.model.impl.FaoInstitute;
...@@ -243,10 +245,14 @@ public class AdminController { ...@@ -243,10 +245,14 @@ public class AdminController {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class); AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter); LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter);
accessionProcessor.apply(filter, (accession) -> { accessionProcessor.apply(filter, (accessions) -> {
// Everything here is executed within a @Transaction(readOnly = false) context // Everything here is executed within a @Transaction(readOnly = false) context
if (accessions == null) {
return accessions;
}
accessions.forEach(accession -> {
AccessionId accessionId = accession.getAccessionId(); AccessionId accessionId = accession.getAccessionId();
PDCI pdci = accessionId.getPdci(); PDCI pdci = accessionId.getPdci();
// create new PDCI if missing // create new PDCI if missing
...@@ -256,15 +262,42 @@ public class AdminController { ...@@ -256,15 +262,42 @@ public class AdminController {
LOG.trace("Assigning new PDCI for {}", accession); LOG.trace("Assigning new PDCI for {}", accession);
resultingPdci.setAccession(accessionId); resultingPdci.setAccession(accessionId);
accessionId.setPdci(resultingPdci); accessionId.setPdci(resultingPdci);
// also updates accession.accessionId.pdci
return accessionRepository.save(accession);
} else { } else {
LOG.trace("Updating PDCI for {}", accession); LOG.trace("Updating PDCI for {}", accession);
accessionId.setPdci(pdciRepository.save(resultingPdci)); accessionId.setPdci(pdciRepository.save(resultingPdci));
return accession;
} }
}); });
LOG.debug("Updated {} PDCI entries", accessions.size());
return accessionRepository.save(accessions);
});
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "action=tile-index")
public String updateTileIndex(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter);
accessionProcessor.apply(filter, (accessions) -> {
if (accessions == null) {
return accessions;
}
accessions.forEach(accession -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
accession.setTileIndex(WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude()));
}
});
LOG.debug("Regenerated {} tileIndexes", accessions.size());
return accessionRepository.save(accessions);
});
return "redirect:/admin/"; return "redirect:/admin/";
} }
......
...@@ -97,6 +97,16 @@ public interface AccessionService { ...@@ -97,6 +97,16 @@ public interface AccessionService {
*/ */
List<Accession> processAccessions(List<Long> accessionIds, IAccessionAction action); List<Accession> processAccessions(List<Long> accessionIds, IAccessionAction action);
/**
* Executes the action on a list of accession in a
* Spring transaction. Spring security context not supported.
*
* @param accessionIds List of accession IDs
* @param action the action to execute on each accession
* @return the list of processed accessions
*/
List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action);
public static interface IAccessionAction { public static interface IAccessionAction {
/** /**
* Run action on Accession * Run action on Accession
...@@ -106,4 +116,14 @@ public interface AccessionService { ...@@ -106,4 +116,14 @@ public interface AccessionService {
*/ */
Accession apply(Accession a) throws Exception; Accession apply(Accession a) throws Exception;
} }
public static interface IAccessionBatchAction {
/**
* Run action on batch of Accessions
*
* @param a the accession
* @return must return the resulting {@link Accession}
*/
List<Accession> apply(List<Accession> a) throws Exception;
}
} }
...@@ -209,4 +209,17 @@ public class AccessionServiceImpl implements AccessionService { ...@@ -209,4 +209,17 @@ public class AccessionServiceImpl implements AccessionService {
return accessions; return accessions;
} }
@Override
@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
public List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action) {
List<Accession> accessions = accessionRepository.findAll(accessionIds);
accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute));
LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size());
try {
return action.apply(accessions);
} catch (Exception e) {
LOG.info("Error processing accession: {}", e.getMessage());
return accessions;
}
}
} }
...@@ -25,7 +25,6 @@ import java.util.List; ...@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.sql.DataSource; import javax.sql.DataSource;
import org.genesys2.server.exception.MaxPageLimitException; import org.genesys2.server.exception.MaxPageLimitException;
...@@ -74,8 +73,8 @@ import org.springframework.transaction.annotation.Transactional; ...@@ -74,8 +73,8 @@ import org.springframework.transaction.annotation.Transactional;
import com.querydsl.core.BooleanBuilder; import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple; import com.querydsl.core.Tuple;
import com.querydsl.jpa.JPQLQuery;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
@Service @Service
@Transactional(readOnly = true) @Transactional(readOnly = true)
...@@ -98,9 +97,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { ...@@ -98,9 +97,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Autowired @Autowired
private GeoService geoService; private GeoService geoService;
@Autowired
private EntityManager entityManager;
private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
@Autowired @Autowired
...@@ -118,6 +114,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { ...@@ -118,6 +114,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Autowired @Autowired
private GeoRegionService geoRegionService; private GeoRegionService geoRegionService;
@Autowired
private JPAQueryFactory jpaQueryFactory;
@Autowired @Autowired
public void setDataSource(final DataSource dataSource) { public void setDataSource(final DataSource dataSource) {
this.jdbcTemplate = new JdbcTemplate(dataSource); this.jdbcTemplate = new JdbcTemplate(dataSource);
...@@ -449,10 +448,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { ...@@ -449,10 +448,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
QAccession accession = QAccession.accession; QAccession accession = QAccession.accession;
QAccessionGeo accessionGeo = accession.accessionId.geo; QAccessionGeo accessionGeo = accession.accessionId.geo;
JPQLQuery<Tuple> query = new JPAQuery<>(entityManager); JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accessionGeo.longitude, accessionGeo.latitude).distinct();
query.select(accessionGeo.longitude, accessionGeo.latitude).distinct().from(accession);
BooleanBuilder filt = new BooleanBuilder(); BooleanBuilder filt = new BooleanBuilder();
// filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull());
filt.and(accession.tileIndex.isNotNull()); filt.and(accession.tileIndex.isNotNull());
if (filter != null) { if (filter != null) {
filt.and(filter.buildQuery()); filt.and(filter.buildQuery());
......
...@@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean { ...@@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class); private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class);
// Interval time (in ms) for scanning the queue for expired updates // Interval time (in ms) for scanning the queue for expired updates
private static final int DELAY_BETWEEN_QUEUESCANS = 2000; private static final int DELAY_BETWEEN_QUEUESCANS = 5000;
// If a request comes before this delay (in ms), reschedule // If a request comes before this delay (in ms), reschedule
private static final long MINTIME_BETWEEN_UPDATES = 10000; private static final long MINTIME_BETWEEN_UPDATES = 30000;
private DelayQueue<DelayedObject<String>> instituteQueue; private DelayQueue<DelayedObject<String>> instituteQueue;
......
...@@ -26,6 +26,7 @@ import org.genesys2.server.model.genesys.QAccession; ...@@ -26,6 +26,7 @@ import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.persistence.AccessionRepository; import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.AccessionService; import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.AccessionService.IAccessionAction; import org.genesys2.server.service.AccessionService.IAccessionAction;
import org.genesys2.server.service.AccessionService.IAccessionBatchAction;
import org.genesys2.server.service.filter.AccessionFilter; import org.genesys2.server.service.filter.AccessionFilter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -121,7 +122,7 @@ public class AccessionProcessor { ...@@ -121,7 +122,7 @@ public class AccessionProcessor {
* @param action the action * @param action the action
*/ */
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW) @Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void apply(AccessionFilter filter, IAccessionAction action) { public void apply(AccessionFilter filter, IAccessionBatchAction action) {
Predicate predicate = filter.buildQuery(); Predicate predicate = filter.buildQuery();
long count = accessionRepository.count(predicate); long count = accessionRepository.count(predicate);
...@@ -163,7 +164,7 @@ public class AccessionProcessor { ...@@ -163,7 +164,7 @@ public class AccessionProcessor {
LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime()); LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime());
} }
private void asyncUpdate(List<Long> accessionIds, IAccessionAction action) { private void asyncUpdate(List<Long> accessionIds, IAccessionBatchAction action) {
if (accessionIds.size() == 0) { if (accessionIds.size() == 0) {
return; return;
} }
......
...@@ -58,7 +58,7 @@ public class AccessionRefAspect { ...@@ -58,7 +58,7 @@ public class AccessionRefAspect {
@Before(value = "execution(* org.genesys2.server.persistence.AccessionRepository.delete(*)) && args(arg)") @Before(value = "execution(* org.genesys2.server.persistence.AccessionRepository.delete(*)) && args(arg)")
public void beforeDeleteAccession(JoinPoint joinPoint, Object arg) throws Throwable { public void beforeDeleteAccession(JoinPoint joinPoint, Object arg) throws Throwable {
LOG.debug("Before delete accession" + arg); LOG.trace("Before delete accession" + arg);
if (arg == null) { if (arg == null) {
return; return;
} }
...@@ -121,7 +121,7 @@ public class AccessionRefAspect { ...@@ -121,7 +121,7 @@ public class AccessionRefAspect {
} }
private void updateDatasets(Accession accession) { private void updateDatasets(Accession accession) {
LOG.debug("After returning saved accession" + accession); LOG.trace("After returning saved accession" + accession);
List<Dataset> datasets = (List<Dataset>) datasetRepository.findAll(QDataset.dataset.accessionRefs.any().acceNumb.eq(accession.getAccessionNumber()).and( List<Dataset> datasets = (List<Dataset>) datasetRepository.findAll(QDataset.dataset.accessionRefs.any().acceNumb.eq(accession.getAccessionNumber()).and(
QDataset.dataset.accessionRefs.any().genus.eq(accession.getTaxonomy().getGenus())).and(QDataset.dataset.accessionRefs.any().instCode.eq(accession.getInstitute() QDataset.dataset.accessionRefs.any().genus.eq(accession.getTaxonomy().getGenus())).and(QDataset.dataset.accessionRefs.any().instCode.eq(accession.getInstitute()
.getCode()))); .getCode())));
......
...@@ -21,6 +21,15 @@ ...@@ -21,6 +21,15 @@
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" /> <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form> </form>
<form method="post" action="<c:url value="/admin/admin-action" />">
<input type="text" name="filter" placeholder="{}" value="{}" />
<button name="action" class="btn btn-default" value="tile-index">Update tile index</button>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>PDCI</h3>
<form method="post" action="<c:url value="/admin/pdci" />"> <form method="post" action="<c:url value="/admin/pdci" />">
<input type="text" name="filter" placeholder="{}" value="{}" /> <input type="text" name="filter" placeholder="{}" value="{}" />
<button name="action" class="btn btn-default" value="institute-pdci">Update PDCI stats for all institutes</button> <button name="action" class="btn btn-default" value="institute-pdci">Update PDCI stats for all institutes</button>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment