Commit a025c927 authored by Matija Obreza's avatar Matija Obreza

AccessionProcessor for processing Lists of Accessions

- Admin: recalculate Accession#tileIndex
- Needed after database upgrade
- Longer delay before updating Institute numbers and PDCI
parent d0e2612d
......@@ -29,6 +29,8 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch;
import org.genesys.catalog.service.DatasetService;
import org.genesys.worldclim.WorldClimUtil;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.impl.FaoInstitute;
......@@ -243,30 +245,61 @@ public class AdminController {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter);
accessionProcessor.apply(filter, (accession) -> {
accessionProcessor.apply(filter, (accessions) -> {
// Everything here is executed within a @Transaction(readOnly = false) context
AccessionId accessionId = accession.getAccessionId();
PDCI pdci = accessionId.getPdci();
// create new PDCI if missing
PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession);
// if PDCI was missing, link it with accession
if (pdci == null) {
LOG.trace("Assigning new PDCI for {}", accession);
resultingPdci.setAccession(accessionId);
accessionId.setPdci(resultingPdci);
// also updates accession.accessionId.pdci
return accessionRepository.save(accession);
} else {
LOG.trace("Updating PDCI for {}", accession);
accessionId.setPdci(pdciRepository.save(resultingPdci));
return accession;
if (accessions == null) {
return accessions;
}
accessions.forEach(accession -> {
AccessionId accessionId = accession.getAccessionId();
PDCI pdci = accessionId.getPdci();
// create new PDCI if missing
PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession);
// if PDCI was missing, link it with accession
if (pdci == null) {
LOG.trace("Assigning new PDCI for {}", accession);
resultingPdci.setAccession(accessionId);
accessionId.setPdci(resultingPdci);
} else {
LOG.trace("Updating PDCI for {}", accession);
accessionId.setPdci(pdciRepository.save(resultingPdci));
}
});
LOG.debug("Updated {} PDCI entries", accessions.size());
return accessionRepository.save(accessions);
});
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "action=tile-index")
public String updateTileIndex(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter);
accessionProcessor.apply(filter, (accessions) -> {
if (accessions == null) {
return accessions;
}
accessions.forEach(accession -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
accession.setTileIndex(WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude()));
}
});
LOG.debug("Regenerated {} tileIndexes", accessions.size());
return accessionRepository.save(accessions);
});
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
......
......@@ -97,6 +97,16 @@ public interface AccessionService {
*/
List<Accession> processAccessions(List<Long> accessionIds, IAccessionAction action);
/**
* Executes the action on a list of accession in a
* Spring transaction. Spring security context not supported.
*
* @param accessionIds List of accession IDs
* @param action the action to execute on each accession
* @return the list of processed accessions
*/
List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action);
public static interface IAccessionAction {
/**
* Run action on Accession
......@@ -106,4 +116,14 @@ public interface AccessionService {
*/
Accession apply(Accession a) throws Exception;
}
public static interface IAccessionBatchAction {
/**
* Run action on batch of Accessions
*
* @param a the accession
* @return must return the resulting {@link Accession}
*/
List<Accession> apply(List<Accession> a) throws Exception;
}
}
......@@ -209,4 +209,17 @@ public class AccessionServiceImpl implements AccessionService {
return accessions;
}
@Override
@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
public List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action) {
List<Accession> accessions = accessionRepository.findAll(accessionIds);
accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute));
LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size());
try {
return action.apply(accessions);
} catch (Exception e) {
LOG.info("Error processing accession: {}", e.getMessage());
return accessions;
}
}
}
......@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.sql.DataSource;
import org.genesys2.server.exception.MaxPageLimitException;
......@@ -74,8 +73,8 @@ import org.springframework.transaction.annotation.Transactional;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.JPQLQuery;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
@Service
@Transactional(readOnly = true)
......@@ -98,9 +97,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Autowired
private GeoService geoService;
@Autowired
private EntityManager entityManager;
private JdbcTemplate jdbcTemplate;
@Autowired
......@@ -117,6 +113,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Autowired
private GeoRegionService geoRegionService;
@Autowired
private JPAQueryFactory jpaQueryFactory;
@Autowired
public void setDataSource(final DataSource dataSource) {
......@@ -449,10 +448,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
QAccession accession = QAccession.accession;
QAccessionGeo accessionGeo = accession.accessionId.geo;
JPQLQuery<Tuple> query = new JPAQuery<>(entityManager);
query.select(accessionGeo.longitude, accessionGeo.latitude).distinct().from(accession);
JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accessionGeo.longitude, accessionGeo.latitude).distinct();
BooleanBuilder filt = new BooleanBuilder();
// filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull());
filt.and(accession.tileIndex.isNotNull());
if (filter != null) {
filt.and(filter.buildQuery());
......
......@@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class);
// Interval time (in ms) for scanning the queue for expired updates
private static final int DELAY_BETWEEN_QUEUESCANS = 2000;
private static final int DELAY_BETWEEN_QUEUESCANS = 5000;
// If a request comes before this delay (in ms), reschedule
private static final long MINTIME_BETWEEN_UPDATES = 10000;
private static final long MINTIME_BETWEEN_UPDATES = 30000;
private DelayQueue<DelayedObject<String>> instituteQueue;
......
......@@ -26,6 +26,7 @@ import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.AccessionService.IAccessionAction;
import org.genesys2.server.service.AccessionService.IAccessionBatchAction;
import org.genesys2.server.service.filter.AccessionFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -121,7 +122,7 @@ public class AccessionProcessor {
* @param action the action
*/
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void apply(AccessionFilter filter, IAccessionAction action) {
public void apply(AccessionFilter filter, IAccessionBatchAction action) {
Predicate predicate = filter.buildQuery();
long count = accessionRepository.count(predicate);
......@@ -163,7 +164,7 @@ public class AccessionProcessor {
LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime());
}
private void asyncUpdate(List<Long> accessionIds, IAccessionAction action) {
private void asyncUpdate(List<Long> accessionIds, IAccessionBatchAction action) {
if (accessionIds.size() == 0) {
return;
}
......
......@@ -58,7 +58,7 @@ public class AccessionRefAspect {
@Before(value = "execution(* org.genesys2.server.persistence.AccessionRepository.delete(*)) && args(arg)")
public void beforeDeleteAccession(JoinPoint joinPoint, Object arg) throws Throwable {
LOG.debug("Before delete accession" + arg);
LOG.trace("Before delete accession" + arg);
if (arg == null) {
return;
}
......@@ -121,7 +121,7 @@ public class AccessionRefAspect {
}
private void updateDatasets(Accession accession) {
LOG.debug("After returning saved accession" + accession);
LOG.trace("After returning saved accession" + accession);
List<Dataset> datasets = (List<Dataset>) datasetRepository.findAll(QDataset.dataset.accessionRefs.any().acceNumb.eq(accession.getAccessionNumber()).and(
QDataset.dataset.accessionRefs.any().genus.eq(accession.getTaxonomy().getGenus())).and(QDataset.dataset.accessionRefs.any().instCode.eq(accession.getInstitute()
.getCode())));
......
......@@ -20,6 +20,15 @@
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/admin-action" />">
<input type="text" name="filter" placeholder="{}" value="{}" />
<button name="action" class="btn btn-default" value="tile-index">Update tile index</button>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>PDCI</h3>
<form method="post" action="<c:url value="/admin/pdci" />">
<input type="text" name="filter" placeholder="{}" value="{}" />
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment