AccessionProcessor for processing Lists of Accessions

- Admin: recalculate Accession#tileIndex
- Needed after database upgrade
- Longer delay before updating Institute numbers and PDCI

src/main/java/org/genesys2/server/mvc/admin/AdminController.java

@@ -29,6 +29,8 @@ import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.commons.lang.time.StopWatch;
 import org.genesys.catalog.service.DatasetService;
+import org.genesys.worldclim.WorldClimUtil;
+import org.genesys2.server.model.genesys.AccessionGeo;
 import org.genesys2.server.model.genesys.AccessionId;
 import org.genesys2.server.model.genesys.PDCI;
 import org.genesys2.server.model.impl.FaoInstitute;
@@ -243,10 +245,14 @@ public class AdminController {
 		AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
 		LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter);
 
-		accessionProcessor.apply(filter, (accession) -> {
+		accessionProcessor.apply(filter, (accessions) -> {
 			
 			// Everything here is executed within a @Transaction(readOnly = false) context
+			if (accessions == null) {
+				return accessions;
+			}
 
+			accessions.forEach(accession -> {
 				AccessionId accessionId = accession.getAccessionId();
 				PDCI pdci = accessionId.getPdci();
 				// create new PDCI if missing
@@ -256,15 +262,42 @@ public class AdminController {
 					LOG.trace("Assigning new PDCI for {}", accession);
 					resultingPdci.setAccession(accessionId);
 					accessionId.setPdci(resultingPdci);
-				// also updates accession.accessionId.pdci
-				return accessionRepository.save(accession);
 				} else {
 					LOG.trace("Updating PDCI for {}", accession);
 					accessionId.setPdci(pdciRepository.save(resultingPdci));
-				return accession;
 				}
 			});
 			
+			LOG.debug("Updated {} PDCI entries", accessions.size());
+			return accessionRepository.save(accessions);
+		});
+		
+		return "redirect:/admin/";
+	}
+	
+
+	@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "action=tile-index")
+	public String updateTileIndex(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException {
+		
+		AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
+		LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter);
+
+		accessionProcessor.apply(filter, (accessions) -> {
+			if (accessions == null) {
+				return accessions;
+			}
+
+			accessions.forEach(accession -> {
+				AccessionGeo geo = accession.getAccessionId().getGeo();
+				if (geo != null) {
+					accession.setTileIndex(WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude()));
+				}
+			});
+			
+			LOG.debug("Regenerated {} tileIndexes", accessions.size());
+			return accessionRepository.save(accessions);
+		});
+
 		return "redirect:/admin/";
 	}
 

src/main/java/org/genesys2/server/service/AccessionService.java

@@ -97,6 +97,16 @@ public interface AccessionService {
 	 */
 	List<Accession> processAccessions(List<Long> accessionIds, IAccessionAction action);
 
+	/**
+	 * Executes the action on a list of accession in a
+	 * Spring transaction. Spring security context not supported.
+	 *
+	 * @param accessionIds List of accession IDs
+	 * @param action the action to execute on each accession
+	 * @return the list of processed accessions
+	 */
+	List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action);
+
 	public static interface IAccessionAction {
 		/**
 		 * Run action on Accession
@@ -106,4 +116,14 @@ public interface AccessionService {
 		 */
 		Accession apply(Accession a) throws Exception;
 	}
+	
+	public static interface IAccessionBatchAction {
+		/**
+		 * Run action on batch of Accessions
+		 *
+		 * @param a the accession
+		 * @return must return the resulting {@link Accession}
+		 */
+		List<Accession> apply(List<Accession> a) throws Exception;
+	}
 }

src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java

@@ -209,4 +209,17 @@ public class AccessionServiceImpl implements AccessionService {
 		return accessions;
 	}
 
+	@Override
+	@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
+	public List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action) {
+		List<Accession> accessions = accessionRepository.findAll(accessionIds);
+		accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute));
+		LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size());
+		try {
+			return action.apply(accessions);
+		} catch (Exception e) {
+			LOG.info("Error processing accession: {}", e.getMessage());
+			return accessions;
+		}
+	}
 }

src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java

@@ -25,7 +25,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import javax.persistence.EntityManager;
 import javax.sql.DataSource;
 
 import org.genesys2.server.exception.MaxPageLimitException;
@@ -74,8 +73,8 @@ import org.springframework.transaction.annotation.Transactional;
 
 import com.querydsl.core.BooleanBuilder;
 import com.querydsl.core.Tuple;
-import com.querydsl.jpa.JPQLQuery;
 import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
 
 @Service
 @Transactional(readOnly = true)
@@ -98,9 +97,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
 	@Autowired
 	private GeoService geoService;
 
-	@Autowired
-	private EntityManager entityManager;
-
 	private JdbcTemplate jdbcTemplate;
 
 	@Autowired
@@ -118,6 +114,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
 	@Autowired
 	private GeoRegionService geoRegionService;
 	
+	@Autowired
+	private JPAQueryFactory jpaQueryFactory;
+
 	@Autowired
 	public void setDataSource(final DataSource dataSource) {
 		this.jdbcTemplate = new JdbcTemplate(dataSource);
@@ -449,10 +448,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
 		QAccession accession = QAccession.accession;
 		QAccessionGeo accessionGeo = accession.accessionId.geo;
 
-		JPQLQuery<Tuple> query = new JPAQuery<>(entityManager);
-		query.select(accessionGeo.longitude, accessionGeo.latitude).distinct().from(accession);
+		JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accessionGeo.longitude, accessionGeo.latitude).distinct();
 
 		BooleanBuilder filt = new BooleanBuilder();
+		// filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull());
 		filt.and(accession.tileIndex.isNotNull());
 		if (filter != null) {
 			filt.and(filter.buildQuery());

src/main/java/org/genesys2/server/service/worker/AccessionCounter.java

@@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
 	private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class);
 
 	// Interval time (in ms) for scanning the queue for expired updates
-	private static final int DELAY_BETWEEN_QUEUESCANS = 2000;
+	private static final int DELAY_BETWEEN_QUEUESCANS = 5000;
 	// If a request comes before this delay (in ms), reschedule
-	private static final long MINTIME_BETWEEN_UPDATES = 10000;
+	private static final long MINTIME_BETWEEN_UPDATES = 30000;
 
 	private DelayQueue<DelayedObject<String>> instituteQueue;
 

src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java

@@ -26,6 +26,7 @@ import org.genesys2.server.model.genesys.QAccession;
 import org.genesys2.server.persistence.AccessionRepository;
 import org.genesys2.server.service.AccessionService;
 import org.genesys2.server.service.AccessionService.IAccessionAction;
+import org.genesys2.server.service.AccessionService.IAccessionBatchAction;
 import org.genesys2.server.service.filter.AccessionFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -121,7 +122,7 @@ public class AccessionProcessor {
 	 * @param action the action
 	 */
 	@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
-	public void apply(AccessionFilter filter, IAccessionAction action) {
+	public void apply(AccessionFilter filter, IAccessionBatchAction action) {
 
 		Predicate predicate = filter.buildQuery();
 		long count = accessionRepository.count(predicate);
@@ -163,7 +164,7 @@ public class AccessionProcessor {
 		LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime());
 	}
 
-	private void asyncUpdate(List<Long> accessionIds, IAccessionAction action) {
+	private void asyncUpdate(List<Long> accessionIds, IAccessionBatchAction action) {
 		if (accessionIds.size() == 0) {
 			return;
 		}

src/main/java/org/genesys2/server/service/worker/AccessionRefAspect.java

@@ -58,7 +58,7 @@ public class AccessionRefAspect {
 	@Before(value = "execution(* org.genesys2.server.persistence.AccessionRepository.delete(*)) && args(arg)")
 	public void beforeDeleteAccession(JoinPoint joinPoint, Object arg) throws Throwable {
 
-		LOG.debug("Before delete accession" + arg);
+		LOG.trace("Before delete accession" + arg);
 		if (arg == null) {
 			return;
 		}
@@ -121,7 +121,7 @@ public class AccessionRefAspect {
 	}
 
 	private void updateDatasets(Accession accession) {
-		LOG.debug("After returning saved accession" + accession);
+		LOG.trace("After returning saved accession" + accession);
 		List<Dataset> datasets = (List<Dataset>) datasetRepository.findAll(QDataset.dataset.accessionRefs.any().acceNumb.eq(accession.getAccessionNumber()).and(
 			QDataset.dataset.accessionRefs.any().genus.eq(accession.getTaxonomy().getGenus())).and(QDataset.dataset.accessionRefs.any().instCode.eq(accession.getInstitute()
 				.getCode())));

src/main/webapp/WEB-INF/jsp/admin/index.jsp

@@ -21,6 +21,15 @@
     <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
   </form>
   
+	<form method="post" action="<c:url value="/admin/admin-action" />">
+		<input type="text" name="filter" placeholder="{}" value="{}" />
+	  	<button name="action" class="btn btn-default" value="tile-index">Update tile index</button>
+	  	<!-- CSRF protection -->
+		<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
+	</form>
+  
+  <h3>PDCI</h3>
+
   <form method="post" action="<c:url value="/admin/pdci" />">
 		<input type="text" name="filter" placeholder="{}" value="{}" />
     <button name="action" class="btn btn-default" value="institute-pdci">Update PDCI stats for all institutes</button>