Merge branch 'forages' into staging

# Conflicts:
#	.gitignore
#	src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
#	src/main/java/org/genesys2/server/model/genesys/AccessionId.java
#	src/main/java/org/genesys2/server/persistence/domain/AccessionIdRepository.java
#	src/main/java/org/genesys2/server/service/GenesysService.java
#	src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
#	src/main/java/org/genesys2/server/service/worker/ElasticUpdaterAspect.java
#	src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java
#	src/main/resources/content/language.properties

src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java

@@ -92,6 +92,7 @@ public class AccessionDetails {
 
 	private Date modifiedDate;
 	private Date acqusitionDate;
+	private Float pdciScore;
 
 	private boolean historic;
 
@@ -127,6 +128,12 @@ public class AccessionDetails {
 		ad.lists = new HashSet<String>();
 		ad.cropName=accession.getCropName();
 		ad.inSgsv = accession.getInSvalbard() != null && accession.getInSvalbard() == true ? true : false;
+		ad.cropName = accession.getCropName();
+
+		if (accession.getAccessionId().getPdci() != null) {
+			ad.pdciScore = accession.getAccessionId().getPdci().getScore();
+		}
+
 		for (AccessionList al : accession.getAccessionId().getLists()) {
 			ad.lists.add(al.getUuid().toString());
 		}
@@ -507,4 +514,11 @@ public class AccessionDetails {
 		this.lists = lists;
 	}
 
+	public void setPdciScore(Float pdciScore) {
+		this.pdciScore = pdciScore;
+	}
+
+	public Float getPdciScore() {
+		return pdciScore;
+	}
 }

src/main/java/org/genesys2/server/model/genesys/AccessionId.java

@@ -87,6 +87,10 @@ public class AccessionId extends VersionedAuditedModel implements IdUUID {
 		this.lists = lists;
 	}
 
+	public PDCI getPdci() {
+		return pdci;
+	}
+
 	//
 	// public List<AccessionTrait> getTraits() {
 	// return traits;

src/main/java/org/genesys2/server/model/genesys/PDCI.java

@@ -21,6 +21,7 @@ import java.util.Set;
 
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
+import javax.persistence.Index;
 import javax.persistence.JoinColumn;
 import javax.persistence.OneToOne;
 import javax.persistence.PrePersist;
@@ -31,14 +32,14 @@ import javax.persistence.Transient;
 import org.genesys2.server.model.VersionedModel;
 
 @Entity
-@Table(name = "pdci")
-public class PDCI extends VersionedModel {
+@Table(name = "pdci", indexes = { @Index(columnList = "score") })
+public class PDCI extends VersionedModel implements AccessionRelated {
 	private static final long serialVersionUID = -1312366054528702261L;
 
-	public static final String[] independentItems = { "genus", "species", "spAuthor", "subTaxa", "subtAuthor", "cropName", "acqDate", "sampStat", "donorCode",
-			"donorNumb", "otherNumb", "duplSite", "storage", "donorName", "duplInstName", "acceUrl", "mlsStat" };
-	public static final String[] dependentItems = { "origCty", "collSite", "latitude", "longitude", "elevation", "collDate", "bredCode", "ancest", "collSrc",
-			"acceName", "collNumb", "collCode", "collName" };
+	public static final String[] independentItems = { "genus", "species", "spAuthor", "subTaxa", "subtAuthor", "cropName", "acqDate", "sampStat", "donorCode", "donorNumb", "otherNumb", "duplSite",
+			"storage", "donorName", "duplInstName", "acceUrl", "mlsStat" };
+	public static final String[] dependentItems = { "origCty", "collSite", "latitude", "longitude", "elevation", "collDate", "bredCode", "ancest", "collSrc", "acceName", "collNumb", "collCode",
+			"collName" };
 
 	@OneToOne(cascade = {}, fetch = FetchType.LAZY, optional = false)
 	@JoinColumn(name = "accessionId")
@@ -86,6 +87,7 @@ public class PDCI extends VersionedModel {
 		this.scoreHist = (float) (Math.ceil(this.score * 2) / 2);
 	}
 
+	@Override
 	public AccessionId getAccession() {
 		return accession;
 	}

src/main/java/org/genesys2/server/persistence/domain/AccessionIdRepository.java

@@ -29,10 +29,10 @@ public interface AccessionIdRepository extends JpaRepository<AccessionId, Long>
 	@Query("select aid from AccessionId aid where aid.uuid is null")
 	List<AccessionId> findMissingUuid(Pageable pageable);
 
-	@Query("select aid from AccessionId aid left join aid.pdci pdci where pdci.score is null")
-	List<AccessionId> findMissingPDCI(Pageable pageable);
+	@Query("select aid.id from AccessionId aid left join aid.pdci pdci where pdci.score is null")
+	Collection<Long> findMissingPDCI();
 
-	@Query("select aid from AccessionId aid left join aid.pdci pdci where pdci.score is null")
-	Collection<AccessionId> findMissingPDCI();
+	@Query("select id from AccessionId")
+	Collection<Long> findAllIds();
 
 }

src/main/java/org/genesys2/server/service/FilterConstants.java

@@ -81,4 +81,6 @@ public interface FilterConstants {
 
 	public static final String LISTS = "lists";
 
+	public static final String PDCI = "pdci";
+
 }

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -18,7 +18,6 @@ package org.genesys2.server.service;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -255,7 +254,7 @@ public interface GenesysService {
 
 	List<PDCI> loadPDCI(List<Long> batch);
 
-	int updatePDCI(Set<Long> ids);
+	List<PDCI> updatePDCI(Set<Long> ids);
 
 	PDCIStatistics statisticsPDCI(Organization organization);
 
@@ -265,6 +264,6 @@ public interface GenesysService {
 
 	List<SvalbardDeposit> saveSvalbards(List<SvalbardDeposit> svalbards);
 	
-	List<Accession> saveAccessionCrops(List<Long> accessionIds);
+	List<Accession> updateAccessionCrops(Set<Long> accessionIds);
 
 }

src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java

@@ -120,6 +120,10 @@ public class DirectMysqlQuery {
 			innerJoin("crop", null, "crop.id=a.cropId");
 		}
 		
+		if (filters.hasFilter(FilterConstants.PDCI)) {
+			innerJoin("pdci", null, "pdci.accessionId=a.id");
+		}
+		
 		// if (filters.hasFilter(FilterConstants.ORGCTY_ISO3)) {
 		// innerJoin("country", "cty", "cty.id=a.orgCtyId");
 		// }
@@ -201,6 +205,7 @@ public class DirectMysqlQuery {
 		createQuery(whereBuffer, "accename.name", filters.get(FilterConstants.ALIAS), params);
 		createQuery(whereBuffer, "col.collMissId", filters.get(FilterConstants.COLLMISSID), params);
 		createQuery(whereBuffer, "storage.storage", filters.get(FilterConstants.STORAGE), params);
+		createQuery(whereBuffer, "pdci.score", filters.get(FilterConstants.PDCI), params);
 
 		for (final AppliedFilter methodFilter : filters.methodFilters()) {
 			// Handle Genesys Method!

src/main/java/org/genesys2/server/service/impl/FilterHandler.java

@@ -70,8 +70,7 @@ public class FilterHandler {
 	private final ArrayList<GenesysFilter> availableFilters;
 
 	/**
-	 * By default we exclude historic records unless the user explicitly
-	 * specifies otherwise.
+	 * By default we exclude historic records unless the user explicitly specifies otherwise.
 	 */
 	public static final AppliedFilter NON_HISTORIC_FILTER = new AppliedFilter().setFilterName(FilterConstants.HISTORIC).addFilterValue(new LiteralValueFilter(false));
 
@@ -82,8 +81,8 @@ public class FilterHandler {
 		this.availableFilters.add(new BasicFilter(FilterConstants.CROPNAME, DataType.STRING));
 		this.availableFilters.add(new BasicFilter(FilterConstants.LISTS, DataType.STRING));
 
-		this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.SAMPSTAT, DataType.NUMERIC).build("accession.sampleStatus", new Integer[] { 100,
-				110, 120, 130, 200, 300, 400, 410, 411, 412, 413, 414, 415, 416, 420, 421, 422, 423, 500, 600, 999 }));
+		this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.SAMPSTAT, DataType.NUMERIC).build("accession.sampleStatus",
+				new Integer[] { 100, 110, 120, 130, 200, 300, 400, 410, 411, 412, 413, 414, 415, 416, 420, 421, 422, 423, 500, 600, 999 }));
 		this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_GENUS, "/explore/ac/" + FilterConstants.TAXONOMY_GENUS));
 		this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_SPECIES, "/explore/ac/" + FilterConstants.TAXONOMY_SPECIES));
 		this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_SUBTAXA, "/explore/ac/" + FilterConstants.TAXONOMY_SUBTAXA));
@@ -106,8 +105,8 @@ public class FilterHandler {
 		this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.HISTORIC, DataType.BOOLEAN).allowsNull(false));
 		this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
-		this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
-				13, 20, 30, 40, 50, 99 }));
+		this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12, 13, 20, 30, 40, 50, 99 }));
+		this.availableFilters.add(new BasicFilter(FilterConstants.PDCI, DataType.NUMERIC).allowsNull(false));
 	}
 
 	public List<GenesysFilter> listAvailableFilters() {
@@ -380,8 +379,7 @@ public class FilterHandler {
 		}
 
 		/**
-		 * Returns true if the filter is listed and has at least one value
-		 * specified
+		 * Returns true if the filter is listed and has at least one value specified
 		 * 
 		 * @param filterName
 		 * @return
@@ -753,7 +751,7 @@ public class FilterHandler {
 		public String toString() {
 			return getClass().getSimpleName() + " " + getType() + ": " + getTo();
 		}
-		
+
 		@Override
 		public int hashCode() {
 			final int prime = 31;
@@ -811,13 +809,11 @@ public class FilterHandler {
 			return from;
 		}
 
-
 		@Override
 		public String toString() {
 			return getClass().getSimpleName() + " " + getType() + ": " + getFrom();
 		}
-		
-		
+
 		@Override
 		public int hashCode() {
 			final int prime = 31;
@@ -875,7 +871,7 @@ public class FilterHandler {
 		public String toString() {
 			return getClass().getSimpleName() + " startsWith " + getStartsWith();
 		}
-				
+
 		@Override
 		public int hashCode() {
 			final int prime = 31;

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -1642,8 +1642,10 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	@Override
 	@Transactional
 	@CacheEvict(value = "statistics", allEntries = true)
-	public int updatePDCI(final Set<Long> ids) {
-		LOG.info("Calculating PDCI for " + ids.size() + " accessions");
+	public List<PDCI> updatePDCI(final Set<Long> ids) {
+		if (LOG.isDebugEnabled())
+			LOG.debug("Calculating PDCI for " + ids.size() + " accessions");
+
 		final List<PDCI> pdcis = Collections.synchronizedList(new ArrayList<PDCI>(ids.size()));
 
 		getAccessionDetails(ids).stream().forEach(ad -> {
@@ -1658,7 +1660,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 
 		LOG.info("Generated " + pdcis.size() + " new PDCIs");
 		repoPdci.save(pdcis);
-		return pdcis.size();
+		return pdcis;
 	}
 
 	@Override
@@ -1799,7 +1801,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	@Override
 	@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
 	// @PreAuthorize("hasRole('ADMINISTRATOR')")
-	public List<Accession> saveAccessionCrops(ArrayList<Long> accessionIds) {
+	public List<Accession> updateAccessionCrops(Set<Long> accessionIds) {
 		List<Accession> accessions = new ArrayList<>(accessionRepository.findAll(accessionIds));
 		for (Accession accession : accessions) {
 			accessionRepository.updateCrop(accession, cropService.getCrop(accession.getCropName()));

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterAspect.java

@@ -31,6 +31,7 @@ import org.aspectj.lang.annotation.AfterReturning;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
 import org.aspectj.lang.annotation.Before;
+import org.aspectj.lang.annotation.Pointcut;
 import org.genesys2.server.model.elastic.AccessionDetails;
 import org.genesys2.server.model.elastic.FullTextDocument;
 import org.genesys2.server.model.genesys.Accession;
@@ -59,6 +60,10 @@ public class ElasticUpdaterAspect {
 	@Autowired
 	private GenesysService genesysService;
 
+	@Pointcut(value = "execution(* org.genesys2.server.service.GenesysService.save*(..)) || execution(* org.genesys2.server.service.GenesysService.update*(..))")
+	public void accessionUpdates() {
+	}
+
 	/**
 	 * Handle updates to Accession and related objects
 	 * 
@@ -67,7 +72,7 @@ public class ElasticUpdaterAspect {
 	 * @return
 	 * @throws Throwable
 	 */
-	@AfterReturning(value = "execution(* org.genesys2.server.service.GenesysService.save*(..))", returning = "result")
+	@AfterReturning(value = "accessionUpdates()", returning = "result")
 	public Object afterSave(final JoinPoint joinPoint, final Object result) throws Throwable {
 		if (LOG.isTraceEnabled()) {
 			LOG.trace("Returning from " + joinPoint.toLongString());
@@ -82,26 +87,30 @@ public class ElasticUpdaterAspect {
 					return result;
 
 				for (Object o : list) {
-					if (LOG.isTraceEnabled()) {
-						LOG.trace("Indexing object " + o);
-					}
-					if (o == null)
-						continue;
-					if (o instanceof Accession) {
-						elasticUpdater.update(Accession.class, ((Accession) o).getId());
-					}
-					if (o instanceof AccessionRelated) {
-						AccessionId acceId = ((AccessionRelated) o).getAccession();
-						if (acceId != null)
-							elasticUpdater.update(Accession.class, acceId.getId());
-					}
+					maybeUpdate(o);
 				}
 
+			} else {
+				maybeUpdate(result);
 			}
 		}
 		return result;
 	}
 
+	private void maybeUpdate(Object o) {
+		if (o == null)
+			return;
+		if (LOG.isTraceEnabled()) {
+			LOG.trace("Maybe indexing object " + o.getClass() + " = " + o);
+		}
+		if (o instanceof Accession) {
+			elasticUpdater.update(Accession.class, ((Accession) o).getId());
+		}
+		if (o instanceof AccessionRelated) {
+			elasticUpdater.update(Accession.class, ((AccessionRelated) o).getAccession().getId());
+		}
+	}
+
 	/**
 	 * Handle removal of accession related objects
 	 * 

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java

@@ -9,10 +9,8 @@ import javax.annotation.Resource;
 import org.apache.commons.lang.time.StopWatch;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.service.ElasticService;
 import org.genesys2.server.service.FullTextSearchService;
-import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -37,9 +35,6 @@ class ElasticUpdaterProcessor {
 	@Autowired
 	private FullTextSearchService fullTextSearchService;
 
-	@Autowired
-	private GenesysService genesysService;
-
 	@Autowired
 	private ThreadPoolTaskExecutor executor;
 
@@ -133,9 +128,6 @@ class ElasticUpdaterProcessor {
 
 				elasticService.updateAll(className, copy);
 				fullTextSearchService.updateAll(className, copy);
-				if (Accession.class.getName().equals(className)) {
-					genesysService.updatePDCI(copy);
-				}
 
 				if (LOG.isTraceEnabled()) {
 					LOG.trace("Reindexing done.");

src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java

@@ -374,17 +374,38 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
-	@RequestMapping(value = "/pdci", method = RequestMethod.POST)
+	@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = { "action=missing-pdci" })
 	public String generatePDCI() {
 		final Set<Long> batch = Collections.synchronizedSet(new HashSet<>());
 
-		Collection<AccessionId> idsMissingPdci = accessionIdRepository.findMissingPDCI();
+		Collection<Long> idsMissingPdci = accessionIdRepository.findMissingPDCI();
 		LOG.info("Emm, " + idsMissingPdci.size() + " are missing PDCI");
 
 		idsMissingPdci.parallelStream().forEach(accessionId -> {
-			batch.add(accessionId.getId());
+			batch.add(accessionId);
+			synchronized (batch) {
+				if (batch.size() >= 1000) {
+					taskExecutor.execute(new PDCIUpdater(new HashSet<Long>(batch)));
+					batch.clear();
+				}
+			}
+		});
+
+		taskExecutor.execute(new PDCIUpdater(batch));
+		return "redirect:/admin/";
+	}
+
+	@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = { "action=full-recalc" })
+	public String recreatePDCI() {
+		final Set<Long> batch = Collections.synchronizedSet(new HashSet<>());
+
+		Collection<Long> idsMissingPdci = accessionIdRepository.findAllIds();
+		LOG.info("Emm, " + idsMissingPdci.size() + " will have their PDCI recalculated");
+
+		idsMissingPdci.parallelStream().forEach(accessionId -> {
+			batch.add(accessionId);
 			synchronized (batch) {
-				if (batch.size() > 1000) {
+				if (batch.size() >= 1000) {
 					taskExecutor.execute(new PDCIUpdater(new HashSet<Long>(batch)));
 					batch.clear();
 				}
@@ -432,27 +453,27 @@ public class AdminController {
 		AtomicLong counter = new AtomicLong(0);
 		StopWatch stopWatch = new StopWatch();
 		stopWatch.start();
-		final List<Long> batch = Collections.synchronizedList(new ArrayList<>());
+		final Set<Long> batch = Collections.synchronizedSet(new HashSet<>(100));
 		List<Long> list = accessionRepository.listAccessionIdsWithCropname();
 		LOG.info("The list has " + list.size() + " elements");
 		list.stream().parallel().forEach(accessionId -> {
 			batch.add(accessionId);
-			ArrayList<Long> copy = null;
+			Set<Long> copy = null;
 			synchronized (batch) {
 				if (batch.size() > 100) {
-					copy = new ArrayList<>(batch);
+					copy = new HashSet<>(batch);
 					batch.clear();
 				}
 			}
 			if (copy != null)
-				genesysService.saveAccessionCrops(copy);
+				genesysService.updateAccessionCrops(copy);
 			if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) {
 				LOG.info("Updated " + counter.get() + " records overall_rate=" + Math.round(1000.0 * counter.get() / (stopWatch.getTime())) + " records/s");
 			}
 		});
 
 		// handle remaining
-		genesysService.saveAccessionCrops(batch);
+		genesysService.updateAccessionCrops(batch);
 
 		LOG.info("Done assigning crops to accessions with CROPNAME.");
 		return "redirect:/admin/";

src/main/resources/content/language.properties

@@ -463,6 +463,7 @@ filter.internal.message.more=More than {0}
 filter.internal.message.less=Less than {0}
 filter.lists=Listed on accession list
 filter.more-filters=More filters
+filter.pdci=PDCI Score
 
 columns.add=Change displayed columns
 columns.apply=Apply

src/main/webapp/WEB-INF/jsp/accession/explore2.jsp

@@ -129,6 +129,10 @@
                     <filters:panel id="storage" title="filter.storage">
                         <filters:filter availableFilters="${availableFilters}" filterMap="${filters}" filterKey="storage" appliedFilters="${appliedFilters}" type="option" />
                     </filters:panel>
+
+                    <filters:panel id="pdci" title="filter.pdci">
+                        <filters:filter availableFilters="${availableFilters}" filterMap="${filters}" filterKey="pdci" appliedFilters="${appliedFilters}" type="range" />
+                    </filters:panel>
 				</filters:group>
 				<!--List-->
             <div class="col-lg-10 col-md-9 col-sm-9 col-xs-12 main-col-header">

src/main/webapp/WEB-INF/jsp/admin/index.jsp

@@ -23,7 +23,8 @@
   </form>
 
   <form method="post" action="<c:url value="/admin/pdci" />">
-    <input type="submit" class="btn btn-default" value="Calculate missing PDCI" />
+    <button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
+    <button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button>
     <!-- CSRF protection -->
     <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
   </form>