CropTaxonomy changes trigger reindexing of affected accessions

src/main/java/org/genesys2/server/model/impl/CropRule.java

@@ -30,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 
 /**
  * Crop rules establish which taxonomies are part of a {@link Crop}.
- *
+ * 
  * @author mobreza
  */
 @Entity
@@ -94,9 +94,20 @@ public class CropRule extends BusinessModel {
 	public void setCrop(Crop crop) {
 		this.crop = crop;
 	}
-	
+
 	@Override
 	public String toString() {
 		return "CropRule included=" + included + " genus=" + genus + " species=" + species + " crop=" + crop.getShortName();
 	}
+
+	public boolean matches(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) {
+		if (this.genus != null && !this.genus.equalsIgnoreCase(genus))
+			return false;
+		if (this.species != null && !this.species.equalsIgnoreCase(species))
+			return false;
+		if (this.subtaxa != null && !this.subtaxa.equalsIgnoreCase(subtaxa))
+			return false;
+
+		return true;
+	}
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java

@@ -128,4 +128,7 @@ public interface AccessionRepository extends JpaRepository<Accession, Long> {
 	@Query("select distinct a.id from Accession a where a.institute = :institute and a.id in ( :ids )")
 	Set<Long> findByInstituteAndIds(@Param("institute") FaoInstitute institute, @Param("ids") Set<Long> accessionIds);
 
+	@Query("select a.id from Accession a where a.taxonomy = ?1")
+	public List<Long> listAccessionsIds(Taxonomy2 taxonomy);
+
 }

src/main/java/org/genesys2/server/persistence/domain/CropTaxonomyRepository.java

@@ -24,7 +24,6 @@ import org.genesys2.server.model.impl.CropTaxonomy;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 
 public interface CropTaxonomyRepository extends JpaRepository<CropTaxonomy, Long> {
@@ -43,9 +42,8 @@ public interface CropTaxonomyRepository extends JpaRepository<CropTaxonomy, Long
 	List<Taxonomy2> findTaxonomiesByCrop(Crop crop);
 
 	@Query("select distinct ct.crop from CropTaxonomy ct where ct.taxonomy= ?1")
-	List<Crop> findByTaxonomy(Taxonomy2 taxonomy);
+	List<Crop> findCropsByTaxonomy(Taxonomy2 taxonomy);
 
-	@Modifying
-	@Query("delete from CropTaxonomy ct where ct.crop = ?1")
-	void clearCrop(Crop crop);
+	@Query("select ct from CropTaxonomy ct where ct.taxonomy= ?1")
+	List<CropTaxonomy> findByTaxonomy(Taxonomy2 taxonomy);
 }

src/main/java/org/genesys2/server/service/CropService.java

@@ -38,6 +38,8 @@ public interface CropService {
 
 	void rebuildTaxonomies();
 
+	void rebuildTaxonomies(Crop crop);
+
 	Page<CropTaxonomy> getCropTaxonomies(Crop crop, Pageable pageable);
 
 	/**
@@ -78,8 +80,15 @@ public interface CropService {
 	 * Remove crop
 	 * 
 	 * @param crop
-	 * @return 
+	 * @return
 	 */
 	Crop delete(Crop crop);
 
+	/**
+	 * Insert Taxonomy2 into correct CropTaxonomy lists
+	 * 
+	 * @param taxonomy
+	 */
+	void updateCropTaxonomyLists(Taxonomy2 taxonomy);
+
 }

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -36,6 +36,7 @@ import org.genesys2.server.model.genesys.ExperimentTrait;
 import org.genesys2.server.model.genesys.Metadata;
 import org.genesys2.server.model.genesys.Method;
 import org.genesys2.server.model.genesys.SvalbardData;
+import org.genesys2.server.model.genesys.Taxonomy2;
 import org.genesys2.server.model.impl.AccessionIdentifier3;
 import org.genesys2.server.model.impl.Country;
 import org.genesys2.server.model.impl.Crop;
@@ -202,4 +203,6 @@ public interface GenesysService {
 
 	Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions);
 
+	List<Long> listAccessionsIds(Taxonomy2 taxonomy);
+
 }

src/main/java/org/genesys2/server/service/impl/CropServiceImpl.java

@@ -19,8 +19,10 @@ package org.genesys2.server.service.impl;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.Predicate;
@@ -75,8 +77,6 @@ public class CropServiceImpl implements CropService {
 	@Override
 	@Transactional
 	public Crop delete(Crop crop) {
-		cropTaxonomyRepository.clearCrop(crop);
-
 		cropRepository.delete(crop);
 		crop.setId(null);
 		return crop;
@@ -101,7 +101,7 @@ public class CropServiceImpl implements CropService {
 
 	@Override
 	public List<Crop> getCrops(Taxonomy2 taxonomy2) {
-		return cropTaxonomyRepository.findByTaxonomy(taxonomy2);
+		return cropTaxonomyRepository.findCropsByTaxonomy(taxonomy2);
 	}
 
 	@Override
@@ -112,38 +112,30 @@ public class CropServiceImpl implements CropService {
 
 		// for each crop
 		for (final Crop crop : cropRepository.findAll()) {
-			LOG.info("Rebuilding crop taxonomy for: " + crop.getName());
-
-			// for all rules
-			final List<CropRule> cropRules = crop.getCropRules();
-			LOG.warn("Using rules: " + cropRules);
-			rebuildCropTaxonomies(crop, cropRules);
+			rebuildTaxonomies(crop);
 		}
 
 	}
 
-	private void rebuildCropTaxonomies(Crop crop, List<CropRule> cropRules) {
-		final List<CropRule> rulesSorted = new ArrayList<CropRule>(cropRules);
-
-		final List<CropRule> includedTaxa = new ArrayList<CropRule>();
-		final List<CropRule> excludedTaxa = new ArrayList<CropRule>();
-
-		for (final CropRule cr : rulesSorted) {
-			if (cr.isIncluded()) {
-				LOG.debug("Included: " + cr);
-				includedTaxa.add(cr);
-			} else {
-				LOG.debug("Excluded: " + cr);
-				excludedTaxa.add(cr);
-			}
-		}
+	@Override
+	@Transactional(readOnly = false)
+	@PreAuthorize("hasRole('ADMINISTRATOR')")
+	public void rebuildTaxonomies(Crop crop) {
+		LOG.info("Rebuilding crop taxonomy for: " + crop.getName());
+		// for all rules
+		final List<CropRule> cropRules = crop.getCropRules();
+		LOG.warn("Using rules: " + cropRules);
+		rebuildCropTaxonomies(crop, cropRules);
+	}
 
-		LOG.info("Included: " + includedTaxa.size());
-		LOG.info("Excluded: " + excludedTaxa.size());
+	private void rebuildCropTaxonomies(Crop crop, List<CropRule> cropRules) {
 
 		final List<Taxonomy2> taxa = new ArrayList<Taxonomy2>();
 
-		for (final CropRule cr : includedTaxa) {
+		for (final CropRule cr : cropRules) {
+			if (!cr.isIncluded())
+				continue;
+
 			if (cr.getSpecies() == null && cr.getSubtaxa() == null) {
 				taxa.addAll(taxonomy2Repository.findByGenus(cr.getGenus()));
 			} else if (cr.getSubtaxa() == null) {
@@ -153,7 +145,10 @@ public class CropServiceImpl implements CropService {
 			}
 		}
 
-		for (final CropRule cr : excludedTaxa) {
+		for (final CropRule cr : cropRules) {
+			if (cr.isIncluded())
+				continue;
+
 			CollectionUtils.filterInverse(taxa, new Predicate<Taxonomy2>() {
 				@Override
 				public boolean evaluate(Taxonomy2 taxonomy) {
@@ -169,51 +164,54 @@ public class CropServiceImpl implements CropService {
 			});
 		}
 
-		LOG.warn("Clearing crop taxonomy for " + crop.getName());
-
 		// To check
 		final List<CropTaxonomy> existing = cropTaxonomyRepository.findByCrop(crop);
+	
+		{
+			final List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
 
-		final List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
-
-		// Remove existing if not in list
-		for (final CropTaxonomy ct : existing) {
-			final long taxonomyId = ct.getTaxonomy().getId();
-			boolean found = false;
+			// Remove existing if not in list
+			for (final CropTaxonomy ct : existing) {
+				final long taxonomyId = ct.getTaxonomy().getId();
+				boolean found = false;
 
-			for (final Taxonomy2 taxonomy : taxa) {
-				if (taxonomyId == taxonomy.getId()) {
-					found = true;
-					break;
+				for (final Taxonomy2 taxonomy : taxa) {
+					if (taxonomyId == taxonomy.getId()) {
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					toRemove.add(ct);
 				}
 			}
-			if (!found) {
-				toRemove.add(ct);
-			}
+			cropTaxonomyRepository.delete(toRemove);
+			LOG.info("Removed crop taxonomy for " + crop.getName() + " size=" + toRemove.size());
 		}
-		cropTaxonomyRepository.delete(toRemove);
-
-		final List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
+		{
+			final List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
 
-		// To add
-		for (final Taxonomy2 taxonomy : taxa) {
-			boolean found = false;
+			// To add
+			for (final Taxonomy2 taxonomy : taxa) {
+				boolean found = false;
 
-			for (final CropTaxonomy ct : existing) {
-				if (ct.getTaxonomy().getId().equals(taxonomy.getId())) {
-					found = true;
-					break;
+				for (final CropTaxonomy ct : existing) {
+					if (ct.getTaxonomy().getId().equals(taxonomy.getId())) {
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					final CropTaxonomy ct = new CropTaxonomy();
+					ct.setCrop(crop);
+					ct.setTaxonomy(taxonomy);
+					toAdd.add(ct);
 				}
 			}
-			if (!found) {
-				final CropTaxonomy ct = new CropTaxonomy();
-				ct.setCrop(crop);
-				ct.setTaxonomy(taxonomy);
-				toAdd.add(ct);
-			}
-		}
 
-		cropTaxonomyRepository.save(toAdd);
+			cropTaxonomyRepository.save(toAdd);
+			LOG.info("Added crop taxonomy for " + crop.getName() + " size=" + toAdd.size());
+		}
 	}
 
 	@Override
@@ -308,4 +306,68 @@ public class CropServiceImpl implements CropService {
 		return crop == null ? null : cropRuleRepository.findByCrop(crop);
 	}
 
+	@Override
+	@Transactional(readOnly = false)
+	public void updateCropTaxonomyLists(Taxonomy2 taxonomy) {
+		// Load all rules
+		List<CropRule> cropRules = cropRuleRepository.findAll();
+
+		Set<Crop> crops = new HashSet<Crop>();
+
+		// Where does it fit
+		for (CropRule cr : cropRules) {
+			if (!cr.isIncluded())
+				continue;
+
+			if (cr.matches(taxonomy.getGenus(), taxonomy.getSpecies(), taxonomy.getSpAuthor(), taxonomy.getSubtaxa(), taxonomy.getSubtAuthor())) {
+				crops.add(cr.getCrop());
+			}
+		}
+
+		// Where doesn't it fit
+		for (CropRule cr : cropRules) {
+			if (cr.isIncluded())
+				continue;
+
+			// Skip crops we don't have
+			if (!crops.contains(cr.getCrop()))
+				continue;
+
+			if (cr.matches(taxonomy.getGenus(), taxonomy.getSpecies(), taxonomy.getSpAuthor(), taxonomy.getSubtaxa(), taxonomy.getSubtAuthor())) {
+				crops.remove(cr.getCrop());
+			}
+		}
+
+		// Ensure we have matching entries in CropTaxonomy
+		List<CropTaxonomy> existing = cropTaxonomyRepository.findByTaxonomy(taxonomy);
+
+		{
+			List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
+			for (CropTaxonomy ct : existing) {
+				if (!crops.contains(ct.getCrop())) {
+					toRemove.add(ct);
+				}
+			}
+			cropTaxonomyRepository.delete(toRemove);
+		}
+
+		{
+			List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
+			for (Crop crop : crops) {
+				boolean found = false;
+				for (CropTaxonomy ct : existing) {
+					if (crop.equals(ct.getCrop())) {
+						found = true;
+					}
+				}
+				if (!found) {
+					CropTaxonomy ct = new CropTaxonomy();
+					ct.setCrop(crop);
+					ct.setTaxonomy(taxonomy);
+					toAdd.add(ct);
+				}
+			}
+			cropTaxonomyRepository.save(toAdd);
+		}
+	}
 }

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -1573,4 +1573,9 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 		return paramMethods;
 	}
 
+	@Override
+	public List<Long> listAccessionsIds(Taxonomy2 taxonomy) {
+		return accessionRepository.listAccessionsIds(taxonomy);
+	}
+
 }

src/main/java/org/genesys2/server/service/impl/TaxonomyServiceImpl.java

@@ -17,6 +17,9 @@
 package org.genesys2.server.service.impl;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.Resource;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.text.WordUtils;
@@ -25,6 +28,7 @@ import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.model.genesys.Taxonomy2;
 import org.genesys2.server.model.impl.Crop;
 import org.genesys2.server.persistence.domain.Taxonomy2Repository;
+import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.TaxonomyService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.PageRequest;
@@ -32,6 +36,8 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 
+import com.hazelcast.core.ILock;
+
 @Service
 @Transactional(readOnly = true)
 public class TaxonomyServiceImpl implements TaxonomyService {
@@ -39,39 +45,49 @@ public class TaxonomyServiceImpl implements TaxonomyService {
 
 	@Autowired
 	private Taxonomy2Repository taxonomy2Repository;
+	
+	@Autowired
+	private CropService cropService;
+
+	/**
+	 * This lock ensures only one member of the cluster can enter taxonomy
+	 * update logic
+	 */
+	@Resource
+	private ILock taxonomyUpdateLock;
 
 	@Override
 	public Taxonomy2 get(Long id) {
 		return taxonomy2Repository.findOne(id);
 	}
 
-    @Override
-    public List<String> autocompleteGenus(String term, Crop crop) {
-        List<String> strings;
-
-        if (crop != null) {
-            strings = taxonomy2Repository.autocompleteGenusByCrop(term + "%", crop, new PageRequest(0, 10));
-        } else {
-            strings = taxonomy2Repository.autocompleteGenus(term + "%", new PageRequest(0, 10));
-        }
+	@Override
+	public List<String> autocompleteGenus(String term, Crop crop) {
+		List<String> strings;
 
-        return strings;
-    }
+		if (crop != null) {
+			strings = taxonomy2Repository.autocompleteGenusByCrop(term + "%", crop, new PageRequest(0, 10));
+		} else {
+			strings = taxonomy2Repository.autocompleteGenus(term + "%", new PageRequest(0, 10));
+		}
 
-    @Override
-    public List<String> autocompleteSpecies(String term, Crop crop, List<String> genus) {
-        List<String> strings;
+		return strings;
+	}
 
-        if (!genus.isEmpty()) {
-            strings = taxonomy2Repository.autocompleteSpeciesByGenus(term + "%", genus, new PageRequest(0, 10));
-        } else if (crop != null) {
-            strings = taxonomy2Repository.autocompleteSpeciesByCrop(term + "%", crop, new PageRequest(0, 10));
-        } else {
-            strings = taxonomy2Repository.autocompleteSpecies(term + "%", new PageRequest(0, 10));
-        }
+	@Override
+	public List<String> autocompleteSpecies(String term, Crop crop, List<String> genus) {
+		List<String> strings;
+
+		if (!genus.isEmpty()) {
+			strings = taxonomy2Repository.autocompleteSpeciesByGenus(term + "%", genus, new PageRequest(0, 10));
+		} else if (crop != null) {
+			strings = taxonomy2Repository.autocompleteSpeciesByCrop(term + "%", crop, new PageRequest(0, 10));
+		} else {
+			strings = taxonomy2Repository.autocompleteSpecies(term + "%", new PageRequest(0, 10));
+		}
 
-        return strings;
-    }
+		return strings;
+	}
 
 	@Override
 	public List<String> autocompleteTaxonomy(String term) {
@@ -107,70 +123,83 @@ public class TaxonomyServiceImpl implements TaxonomyService {
 
 		final Taxonomy2 existing = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
 		if (existing == null) {
-			return internalEnsure(genus, species, spAuthor, subtaxa, subtAuthor);
+			try {
+				Taxonomy2 taxonomy = internalEnsure(genus, species, spAuthor, subtaxa, subtAuthor);
+				return taxonomy;
+			} catch (InterruptedException e) {
+				LOG.warn("Thread interrupted while waiting for taxonomy lock.", e);
+				throw new RuntimeException(e);
+			}
 		}
 		return existing;
 	}
 
-	private synchronized Taxonomy2 internalEnsure(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) {
+	private synchronized Taxonomy2 internalEnsure(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) throws InterruptedException {
 		Long taxSpeciesId = null, taxGenusId = null;
 
-		if (!StringUtils.equals(species, "sp.") && !(subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals(""))) {
-			final Taxonomy2 speciesTaxa = internalEnsure(genus, species, "", "", "");
-			taxSpeciesId = speciesTaxa.getId();
-			taxGenusId = speciesTaxa.getTaxGenus();
-		} else if (!StringUtils.equals(species, "sp.") && subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals("")) {
-			final Taxonomy2 genusTaxa = internalEnsure(genus, "sp.", "", "", "");
-			taxGenusId = genusTaxa.getId();
-			taxSpeciesId = genusTaxa.getId();
-		}
-
-		// Loop it a bit if required
-		for (int i = 5; i > 0; i--) {
-			Taxonomy2 taxonomy = null;
+		// LOCK
+		if (taxonomyUpdateLock.tryLock(10, TimeUnit.SECONDS)) {
 			try {
-				taxonomy = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
-			} catch (final Throwable e) {
-				// is null
-				e.printStackTrace();
-			}
-			if (taxonomy == null) {
-				LOG.info("Adding new taxonomic name: " + genus + " " + species + " " + spAuthor + " " + subtaxa + " " + subtAuthor);
-				taxonomy = new Taxonomy2();
-				taxonomy.setGenus(genus);
-				taxonomy.setSpecies(species);
-				taxonomy.setSpAuthor(spAuthor);
-				taxonomy.setSubtaxa(subtaxa);
-				taxonomy.setSubtAuthor(subtAuthor);
-				taxonomy.setTaxGenus(taxGenusId);
-				taxonomy.setTaxSpecies(taxSpeciesId);
 
-				try {
-					taxonomy = taxonomy2Repository.save(taxonomy);
+				if (!StringUtils.equals(species, "sp.") && !(subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals(""))) {
+					final Taxonomy2 speciesTaxa = internalEnsure(genus, species, "", "", "");
+					taxSpeciesId = speciesTaxa.getId();
+					taxGenusId = speciesTaxa.getTaxGenus();
+				} else if (!StringUtils.equals(species, "sp.") && subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals("")) {
+					final Taxonomy2 genusTaxa = internalEnsure(genus, "sp.", "", "", "");
+					taxGenusId = genusTaxa.getId();
+					taxSpeciesId = genusTaxa.getId();
+				}
 
-					if (taxGenusId == null) {
-						taxonomy.setTaxGenus(taxonomy.getId());
-						taxonomy = taxonomy2Repository.save(taxonomy);
-					}
-					if (taxSpeciesId == null) {
-						taxonomy.setTaxSpecies(taxonomy.getId());
-						taxonomy = taxonomy2Repository.save(taxonomy);
-					}
+				Taxonomy2 taxonomy = null;
+				try {
+					taxonomy = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
+				} catch (final Throwable e) {
+					LOG.info("Taxonomy not found: " + e.getMessage());
+				}
 
+				if (taxonomy != null) {
 					return taxonomy;
+				} else {
+					LOG.info("Adding new taxonomic name: " + genus + " " + species + " " + spAuthor + " " + subtaxa + " " + subtAuthor);
+					taxonomy = new Taxonomy2();
+					taxonomy.setGenus(genus);
+					taxonomy.setSpecies(species);
+					taxonomy.setSpAuthor(spAuthor);
+					taxonomy.setSubtaxa(subtaxa);
+					taxonomy.setSubtAuthor(subtAuthor);
+					taxonomy.setTaxGenus(taxGenusId);
+					taxonomy.setTaxSpecies(taxSpeciesId);
+
+					try {
+						taxonomy = taxonomy2Repository.save(taxonomy);
 
-					// TODO Should update crop taxonomy lists?
-
-				} catch (final Throwable e) {
-					LOG.warn("Error " + e.getMessage() + " :" + taxonomy);
+						if (taxGenusId == null) {
+							taxonomy.setTaxGenus(taxonomy.getId());
+							taxonomy = taxonomy2Repository.save(taxonomy);
+						}
+						if (taxSpeciesId == null) {
+							taxonomy.setTaxSpecies(taxonomy.getId());
+							taxonomy = taxonomy2Repository.save(taxonomy);
+						}
+
+						// Update crop taxonomy lists
+						cropService.updateCropTaxonomyLists(taxonomy);
+						
+						return taxonomy;
+
+					} catch (final Throwable e) {
+						LOG.warn("Error " + e.getMessage() + " :" + taxonomy);
+						throw new RuntimeException(e.getMessage());
+					}
 				}
-			} else {
-				return taxonomy;