Skip to content
GitLab
Commit 40febb8c authored by Matija Obreza's avatar Matija Obreza
Browse files

CropTaxonomy changes trigger reindexing of affected accessions

parent e7b55eb3
Hide whitespace changes
Inline Side-by-side
src/main/java/org/genesys2/server/model/impl/CropRule.java
View file @ 40febb8c
......@@ -30,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* Crop rules establish which taxonomies are part of a {@link Crop}.
*
*
* @author mobreza
*/
@Entity
......@@ -94,9 +94,20 @@ public class CropRule extends BusinessModel {
public void setCrop(Crop crop) {
this.crop = crop;
}
@Override
public String toString() {
return "CropRule included=" + included + " genus=" + genus + " species=" + species + " crop=" + crop.getShortName();
}
public boolean matches(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) {
if (this.genus != null && !this.genus.equalsIgnoreCase(genus))
return false;
if (this.species != null && !this.species.equalsIgnoreCase(species))
return false;
if (this.subtaxa != null && !this.subtaxa.equalsIgnoreCase(subtaxa))
return false;
return true;
}
}
src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java
View file @ 40febb8c
......@@ -128,4 +128,7 @@ public interface AccessionRepository extends JpaRepository<Accession, Long> {
@Query("select distinct a.id from Accession a where a.institute = :institute and a.id in ( :ids )")
Set<Long> findByInstituteAndIds(@Param("institute") FaoInstitute institute, @Param("ids") Set<Long> accessionIds);
@Query("select a.id from Accession a where a.taxonomy = ?1")
public List<Long> listAccessionsIds(Taxonomy2 taxonomy);
}
src/main/java/org/genesys2/server/persistence/domain/CropTaxonomyRepository.java
View file @ 40febb8c
......@@ -24,7 +24,6 @@ import org.genesys2.server.model.impl.CropTaxonomy;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface CropTaxonomyRepository extends JpaRepository<CropTaxonomy, Long> {
......@@ -43,9 +42,8 @@ public interface CropTaxonomyRepository extends JpaRepository<CropTaxonomy, Long
List<Taxonomy2> findTaxonomiesByCrop(Crop crop);
@Query("select distinct ct.crop from CropTaxonomy ct where ct.taxonomy= ?1")
List<Crop> findByTaxonomy(Taxonomy2 taxonomy);
List<Crop> findCropsByTaxonomy(Taxonomy2 taxonomy);
@Modifying
@Query("delete from CropTaxonomy ct where ct.crop = ?1")
void clearCrop(Crop crop);
@Query("select ct from CropTaxonomy ct where ct.taxonomy= ?1")
List<CropTaxonomy> findByTaxonomy(Taxonomy2 taxonomy);
}
src/main/java/org/genesys2/server/service/CropService.java
View file @ 40febb8c
......@@ -38,6 +38,8 @@ public interface CropService {
void rebuildTaxonomies();
void rebuildTaxonomies(Crop crop);
Page<CropTaxonomy> getCropTaxonomies(Crop crop, Pageable pageable);
/**
......@@ -78,8 +80,15 @@ public interface CropService {
* Remove crop
*
* @param crop
* @return
* @return
*/
Crop delete(Crop crop);
/**
* Insert Taxonomy2 into correct CropTaxonomy lists
*
* @param taxonomy
*/
void updateCropTaxonomyLists(Taxonomy2 taxonomy);
}
src/main/java/org/genesys2/server/service/GenesysService.java
View file @ 40febb8c
......@@ -36,6 +36,7 @@ import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.SvalbardData;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
......@@ -202,4 +203,6 @@ public interface GenesysService {
Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions);
List<Long> listAccessionsIds(Taxonomy2 taxonomy);
}
src/main/java/org/genesys2/server/service/impl/CropServiceImpl.java
View file @ 40febb8c
......@@ -19,8 +19,10 @@ package org.genesys2.server.service.impl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
......@@ -75,8 +77,6 @@ public class CropServiceImpl implements CropService {
@Override
@Transactional
public Crop delete(Crop crop) {
cropTaxonomyRepository.clearCrop(crop);
cropRepository.delete(crop);
crop.setId(null);
return crop;
......@@ -101,7 +101,7 @@ public class CropServiceImpl implements CropService {
@Override
public List<Crop> getCrops(Taxonomy2 taxonomy2) {
return cropTaxonomyRepository.findByTaxonomy(taxonomy2);
return cropTaxonomyRepository.findCropsByTaxonomy(taxonomy2);
}
@Override
......@@ -112,38 +112,30 @@ public class CropServiceImpl implements CropService {
// for each crop
for (final Crop crop : cropRepository.findAll()) {
LOG.info("Rebuilding crop taxonomy for: " + crop.getName());
// for all rules
final List<CropRule> cropRules = crop.getCropRules();
LOG.warn("Using rules: " + cropRules);
rebuildCropTaxonomies(crop, cropRules);
rebuildTaxonomies(crop);
}
}
private void rebuildCropTaxonomies(Crop crop, List<CropRule> cropRules) {
final List<CropRule> rulesSorted = new ArrayList<CropRule>(cropRules);
final List<CropRule> includedTaxa = new ArrayList<CropRule>();
final List<CropRule> excludedTaxa = new ArrayList<CropRule>();
for (final CropRule cr : rulesSorted) {
if (cr.isIncluded()) {
LOG.debug("Included: " + cr);
includedTaxa.add(cr);
} else {
LOG.debug("Excluded: " + cr);
excludedTaxa.add(cr);
}
}
@Override
@Transactional(readOnly = false)
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void rebuildTaxonomies(Crop crop) {
LOG.info("Rebuilding crop taxonomy for: " + crop.getName());
// for all rules
final List<CropRule> cropRules = crop.getCropRules();
LOG.warn("Using rules: " + cropRules);
rebuildCropTaxonomies(crop, cropRules);
}
LOG.info("Included: " + includedTaxa.size());
LOG.info("Excluded: " + excludedTaxa.size());
private void rebuildCropTaxonomies(Crop crop, List<CropRule> cropRules) {
final List<Taxonomy2> taxa = new ArrayList<Taxonomy2>();
for (final CropRule cr : includedTaxa) {
for (final CropRule cr : cropRules) {
if (!cr.isIncluded())
continue;
if (cr.getSpecies() == null && cr.getSubtaxa() == null) {
taxa.addAll(taxonomy2Repository.findByGenus(cr.getGenus()));
} else if (cr.getSubtaxa() == null) {
......@@ -153,7 +145,10 @@ public class CropServiceImpl implements CropService {
}
}
for (final CropRule cr : excludedTaxa) {
for (final CropRule cr : cropRules) {
if (cr.isIncluded())
continue;
CollectionUtils.filterInverse(taxa, new Predicate<Taxonomy2>() {
@Override
public boolean evaluate(Taxonomy2 taxonomy) {
......@@ -169,51 +164,54 @@ public class CropServiceImpl implements CropService {
});
}
LOG.warn("Clearing crop taxonomy for " + crop.getName());
// To check
final List<CropTaxonomy> existing = cropTaxonomyRepository.findByCrop(crop);
{
final List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
final List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
// Remove existing if not in list
for (final CropTaxonomy ct : existing) {
final long taxonomyId = ct.getTaxonomy().getId();
boolean found = false;
// Remove existing if not in list
for (final CropTaxonomy ct : existing) {
final long taxonomyId = ct.getTaxonomy().getId();
boolean found = false;
for (final Taxonomy2 taxonomy : taxa) {
if (taxonomyId == taxonomy.getId()) {
found = true;
break;
for (final Taxonomy2 taxonomy : taxa) {
if (taxonomyId == taxonomy.getId()) {
found = true;
break;
}
}
if (!found) {
toRemove.add(ct);
}
}
if (!found) {
toRemove.add(ct);
}
cropTaxonomyRepository.delete(toRemove);
LOG.info("Removed crop taxonomy for " + crop.getName() + " size=" + toRemove.size());
}
cropTaxonomyRepository.delete(toRemove);
final List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
{
final List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
// To add
for (final Taxonomy2 taxonomy : taxa) {
boolean found = false;
// To add
for (final Taxonomy2 taxonomy : taxa) {
boolean found = false;
for (final CropTaxonomy ct : existing) {
if (ct.getTaxonomy().getId().equals(taxonomy.getId())) {
found = true;
break;
for (final CropTaxonomy ct : existing) {
if (ct.getTaxonomy().getId().equals(taxonomy.getId())) {
found = true;
break;
}
}
if (!found) {
final CropTaxonomy ct = new CropTaxonomy();
ct.setCrop(crop);
ct.setTaxonomy(taxonomy);
toAdd.add(ct);
}
}
if (!found) {
final CropTaxonomy ct = new CropTaxonomy();
ct.setCrop(crop);
ct.setTaxonomy(taxonomy);
toAdd.add(ct);
}
}
cropTaxonomyRepository.save(toAdd);
cropTaxonomyRepository.save(toAdd);
LOG.info("Added crop taxonomy for " + crop.getName() + " size=" + toAdd.size());
}
}
@Override
......@@ -308,4 +306,68 @@ public class CropServiceImpl implements CropService {
return crop == null ? null : cropRuleRepository.findByCrop(crop);
}
@Override
@Transactional(readOnly = false)
public void updateCropTaxonomyLists(Taxonomy2 taxonomy) {
// Load all rules
List<CropRule> cropRules = cropRuleRepository.findAll();
Set<Crop> crops = new HashSet<Crop>();
// Where does it fit
for (CropRule cr : cropRules) {
if (!cr.isIncluded())
continue;
if (cr.matches(taxonomy.getGenus(), taxonomy.getSpecies(), taxonomy.getSpAuthor(), taxonomy.getSubtaxa(), taxonomy.getSubtAuthor())) {
crops.add(cr.getCrop());
}
}
// Where doesn't it fit
for (CropRule cr : cropRules) {
if (cr.isIncluded())
continue;
// Skip crops we don't have
if (!crops.contains(cr.getCrop()))
continue;
if (cr.matches(taxonomy.getGenus(), taxonomy.getSpecies(), taxonomy.getSpAuthor(), taxonomy.getSubtaxa(), taxonomy.getSubtAuthor())) {
crops.remove(cr.getCrop());
}
}
// Ensure we have matching entries in CropTaxonomy
List<CropTaxonomy> existing = cropTaxonomyRepository.findByTaxonomy(taxonomy);
{
List<CropTaxonomy> toRemove = new ArrayList<CropTaxonomy>();
for (CropTaxonomy ct : existing) {
if (!crops.contains(ct.getCrop())) {
toRemove.add(ct);
}
}
cropTaxonomyRepository.delete(toRemove);
}
{
List<CropTaxonomy> toAdd = new ArrayList<CropTaxonomy>();
for (Crop crop : crops) {
boolean found = false;
for (CropTaxonomy ct : existing) {
if (crop.equals(ct.getCrop())) {
found = true;
}
}
if (!found) {
CropTaxonomy ct = new CropTaxonomy();
ct.setCrop(crop);
ct.setTaxonomy(taxonomy);
toAdd.add(ct);
}
}
cropTaxonomyRepository.save(toAdd);
}
}
}
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @ 40febb8c
......@@ -1573,4 +1573,9 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
return paramMethods;
}
@Override
public List<Long> listAccessionsIds(Taxonomy2 taxonomy) {
return accessionRepository.listAccessionsIds(taxonomy);
}
}
src/main/java/org/genesys2/server/service/impl/TaxonomyServiceImpl.java
View file @ 40febb8c
......@@ -17,6 +17,9 @@
package org.genesys2.server.service.impl;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.annotation.Resource;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.text.WordUtils;
......@@ -25,6 +28,7 @@ import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.persistence.domain.Taxonomy2Repository;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.TaxonomyService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
......@@ -32,6 +36,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import com.hazelcast.core.ILock;
@Service
@Transactional(readOnly = true)
public class TaxonomyServiceImpl implements TaxonomyService {
......@@ -39,39 +45,49 @@ public class TaxonomyServiceImpl implements TaxonomyService {
@Autowired
private Taxonomy2Repository taxonomy2Repository;
@Autowired
private CropService cropService;
/**
* This lock ensures only one member of the cluster can enter taxonomy
* update logic
*/
@Resource
private ILock taxonomyUpdateLock;
@Override
public Taxonomy2 get(Long id) {
return taxonomy2Repository.findOne(id);
}
@Override
public List<String> autocompleteGenus(String term, Crop crop) {
List<String> strings;
if (crop != null) {
strings = taxonomy2Repository.autocompleteGenusByCrop(term + "%", crop, new PageRequest(0, 10));
} else {
strings = taxonomy2Repository.autocompleteGenus(term + "%", new PageRequest(0, 10));
}
@Override
public List<String> autocompleteGenus(String term, Crop crop) {
List<String> strings;
return strings;
}
if (crop != null) {
strings = taxonomy2Repository.autocompleteGenusByCrop(term + "%", crop, new PageRequest(0, 10));
} else {
strings = taxonomy2Repository.autocompleteGenus(term + "%", new PageRequest(0, 10));
}
@Override
public List<String> autocompleteSpecies(String term, Crop crop, List<String> genus) {
List<String> strings;
return strings;
}
if (!genus.isEmpty()) {
strings = taxonomy2Repository.autocompleteSpeciesByGenus(term + "%", genus, new PageRequest(0, 10));
} else if (crop != null) {
strings = taxonomy2Repository.autocompleteSpeciesByCrop(term + "%", crop, new PageRequest(0, 10));
} else {
strings = taxonomy2Repository.autocompleteSpecies(term + "%", new PageRequest(0, 10));
}
@Override
public List<String> autocompleteSpecies(String term, Crop crop, List<String> genus) {
List<String> strings;
if (!genus.isEmpty()) {
strings = taxonomy2Repository.autocompleteSpeciesByGenus(term + "%", genus, new PageRequest(0, 10));
} else if (crop != null) {
strings = taxonomy2Repository.autocompleteSpeciesByCrop(term + "%", crop, new PageRequest(0, 10));
} else {
strings = taxonomy2Repository.autocompleteSpecies(term + "%", new PageRequest(0, 10));
}
return strings;
}
return strings;
}
@Override
public List<String> autocompleteTaxonomy(String term) {
......@@ -107,70 +123,83 @@ public class TaxonomyServiceImpl implements TaxonomyService {
final Taxonomy2 existing = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
if (existing == null) {
return internalEnsure(genus, species, spAuthor, subtaxa, subtAuthor);
try {
Taxonomy2 taxonomy = internalEnsure(genus, species, spAuthor, subtaxa, subtAuthor);
return taxonomy;
} catch (InterruptedException e) {
LOG.warn("Thread interrupted while waiting for taxonomy lock.", e);
throw new RuntimeException(e);
}
}
return existing;
}
private synchronized Taxonomy2 internalEnsure(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) {
private synchronized Taxonomy2 internalEnsure(String genus, String species, String spAuthor, String subtaxa, String subtAuthor) throws InterruptedException {
Long taxSpeciesId = null, taxGenusId = null;
if (!StringUtils.equals(species, "sp.") && !(subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals(""))) {
final Taxonomy2 speciesTaxa = internalEnsure(genus, species, "", "", "");
taxSpeciesId = speciesTaxa.getId();
taxGenusId = speciesTaxa.getTaxGenus();
} else if (!StringUtils.equals(species, "sp.") && subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals("")) {
final Taxonomy2 genusTaxa = internalEnsure(genus, "sp.", "", "", "");
taxGenusId = genusTaxa.getId();
taxSpeciesId = genusTaxa.getId();
}
// Loop it a bit if required
for (int i = 5; i > 0; i--) {
Taxonomy2 taxonomy = null;
// LOCK
if (taxonomyUpdateLock.tryLock(10, TimeUnit.SECONDS)) {
try {
taxonomy = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
} catch (final Throwable e) {
// is null
e.printStackTrace();
}
if (taxonomy == null) {
LOG.info("Adding new taxonomic name: " + genus + " " + species + " " + spAuthor + " " + subtaxa + " " + subtAuthor);
taxonomy = new Taxonomy2();
taxonomy.setGenus(genus);
taxonomy.setSpecies(species);
taxonomy.setSpAuthor(spAuthor);
taxonomy.setSubtaxa(subtaxa);
taxonomy.setSubtAuthor(subtAuthor);
taxonomy.setTaxGenus(taxGenusId);
taxonomy.setTaxSpecies(taxSpeciesId);
try {
taxonomy = taxonomy2Repository.save(taxonomy);
if (!StringUtils.equals(species, "sp.") && !(subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals(""))) {
final Taxonomy2 speciesTaxa = internalEnsure(genus, species, "", "", "");
taxSpeciesId = speciesTaxa.getId();
taxGenusId = speciesTaxa.getTaxGenus();
} else if (!StringUtils.equals(species, "sp.") && subtaxa.equals("") && spAuthor.equals("") && subtAuthor.equals("")) {
final Taxonomy2 genusTaxa = internalEnsure(genus, "sp.", "", "", "");
taxGenusId = genusTaxa.getId();
taxSpeciesId = genusTaxa.getId();
}
if (taxGenusId == null) {
taxonomy.setTaxGenus(taxonomy.getId());
taxonomy = taxonomy2Repository.save(taxonomy);
}
if (taxSpeciesId == null) {
taxonomy.setTaxSpecies(taxonomy.getId());
taxonomy = taxonomy2Repository.save(taxonomy);
}
Taxonomy2 taxonomy = null;
try {
taxonomy = taxonomy2Repository.findByGenusAndSpeciesAndSpAuthorAndSubtaxaAndSubtAuthor(genus, species, spAuthor, subtaxa, subtAuthor);
} catch (final Throwable e) {
LOG.info("Taxonomy not found: " + e.getMessage());
}
if (taxonomy != null) {
return taxonomy;
} else {
LOG.info("Adding new taxonomic name: " + genus + " " + species + " " + spAuthor + " " + subtaxa + " " + subtAuthor);
taxonomy = new Taxonomy2();
taxonomy.setGenus(genus);
taxonomy.setSpecies(species);
taxonomy.setSpAuthor(spAuthor);
taxonomy.setSubtaxa(subtaxa);
taxonomy.setSubtAuthor(subtAuthor);
taxonomy.setTaxGenus(taxGenusId);
taxonomy.setTaxSpecies(taxSpeciesId);
try {
taxonomy = taxonomy2Repository.save(taxonomy);
// TODO Should update crop taxonomy lists?
} catch (final Throwable e) {
LOG.warn("Error " + e.getMessage() + " :" + taxonomy);
if (taxGenusId == null) {
taxonomy.setTaxGenus(taxonomy.getId());
taxonomy = taxonomy2Repository.save(taxonomy);
}
if (taxSpeciesId == null) {
taxonomy.setTaxSpecies(taxonomy.getId());
taxonomy = taxonomy2Repository.save(taxonomy);
}
// Update crop taxonomy lists
cropService.updateCropTaxonomyLists(taxonomy);
return taxonomy;
} catch (final Throwable e) {
LOG.warn("Error " + e.getMessage() + " :" + taxonomy);
throw new RuntimeException(e.getMessage());
}
}