Commit d8c98197 authored by Matija Obreza's avatar Matija Obreza

Crops: relink accessions with Crop by #cropName

- unlinks crop name mismatches
- links by crop shortName and otherNames
parent d5e87016
...@@ -31,7 +31,6 @@ import org.genesys2.server.model.impl.Crop; ...@@ -31,7 +31,6 @@ import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.CropRule; import org.genesys2.server.model.impl.CropRule;
import org.genesys2.server.model.impl.CropTaxonomy; import org.genesys2.server.model.impl.CropTaxonomy;
import org.genesys2.server.service.CropService; import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.TraitService; import org.genesys2.server.service.TraitService;
import org.genesys2.spring.ResourceNotFoundException; import org.genesys2.spring.ResourceNotFoundException;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -39,6 +38,7 @@ import org.springframework.context.i18n.LocaleContextHolder; ...@@ -39,6 +38,7 @@ import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
...@@ -61,13 +61,13 @@ public class CropsController extends ApiBaseController { ...@@ -61,13 +61,13 @@ public class CropsController extends ApiBaseController {
public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/crops"; public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/crops";
@Autowired @Autowired
GenesysService genesysService; private TraitService traitService;
@Autowired @Autowired
TraitService traitService; private CropService cropService;
@Autowired @Autowired
CropService cropService; private ThreadPoolTaskExecutor threadPoolTaskExecutor;
/** /**
* List of Crop details * List of Crop details
...@@ -258,4 +258,24 @@ public class CropsController extends ApiBaseController { ...@@ -258,4 +258,24 @@ public class CropsController extends ApiBaseController {
cropService.rebuildTaxonomies(crop); cropService.rebuildTaxonomies(crop);
} }
/**
* Link accession#cropName with crop
*
* @return
* @throws AuthorizationException
*/
@PreAuthorize("hasRole('ADMINISTRATOR')")
@RequestMapping(value = "{shortName}/relink", params= { "accessions" }, method = RequestMethod.POST, produces = { MediaType.APPLICATION_JSON_VALUE })
public void relinkAccessions(@PathVariable("shortName") String shortName) {
LOG.info("Linking accessions to crop {}", shortName);
final Crop crop = cropService.getCrop(shortName);
if (crop == null)
throw new ResourceNotFoundException("No crop " + shortName);
threadPoolTaskExecutor.execute(() -> {
cropService.unlinkAccessionsForCrop(crop);
cropService.linkAccessionsWithCrop(crop);
});
}
} }
...@@ -35,29 +35,32 @@ import javax.persistence.JoinColumn; ...@@ -35,29 +35,32 @@ import javax.persistence.JoinColumn;
import javax.persistence.Lob; import javax.persistence.Lob;
import javax.persistence.OneToMany; import javax.persistence.OneToMany;
import javax.persistence.OrderBy; import javax.persistence.OrderBy;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Transient; import javax.persistence.Transient;
import javax.persistence.UniqueConstraint; import javax.persistence.UniqueConstraint;
import javax.validation.constraints.NotNull; import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size; import javax.validation.constraints.Size;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.JsonViews; import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.model.SelfCleaning;
import org.genesys.blocks.security.model.AclAwareModel; import org.genesys.blocks.security.model.AclAwareModel;
import org.genesys2.server.model.GlobalVersionedAuditedModel; import org.genesys2.server.model.GlobalVersionedAuditedModel;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@Cacheable @Cacheable
@Entity @Entity
@Table(name = "crop") @Table(name = "crop")
public class Crop extends GlobalVersionedAuditedModel implements AclAwareModel { public class Crop extends GlobalVersionedAuditedModel implements SelfCleaning, AclAwareModel {
private static final long serialVersionUID = -2686341831839109257L; private static final long serialVersionUID = -2686341831839109257L;
...@@ -117,6 +120,13 @@ public class Crop extends GlobalVersionedAuditedModel implements AclAwareModel { ...@@ -117,6 +120,13 @@ public class Crop extends GlobalVersionedAuditedModel implements AclAwareModel {
@JsonIgnore @JsonIgnore
private transient JsonNode i18nJU; private transient JsonNode i18nJU;
@PrePersist
@PreUpdate
private void prePersist() {
trimStringsToNull();
this.shortName = this.shortName.toLowerCase();
}
public String getName() { public String getName() {
return name; return name;
......
/** /*
* Copyright 2014 Global Crop Diversity Trust * Copyright 2018 Global Crop Diversity Trust
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
**/ */
package org.genesys2.server.service; package org.genesys2.server.service;
...@@ -128,6 +128,21 @@ public interface CropService { ...@@ -128,6 +128,21 @@ public interface CropService {
*/ */
Crop updateAliases(Crop crop, List<String> otherNames); Crop updateAliases(Crop crop, List<String> otherNames);
/**
* Unlink accessions from crop where accession#cropName doesn't match
*
* @param crop
*/
void unlinkAccessionsForCrop(Crop crop);
/**
* Link accessions to Crop by accession#cropName
*
* @param crop
*/
void linkAccessionsWithCrop(Crop crop);
public static class CropDetails implements Serializable { public static class CropDetails implements Serializable {
private static final long serialVersionUID = 1622015024393351713L; private static final long serialVersionUID = 1622015024393351713L;
...@@ -152,4 +167,6 @@ public interface CropService { ...@@ -152,4 +167,6 @@ public interface CropService {
return cropDetails; return cropDetails;
} }
} }
} }
...@@ -27,6 +27,8 @@ import java.util.Map; ...@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.validation.Valid;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate; import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
...@@ -35,12 +37,15 @@ import org.genesys.filerepository.InvalidRepositoryPathException; ...@@ -35,12 +37,15 @@ import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.model.ImageGallery; import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryImage; import org.genesys.filerepository.model.RepositoryImage;
import org.genesys.filerepository.service.ImageGalleryService; import org.genesys.filerepository.service.ImageGalleryService;
import org.genesys2.server.component.security.AsAdminInvoker;
import org.genesys2.server.model.genesys.Accession; import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.genesys.Taxonomy2; import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Article; import org.genesys2.server.model.impl.Article;
import org.genesys2.server.model.impl.Crop; import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.CropRule; import org.genesys2.server.model.impl.CropRule;
import org.genesys2.server.model.impl.CropTaxonomy; import org.genesys2.server.model.impl.CropTaxonomy;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.persistence.CropRepository; import org.genesys2.server.persistence.CropRepository;
import org.genesys2.server.persistence.CropRuleRepository; import org.genesys2.server.persistence.CropRuleRepository;
import org.genesys2.server.persistence.CropTaxonomyRepository; import org.genesys2.server.persistence.CropTaxonomyRepository;
...@@ -52,6 +57,7 @@ import org.genesys2.server.service.CropService; ...@@ -52,6 +57,7 @@ import org.genesys2.server.service.CropService;
import org.genesys2.server.service.ElasticsearchService; import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.HtmlSanitizer; import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.filter.AccessionFilter; import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -65,7 +71,8 @@ import org.springframework.transaction.annotation.Propagation; ...@@ -65,7 +71,8 @@ import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import javax.validation.Valid; import com.google.common.collect.Sets;
import com.querydsl.core.BooleanBuilder;
@Service @Service
@Transactional(readOnly = true) @Transactional(readOnly = true)
...@@ -103,26 +110,49 @@ public class CropServiceImpl implements CropService { ...@@ -103,26 +110,49 @@ public class CropServiceImpl implements CropService {
@Autowired @Autowired
private Taxonomy2Repository taxonomy2Repository; private Taxonomy2Repository taxonomy2Repository;
@Autowired
private AccessionProcessor accessionProcessor;
@Autowired
private AccessionRepository accessionRepository;
@Autowired
protected AsAdminInvoker asAdminInvoker;
/**
* @param shortName the crop name ("groundnuts") as provided by partners, but
* can also be a list of names ("peanut,goober,groundnut")
*/
@Override @Override
@Cacheable(value = CACHE_CROPS, key = "#shortName", unless = "#result == null || #shortName == null") @Cacheable(value = CACHE_CROPS, key = "#shortName", unless = "#result == null || #shortName == null")
public Crop getCrop(String shortName) { public Crop getCrop(String shortName) {
if (StringUtils.isBlank(shortName)) { if (StringUtils.isBlank(shortName)) {
return null; return null;
} }
Crop crop = cropRepository.findByShortName(shortName);
for (String cropName : shortName.split(",;")) {
// Find crop by alias when null cropName = StringUtils.trimToNull(cropName);
if (crop == null) { if (cropName != null) {
crop = cropRepository.findByOtherNames(shortName); Crop crop = cropRepository.findByShortName(cropName);
// Find crop by alias when null
if (crop == null) {
crop = cropRepository.findByOtherNames(cropName);
}
if (crop != null) {
// Lazy load otherNames
if (crop.getOtherNames() != null) {
crop.getOtherNames().size();
}
return crop;
}
}
} }
// Lazy load otherNames return null;
if (crop != null && crop.getOtherNames() != null)
crop.getOtherNames().size();
return crop;
} }
@Override @Override
...@@ -134,7 +164,7 @@ public class CropServiceImpl implements CropService { ...@@ -134,7 +164,7 @@ public class CropServiceImpl implements CropService {
Article article = contentService.getArticle(crop, "blurp", locale); Article article = contentService.getArticle(crop, "blurp", locale);
LOG.trace("got article after {}ms", stopWatch.getTime()); LOG.trace("got article after {}ms", stopWatch.getTime());
AccessionFilter byCrop = new AccessionFilter(); AccessionFilter byCrop = new AccessionFilter();
byCrop.cropName = crop.getShortName(); byCrop.crop = Sets.newHashSet(crop.getShortName());
long accessionCount = accessionService.countAccessions(byCrop); long accessionCount = accessionService.countAccessions(byCrop);
LOG.trace("got count after {}ms", stopWatch.getTime()); LOG.trace("got count after {}ms", stopWatch.getTime());
...@@ -535,6 +565,67 @@ public class CropServiceImpl implements CropService { ...@@ -535,6 +565,67 @@ public class CropServiceImpl implements CropService {
cropTaxonomyRepository.save(toAdd); cropTaxonomyRepository.save(toAdd);
} }
} }
@Override
public void unlinkAccessionsForCrop(Crop crop) {
final BooleanBuilder predicate = new BooleanBuilder();
// accessions with selected crop
predicate.and(QAccession.accession.crop.eq(crop));
final BooleanBuilder cropNames = new BooleanBuilder();
cropNames.or(QAccession.accession.cropName.equalsIgnoreCase(crop.getShortName()));
cropNames.or(QAccession.accession.cropName.likeIgnoreCase(crop.getShortName()));
crop.getOtherNames().forEach(otherName -> {
cropNames.or(QAccession.accession.cropName.equalsIgnoreCase(otherName));
cropNames.or(QAccession.accession.cropName.likeIgnoreCase(otherName));
});
// accession#cropName not one of valid names
predicate.andNot(cropNames);
accessionProcessor.apply(predicate, (accessions) -> {
accessions.forEach(accession -> {
Crop newCrop = getCrop(accession.getCropName());
LOG.debug("{} from {}: {} -> {}", accession.getUuid(), accession.getInstCode(), accession.getCropName(), newCrop == null ? "null" : newCrop.getShortName());
// assign new crop
accession.setCrop(newCrop);
});
asAdminInvoker.invoke(() -> {
accessionRepository.save(accessions);
return true;
});
return accessions;
});
}
@Override
public void linkAccessionsWithCrop(Crop crop) {
final BooleanBuilder predicate = new BooleanBuilder();
// accepted crop names
predicate.or(QAccession.accession.cropName.equalsIgnoreCase(crop.getShortName()));
predicate.or(QAccession.accession.cropName.likeIgnoreCase(crop.getShortName()));
crop.getOtherNames().forEach(otherName -> {
predicate.or(QAccession.accession.cropName.equalsIgnoreCase(otherName));
predicate.or(QAccession.accession.cropName.likeIgnoreCase(otherName));
});
// but not assigned to this crop
predicate.andAnyOf(QAccession.accession.crop.isNull(), QAccession.accession.crop.ne(crop));
accessionProcessor.apply(predicate, (accessions) -> {
accessions.forEach(accession -> {
Crop newCrop = getCrop(accession.getCropName());
LOG.debug("{} from {}: {} -> {}", accession.getUuid(), accession.getInstCode(), accession.getCropName(), newCrop == null ? "null" : newCrop.getShortName());
// assign crop
accession.setCrop(newCrop);
});
asAdminInvoker.invoke(() -> {
accessionRepository.save(accessions);
return true;
});
return accessions;
});
}
private Map<String, ElasticsearchService.TermResult> getOverviewData(AccessionFilter byCropFilter) { private Map<String, ElasticsearchService.TermResult> getOverviewData(AccessionFilter byCropFilter) {
String[] terms = new String[] {"taxonomy.genus", "taxonomy.genusSpecies", "institute.code", String[] terms = new String[] {"taxonomy.genus", "taxonomy.genusSpecies", "institute.code",
...@@ -558,7 +649,8 @@ public class CropServiceImpl implements CropService { ...@@ -558,7 +649,8 @@ public class CropServiceImpl implements CropService {
if (source.getOtherNames() != null) { if (source.getOtherNames() != null) {
target.setOtherNames(source.getOtherNames().stream().distinct().map(otherName -> StringUtils.trim(otherName)).filter(otherName -> StringUtils.isNotBlank(otherName)).sorted().collect(Collectors.toList())); target.setOtherNames(source.getOtherNames().stream().distinct().map(otherName -> StringUtils.trim(otherName)).filter(otherName -> StringUtils.isNotBlank(otherName)).sorted().collect(Collectors.toList()));
} }
if (source.getCropRules() != null) if (source.getCropRules() != null) {
target.getCropRules().addAll(source.getCropRules()); target.getCropRules().addAll(source.getCropRules());
}
} }
} }
...@@ -127,7 +127,6 @@ public class AccessionProcessor { ...@@ -127,7 +127,6 @@ public class AccessionProcessor {
action.apply(a); action.apply(a);
} }
} }
/** /**
* Apply action on accessions matching the provided filter. * Apply action on accessions matching the provided filter.
* *
...@@ -137,8 +136,20 @@ public class AccessionProcessor { ...@@ -137,8 +136,20 @@ public class AccessionProcessor {
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW) @Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void apply(AccessionFilter filter, IAccessionBatchAction action) { public void apply(AccessionFilter filter, IAccessionBatchAction action) {
Predicate predicate = filter.buildQuery(); apply(filter.buildQuery(), action);
}
/**
* Apply action on accessions matching the provided filter.
*
* @param predicate JPA query predicate on Accession
* @param action the action
*/
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void apply(Predicate predicate, IAccessionBatchAction action) {
long count = accessionRepository.count(predicate); long count = accessionRepository.count(predicate);
LOG.info("{} accessions match the query", count);
PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class); PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
Querydsl querydsl = new Querydsl(em, builder); Querydsl querydsl = new Querydsl(em, builder);
...@@ -174,7 +185,7 @@ public class AccessionProcessor { ...@@ -174,7 +185,7 @@ public class AccessionProcessor {
} while (results.size() > 0); } while (results.size() > 0);
stopWatch.stop(); stopWatch.stop();
LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime()); LOG.info("Processing Accessions took {}ms", stopWatch.getTime());
} }
private void asyncUpdate(List<Long> accessionIds, IAccessionBatchAction action) { private void asyncUpdate(List<Long> accessionIds, IAccessionBatchAction action) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment