Commit 197ac55b authored by Matija Obreza's avatar Matija Obreza

Reassign accession#crop based on #cropName

parent 496e5977
......@@ -576,7 +576,7 @@
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>0.0.26</version>
<version>1.0</version>
<executions>
<execution>
<id>install node and npm</id>
......@@ -611,8 +611,9 @@
<goals>
<goal>grunt</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<arguments>build</arguments>
<arguments>build --no-color</arguments>
<srcdir>${basedir}/src/main/sourceapp</srcdir>
<outputdir>${basedir}/src/main/webapp/html</outputdir>
<triggerfiles>
......
......@@ -64,7 +64,7 @@ public class Crop extends GlobalVersionedAuditedModel implements AclAwareModel {
@Column(nullable = false, length = 50, unique = true)
private String shortName;
@Column(name = "otherName", nullable = false)
@Column(name = "otherName", nullable = false, unique = true)
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "cropname", joinColumns = @JoinColumn(name = "cropId", referencedColumnName = "id"), uniqueConstraints = { @UniqueConstraint(columnNames = "otherName") })
@OrderBy("otherName")
......
......@@ -25,6 +25,7 @@ import java.util.stream.Stream;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
......@@ -126,4 +127,11 @@ public interface AccessionRepository extends JpaRepository<Accession, Long> {
Stream<Accession> streamAll();
public long countByHistoric(boolean historic);
@Query("select a from Accession a where a.cropName != null")
Stream<Accession> streamWithCropname();
@Query("update Accession a set a.crop = ?2 where a = ?1")
@Modifying
public void updateCrop(Accession accession, Crop crop);
}
......@@ -98,4 +98,13 @@ public interface CropService {
*/
List<Crop> getCrops(List<String> cropNames);
/**
* Update crop aliases
*
* @param crop
* @param otherNames
* @return updated crop
*/
Crop updateAliases(Crop crop, List<String> otherNames);
}
......@@ -18,11 +18,13 @@ package org.genesys2.server.service;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Stream;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
......@@ -266,5 +268,8 @@ public interface GenesysService {
void regenerateAccessionSequentialNumber();
Stream<Accession> streamWithCropname();
List<Accession> saveAccessionCrops(ArrayList<Accession> copy);
}
......@@ -42,6 +42,7 @@ import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.AccessionRemark;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.json.Api1Constants;
import org.genesys2.server.persistence.domain.AccessionCustomRepository;
......@@ -571,12 +572,27 @@ public class BatchRESTServiceImpl implements BatchRESTService {
throw new RESTApiDataTypeException("cropName must be a String");
}
final String cropName = value.textValue().trim();
if (!StringUtils.equals(cropName, accession.getCropName())) {
final String cropName = StringUtils.trimToNull(value.textValue());
if (!StringUtils.equals(cropName, accession.getCropName()) || accession.getCrop() == null) {
// existing cropName doesn't match or current crop assigned is null
accession.setCropName(cropName);
accession.setCrop(cropService.getCrop(cropName));
return true;
}
if (StringUtils.isNotBlank(cropName)) {
// has cropName, check if needs updating
Crop newCrop = cropService.getCrop(cropName);
if (accession.getCrop() == null || (newCrop != null && accession.getCrop().getId() != newCrop.getId())) {
accession.setCrop(newCrop);
return true;
}
} else if (accession.getCrop() != null) {
// blank but has crop assigned
accession.setCrop(null);
return true;
}
}
return false;
}
......
......@@ -16,6 +16,15 @@
package org.genesys2.server.service.impl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.StringUtils;
......@@ -42,8 +51,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
@Service
@Transactional(readOnly = true)
public class CropServiceImpl implements CropService {
......@@ -72,19 +79,25 @@ public class CropServiceImpl implements CropService {
private Taxonomy2Repository taxonomy2Repository;
@Override
@Cacheable(value = CACHE_CROPS, key = "#shortName")
public Crop getCrop(String shortName) {
Crop crop = cropRepository.findByShortName(shortName);
if (crop == null) {
// Find crop by alias
// TODO Catch multiple results exception here!
crop = cropRepository.findByOtherNames(shortName);
}
if (crop != null) {
// lazy load
crop.getOtherNames().size();
}
return crop;
}
@PreAuthorize("hasRole('ADMINISTRATOR')")
@Override
@Transactional
@CacheEvict(allEntries=true, value=CACHE_CROPS)
@CacheEvict(allEntries = true, value = CACHE_CROPS)
public Crop delete(Crop crop) {
cropRepository.delete(crop);
crop.setId(null);
......@@ -98,6 +111,18 @@ public class CropServiceImpl implements CropService {
contentService.updateArticle(crop, "blurp", null, textBody, summary, locale);
}
@Override
@PreAuthorize("hasRole('ADMINISTRATOR') or hasRole('CONTENTMANAGER') or hasPermission(#crop, 'ADMINISTRATION')")
@Transactional(readOnly = false)
@CacheEvict(value = CACHE_CROPS, allEntries = true)
public Crop updateAliases(Crop crop, List<String> otherNames) {
crop.setOtherNames(otherNames.stream().distinct().map(otherName -> StringUtils.trim(otherName)).filter(otherName -> StringUtils.isNotBlank(otherName))
.sorted().collect(Collectors.toList()));
return cropRepository.save(crop);
}
@Override
@Cacheable(value = CACHE_CROPS, key = "'findAll'")
public List<Crop> listCrops() {
......@@ -260,7 +285,7 @@ public class CropServiceImpl implements CropService {
@Override
@PreAuthorize("hasRole('ADMINISTRATOR')")
@Transactional(readOnly = false)
@CacheEvict(allEntries=true, value=CACHE_CROPS)
@CacheEvict(allEntries = true, value = CACHE_CROPS)
public Crop addCrop(String shortName, String name, String description, String i18n) {
LOG.info("Adding crop " + shortName);
final Crop crop = new Crop();
......@@ -281,7 +306,7 @@ public class CropServiceImpl implements CropService {
@Override
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#crop, 'ADMINISTRATION')")
@Transactional(readOnly = false)
@CacheEvict(allEntries=true, value=CACHE_CROPS)
@CacheEvict(allEntries = true, value = CACHE_CROPS)
public Crop updateCrop(Crop crop, String name, String description, String i18n) {
LOG.info("Updating crop " + crop);
......
......@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
......@@ -117,6 +118,7 @@ import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.acls.domain.BasePermission;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import com.opencsv.CSVWriter;
......@@ -789,7 +791,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
public List<Accession> saveAccessions(Iterable<Accession> accessions) {
Set<FaoInstitute> institutes = new HashSet<FaoInstitute>();
for (Accession accession : accessions) {
System.out.println("Saving " + accession + " STO=" + accession.getStoRage() + " ST=" + accession.getStorage());
// System.out.println("Saving " + accession + " STO=" + accession.getStoRage() + " ST=" + accession.getStorage());
institutes.add(accession.getInstitute());
}
List<Accession> res = accessionRepository.save(accessions);
......@@ -1111,7 +1113,8 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
zos.putNextEntry(metaEntry);
final BufferedWriter osw = new BufferedWriter(new OutputStreamWriter(zos));
osw.write("<?xml version='1.0' encoding='utf-8'?>\n");
osw.write("<archive xmlns=\"http://rs.tdwg.org/dwc/text/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://rs.tdwg.org/dwc/text/ http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd\">\n");
osw.write(
"<archive xmlns=\"http://rs.tdwg.org/dwc/text/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://rs.tdwg.org/dwc/text/ http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd\">\n");
osw.write("<core encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write("\t<files><location>core.csv</location></files>\n");
osw.write("\t<id index=\"0\" />\n");
......@@ -1378,7 +1381,8 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
zos.putNextEntry(metaEntry);
final BufferedWriter osw = new BufferedWriter(new OutputStreamWriter(zos));
osw.write("<?xml version='1.0' encoding='utf-8'?>\n");
osw.write("<archive xmlns=\"http://rs.tdwg.org/dwc/text/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://rs.tdwg.org/dwc/text/ http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd\">\n");
osw.write(
"<archive xmlns=\"http://rs.tdwg.org/dwc/text/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://rs.tdwg.org/dwc/text/ http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd\">\n");
osw.write("<core encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write("\t<files><location>core.csv</location></files>\n");
osw.write("\t<id index=\"0\" />\n");
......@@ -1390,7 +1394,8 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
for (int i = 0; i < metadataMethods.size(); i++) {
final Method method = metadataMethods.get(i);
osw.write("<extension encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write(
"<extension encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write("\t<files><location>");
osw.write(method.getFieldName().toLowerCase());
osw.write(".csv</location></files>\n");
......@@ -1783,4 +1788,19 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
genesysLowlevelRepository.updateAccessionSequentialNumber(batch);
}
@Override
public Stream<Accession> streamWithCropname() {
return accessionRepository.streamWithCropname();
}
@Override
@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
// @PreAuthorize("hasRole('ADMINISTRATOR')")
public List<Accession> saveAccessionCrops(ArrayList<Accession> accessions) {
for (Accession accession : accessions)
accessionRepository.updateCrop(accession, cropService.getCrop(accession.getCropName()));
return accessions;
}
}
......@@ -16,6 +16,10 @@
package org.genesys2.server.servlet.controller;
import java.util.Arrays;
import java.util.stream.Collectors;
import org.elasticsearch.common.lang3.StringUtils;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.CropTaxonomy;
import org.genesys2.server.service.ContentService;
......@@ -31,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.PathVariable;
......@@ -92,19 +97,34 @@ public class CropController extends BaseController {
}
@RequestMapping("/{shortName}/edit")
@PreAuthorize("hasRole('ADMINISTRATOR') or hasRole('CONTENTMANAGER') or hasPermission(#crop, 'ADMINISTRATION')")
public String edit(ModelMap model, @PathVariable(value = "shortName") String shortName) {
view(model, shortName);
return "/crop/edit";
}
@RequestMapping("/{shortName}/new")
@PreAuthorize("hasRole('ADMINISTRATOR') or hasRole('CONTENTMANAGER') or hasPermission(#crop, 'ADMINISTRATION')")
public String newCrop(ModelMap model, @PathVariable(value = "shortName") String shortName) throws Exception {
_logger.debug("Making crop " + shortName);
if (cropService.getCrop(shortName) != null) {
throw new Exception("Crop exists.");
}
Crop crop = new Crop();
crop.setShortName(shortName);
model.addAttribute("crop", crop);
return "/crop/edit";
}
@RequestMapping("/{shortName}/update")
public String update(ModelMap model, @PathVariable(value = "shortName") String shortName, @RequestParam("blurp") String aboutBody,
@RequestParam(value = "summary", required = false) String summary) {
public String update(ModelMap model, @PathVariable(value = "shortName") String shortName, @RequestParam("cropName") String cropName,
@RequestParam("blurp") String aboutBody, @RequestParam(value = "summary", required = false) String summary) {
_logger.debug("Updating crop " + shortName);
final Crop crop = cropService.getCrop(shortName);
Crop crop = cropService.getCrop(shortName);
if (crop == null) {
throw new ResourceNotFoundException();
crop = cropService.addCrop(shortName, cropName, null, null);
}
cropService.updateBlurp(crop, aboutBody, summary, getLocale());
......@@ -112,6 +132,21 @@ public class CropController extends BaseController {
return "redirect:/c/" + shortName;
}
@RequestMapping(value = "/{shortName}/update", params = { "otherNames" })
public String updateOtherNames(ModelMap model, @PathVariable(value = "shortName") String shortName, @RequestParam("otherNames") String otherNames) {
_logger.debug("Updating crop aliases for " + shortName);
final Crop crop = cropService.getCrop(shortName);
if (crop == null) {
throw new ResourceNotFoundException();
}
cropService.updateAliases(crop, Arrays.stream(otherNames.split("\\n|,|;")).distinct().map(otherName -> StringUtils.trim(otherName))
.filter(otherName -> StringUtils.isNotBlank(otherName)).sorted().collect(Collectors.toList()));
return edit(model, shortName);
}
@RequestMapping("/{shortName}/ajax/taxonomies")
public String ajaxTaxonomies(ModelMap model, @PathVariable(value = "shortName") String shortName,
@RequestParam(value = "page", required = false, defaultValue = "1") int page) {
......@@ -128,13 +163,6 @@ public class CropController extends BaseController {
return "/crop/ajax.taxonomies";
}
@RequestMapping(value = "/rebuild", method = RequestMethod.POST)
public String rebuild() {
_logger.info("Rebuilding taxonomies");
cropService.rebuildTaxonomies();
return "redirect:/";
}
@RequestMapping("/{shortName}/data")
public String viewData(ModelMap model, @PathVariable(value = "shortName") String shortName,
@RequestParam(value = "page", required = false, defaultValue = "1") int page) {
......
......@@ -20,20 +20,31 @@ import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.*;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.impl.ContentSanitizer;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.worker.ElasticUpdater;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
......@@ -43,10 +54,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.xml.sax.SAXException;
import com.fasterxml.jackson.databind.ObjectMapper;
......@@ -98,6 +110,12 @@ public class AdminController {
@Autowired
private TaxonomyService taxonomyService;
@Autowired
private CropService cropService;
@PersistenceContext
private EntityManager entityManager;
ObjectMapper mapper = new ObjectMapper();
@RequestMapping("/")
......@@ -356,10 +374,42 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/rebuild-taxonomies", method = RequestMethod.POST)
public String rebuild() {
LOG.info("Rebuilding taxonomies");
cropService.rebuildTaxonomies();
return "redirect:/admin/";
}
@RequestMapping(value = "/cleanup-taxonomies", method = RequestMethod.POST)
public String cleanupTaxonomies() {
taxonomyService.cleanupTaxonomies();
return "redirect:/admin/";
}
@Transactional(readOnly=true, propagation=Propagation.REQUIRES_NEW)
@RequestMapping(value = "/cropname-crop", method = RequestMethod.POST)
public String assignCropWithCropname() {
LOG.info("Assigning crops to accessions with CROPNAME.");
AtomicLong counter = new AtomicLong(0);
List<Accession> batch = Collections.synchronizedList(new ArrayList<>());
genesysService.streamWithCropname().parallel().forEach(accession -> {
batch.add(accession);
ArrayList<Accession> copy = null;
synchronized (batch) {
if (batch.size() > 100) {
copy = new ArrayList<>(batch);
batch.clear();
}
}
if (copy != null) {
genesysService.saveAccessionCrops(copy);
}
if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) {
LOG.info("Updated " + counter.get() + " records");
}
});
LOG.info("Done assigning crops to accessions with CROPNAME.");
return "redirect:/admin/";
}
}
......@@ -139,6 +139,8 @@ crop.taxonomy-rules=Taxonomic rules
crop.view-descriptors=View crop descriptors...
crop.page.edit.title=Edit {0}
crop.summary=Summary (HTML metadata)
crop.name-en=Crop name (in English)
crop.cropname-aliases=Other accepted names
activity.recent-activity=Recent activity
......
......@@ -128,5 +128,12 @@
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>Crops</h3>
<form method="post" action="<c:url value="/admin/cropname-crop" />">
<input type="submit" class="btn btn-default" value="Assign Crop for CROPNAME" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
</body>
</html>
\ No newline at end of file
......@@ -13,6 +13,16 @@
</h1>
<form role="form" class="form-horizontal" action="<c:url value="/c/${crop.shortName}/update" />" method="post">
<c:if test="${crop.name eq null}">
<div class="form-group">
<label for="blurp-body" class="col-lg-12 control-label"><spring:message code="crop.name-en" /></label>
<div class="controls col-lg-12">
<input name="cropName" class="form-control" value="<c:out value="${crop.name}" />" />
</div>
</div>
</c:if>
<div class="form-group">
<label for="blurp-body" class="col-lg-12 control-label"><spring:message code="blurp.blurp-body" /></label>
<div class="controls col-lg-12">
......@@ -38,7 +48,22 @@
</form>
<content tag="javascript">
<form role="form" class="form-horizontal" action="<c:url value="/c/${crop.shortName}/update" />" method="post">
<div class="form-group">
<label class="col-lg-12 control-label">
<spring:message code="crop.cropname-aliases" />
</label>
<div class="controls col-lg-12">
<textarea name="otherNames" class="form-control"><c:forEach items="${crop.otherNames}" var="otherName"><c:out value="${otherName}" />
</c:forEach></textarea>
</div>
</div>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
<input type="submit" value="<spring:message code="save"/>" class="btn btn-primary" />
</form>
<content tag="javascript">
<script type="text/javascript">
<local:tinyMCE selector=".html-editor" />
</script>
......
......@@ -88,6 +88,21 @@
</div>
</div>
<div class="content-section-2015">
<h3>
<span>
<spring:message code="crop.cropname-aliases" />
</span>
</h3>
<div class="row">
<div class="col-md-offset-2 col-md-10">
<c:forEach items="${crop.otherNames}" var="otherName">
<c:out value="${otherName}" />
</c:forEach>
</div>
</div>
</div>
<div class="content-section-2015">
<h3>
<span>
......@@ -98,7 +113,7 @@
<div class="col-md-offset-2 col-md-10">
<ul class="funny-list">
<c:forEach items="${cropRules}" var="rule">
<li class="${rule.included ? '' : 'excluded'}"/>
<li class="${rule.included ? '' : 'excluded'}">
<b>${rule.included ? '+' : '-'}</b> <c:out value="${rule.genus}"/> <c:out
value="${rule.species eq null ? '*' : rule.species}"/></li>
</c:forEach>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment