Commit 7fd34073 authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch 'forages' into staging

# Conflicts:
#	.gitignore
#	src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
#	src/main/java/org/genesys2/server/model/genesys/AccessionId.java
#	src/main/java/org/genesys2/server/persistence/domain/AccessionIdRepository.java
#	src/main/java/org/genesys2/server/service/GenesysService.java
#	src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
#	src/main/java/org/genesys2/server/service/worker/ElasticUpdaterAspect.java
#	src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java
#	src/main/resources/content/language.properties
parents e6f5d2f3 c058f638
......@@ -92,6 +92,7 @@ public class AccessionDetails {
private Date modifiedDate;
private Date acqusitionDate;
private Float pdciScore;
private boolean historic;
......@@ -127,6 +128,12 @@ public class AccessionDetails {
ad.lists = new HashSet<String>();
ad.cropName=accession.getCropName();
ad.inSgsv = accession.getInSvalbard() != null && accession.getInSvalbard() == true ? true : false;
ad.cropName = accession.getCropName();
if (accession.getAccessionId().getPdci() != null) {
ad.pdciScore = accession.getAccessionId().getPdci().getScore();
}
for (AccessionList al : accession.getAccessionId().getLists()) {
ad.lists.add(al.getUuid().toString());
}
......@@ -507,4 +514,11 @@ public class AccessionDetails {
this.lists = lists;
}
public void setPdciScore(Float pdciScore) {
this.pdciScore = pdciScore;
}
public Float getPdciScore() {
return pdciScore;
}
}
......@@ -87,6 +87,10 @@ public class AccessionId extends VersionedAuditedModel implements IdUUID {
this.lists = lists;
}
public PDCI getPdci() {
return pdci;
}
//
// public List<AccessionTrait> getTraits() {
// return traits;
......
......@@ -21,6 +21,7 @@ import java.util.Set;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.OneToOne;
import javax.persistence.PrePersist;
......@@ -31,14 +32,14 @@ import javax.persistence.Transient;
import org.genesys2.server.model.VersionedModel;
@Entity
@Table(name = "pdci")
public class PDCI extends VersionedModel {
@Table(name = "pdci", indexes = { @Index(columnList = "score") })
public class PDCI extends VersionedModel implements AccessionRelated {
private static final long serialVersionUID = -1312366054528702261L;
public static final String[] independentItems = { "genus", "species", "spAuthor", "subTaxa", "subtAuthor", "cropName", "acqDate", "sampStat", "donorCode",
"donorNumb", "otherNumb", "duplSite", "storage", "donorName", "duplInstName", "acceUrl", "mlsStat" };
public static final String[] dependentItems = { "origCty", "collSite", "latitude", "longitude", "elevation", "collDate", "bredCode", "ancest", "collSrc",
"acceName", "collNumb", "collCode", "collName" };
public static final String[] independentItems = { "genus", "species", "spAuthor", "subTaxa", "subtAuthor", "cropName", "acqDate", "sampStat", "donorCode", "donorNumb", "otherNumb", "duplSite",
"storage", "donorName", "duplInstName", "acceUrl", "mlsStat" };
public static final String[] dependentItems = { "origCty", "collSite", "latitude", "longitude", "elevation", "collDate", "bredCode", "ancest", "collSrc", "acceName", "collNumb", "collCode",
"collName" };
@OneToOne(cascade = {}, fetch = FetchType.LAZY, optional = false)
@JoinColumn(name = "accessionId")
......@@ -86,6 +87,7 @@ public class PDCI extends VersionedModel {
this.scoreHist = (float) (Math.ceil(this.score * 2) / 2);
}
@Override
public AccessionId getAccession() {
return accession;
}
......
......@@ -29,10 +29,10 @@ public interface AccessionIdRepository extends JpaRepository<AccessionId, Long>
@Query("select aid from AccessionId aid where aid.uuid is null")
List<AccessionId> findMissingUuid(Pageable pageable);
@Query("select aid from AccessionId aid left join aid.pdci pdci where pdci.score is null")
List<AccessionId> findMissingPDCI(Pageable pageable);
@Query("select aid.id from AccessionId aid left join aid.pdci pdci where pdci.score is null")
Collection<Long> findMissingPDCI();
@Query("select aid from AccessionId aid left join aid.pdci pdci where pdci.score is null")
Collection<AccessionId> findMissingPDCI();
@Query("select id from AccessionId")
Collection<Long> findAllIds();
}
......@@ -81,4 +81,6 @@ public interface FilterConstants {
public static final String LISTS = "lists";
public static final String PDCI = "pdci";
}
......@@ -18,7 +18,6 @@ package org.genesys2.server.service;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
......@@ -255,7 +254,7 @@ public interface GenesysService {
List<PDCI> loadPDCI(List<Long> batch);
int updatePDCI(Set<Long> ids);
List<PDCI> updatePDCI(Set<Long> ids);
PDCIStatistics statisticsPDCI(Organization organization);
......@@ -265,6 +264,6 @@ public interface GenesysService {
List<SvalbardDeposit> saveSvalbards(List<SvalbardDeposit> svalbards);
List<Accession> saveAccessionCrops(List<Long> accessionIds);
List<Accession> updateAccessionCrops(Set<Long> accessionIds);
}
......@@ -120,6 +120,10 @@ public class DirectMysqlQuery {
innerJoin("crop", null, "crop.id=a.cropId");
}
if (filters.hasFilter(FilterConstants.PDCI)) {
innerJoin("pdci", null, "pdci.accessionId=a.id");
}
// if (filters.hasFilter(FilterConstants.ORGCTY_ISO3)) {
// innerJoin("country", "cty", "cty.id=a.orgCtyId");
// }
......@@ -201,6 +205,7 @@ public class DirectMysqlQuery {
createQuery(whereBuffer, "accename.name", filters.get(FilterConstants.ALIAS), params);
createQuery(whereBuffer, "col.collMissId", filters.get(FilterConstants.COLLMISSID), params);
createQuery(whereBuffer, "storage.storage", filters.get(FilterConstants.STORAGE), params);
createQuery(whereBuffer, "pdci.score", filters.get(FilterConstants.PDCI), params);
for (final AppliedFilter methodFilter : filters.methodFilters()) {
// Handle Genesys Method!
......
......@@ -70,8 +70,7 @@ public class FilterHandler {
private final ArrayList<GenesysFilter> availableFilters;
/**
* By default we exclude historic records unless the user explicitly
* specifies otherwise.
* By default we exclude historic records unless the user explicitly specifies otherwise.
*/
public static final AppliedFilter NON_HISTORIC_FILTER = new AppliedFilter().setFilterName(FilterConstants.HISTORIC).addFilterValue(new LiteralValueFilter(false));
......@@ -82,8 +81,8 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.CROPNAME, DataType.STRING));
this.availableFilters.add(new BasicFilter(FilterConstants.LISTS, DataType.STRING));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.SAMPSTAT, DataType.NUMERIC).build("accession.sampleStatus", new Integer[] { 100,
110, 120, 130, 200, 300, 400, 410, 411, 412, 413, 414, 415, 416, 420, 421, 422, 423, 500, 600, 999 }));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.SAMPSTAT, DataType.NUMERIC).build("accession.sampleStatus",
new Integer[] { 100, 110, 120, 130, 200, 300, 400, 410, 411, 412, 413, 414, 415, 416, 420, 421, 422, 423, 500, 600, 999 }));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_GENUS, "/explore/ac/" + FilterConstants.TAXONOMY_GENUS));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_SPECIES, "/explore/ac/" + FilterConstants.TAXONOMY_SPECIES));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_SUBTAXA, "/explore/ac/" + FilterConstants.TAXONOMY_SUBTAXA));
......@@ -106,8 +105,8 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.HISTORIC, DataType.BOOLEAN).allowsNull(false));
this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
13, 20, 30, 40, 50, 99 }));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12, 13, 20, 30, 40, 50, 99 }));
this.availableFilters.add(new BasicFilter(FilterConstants.PDCI, DataType.NUMERIC).allowsNull(false));
}
public List<GenesysFilter> listAvailableFilters() {
......@@ -380,8 +379,7 @@ public class FilterHandler {
}
/**
* Returns true if the filter is listed and has at least one value
* specified
* Returns true if the filter is listed and has at least one value specified
*
* @param filterName
* @return
......@@ -753,7 +751,7 @@ public class FilterHandler {
public String toString() {
return getClass().getSimpleName() + " " + getType() + ": " + getTo();
}
@Override
public int hashCode() {
final int prime = 31;
......@@ -811,13 +809,11 @@ public class FilterHandler {
return from;
}
@Override
public String toString() {
return getClass().getSimpleName() + " " + getType() + ": " + getFrom();
}
@Override
public int hashCode() {
final int prime = 31;
......@@ -875,7 +871,7 @@ public class FilterHandler {
public String toString() {
return getClass().getSimpleName() + " startsWith " + getStartsWith();
}
@Override
public int hashCode() {
final int prime = 31;
......
......@@ -1642,8 +1642,10 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Override
@Transactional
@CacheEvict(value = "statistics", allEntries = true)
public int updatePDCI(final Set<Long> ids) {
LOG.info("Calculating PDCI for " + ids.size() + " accessions");
public List<PDCI> updatePDCI(final Set<Long> ids) {
if (LOG.isDebugEnabled())
LOG.debug("Calculating PDCI for " + ids.size() + " accessions");
final List<PDCI> pdcis = Collections.synchronizedList(new ArrayList<PDCI>(ids.size()));
getAccessionDetails(ids).stream().forEach(ad -> {
......@@ -1658,7 +1660,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
LOG.info("Generated " + pdcis.size() + " new PDCIs");
repoPdci.save(pdcis);
return pdcis.size();
return pdcis;
}
@Override
......@@ -1799,7 +1801,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Override
@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
// @PreAuthorize("hasRole('ADMINISTRATOR')")
public List<Accession> saveAccessionCrops(ArrayList<Long> accessionIds) {
public List<Accession> updateAccessionCrops(Set<Long> accessionIds) {
List<Accession> accessions = new ArrayList<>(accessionRepository.findAll(accessionIds));
for (Accession accession : accessions) {
accessionRepository.updateCrop(accession, cropService.getCrop(accession.getCropName()));
......
......@@ -31,6 +31,7 @@ import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.elastic.FullTextDocument;
import org.genesys2.server.model.genesys.Accession;
......@@ -59,6 +60,10 @@ public class ElasticUpdaterAspect {
@Autowired
private GenesysService genesysService;
@Pointcut(value = "execution(* org.genesys2.server.service.GenesysService.save*(..)) || execution(* org.genesys2.server.service.GenesysService.update*(..))")
public void accessionUpdates() {
}
/**
* Handle updates to Accession and related objects
*
......@@ -67,7 +72,7 @@ public class ElasticUpdaterAspect {
* @return
* @throws Throwable
*/
@AfterReturning(value = "execution(* org.genesys2.server.service.GenesysService.save*(..))", returning = "result")
@AfterReturning(value = "accessionUpdates()", returning = "result")
public Object afterSave(final JoinPoint joinPoint, final Object result) throws Throwable {
if (LOG.isTraceEnabled()) {
LOG.trace("Returning from " + joinPoint.toLongString());
......@@ -82,26 +87,30 @@ public class ElasticUpdaterAspect {
return result;
for (Object o : list) {
if (LOG.isTraceEnabled()) {
LOG.trace("Indexing object " + o);
}
if (o == null)
continue;
if (o instanceof Accession) {
elasticUpdater.update(Accession.class, ((Accession) o).getId());
}
if (o instanceof AccessionRelated) {
AccessionId acceId = ((AccessionRelated) o).getAccession();
if (acceId != null)
elasticUpdater.update(Accession.class, acceId.getId());
}
maybeUpdate(o);
}
} else {
maybeUpdate(result);
}
}
return result;
}
private void maybeUpdate(Object o) {
if (o == null)
return;
if (LOG.isTraceEnabled()) {
LOG.trace("Maybe indexing object " + o.getClass() + " = " + o);
}
if (o instanceof Accession) {
elasticUpdater.update(Accession.class, ((Accession) o).getId());
}
if (o instanceof AccessionRelated) {
elasticUpdater.update(Accession.class, ((AccessionRelated) o).getAccession().getId());
}
}
/**
* Handle removal of accession related objects
*
......
......@@ -9,10 +9,8 @@ import javax.annotation.Resource;
import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FullTextSearchService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
......@@ -37,9 +35,6 @@ class ElasticUpdaterProcessor {
@Autowired
private FullTextSearchService fullTextSearchService;
@Autowired
private GenesysService genesysService;
@Autowired
private ThreadPoolTaskExecutor executor;
......@@ -133,9 +128,6 @@ class ElasticUpdaterProcessor {
elasticService.updateAll(className, copy);
fullTextSearchService.updateAll(className, copy);
if (Accession.class.getName().equals(className)) {
genesysService.updatePDCI(copy);
}
if (LOG.isTraceEnabled()) {
LOG.trace("Reindexing done.");
......
......@@ -374,17 +374,38 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/pdci", method = RequestMethod.POST)
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = { "action=missing-pdci" })
public String generatePDCI() {
final Set<Long> batch = Collections.synchronizedSet(new HashSet<>());
Collection<AccessionId> idsMissingPdci = accessionIdRepository.findMissingPDCI();
Collection<Long> idsMissingPdci = accessionIdRepository.findMissingPDCI();
LOG.info("Emm, " + idsMissingPdci.size() + " are missing PDCI");
idsMissingPdci.parallelStream().forEach(accessionId -> {
batch.add(accessionId.getId());
batch.add(accessionId);
synchronized (batch) {
if (batch.size() >= 1000) {
taskExecutor.execute(new PDCIUpdater(new HashSet<Long>(batch)));
batch.clear();
}
}
});
taskExecutor.execute(new PDCIUpdater(batch));
return "redirect:/admin/";
}
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = { "action=full-recalc" })
public String recreatePDCI() {
final Set<Long> batch = Collections.synchronizedSet(new HashSet<>());
Collection<Long> idsMissingPdci = accessionIdRepository.findAllIds();
LOG.info("Emm, " + idsMissingPdci.size() + " will have their PDCI recalculated");
idsMissingPdci.parallelStream().forEach(accessionId -> {
batch.add(accessionId);
synchronized (batch) {
if (batch.size() > 1000) {
if (batch.size() >= 1000) {
taskExecutor.execute(new PDCIUpdater(new HashSet<Long>(batch)));
batch.clear();
}
......@@ -432,27 +453,27 @@ public class AdminController {
AtomicLong counter = new AtomicLong(0);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
final List<Long> batch = Collections.synchronizedList(new ArrayList<>());
final Set<Long> batch = Collections.synchronizedSet(new HashSet<>(100));
List<Long> list = accessionRepository.listAccessionIdsWithCropname();
LOG.info("The list has " + list.size() + " elements");
list.stream().parallel().forEach(accessionId -> {
batch.add(accessionId);
ArrayList<Long> copy = null;
Set<Long> copy = null;
synchronized (batch) {
if (batch.size() > 100) {
copy = new ArrayList<>(batch);
copy = new HashSet<>(batch);
batch.clear();
}
}
if (copy != null)
genesysService.saveAccessionCrops(copy);
genesysService.updateAccessionCrops(copy);
if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) {
LOG.info("Updated " + counter.get() + " records overall_rate=" + Math.round(1000.0 * counter.get() / (stopWatch.getTime())) + " records/s");
}
});
// handle remaining
genesysService.saveAccessionCrops(batch);
genesysService.updateAccessionCrops(batch);
LOG.info("Done assigning crops to accessions with CROPNAME.");
return "redirect:/admin/";
......
......@@ -463,6 +463,7 @@ filter.internal.message.more=More than {0}
filter.internal.message.less=Less than {0}
filter.lists=Listed on accession list
filter.more-filters=More filters
filter.pdci=PDCI Score
columns.add=Change displayed columns
columns.apply=Apply
......
......@@ -129,6 +129,10 @@
<filters:panel id="storage" title="filter.storage">
<filters:filter availableFilters="${availableFilters}" filterMap="${filters}" filterKey="storage" appliedFilters="${appliedFilters}" type="option" />
</filters:panel>
<filters:panel id="pdci" title="filter.pdci">
<filters:filter availableFilters="${availableFilters}" filterMap="${filters}" filterKey="pdci" appliedFilters="${appliedFilters}" type="range" />
</filters:panel>
</filters:group>
<!--List-->
<div class="col-lg-10 col-md-9 col-sm-9 col-xs-12 main-col-header">
......
......@@ -23,7 +23,8 @@
</form>
<form method="post" action="<c:url value="/admin/pdci" />">
<input type="submit" class="btn btn-default" value="Calculate missing PDCI" />
<button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
<button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment