Commit 47537a5e authored by Alexander Prendetskiy's avatar Alexander Prendetskiy Committed by Matija Obreza

PDCI statistics for Institute

- add button on admin page for updating pdci stats
- add unit test for updateHistogram method
- add PDCIStatistics to FaoInstitute
parent 31af4688
......@@ -138,7 +138,7 @@ public class InstituteController {
InstituteDetails details = new InstituteDetails();
details.details = faoInstitute;
details.blurb = contentService.getArticle(faoInstitute, "blurp", getLocale());
details.pdciStats = statisticsService.statisticsPDCI(faoInstitute);
details.pdciStats = faoInstitute.getStatisticsPDCI();
details.lastUpdates = genesysService.getLastUpdatedStatistics(faoInstitute);
details.overview= getOverviewData(byInstituteFilter);
......
......@@ -69,6 +69,10 @@ public class PDCIStatistics implements Serializable {
return histogram;
}
public void setHistogram(long[] histogram) {
this.histogram = histogram;
}
/**
* Converts {@link #histogram} to flot.categories compatible JSON
*
......@@ -120,6 +124,16 @@ public class PDCIStatistics implements Serializable {
this.count = this.count==null ? count.longValue() : this.count + count.longValue();
}
public void updateHistogram(long[] updateHistogram) {
if (updateHistogram == null) {
return;
}
for (int i = 0; i < this.histogram.length; i++) {
this.histogram[i] += updateHistogram[i];
}
}
/**
* Return object array to be used for i18n
*
......@@ -128,4 +142,13 @@ public class PDCIStatistics implements Serializable {
public Object[] getElStats() {
return new Object[] { count, avg, min, max };
}
public PDCIStatistics merge(PDCIStatistics other) {
updateMin(other.getMin());
updateMax(other.getMax());
updateCountAndAvg(other.getCount(), other.getAvg());
updateHistogram(other.getHistogram());
return this;
}
}
......@@ -16,31 +16,22 @@
package org.genesys2.server.model.impl;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.persistence.Cacheable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.MapKey;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import java.util.*;
import javax.persistence.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.EntityId;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.model.AclAwareModel;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -55,6 +46,8 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private static final long serialVersionUID = -8773002513838748431L;
private static final int LEN_ACRONYM = 50;
private static final ObjectMapper mapper = new ObjectMapper();
@Column(unique = true, nullable = false, length = 10)
@Field(index = FieldIndex.not_analyzed)
......@@ -109,6 +102,12 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private Double elevation;
private boolean uniqueAcceNumbs = false;
private boolean allowMaterialRequests = false;
private Double pdciMin;
private Double pdciMax;
private Double pdciAvg;
@Column(name = "pdciHistogram")
private String pdciHistogram;
public FaoInstitute() {
}
......@@ -278,6 +277,65 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
this.codeSGSV = codeSGSV;
}
public Double getPdciMin() {
return pdciMin;
}
public void setPdciMin(Double pdciMin) {
this.pdciMin = pdciMin;
}
public Double getPdciMax() {
return pdciMax;
}
public void setPdciMax(Double pdciMax) {
this.pdciMax = pdciMax;
}
public Double getPdciAvg() {
return pdciAvg;
}
public void setPdciAvg(Double pdciAvg) {
this.pdciAvg = pdciAvg;
}
public String getPdciHistogram() {
return pdciHistogram;
}
public void setPdciHistogram(String pdciHistogram) {
this.pdciHistogram = pdciHistogram;
}
public PDCIStatistics getStatisticsPDCI() {
PDCIStatistics stats = new PDCIStatistics();
stats.setMin(this.pdciMin);
stats.setMax(this.pdciMax);
stats.setCount(this.accessionCount);
stats.setAvg(this.pdciAvg);
stats.setHistogram(parsePdciHistogram());
return stats;
}
public long[] parsePdciHistogram() {
long[] histogram = new long[21];
if (StringUtils.isBlank(this.pdciHistogram)) {
return histogram;
}
try {
histogram = mapper.readValue(this.pdciHistogram, long[].class);
} catch (IOException e) {
e.printStackTrace();
}
return histogram;
}
/**
* Transitive
*
......
......@@ -47,7 +47,6 @@ import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.InstituteFilesService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.StatisticsService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TraitService;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
......@@ -89,9 +88,6 @@ public class AccessionController extends BaseController {
@Autowired
private DSService dsService;
@Autowired
private StatisticsService statisticsService;
@Autowired
private InstituteFilesService instituteFilesService;
......@@ -200,7 +196,7 @@ public class AccessionController extends BaseController {
}
model.addAttribute("pdci", accessionId.getPdci());
model.addAttribute("institutePDCI", statisticsService.statisticsPDCI(accession.getInstitute()));
model.addAttribute("institutePDCI", accession.getInstitute().getStatisticsPDCI());
ImageGallery imageGallery = null;
try {
......@@ -279,7 +275,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy
*
* @param model
* @param wiewsCode
* @param genus
* @param page
* @return
......@@ -304,7 +299,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy
*
* @param model
* @param wiewsCode
* @param genus
* @param species
* @param page
......
......@@ -193,7 +193,7 @@ public class WiewsController extends BaseController {
model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsPDCI", statisticsService.statisticsPDCI(faoInstitute));
model.addAttribute("statisticsPDCI", faoInstitute.getStatisticsPDCI());
model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute));
if (datasetCount > 0) {
......@@ -211,7 +211,7 @@ public class WiewsController extends BaseController {
if (institute == null) {
throw new Exception("Institute with instCode=" + instCode + " doesn't exist.");
}
final PDCIStatistics pdciStatistics = statisticsService.statisticsPDCI(institute);
final PDCIStatistics pdciStatistics = institute.getStatisticsPDCI();
final List<Object[]> updatedStats = genesysService.getLastUpdatedStatistics(institute, false);
final Long activeAccessions = institute.getAccessionCount();
final Long activeAccessionsWithDoi = genesysService.countByInstituteWithDoi(institute);
......
......@@ -29,6 +29,7 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch;
import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.CountryNamesUpdater;
......@@ -45,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
......@@ -210,6 +212,15 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "updatePdciStats")
public String updatePDCI() {
for (FaoInstitute institute: instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
genesysService.updatePDCI(institute);
}
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
geoRegionService.updateGeoRegionData();
......
......@@ -144,6 +144,8 @@ public interface GenesysService {
void updateAccessionCount(FaoInstitute institute);
void updatePDCI(FaoInstitute institute);
List<SvalbardDeposit> getSvalbardData(AccessionId accession);
void refreshMetadataMethods();
......@@ -225,8 +227,6 @@ public interface GenesysService {
AccessionHistoric getHistoricAccession(UUID uuid);
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
List<PDCI> loadPDCI(List<Long> batch);
PDCIStatistics statisticsPDCI(Organization organization);
......
......@@ -35,8 +35,6 @@ public interface StatisticsService {
long numberOfPublishedDatasets();
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
PDCIStatistics statisticsPDCI(Organization organization);
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
......
......@@ -22,16 +22,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
......@@ -1426,45 +1417,12 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return repoPdci.findByAccessionId(accessionIds);
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
PDCIStatistics stats = new PDCIStatistics();
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", faoInstitute);
return null;
}
stats.setMin((Number) overall[0]);
stats.setMax((Number) overall[1]);
stats.setAvg((Number) overall[2]);
stats.setCount((Number) overall[3]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
return stats;
}
@Override
public PDCIStatistics statisticsPDCI(Organization organization) {
PDCIStatistics stats = new PDCIStatistics();
for (FaoInstitute faoInstitute : organizationService.getMembers(organization)) {
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", organization);
continue;
}
stats.updateMin((Number) overall[0]);
stats.updateMax((Number) overall[1]);
stats.updateCountAndAvg((Number) overall[3], (Number) overall[2]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
stats.merge(faoInstitute.getStatisticsPDCI());
}
return stats;
}
......@@ -1507,6 +1465,28 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return stats;
}
@Transactional
@Override
public void updatePDCI(FaoInstitute faoInstitute) {
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", faoInstitute);
return;
}
List<Object[]> histogram = repoPdci.histogram(faoInstitute);
PDCIStatistics pdciStatistics = new PDCIStatistics();
pdciStatistics.makeHistogram(histogram);
faoInstitute.setPdciMin((Double) overall[0]);
faoInstitute.setPdciMax((Double) overall[1]);
faoInstitute.setPdciAvg((Double) overall[2]);
faoInstitute.setPdciHistogram(Arrays.toString(pdciStatistics.getHistogram()));
instituteRepository.save(faoInstitute);
}
@Transactional
@Override
public void regenerateAccessionSequentialNumber() {
......
......@@ -21,6 +21,7 @@ import org.genesys2.server.model.genesys.PhenoStatistics;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.persistence.PDCIRepository;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
......@@ -32,6 +33,8 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@Service
@Transactional(readOnly = true)
public class StatisticsServiceImpl implements StatisticsService {
......@@ -49,6 +52,9 @@ public class StatisticsServiceImpl implements StatisticsService {
@Autowired
private DatasetService datasetService;
@Autowired
private PDCIRepository repoPdci;
@Override
@Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
public long numberOfCountries() {
......@@ -85,17 +91,6 @@ public class StatisticsServiceImpl implements StatisticsService {
return datasetService.countPublished();
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
if (LOG.isDebugEnabled()) {
LOG.debug("Regenerating PDCI statistics for {}", faoInstitute);
}
synchronized (this) {
return genesysService.statisticsPDCI(faoInstitute);
}
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-org' + #organization.id")
public PDCIStatistics statisticsPDCI(Organization organization) {
......
......@@ -111,9 +111,13 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
}
FaoInstitute institute = instituteService.findInstitute(forProcessing.getObj());
if (institute != null) {
LOG.info("Updating count for {}", institute.getCode());
genesysService.updateAccessionCount(institute);
LOG.info("Updating PDCI for {}", institute.getCode());
genesysService.updatePDCI(institute);
}
}
......
......@@ -3767,3 +3767,24 @@ databaseChangeLog:
tableName: subset
columnName: published
- changeSet:
id: 1533127815588-66
author: aprendetskiy
comment: Add new columns to faoinstitute table
changes:
- addColumn:
tableName: faoinstitute
columns:
- column:
name: pdciMin
type: double
- column:
name: pdciMax
type: double
- column:
name: pdciAvg
type: double
- column:
name: pdciHistogram
type: VARCHAR(255)
\ No newline at end of file
......@@ -998,8 +998,8 @@
<spring:message code="accession.pdci.jumbo" arguments="${pdci.score}"/>
</h3>
<small>
<c:if test="${institutePDCI ne null and institutePDCI.count > 0}">
<spring:message code="accession.pdci.institute-avg" arguments="${institutePDCI.avg}"/>
<c:if test="${institutePDCI ne null and institutePDCI.accessionCount > 0}">
<spring:message code="accession.pdci.institute-avg" arguments="${institutePDCI.pdciAvg}"/>
</c:if>
<a href="<c:url value=" /content/passport-data-completeness-index" />"><spring:message code="accession.pdci.about-link"/></a>
</small>
......
......@@ -22,6 +22,7 @@
</form>
<form method="post" action="<c:url value="/admin/pdci" />">
<input type="submit" class="btn btn-default" name="updatePdciStats" value="Update all institute PDCI stats" />
<button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
<button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button>
<!-- CSRF protection -->
......
......@@ -149,4 +149,24 @@ public class PDCIStatisticsTest {
assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100]"));
}
/**
* All values from updateHistogram should be added to current values in histogram
*/
@Test
public void testUpdateHistogram() {
PDCIStatistics stats = new PDCIStatistics();
long[] histogram = new long[] {1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100};
stats.setHistogram(histogram);
long[] updatedHistogram = new long[] {0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0};
stats.updateHistogram(updatedHistogram);
// check merged histogram value
assertThat(stats.getHistogram().length, is(21));
assertThat(stats.getHistogram()[0], is(1l));
assertThat(stats.getHistogram()[8], is(45l));
assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 45, 0, 55, 0, 62, 0, 70, 0, 80, 0, 90, 0, 100]"));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment