Commit 8e3ee85b authored by Matija Obreza's avatar Matija Obreza

Merge branch '289-keep-average-pdci-in-faoinstitute' into 'master'

Resolve "Keep average PDCI in FaoInstitute"

Closes #289

See merge request genesys-pgr/genesys-server!206
parents 31af4688 9eb1dc7f
......@@ -138,7 +138,7 @@ public class InstituteController {
InstituteDetails details = new InstituteDetails();
details.details = faoInstitute;
details.blurb = contentService.getArticle(faoInstitute, "blurp", getLocale());
details.pdciStats = statisticsService.statisticsPDCI(faoInstitute);
details.pdciStats = faoInstitute.getStatisticsPDCI();
details.lastUpdates = genesysService.getLastUpdatedStatistics(faoInstitute);
details.overview= getOverviewData(byInstituteFilter);
......
......@@ -69,6 +69,10 @@ public class PDCIStatistics implements Serializable {
return histogram;
}
public void setHistogram(long[] histogram) {
this.histogram = histogram;
}
/**
* Converts {@link #histogram} to flot.categories compatible JSON
*
......@@ -97,7 +101,7 @@ public class PDCIStatistics implements Serializable {
int index = (int) Math.round(((Number) h[0]).doubleValue() * 2);
long count = ((Number) h[1]).longValue();
histogram[index] += count;
histogram[index] = count;
}
}
......@@ -120,6 +124,16 @@ public class PDCIStatistics implements Serializable {
this.count = this.count==null ? count.longValue() : this.count + count.longValue();
}
public void updateHistogram(long[] updateHistogram) {
if (updateHistogram == null) {
return;
}
for (int i = 0; i < this.histogram.length; i++) {
this.histogram[i] += updateHistogram[i];
}
}
/**
* Return object array to be used for i18n
*
......@@ -128,4 +142,13 @@ public class PDCIStatistics implements Serializable {
public Object[] getElStats() {
return new Object[] { count, avg, min, max };
}
public PDCIStatistics merge(PDCIStatistics other) {
updateMin(other.getMin());
updateMax(other.getMax());
updateCountAndAvg(other.getCount(), other.getAvg());
updateHistogram(other.getHistogram());
return this;
}
}
......@@ -16,31 +16,22 @@
package org.genesys2.server.model.impl;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.persistence.Cacheable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.MapKey;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import java.util.*;
import javax.persistence.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.EntityId;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.model.AclAwareModel;
import org.genesys.custom.elasticsearch.IgnoreField;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -55,6 +46,8 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private static final long serialVersionUID = -8773002513838748431L;
private static final int LEN_ACRONYM = 50;
private static final ObjectMapper mapper = new ObjectMapper();
@Column(unique = true, nullable = false, length = 10)
@Field(index = FieldIndex.not_analyzed)
......@@ -109,6 +102,12 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private Double elevation;
private boolean uniqueAcceNumbs = false;
private boolean allowMaterialRequests = false;
private Double pdciMin;
private Double pdciMax;
private Double pdciAvg;
@Column(name = "pdciHistogram")
private String pdciHistogram;
public FaoInstitute() {
}
......@@ -278,6 +277,69 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
this.codeSGSV = codeSGSV;
}
public Double getPdciMin() {
return pdciMin;
}
public void setPdciMin(Double pdciMin) {
this.pdciMin = pdciMin;
}
public Double getPdciMax() {
return pdciMax;
}
public void setPdciMax(Double pdciMax) {
this.pdciMax = pdciMax;
}
public Double getPdciAvg() {
return pdciAvg;
}
public void setPdciAvg(Double pdciAvg) {
this.pdciAvg = pdciAvg;
}
public String getPdciHistogram() {
return pdciHistogram;
}
public void setPdciHistogram(String pdciHistogram) {
this.pdciHistogram = pdciHistogram;
}
public PDCIStatistics getStatisticsPDCI() {
if (this.pdciMin == null || this.pdciMax == null || this.pdciAvg == null) {
return null;
}
PDCIStatistics stats = new PDCIStatistics();
stats.setMin(this.pdciMin);
stats.setMax(this.pdciMax);
stats.setCount(this.accessionCount);
stats.setAvg(this.pdciAvg);
stats.setHistogram(parsePdciHistogram());
return stats;
}
public long[] parsePdciHistogram() {
long[] histogram = new long[21];
if (StringUtils.isBlank(this.pdciHistogram)) {
return histogram;
}
try {
histogram = mapper.readValue(this.pdciHistogram, long[].class);
} catch (IOException e) {
e.printStackTrace();
}
return histogram;
}
/**
* Transitive
*
......
......@@ -47,7 +47,6 @@ import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.InstituteFilesService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.StatisticsService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TraitService;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
......@@ -89,9 +88,6 @@ public class AccessionController extends BaseController {
@Autowired
private DSService dsService;
@Autowired
private StatisticsService statisticsService;
@Autowired
private InstituteFilesService instituteFilesService;
......@@ -200,7 +196,7 @@ public class AccessionController extends BaseController {
}
model.addAttribute("pdci", accessionId.getPdci());
model.addAttribute("institutePDCI", statisticsService.statisticsPDCI(accession.getInstitute()));
model.addAttribute("institutePDCI", accession.getInstitute().getStatisticsPDCI());
ImageGallery imageGallery = null;
try {
......@@ -279,7 +275,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy
*
* @param model
* @param wiewsCode
* @param genus
* @param page
* @return
......@@ -304,7 +299,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy
*
* @param model
* @param wiewsCode
* @param genus
* @param species
* @param page
......
......@@ -193,7 +193,7 @@ public class WiewsController extends BaseController {
model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsPDCI", statisticsService.statisticsPDCI(faoInstitute));
model.addAttribute("statisticsPDCI", faoInstitute.getStatisticsPDCI());
model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute));
if (datasetCount > 0) {
......@@ -211,7 +211,7 @@ public class WiewsController extends BaseController {
if (institute == null) {
throw new Exception("Institute with instCode=" + instCode + " doesn't exist.");
}
final PDCIStatistics pdciStatistics = statisticsService.statisticsPDCI(institute);
final PDCIStatistics pdciStatistics = institute.getStatisticsPDCI();
final List<Object[]> updatedStats = genesysService.getLastUpdatedStatistics(institute, false);
final Long activeAccessions = institute.getAccessionCount();
final Long activeAccessionsWithDoi = genesysService.countByInstituteWithDoi(institute);
......
......@@ -29,6 +29,7 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch;
import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.CountryNamesUpdater;
......@@ -45,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
......@@ -210,6 +212,16 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "updatePdciStats")
public String updatePDCI() {
for (FaoInstitute institute: instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
LOG.info("Updating PDCI for {}", institute.getCode());
genesysService.updatePDCI(institute);
}
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
geoRegionService.updateGeoRegionData();
......
......@@ -40,7 +40,7 @@ public interface PDCIRepository extends JpaRepository<PDCI, Long> {
* @return [ Float min(pdci.score), Float max(pdci.score), Double
* avg(pdci.score), Long count(pdci) ]
*/
@Query("select min(pdci.score), max(pdci.score), avg(pdci.score), count(pdci) from Accession a, PDCI pdci where a.institute = ?1 and a.historic = false and pdci.accession=a")
@Query("select min(pdci.score), max(pdci.score), avg(pdci.score), count(pdci) from Accession a inner join a.accessionId.pdci pdci where a.institute = ?1 and a.historic = false")
Object statistics(FaoInstitute faoInstitute);
/**
......@@ -50,7 +50,7 @@ public interface PDCIRepository extends JpaRepository<PDCI, Long> {
* @param faoInstitute
* @return List of [ scoreHist, count ]
*/
@Query("select pdci.scoreHist, count(pdci) from Accession a, PDCI pdci where a.institute = ?1 and pdci.accession=a group by pdci.scoreHist")
@Query("select pdci.scoreHist, count(pdci) from Accession a inner join a.accessionId.pdci pdci where a.institute = ?1 and a.historic = false group by pdci.scoreHist")
List<Object[]> histogram(FaoInstitute faoInstitute);
}
......@@ -144,6 +144,8 @@ public interface GenesysService {
void updateAccessionCount(FaoInstitute institute);
FaoInstitute updatePDCI(FaoInstitute institute);
List<SvalbardDeposit> getSvalbardData(AccessionId accession);
void refreshMetadataMethods();
......@@ -225,8 +227,6 @@ public interface GenesysService {
AccessionHistoric getHistoricAccession(UUID uuid);
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
List<PDCI> loadPDCI(List<Long> batch);
PDCIStatistics statisticsPDCI(Organization organization);
......
......@@ -35,8 +35,6 @@ public interface StatisticsService {
long numberOfPublishedDatasets();
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
PDCIStatistics statisticsPDCI(Organization organization);
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
......
......@@ -22,16 +22,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
......@@ -1426,45 +1417,12 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return repoPdci.findByAccessionId(accessionIds);
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
PDCIStatistics stats = new PDCIStatistics();
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", faoInstitute);
return null;
}
stats.setMin((Number) overall[0]);
stats.setMax((Number) overall[1]);
stats.setAvg((Number) overall[2]);
stats.setCount((Number) overall[3]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
return stats;
}
@Override
public PDCIStatistics statisticsPDCI(Organization organization) {
PDCIStatistics stats = new PDCIStatistics();
for (FaoInstitute faoInstitute : organizationService.getMembers(organization)) {
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", organization);
continue;
}
stats.updateMin((Number) overall[0]);
stats.updateMax((Number) overall[1]);
stats.updateCountAndAvg((Number) overall[3], (Number) overall[2]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
stats.merge(faoInstitute.getStatisticsPDCI());
}
return stats;
}
......@@ -1507,6 +1465,35 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return stats;
}
@Transactional(readOnly = false)
@Override
public FaoInstitute updatePDCI(FaoInstitute faoInstitute) {
// Reload
faoInstitute = instituteRepository.findOne(faoInstitute.getId());
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null || overall[3] == null || ((Number) overall[3]).longValue() == 0) {
LOG.warn("Clearing PDCI statistics for {}", faoInstitute);
faoInstitute.setPdciMin(null);
faoInstitute.setPdciMax(null);
faoInstitute.setPdciAvg(null);
faoInstitute.setPdciHistogram(null);
} else {
faoInstitute.setPdciMin(((Number) overall[0]).doubleValue());
faoInstitute.setPdciMax(((Number) overall[1]).doubleValue());
faoInstitute.setPdciAvg(((Number) overall[2]).doubleValue());
PDCIStatistics pdciStatistics = new PDCIStatistics();
List<Object[]> histogram = repoPdci.histogram(faoInstitute);
pdciStatistics.makeHistogram(histogram);
faoInstitute.setPdciHistogram(Arrays.toString(pdciStatistics.getHistogram()));
}
return instituteRepository.save(faoInstitute);
}
@Transactional
@Override
public void regenerateAccessionSequentialNumber() {
......
......@@ -21,6 +21,7 @@ import org.genesys2.server.model.genesys.PhenoStatistics;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.persistence.PDCIRepository;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
......@@ -32,6 +33,8 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@Service
@Transactional(readOnly = true)
public class StatisticsServiceImpl implements StatisticsService {
......@@ -49,6 +52,9 @@ public class StatisticsServiceImpl implements StatisticsService {
@Autowired
private DatasetService datasetService;
@Autowired
private PDCIRepository repoPdci;
@Override
@Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
public long numberOfCountries() {
......@@ -85,17 +91,6 @@ public class StatisticsServiceImpl implements StatisticsService {
return datasetService.countPublished();
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
if (LOG.isDebugEnabled()) {
LOG.debug("Regenerating PDCI statistics for {}", faoInstitute);
}
synchronized (this) {
return genesysService.statisticsPDCI(faoInstitute);
}
}
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-org' + #organization.id")
public PDCIStatistics statisticsPDCI(Organization organization) {
......
......@@ -111,9 +111,13 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
}
FaoInstitute institute = instituteService.findInstitute(forProcessing.getObj());
if (institute != null) {
LOG.info("Updating count for {}", institute.getCode());
genesysService.updateAccessionCount(institute);
LOG.info("Updating PDCI for {}", institute.getCode());
genesysService.updatePDCI(institute);
}
}
......
......@@ -3767,3 +3767,24 @@ databaseChangeLog:
tableName: subset
columnName: published
- changeSet:
id: 1533127815588-66
author: aprendetskiy
comment: Add new columns to faoinstitute table
changes:
- addColumn:
tableName: faoinstitute
columns:
- column:
name: pdciMin
type: double
- column:
name: pdciMax
type: double
- column:
name: pdciAvg
type: double
- column:
name: pdciHistogram
type: VARCHAR(255)
\ No newline at end of file
......@@ -22,8 +22,9 @@
</form>
<form method="post" action="<c:url value="/admin/pdci" />">
<button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
<button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button>
<input type="submit" class="btn btn-default" name="updatePdciStats" value="Update all institute PDCI stats" />
<%-- <button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
<button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button> --%>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
......
......@@ -129,24 +129,41 @@ public class PDCIStatisticsTest {
PDCIStatistics stats = new PDCIStatistics();
/*@formatter:off*/
List<Object[]> hist = Lists.newArrayList(
new Object[] { .2, 1d }, new Object[] { .5, 5d },
new Object[] { .5, 1d }, new Object[] { 1, 5d },
new Object[] { 1, 10 }, new Object[] { 2, 20 },
new Object[] { 3f, 30 }, new Object[] { 4d, 40d },
new Object[] { 5d, 50 }, new Object[] { 6f, 60f },
new Object[] { 7, 70f }, new Object[] { 8d, 80f },
new Object[] { 9f, 40d }, new Object[] { 10, 50 },
// extras
new Object[] { 9.1f, 50d }, new Object[] { 9.9, 50 }
new Object[] { 9f, 40d }, new Object[] { 10, 50 }
);
/*@formatter:on*/
stats.makeHistogram(hist);
assertThat(stats.getHistogram().length, is(21));
assertThat(stats.getHistogram()[9*2], is(90l));
assertThat(stats.getHistogram()[9*2], is(40l));
assertThat(stats.getHistogram()[(int) (2*9.5)], is(0l));
assertThat(stats.getHistogram()[10*2], is(100l));
assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100]"));
assertThat(stats.getHistogram()[10*2], is(50l));
assertThat(Arrays.toString(stats.getHistogram()), is("[0, 1, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 40, 0, 50]"));
}
/**
* All values from updateHistogram should be added to current values in histogram
*/
@Test
public void testUpdateHistogram() {
PDCIStatistics stats = new PDCIStatistics();
long[] histogram = new long[] {1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100};
stats.setHistogram(histogram);
long[] updatedHistogram = new long[] {0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0};
stats.updateHistogram(updatedHistogram);
// check merged histogram value
assertThat(stats.getHistogram().length, is(21));
assertThat(stats.getHistogram()[0], is(1l));
assertThat(stats.getHistogram()[8], is(45l));
assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 45, 0, 55, 0, 62, 0, 70, 0, 80, 0, 90, 0, 100]"));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment