Commit 47537a5e authored by Alexander Prendetskiy's avatar Alexander Prendetskiy Committed by Matija Obreza

PDCI statistics for Institute

- add button on admin page for updating pdci stats
- add unit test for updateHistogram method
- add PDCIStatistics to FaoInstitute
parent 31af4688
...@@ -138,7 +138,7 @@ public class InstituteController { ...@@ -138,7 +138,7 @@ public class InstituteController {
InstituteDetails details = new InstituteDetails(); InstituteDetails details = new InstituteDetails();
details.details = faoInstitute; details.details = faoInstitute;
details.blurb = contentService.getArticle(faoInstitute, "blurp", getLocale()); details.blurb = contentService.getArticle(faoInstitute, "blurp", getLocale());
details.pdciStats = statisticsService.statisticsPDCI(faoInstitute); details.pdciStats = faoInstitute.getStatisticsPDCI();
details.lastUpdates = genesysService.getLastUpdatedStatistics(faoInstitute); details.lastUpdates = genesysService.getLastUpdatedStatistics(faoInstitute);
details.overview= getOverviewData(byInstituteFilter); details.overview= getOverviewData(byInstituteFilter);
......
...@@ -69,6 +69,10 @@ public class PDCIStatistics implements Serializable { ...@@ -69,6 +69,10 @@ public class PDCIStatistics implements Serializable {
return histogram; return histogram;
} }
public void setHistogram(long[] histogram) {
this.histogram = histogram;
}
/** /**
* Converts {@link #histogram} to flot.categories compatible JSON * Converts {@link #histogram} to flot.categories compatible JSON
* *
...@@ -120,6 +124,16 @@ public class PDCIStatistics implements Serializable { ...@@ -120,6 +124,16 @@ public class PDCIStatistics implements Serializable {
this.count = this.count==null ? count.longValue() : this.count + count.longValue(); this.count = this.count==null ? count.longValue() : this.count + count.longValue();
} }
public void updateHistogram(long[] updateHistogram) {
if (updateHistogram == null) {
return;
}
for (int i = 0; i < this.histogram.length; i++) {
this.histogram[i] += updateHistogram[i];
}
}
/** /**
* Return object array to be used for i18n * Return object array to be used for i18n
* *
...@@ -128,4 +142,13 @@ public class PDCIStatistics implements Serializable { ...@@ -128,4 +142,13 @@ public class PDCIStatistics implements Serializable {
public Object[] getElStats() { public Object[] getElStats() {
return new Object[] { count, avg, min, max }; return new Object[] { count, avg, min, max };
} }
public PDCIStatistics merge(PDCIStatistics other) {
updateMin(other.getMin());
updateMax(other.getMax());
updateCountAndAvg(other.getCount(), other.getAvg());
updateHistogram(other.getHistogram());
return this;
}
} }
...@@ -16,31 +16,22 @@ ...@@ -16,31 +16,22 @@
package org.genesys2.server.model.impl; package org.genesys2.server.model.impl;
import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.persistence.Cacheable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.MapKey;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import javax.persistence.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.BasicModel; import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.EntityId; import org.genesys.blocks.model.EntityId;
import org.genesys.blocks.model.JsonViews; import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.model.AclAwareModel; import org.genesys.blocks.security.model.AclAwareModel;
import org.genesys.custom.elasticsearch.IgnoreField; import org.genesys.custom.elasticsearch.IgnoreField;
import org.genesys2.server.model.genesys.PDCIStatistics;
import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex; import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.FieldType;
...@@ -55,6 +46,8 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl ...@@ -55,6 +46,8 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private static final long serialVersionUID = -8773002513838748431L; private static final long serialVersionUID = -8773002513838748431L;
private static final int LEN_ACRONYM = 50; private static final int LEN_ACRONYM = 50;
private static final ObjectMapper mapper = new ObjectMapper();
@Column(unique = true, nullable = false, length = 10) @Column(unique = true, nullable = false, length = 10)
@Field(index = FieldIndex.not_analyzed) @Field(index = FieldIndex.not_analyzed)
...@@ -109,6 +102,12 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl ...@@ -109,6 +102,12 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
private Double elevation; private Double elevation;
private boolean uniqueAcceNumbs = false; private boolean uniqueAcceNumbs = false;
private boolean allowMaterialRequests = false; private boolean allowMaterialRequests = false;
private Double pdciMin;
private Double pdciMax;
private Double pdciAvg;
@Column(name = "pdciHistogram")
private String pdciHistogram;
public FaoInstitute() { public FaoInstitute() {
} }
...@@ -278,6 +277,65 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl ...@@ -278,6 +277,65 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
this.codeSGSV = codeSGSV; this.codeSGSV = codeSGSV;
} }
public Double getPdciMin() {
return pdciMin;
}
public void setPdciMin(Double pdciMin) {
this.pdciMin = pdciMin;
}
public Double getPdciMax() {
return pdciMax;
}
public void setPdciMax(Double pdciMax) {
this.pdciMax = pdciMax;
}
public Double getPdciAvg() {
return pdciAvg;
}
public void setPdciAvg(Double pdciAvg) {
this.pdciAvg = pdciAvg;
}
public String getPdciHistogram() {
return pdciHistogram;
}
public void setPdciHistogram(String pdciHistogram) {
this.pdciHistogram = pdciHistogram;
}
public PDCIStatistics getStatisticsPDCI() {
PDCIStatistics stats = new PDCIStatistics();
stats.setMin(this.pdciMin);
stats.setMax(this.pdciMax);
stats.setCount(this.accessionCount);
stats.setAvg(this.pdciAvg);
stats.setHistogram(parsePdciHistogram());
return stats;
}
public long[] parsePdciHistogram() {
long[] histogram = new long[21];
if (StringUtils.isBlank(this.pdciHistogram)) {
return histogram;
}
try {
histogram = mapper.readValue(this.pdciHistogram, long[].class);
} catch (IOException e) {
e.printStackTrace();
}
return histogram;
}
/** /**
* Transitive * Transitive
* *
......
...@@ -47,7 +47,6 @@ import org.genesys2.server.service.FilterConstants; ...@@ -47,7 +47,6 @@ import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.InstituteFilesService; import org.genesys2.server.service.InstituteFilesService;
import org.genesys2.server.service.InstituteService; import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.StatisticsService;
import org.genesys2.server.service.TaxonomyService; import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TraitService; import org.genesys2.server.service.TraitService;
import org.genesys2.server.service.impl.NonUniqueAccessionException; import org.genesys2.server.service.impl.NonUniqueAccessionException;
...@@ -89,9 +88,6 @@ public class AccessionController extends BaseController { ...@@ -89,9 +88,6 @@ public class AccessionController extends BaseController {
@Autowired @Autowired
private DSService dsService; private DSService dsService;
@Autowired
private StatisticsService statisticsService;
@Autowired @Autowired
private InstituteFilesService instituteFilesService; private InstituteFilesService instituteFilesService;
...@@ -200,7 +196,7 @@ public class AccessionController extends BaseController { ...@@ -200,7 +196,7 @@ public class AccessionController extends BaseController {
} }
model.addAttribute("pdci", accessionId.getPdci()); model.addAttribute("pdci", accessionId.getPdci());
model.addAttribute("institutePDCI", statisticsService.statisticsPDCI(accession.getInstitute())); model.addAttribute("institutePDCI", accession.getInstitute().getStatisticsPDCI());
ImageGallery imageGallery = null; ImageGallery imageGallery = null;
try { try {
...@@ -279,7 +275,6 @@ public class AccessionController extends BaseController { ...@@ -279,7 +275,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy * View by Taxonomy
* *
* @param model * @param model
* @param wiewsCode
* @param genus * @param genus
* @param page * @param page
* @return * @return
...@@ -304,7 +299,6 @@ public class AccessionController extends BaseController { ...@@ -304,7 +299,6 @@ public class AccessionController extends BaseController {
* View by Taxonomy * View by Taxonomy
* *
* @param model * @param model
* @param wiewsCode
* @param genus * @param genus
* @param species * @param species
* @param page * @param page
......
...@@ -193,7 +193,7 @@ public class WiewsController extends BaseController { ...@@ -193,7 +193,7 @@ public class WiewsController extends BaseController {
model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5))); model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5))); model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5))); model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5)));
model.addAttribute("statisticsPDCI", statisticsService.statisticsPDCI(faoInstitute)); model.addAttribute("statisticsPDCI", faoInstitute.getStatisticsPDCI());
model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute)); model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute));
if (datasetCount > 0) { if (datasetCount > 0) {
...@@ -211,7 +211,7 @@ public class WiewsController extends BaseController { ...@@ -211,7 +211,7 @@ public class WiewsController extends BaseController {
if (institute == null) { if (institute == null) {
throw new Exception("Institute with instCode=" + instCode + " doesn't exist."); throw new Exception("Institute with instCode=" + instCode + " doesn't exist.");
} }
final PDCIStatistics pdciStatistics = statisticsService.statisticsPDCI(institute); final PDCIStatistics pdciStatistics = institute.getStatisticsPDCI();
final List<Object[]> updatedStats = genesysService.getLastUpdatedStatistics(institute, false); final List<Object[]> updatedStats = genesysService.getLastUpdatedStatistics(institute, false);
final Long activeAccessions = institute.getAccessionCount(); final Long activeAccessions = institute.getAccessionCount();
final Long activeAccessionsWithDoi = genesysService.countByInstituteWithDoi(institute); final Long activeAccessionsWithDoi = genesysService.countByInstituteWithDoi(institute);
......
...@@ -29,6 +29,7 @@ import javax.xml.parsers.ParserConfigurationException; ...@@ -29,6 +29,7 @@ import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch; import org.apache.commons.lang.time.StopWatch;
import org.genesys.catalog.service.DatasetService; import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionRepository; import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.ContentService; import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.CountryNamesUpdater; import org.genesys2.server.service.CountryNamesUpdater;
...@@ -45,6 +46,7 @@ import org.slf4j.Logger; ...@@ -45,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.ui.Model; import org.springframework.ui.Model;
...@@ -210,6 +212,15 @@ public class AdminController { ...@@ -210,6 +212,15 @@ public class AdminController {
return "redirect:/admin/"; return "redirect:/admin/";
} }
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "updatePdciStats")
public String updatePDCI() {
for (FaoInstitute institute: instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
genesysService.updatePDCI(institute);
}
return "redirect:/admin/";
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion") @RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException { public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
geoRegionService.updateGeoRegionData(); geoRegionService.updateGeoRegionData();
......
...@@ -144,6 +144,8 @@ public interface GenesysService { ...@@ -144,6 +144,8 @@ public interface GenesysService {
void updateAccessionCount(FaoInstitute institute); void updateAccessionCount(FaoInstitute institute);
void updatePDCI(FaoInstitute institute);
List<SvalbardDeposit> getSvalbardData(AccessionId accession); List<SvalbardDeposit> getSvalbardData(AccessionId accession);
void refreshMetadataMethods(); void refreshMetadataMethods();
...@@ -225,8 +227,6 @@ public interface GenesysService { ...@@ -225,8 +227,6 @@ public interface GenesysService {
AccessionHistoric getHistoricAccession(UUID uuid); AccessionHistoric getHistoricAccession(UUID uuid);
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
List<PDCI> loadPDCI(List<Long> batch); List<PDCI> loadPDCI(List<Long> batch);
PDCIStatistics statisticsPDCI(Organization organization); PDCIStatistics statisticsPDCI(Organization organization);
......
...@@ -35,8 +35,6 @@ public interface StatisticsService { ...@@ -35,8 +35,6 @@ public interface StatisticsService {
long numberOfPublishedDatasets(); long numberOfPublishedDatasets();
PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
PDCIStatistics statisticsPDCI(Organization organization); PDCIStatistics statisticsPDCI(Organization organization);
PhenoStatistics statisticsPheno(FaoInstitute faoInstitute); PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);
......
...@@ -22,16 +22,7 @@ import java.io.OutputStream; ...@@ -22,16 +22,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
...@@ -1426,45 +1417,12 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { ...@@ -1426,45 +1417,12 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return repoPdci.findByAccessionId(accessionIds); return repoPdci.findByAccessionId(accessionIds);
} }
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
PDCIStatistics stats = new PDCIStatistics();
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", faoInstitute);
return null;
}
stats.setMin((Number) overall[0]);
stats.setMax((Number) overall[1]);
stats.setAvg((Number) overall[2]);
stats.setCount((Number) overall[3]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
return stats;
}
@Override @Override
public PDCIStatistics statisticsPDCI(Organization organization) { public PDCIStatistics statisticsPDCI(Organization organization) {
PDCIStatistics stats = new PDCIStatistics(); PDCIStatistics stats = new PDCIStatistics();
for (FaoInstitute faoInstitute : organizationService.getMembers(organization)) { for (FaoInstitute faoInstitute : organizationService.getMembers(organization)) {
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute); stats.merge(faoInstitute.getStatisticsPDCI());
if (overall == null) {
LOG.warn("No PDCI statistics for {}", organization);
continue;
}
stats.updateMin((Number) overall[0]);
stats.updateMax((Number) overall[1]);
stats.updateCountAndAvg((Number) overall[3], (Number) overall[2]);
List<Object[]> hist = repoPdci.histogram(faoInstitute);
stats.makeHistogram(hist);
} }
return stats; return stats;
} }
...@@ -1507,6 +1465,28 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { ...@@ -1507,6 +1465,28 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return stats; return stats;
} }
@Transactional
@Override
public void updatePDCI(FaoInstitute faoInstitute) {
Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
if (overall == null) {
LOG.warn("No PDCI statistics for {}", faoInstitute);
return;
}
List<Object[]> histogram = repoPdci.histogram(faoInstitute);
PDCIStatistics pdciStatistics = new PDCIStatistics();
pdciStatistics.makeHistogram(histogram);
faoInstitute.setPdciMin((Double) overall[0]);
faoInstitute.setPdciMax((Double) overall[1]);
faoInstitute.setPdciAvg((Double) overall[2]);
faoInstitute.setPdciHistogram(Arrays.toString(pdciStatistics.getHistogram()));
instituteRepository.save(faoInstitute);
}
@Transactional @Transactional
@Override @Override
public void regenerateAccessionSequentialNumber() { public void regenerateAccessionSequentialNumber() {
......
...@@ -21,6 +21,7 @@ import org.genesys2.server.model.genesys.PhenoStatistics; ...@@ -21,6 +21,7 @@ import org.genesys2.server.model.genesys.PhenoStatistics;
import org.genesys2.server.model.impl.FaoInstitute; import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization; import org.genesys2.server.model.impl.Organization;
import org.genesys.catalog.service.DatasetService; import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.persistence.PDCIRepository;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService; import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService; import org.genesys2.server.service.InstituteService;
...@@ -32,6 +33,8 @@ import org.springframework.cache.annotation.Cacheable; ...@@ -32,6 +33,8 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@Service @Service
@Transactional(readOnly = true) @Transactional(readOnly = true)
public class StatisticsServiceImpl implements StatisticsService { public class StatisticsServiceImpl implements StatisticsService {
...@@ -49,6 +52,9 @@ public class StatisticsServiceImpl implements StatisticsService { ...@@ -49,6 +52,9 @@ public class StatisticsServiceImpl implements StatisticsService {
@Autowired @Autowired
private DatasetService datasetService; private DatasetService datasetService;
@Autowired
private PDCIRepository repoPdci;
@Override @Override
@Cacheable(value = "statistics", key = "'stats.' + #root.methodName") @Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
public long numberOfCountries() { public long numberOfCountries() {
...@@ -85,17 +91,6 @@ public class StatisticsServiceImpl implements StatisticsService { ...@@ -85,17 +91,6 @@ public class StatisticsServiceImpl implements StatisticsService {
return datasetService.countPublished(); return datasetService.countPublished();
} }
@Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
if (LOG.isDebugEnabled()) {
LOG.debug("Regenerating PDCI statistics for {}", faoInstitute);
}
synchronized (this) {
return genesysService.statisticsPDCI(faoInstitute);
}
}
@Override @Override
@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-org' + #organization.id") @Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-org' + #organization.id")
public PDCIStatistics statisticsPDCI(Organization organization) { public PDCIStatistics statisticsPDCI(Organization organization) {
......
...@@ -111,9 +111,13 @@ public class AccessionCounter implements InitializingBean, DisposableBean { ...@@ -111,9 +111,13 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
} }
FaoInstitute institute = instituteService.findInstitute(forProcessing.getObj()); FaoInstitute institute = instituteService.findInstitute(forProcessing.getObj());
if (institute != null) { if (institute != null) {
LOG.info("Updating count for {}", institute.getCode()); LOG.info("Updating count for {}", institute.getCode());
genesysService.updateAccessionCount(institute); genesysService.updateAccessionCount(institute);
LOG.info("Updating PDCI for {}", institute.getCode());
genesysService.updatePDCI(institute);
} }
} }
......
...@@ -3767,3 +3767,24 @@ databaseChangeLog: ...@@ -3767,3 +3767,24 @@ databaseChangeLog:
tableName: subset tableName: subset
columnName: published columnName: published
- changeSet: