PDCI statistics for Institute

- add button on admin page for updating pdci stats
- add unit test for updateHistogram method
- add PDCIStatistics to FaoInstitute

src/main/java/org/genesys2/server/api/v1/InstituteController.java

@@ -138,7 +138,7 @@ public class InstituteController {
         InstituteDetails details = new InstituteDetails();
         details.details = faoInstitute;
         details.blurb = contentService.getArticle(faoInstitute, "blurp", getLocale());
-        details.pdciStats = statisticsService.statisticsPDCI(faoInstitute);
+        details.pdciStats = faoInstitute.getStatisticsPDCI();
         details.lastUpdates = genesysService.getLastUpdatedStatistics(faoInstitute);
 		details.overview= getOverviewData(byInstituteFilter);
 

src/main/java/org/genesys2/server/model/genesys/PDCIStatistics.java

@@ -69,6 +69,10 @@ public class PDCIStatistics implements Serializable {
 		return histogram;
 	}
 
+	public void setHistogram(long[] histogram) {
+		this.histogram = histogram;
+	}
+
 	/**
 	 * Converts {@link #histogram} to flot.categories compatible JSON
 	 * 
@@ -120,6 +124,16 @@ public class PDCIStatistics implements Serializable {
 		this.count = this.count==null ? count.longValue() : this.count + count.longValue();
 	}
 
+	public void updateHistogram(long[] updateHistogram) {
+		if (updateHistogram == null) {
+			return;
+		}
+
+		for (int i = 0; i < this.histogram.length; i++) {
+			this.histogram[i] += updateHistogram[i];
+		}
+	}
+
 	/**
 	 * Return object array to be used for i18n
 	 * 
@@ -128,4 +142,13 @@ public class PDCIStatistics implements Serializable {
 	public Object[] getElStats() {
 		return new Object[] { count, avg, min, max };
 	}
+
+	public PDCIStatistics merge(PDCIStatistics other) {
+		updateMin(other.getMin());
+		updateMax(other.getMax());
+		updateCountAndAvg(other.getCount(), other.getAvg());
+		updateHistogram(other.getHistogram());
+
+		return this;
+	}
 }

src/main/java/org/genesys2/server/model/impl/FaoInstitute.java

@@ -16,31 +16,22 @@
 
 package org.genesys2.server.model.impl;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.persistence.Cacheable;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Index;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.MapKey;
-import javax.persistence.OneToMany;
-import javax.persistence.Table;
-import javax.persistence.UniqueConstraint;
+import java.util.*;
 
+import javax.persistence.*;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.lang3.StringUtils;
 import org.genesys.blocks.model.BasicModel;
 import org.genesys.blocks.model.EntityId;
 import org.genesys.blocks.model.JsonViews;
 import org.genesys.blocks.security.model.AclAwareModel;
 import org.genesys.custom.elasticsearch.IgnoreField;
+import org.genesys2.server.model.genesys.PDCIStatistics;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldIndex;
 import org.springframework.data.elasticsearch.annotations.FieldType;
@@ -55,6 +46,8 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
 	private static final long serialVersionUID = -8773002513838748431L;
 
 	private static final int LEN_ACRONYM = 50;
+	
+	private static final ObjectMapper mapper = new ObjectMapper();
 
 	@Column(unique = true, nullable = false, length = 10)
 	@Field(index = FieldIndex.not_analyzed)
@@ -109,6 +102,12 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
 	private Double elevation;
 	private boolean uniqueAcceNumbs = false;
 	private boolean allowMaterialRequests = false;
+	private Double pdciMin;
+	private Double pdciMax;
+	private Double pdciAvg;
+	@Column(name = "pdciHistogram")
+	private String pdciHistogram;
+
 
 	public FaoInstitute() {
 	}
@@ -278,6 +277,65 @@ public class FaoInstitute extends BasicModel implements GeoReferencedEntity, Acl
 		this.codeSGSV = codeSGSV;
 	}
 
+	public Double getPdciMin() {
+		return pdciMin;
+	}
+
+	public void setPdciMin(Double pdciMin) {
+		this.pdciMin = pdciMin;
+	}
+
+	public Double getPdciMax() {
+		return pdciMax;
+	}
+
+	public void setPdciMax(Double pdciMax) {
+		this.pdciMax = pdciMax;
+	}
+
+	public Double getPdciAvg() {
+		return pdciAvg;
+	}
+
+	public void setPdciAvg(Double pdciAvg) {
+		this.pdciAvg = pdciAvg;
+	}
+
+	public String getPdciHistogram() {
+		return pdciHistogram;
+	}
+
+	public void setPdciHistogram(String pdciHistogram) {
+		this.pdciHistogram = pdciHistogram;
+	}
+
+	public PDCIStatistics getStatisticsPDCI() {
+		PDCIStatistics stats = new PDCIStatistics();
+		stats.setMin(this.pdciMin);
+		stats.setMax(this.pdciMax);
+		stats.setCount(this.accessionCount);
+		stats.setAvg(this.pdciAvg);
+		stats.setHistogram(parsePdciHistogram());
+
+		return stats;
+	}
+
+	public long[] parsePdciHistogram() {
+		long[] histogram = new long[21];
+
+		if (StringUtils.isBlank(this.pdciHistogram)) {
+			return histogram;
+		}
+
+		try {
+			histogram = mapper.readValue(this.pdciHistogram, long[].class);
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		return histogram;
+	}
+
 	/**
 	 * Transitive
 	 * 

src/main/java/org/genesys2/server/mvc/AccessionController.java

@@ -47,7 +47,6 @@ import org.genesys2.server.service.FilterConstants;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.InstituteFilesService;
 import org.genesys2.server.service.InstituteService;
-import org.genesys2.server.service.StatisticsService;
 import org.genesys2.server.service.TaxonomyService;
 import org.genesys2.server.service.TraitService;
 import org.genesys2.server.service.impl.NonUniqueAccessionException;
@@ -89,9 +88,6 @@ public class AccessionController extends BaseController {
 	@Autowired
 	private DSService dsService;
 
-	@Autowired
-	private StatisticsService statisticsService;
-
 	@Autowired
 	private InstituteFilesService instituteFilesService;
 
@@ -200,7 +196,7 @@ public class AccessionController extends BaseController {
 		}
 
 		model.addAttribute("pdci", accessionId.getPdci());
-		model.addAttribute("institutePDCI", statisticsService.statisticsPDCI(accession.getInstitute()));
+		model.addAttribute("institutePDCI", accession.getInstitute().getStatisticsPDCI());
 
 		ImageGallery imageGallery = null;
 		try {
@@ -279,7 +275,6 @@ public class AccessionController extends BaseController {
 	 * View by Taxonomy
 	 * 
 	 * @param model
-	 * @param wiewsCode
 	 * @param genus
 	 * @param page
 	 * @return
@@ -304,7 +299,6 @@ public class AccessionController extends BaseController {
 	 * View by Taxonomy
 	 * 
 	 * @param model
-	 * @param wiewsCode
 	 * @param genus
 	 * @param species
 	 * @param page

src/main/java/org/genesys2/server/mvc/WiewsController.java

@@ -193,7 +193,7 @@ public class WiewsController extends BaseController {
 		model.addAttribute("statisticsCropName", genesysService.statisticsCropNameByInstitute(faoInstitute, new PageRequest(0, 5)));
 		model.addAttribute("statisticsGenus", genesysService.statisticsGenusByInstitute(faoInstitute, new PageRequest(0, 5)));
 		model.addAttribute("statisticsTaxonomy", genesysService.statisticsSpeciesByInstitute(faoInstitute, new PageRequest(0, 5)));
-		model.addAttribute("statisticsPDCI", statisticsService.statisticsPDCI(faoInstitute));
+		model.addAttribute("statisticsPDCI", faoInstitute.getStatisticsPDCI());
 		model.addAttribute("updates", genesysService.getLastUpdatedStatistics(faoInstitute));
 
 		if (datasetCount > 0) {
@@ -211,7 +211,7 @@ public class WiewsController extends BaseController {
       if (institute == null) {
           throw new Exception("Institute with instCode=" + instCode + " doesn't exist.");
       }
-      final PDCIStatistics pdciStatistics = statisticsService.statisticsPDCI(institute);
+      final PDCIStatistics pdciStatistics = institute.getStatisticsPDCI();
       final List<Object[]> updatedStats = genesysService.getLastUpdatedStatistics(institute, false);
       final Long activeAccessions = institute.getAccessionCount();
       final Long activeAccessionsWithDoi = genesysService.countByInstituteWithDoi(institute);

src/main/java/org/genesys2/server/mvc/admin/AdminController.java

@@ -29,6 +29,7 @@ import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.commons.lang.time.StopWatch;
 import org.genesys.catalog.service.DatasetService;
+import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.persistence.AccessionRepository;
 import org.genesys2.server.service.ContentService;
 import org.genesys2.server.service.CountryNamesUpdater;
@@ -45,6 +46,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.task.TaskExecutor;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
@@ -210,6 +212,15 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
+	@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "updatePdciStats")
+	public String updatePDCI() {
+		for (FaoInstitute institute: instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
+			genesysService.updatePDCI(institute);
+		}
+
+		return "redirect:/admin/";
+	}
+
 	@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
 	public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
 		geoRegionService.updateGeoRegionData();

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -144,6 +144,8 @@ public interface GenesysService {
 
 	void updateAccessionCount(FaoInstitute institute);
 
+	void updatePDCI(FaoInstitute institute);
+
 	List<SvalbardDeposit> getSvalbardData(AccessionId accession);
 
 	void refreshMetadataMethods();
@@ -225,8 +227,6 @@ public interface GenesysService {
 
 	AccessionHistoric getHistoricAccession(UUID uuid);
 
-	PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
-
 	List<PDCI> loadPDCI(List<Long> batch);
 
 	PDCIStatistics statisticsPDCI(Organization organization);

src/main/java/org/genesys2/server/service/StatisticsService.java

@@ -35,8 +35,6 @@ public interface StatisticsService {
 
 	long numberOfPublishedDatasets();
 
-	PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute);
-
 	PDCIStatistics statisticsPDCI(Organization organization);
 
 	PhenoStatistics statisticsPheno(FaoInstitute faoInstitute);

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -22,16 +22,7 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -1426,45 +1417,12 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		return repoPdci.findByAccessionId(accessionIds);
 	}
 
-	@Override
-	@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
-	public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
-		PDCIStatistics stats = new PDCIStatistics();
-
-		Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
-		if (overall == null) {
-			LOG.warn("No PDCI statistics for {}", faoInstitute);
-			return null;
-		}
-
-		stats.setMin((Number) overall[0]);
-		stats.setMax((Number) overall[1]);
-		stats.setAvg((Number) overall[2]);
-		stats.setCount((Number) overall[3]);
-
-		List<Object[]> hist = repoPdci.histogram(faoInstitute);
-		stats.makeHistogram(hist);
-
-		return stats;
-	}
-
 	@Override
 	public PDCIStatistics statisticsPDCI(Organization organization) {
 		PDCIStatistics stats = new PDCIStatistics();
 
 		for (FaoInstitute faoInstitute : organizationService.getMembers(organization)) {
-			Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
-			if (overall == null) {
-				LOG.warn("No PDCI statistics for {}", organization);
-				continue;
-			}
-
-			stats.updateMin((Number) overall[0]);
-			stats.updateMax((Number) overall[1]);
-			stats.updateCountAndAvg((Number) overall[3], (Number) overall[2]);
-
-			List<Object[]> hist = repoPdci.histogram(faoInstitute);
-			stats.makeHistogram(hist);
+			stats.merge(faoInstitute.getStatisticsPDCI());
 		}
 		return stats;
 	}
@@ -1507,6 +1465,28 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		return stats;
 	}
 
+	@Transactional
+	@Override
+	public void updatePDCI(FaoInstitute faoInstitute) {
+		Object[] overall = (Object[]) repoPdci.statistics(faoInstitute);
+		if (overall == null) {
+			LOG.warn("No PDCI statistics for {}", faoInstitute);
+			return;
+		}
+
+		List<Object[]> histogram = repoPdci.histogram(faoInstitute);
+
+		PDCIStatistics pdciStatistics = new PDCIStatistics();
+		pdciStatistics.makeHistogram(histogram);
+
+		faoInstitute.setPdciMin((Double) overall[0]);
+		faoInstitute.setPdciMax((Double) overall[1]);
+		faoInstitute.setPdciAvg((Double) overall[2]);
+		faoInstitute.setPdciHistogram(Arrays.toString(pdciStatistics.getHistogram()));
+
+		instituteRepository.save(faoInstitute);
+	}
+
 	@Transactional
 	@Override
 	public void regenerateAccessionSequentialNumber() {

src/main/java/org/genesys2/server/service/impl/StatisticsServiceImpl.java

@@ -21,6 +21,7 @@ import org.genesys2.server.model.genesys.PhenoStatistics;
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.model.impl.Organization;
 import org.genesys.catalog.service.DatasetService;
+import org.genesys2.server.persistence.PDCIRepository;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.GeoService;
 import org.genesys2.server.service.InstituteService;
@@ -32,6 +33,8 @@ import org.springframework.cache.annotation.Cacheable;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
+import java.util.List;
+
 @Service
 @Transactional(readOnly = true)
 public class StatisticsServiceImpl implements StatisticsService {
@@ -49,6 +52,9 @@ public class StatisticsServiceImpl implements StatisticsService {
 	@Autowired
 	private DatasetService datasetService;
 
+    @Autowired
+    private PDCIRepository repoPdci;
+
 	@Override
 	@Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
 	public long numberOfCountries() {
@@ -85,17 +91,6 @@ public class StatisticsServiceImpl implements StatisticsService {
 		return datasetService.countPublished();
 	}
 
-	@Override
-	@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-' + #faoInstitute.id")
-	public PDCIStatistics statisticsPDCI(FaoInstitute faoInstitute) {
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Regenerating PDCI statistics for {}", faoInstitute);
-		}
-		synchronized (this) {
-			return genesysService.statisticsPDCI(faoInstitute);
-		}
-	}
-
 	@Override
 	@Cacheable(unless = "#result == null", value = "statistics", key = "'stats.' + #root.methodName + '-org' + #organization.id")
 	public PDCIStatistics statisticsPDCI(Organization organization) {

src/main/java/org/genesys2/server/service/worker/AccessionCounter.java

@@ -111,9 +111,13 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
 		}
 
 		FaoInstitute institute = instituteService.findInstitute(forProcessing.getObj());
+
 		if (institute != null) {
 			LOG.info("Updating count for {}", institute.getCode());
 			genesysService.updateAccessionCount(institute);
+			
+			LOG.info("Updating PDCI for {}", institute.getCode());
+			genesysService.updatePDCI(institute);
 		}
 	}
 

src/main/resources/liquibase/liquibase-changeLog.yml

@@ -3767,3 +3767,24 @@ databaseChangeLog:
             tableName: subset
             columnName:  published
 
+
+- changeSet:
+    id:                    1533127815588-66
+    author:                aprendetskiy
+    comment:               Add new columns to faoinstitute table
+    changes:
+    - addColumn:
+        tableName:       faoinstitute
+        columns:
+        - column:
+            name:        pdciMin
+            type:        double
+        - column:
+            name:        pdciMax
+            type:        double
+        - column:
+            name:        pdciAvg
+            type:        double
+        - column:
+            name:        pdciHistogram
+            type:        VARCHAR(255)
\ No newline at end of file

src/main/webapp/WEB-INF/jsp/accession/details.jsp

@@ -998,8 +998,8 @@
                       <spring:message code="accession.pdci.jumbo" arguments="${pdci.score}"/>
                     </h3>
                     <small>
-                      <c:if test="${institutePDCI ne null and institutePDCI.count > 0}">
-                        <spring:message code="accession.pdci.institute-avg" arguments="${institutePDCI.avg}"/>
+                      <c:if test="${institutePDCI ne null and institutePDCI.accessionCount > 0}">
+                        <spring:message code="accession.pdci.institute-avg" arguments="${institutePDCI.pdciAvg}"/>
                       </c:if>
                       <a href="<c:url value=" /content/passport-data-completeness-index" />"><spring:message code="accession.pdci.about-link"/></a>
                     </small>

src/main/webapp/WEB-INF/jsp/admin/index.jsp

@@ -22,6 +22,7 @@
   </form>
 
   <form method="post" action="<c:url value="/admin/pdci" />">
+    <input type="submit" class="btn btn-default" name="updatePdciStats" value="Update all institute PDCI stats" />
     <button name="action" value="missing-pdci" class="btn btn-default">Calculate missing PDCI</button>
     <button name="action" value="full-recalc" class="btn btn-default">Clean and recalculate</button>
     <!-- CSRF protection -->

src/test/java/org/genesys/test/simpletest/PDCIStatisticsTest.java

@@ -149,4 +149,24 @@ public class PDCIStatisticsTest {
 		assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100]"));
 	}
 
+	/**
+	 *  All values from updateHistogram should be added to current values in histogram
+	 */
+	@Test
+	public void testUpdateHistogram() {
+		PDCIStatistics stats = new PDCIStatistics();
+
+		long[] histogram = new long[] {1, 5, 10, 0, 20, 0, 30, 0, 40, 0, 50, 0, 60, 0, 70, 0, 80, 0, 90, 0, 100};
+		stats.setHistogram(histogram);
+
+		long[] updatedHistogram = new long[] {0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0};
+		stats.updateHistogram(updatedHistogram);
+
+		// check merged histogram value
+		assertThat(stats.getHistogram().length, is(21));
+		assertThat(stats.getHistogram()[0], is(1l));
+		assertThat(stats.getHistogram()[8], is(45l));
+		assertThat(Arrays.toString(stats.getHistogram()), is("[1, 5, 10, 0, 20, 0, 30, 0, 45, 0, 55, 0, 62, 0, 70, 0, 80, 0, 90, 0, 100]"));
+	}
+
 }