From 41b04a60f3d9ed4e9ae2f4756c7485d41dfee917 Mon Sep 17 00:00:00 2001
From: Maxym Borodenko <m.borodenko@aimprosoft.com>
Date: Fri, 3 Aug 2018 19:26:04 +0300
Subject: [PATCH] Extend Recent activity.

- Updated query
- Fetch updates data using ES
---
 .../genesys2/server/mvc/HtmlController.java   | 25 ++++++--
 .../server/service/ElasticsearchService.java  | 14 +++-
 .../service/filter/AccessionFilter.java       | 59 ++++++++++++++++-
 .../impl/ElasticsearchServiceImpl.java        | 64 +++++++++++++++++++
 .../resources/content/language.properties     |  2 +-
 src/main/webapp/WEB-INF/jsp/index.jsp         |  7 +-
 6 files changed, 155 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/genesys2/server/mvc/HtmlController.java b/src/main/java/org/genesys2/server/mvc/HtmlController.java
index 42603c1b0..59c439157 100644
--- a/src/main/java/org/genesys2/server/mvc/HtmlController.java
+++ b/src/main/java/org/genesys2/server/mvc/HtmlController.java
@@ -16,16 +16,21 @@
 
 package org.genesys2.server.mvc;
 
+import java.util.Calendar;
 import java.util.Locale;
 
 import javax.servlet.http.HttpServletRequest;
 
+import org.genesys.blocks.model.filters.DateFilter;
 import org.genesys.catalog.service.DatasetService;
+import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.service.ContentService;
-import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.ElasticsearchService;
 import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.OrganizationService;
 import org.genesys2.server.service.StatisticsService;
+import org.genesys2.server.service.filter.AccessionFilter;
 import org.genesys2.server.servlet.filter.LocaleURLFilter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -61,10 +66,10 @@ public class HtmlController extends BaseController {
 	private OrganizationService organizationService;
 
 	@Autowired
-	private GenesysService genesysService;
+	private DatasetService datasetService;
 
 	@Autowired
-	private DatasetService datasetService;
+	private ElasticsearchService elasticsearchService;
 
 	@RequestMapping("/")
 	public String index(HttpServletRequest request) {
@@ -90,16 +95,24 @@ public class HtmlController extends BaseController {
 		model.addAttribute("welcomeBlurp", contentService.getGlobalArticle(ContentService.WELCOME, getLocale()));
 		model.addAttribute("sideBlurp", contentService.getGlobalArticle(ContentService.SIDE_BLURP, getLocale()));
 		model.addAttribute("organizationsBlurb", contentService.getGlobalArticle(ContentService.WELCOME_ORGANIZATIONS, getLocale()));
-
 		model.addAttribute("numberOfCountries", statisticsService.numberOfCountries());
 		model.addAttribute("numberOfInstitutes", statisticsService.numberOfInstitutes());
 		model.addAttribute("numberOfAccessions", statisticsService.numberOfAccessions());
 		model.addAttribute("numberOfActiveAccessions", statisticsService.numberOfActiveAccessions());
 		model.addAttribute("numberOfHistoricAccessions", statisticsService.numberOfHistoricAccessions());
 		model.addAttribute("numberOfPublishedDatasets", statisticsService.numberOfPublishedDatasets());
-		model.addAttribute("accessionsUpdates", genesysService.getLastUpdatedStatistics());
-		model.addAttribute("lastDatasetsUpdates", datasetService.lastPublished());
 
+		{
+			final Calendar date = Calendar.getInstance();
+			//substract 6 months from current date
+			date.add(Calendar.MONTH, -6);
+			final AccessionFilter filter = new AccessionFilter();
+			filter.lastModifiedDate = new DateFilter();
+			filter.lastModifiedDate.gt = date.getTime();
+			model.addAttribute("lastUpdatesOfInst", elasticsearchService.aggregateDate(5, FaoInstitute.class, Accession.class, "lastModifiedDate", "institute.id", filter));
+		}
+
+		model.addAttribute("lastDatasetsUpdates", datasetService.lastPublished());
 		model.addAttribute("organizations", organizationService.list(new PageRequest(0, 5)));
 
 		return "/index";
diff --git a/src/main/java/org/genesys2/server/service/ElasticsearchService.java b/src/main/java/org/genesys2/server/service/ElasticsearchService.java
index 2d6fe156f..80bf68de8 100644
--- a/src/main/java/org/genesys2/server/service/ElasticsearchService.java
+++ b/src/main/java/org/genesys2/server/service/ElasticsearchService.java
@@ -116,10 +116,18 @@ public interface ElasticsearchService {
 
 	long count(Class<? extends BasicModel> clazz, BasicModelFilter filter);
 
+	/**
+	 * Aggregate by date
+	 * @param size max size of results to be returned
+	 * @param targetClass the target entity class
+	 * @param indexClass the index class
+	 * @param aggregatedDateField the name of date field to aggregate
+	 * @param groupingByField the name of field to grouping
+	 * @param filter the BasicModelFilter filter
+	 * @return the result list
+	 */
+	List<Object[]> aggregateDate(int size, Class<? extends BasicModel> targetClass, Class<? extends BasicModel> indexClass, String aggregatedDateField, String groupingByField, BasicModelFilter filter);
 
-
-	
-	
 	public static class TermResult {
 
 		private List<Term> terms;
diff --git a/src/main/java/org/genesys2/server/service/filter/AccessionFilter.java b/src/main/java/org/genesys2/server/service/filter/AccessionFilter.java
index 1eac84d37..ce47d34b6 100644
--- a/src/main/java/org/genesys2/server/service/filter/AccessionFilter.java
+++ b/src/main/java/org/genesys2/server/service/filter/AccessionFilter.java
@@ -18,6 +18,7 @@ package org.genesys2.server.service.filter;
 import java.util.Set;
 import java.util.UUID;
 
+import org.genesys.blocks.model.filters.DateFilter;
 import org.genesys.blocks.model.filters.NumberFilter;
 import org.genesys.blocks.model.filters.StringFilter;
 import org.genesys.blocks.model.filters.UuidModelFilter;
@@ -29,13 +30,13 @@ import com.hazelcast.util.CollectionUtil;
 import com.querydsl.core.BooleanBuilder;
 import com.querydsl.core.types.Predicate;
 
+// TODO: Auto-generated Javadoc
 /**
- * Filters for {@link Accession}
+ * Filters for {@link Accession}.
  */
 public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession> {
 
 	/** The historic. */
-	/// Include historic
 	public Boolean historic;
 
 	/** The mls status. */
@@ -71,21 +72,34 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 	/** The origin. */
 	public CountryFilter origin;
 
+	/** The geo. */
 	public AccessionGeoFilter geo;
 
+	/** The coll. */
 	public AccessionCollectFilter coll;
 
+	/** The storage. */
 	public Set<Integer> storage;
 
+	/** The sgsv. */
 	public Boolean sgsv;
 
+	/** The art 15. */
 	public Boolean art15;
 
+	/** The pdci. */
 	public NumberFilter<Double> pdci;
 
 	/** acce.lists uuid */
 	public Set<UUID> list;
 
+	/** The lastModifiedDate. */
+	public DateFilter lastModifiedDate;
+
+
+	/* (non-Javadoc)
+	 * @see org.genesys.blocks.model.filters.BasicModelFilter#buildQuery()
+	 */
 	public Predicate buildQuery() {
 		return buildQuery(QAccession.accession);
 	}
@@ -93,6 +107,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 	/**
 	 * Builds the query.
 	 *
+	 * @param accession the accession
 	 * @return the predicate
 	 */
 	public Predicate buildQuery(QAccession accession) {
@@ -123,6 +138,9 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		if (seqNo != null) {
 			and.and(seqNo.buildQuery(accession.seqNo));
 		}
+		if (lastModifiedDate != null) {
+			and.and(lastModifiedDate.buildQuery(accession.lastModifiedDate));
+		}
 		if (CollectionUtil.isNotEmpty(sampStat)) {
 			and.and(accession.sampStat.in(sampStat));
 		}
@@ -164,6 +182,12 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		return and;
 	}
 
+	/**
+	 * Convert.
+	 *
+	 * @param filters the filters
+	 * @return the accession filter
+	 */
 	public static AccessionFilter convert(AppliedFilters filters) {
 		try {
 			return AppliedFiltersConverter.convert(filters);
@@ -172,11 +196,22 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		}
 	}
 
+	/**
+	 * Historic.
+	 *
+	 * @param historic the historic
+	 * @return the accession filter
+	 */
 	public AccessionFilter historic(Boolean historic) {
 		this.historic = historic;
 		return this;
 	}
 
+	/**
+	 * Holder.
+	 *
+	 * @return the institute filter
+	 */
 	public InstituteFilter holder() {
 		if (this.holder==null) {
 			return this.holder = new InstituteFilter();
@@ -185,6 +220,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		}
 	}
 
+	/**
+	 * Origin.
+	 *
+	 * @return the country filter
+	 */
 	public CountryFilter origin() {
 		if (this.origin == null) {
 			return this.origin = new CountryFilter();
@@ -192,6 +232,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		return this.origin;
 	}
 
+	/**
+	 * Geo.
+	 *
+	 * @return the accession geo filter
+	 */
 	public AccessionGeoFilter geo() {
 		if (this.geo == null) {
 			return this.geo = new AccessionGeoFilter();
@@ -199,6 +244,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		return this.geo;
 	}
 
+	/**
+	 * Coll.
+	 *
+	 * @return the accession collect filter
+	 */
 	public AccessionCollectFilter coll() {
 		if (this.coll == null) {
 			return this.coll = new AccessionCollectFilter();
@@ -206,6 +256,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
 		return this.coll;
 	}
 
+	/**
+	 * Taxa.
+	 *
+	 * @return the taxonomy filter
+	 */
 	public TaxonomyFilter taxa() {
 		if (this.taxa == null) {
 			return this.taxa = new TaxonomyFilter();
diff --git a/src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java
index 57c26d6b2..adc615b09 100644
--- a/src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java
+++ b/src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java
@@ -18,10 +18,12 @@ package org.genesys2.server.service.impl;
 import static org.elasticsearch.index.query.QueryBuilders.*;
 
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -45,8 +47,14 @@ import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesNotFoundException;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
@@ -55,6 +63,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
+import org.elasticsearch.search.sort.SortOrder;
 import org.genesys.blocks.model.BasicModel;
 import org.genesys.blocks.model.Publishable;
 import org.genesys.blocks.model.VersionedModel;
@@ -66,6 +76,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.Cacheable;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
@@ -726,6 +737,59 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
 		return hits.getHits().getTotalHits();
 	}
 
+	@Override
+	@Cacheable(value = "statistics", unless = "#result == null", key = "'stats.' + #root.methodName + '-' + #size + '-' + #targetClass.name + '-' + #indexClass.name")
+	public List<Object[]> aggregateDate(final int size, final Class<? extends BasicModel> targetClass, final Class<? extends BasicModel> indexClass,
+			final String aggregatedDateField, final String groupingByField, final BasicModelFilter filter) {
+		final String DATE_HISTOGRAM_AGG = "updates_over_time";
+		final String TERMS_AGG = "institute_terms";
+		final String TOP_HITS_AGG = "top_tag_hits";
+		final String indexName = toIndexName(indexClass) + INDEX_READ;
+
+		AggregationBuilder aggregation =
+			AggregationBuilders.dateHistogram(DATE_HISTOGRAM_AGG).field(aggregatedDateField).order(Histogram.Order.KEY_DESC).minDocCount(1).interval(DateHistogramInterval.DAY)
+				.subAggregation(AggregationBuilders.terms(TERMS_AGG).field(groupingByField).size(size)
+					.subAggregation(AggregationBuilders.topHits(TOP_HITS_AGG).addSort(aggregatedDateField, SortOrder.DESC).addFieldDataField(groupingByField).addFieldDataField(aggregatedDateField).setSize(1)));
+
+		SearchRequestBuilder esQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setSize(0).addAggregation(aggregation).setQuery(toEsQuery(indexClass, filter));
+		SearchResponse response = esQuery.get();
+
+		InternalHistogram internalHistogram = response.getAggregations().get(DATE_HISTOGRAM_AGG);
+		List<? extends InternalHistogram.Bucket> buckets = internalHistogram.getBuckets();
+		final Map<Long, Long> entities = new LinkedHashMap<>(size);
+		mainLoop:
+		for (InternalHistogram.Bucket bucket : buckets) {
+			Terms terms = bucket.getAggregations().get(TERMS_AGG);
+			for (Bucket instBucket: terms.getBuckets()) {
+				TopHits topHits = instBucket.getAggregations().get(TOP_HITS_AGG);
+				SearchHit searchHit = topHits.getHits().getHits()[0];
+				Map<String, SearchHitField> result = searchHit.getFields();
+				List<Object> datesList = result.get(aggregatedDateField).getValues();
+				Long maxDate = datesList.stream().mapToLong(v -> (Long)v).max().orElse((Long)datesList.get(0));
+				Long instId = Long.valueOf(result.get(groupingByField).getValues().get(0).toString());
+				entities.put(instId, maxDate);
+				if (entities.size() == size) {
+					break mainLoop;
+				}
+			}
+		}
+
+		List<Object[]> resultList = new ArrayList<>(size);
+
+		entities.keySet().forEach(key -> {
+			final Object itemInfo[] = new Object[4];
+			Calendar cal = Calendar.getInstance();
+			cal.setTimeInMillis(entities.get(key));
+			itemInfo[0] = cal.get(Calendar.YEAR);
+			itemInfo[1] = cal.get(Calendar.MONTH) + 1;
+			itemInfo[2] = cal.get(Calendar.DAY_OF_MONTH);
+			itemInfo[3] = loadEntity(targetClass, key);
+			resultList.add(itemInfo);
+		});
+
+		return resultList;
+	}
+
 	@Override
 	public <T extends BasicModel> List<T> find(Class<T> clazz, BasicModelFilter filter) {
 		String indexName = toIndexName(clazz) + INDEX_READ;
diff --git a/src/main/resources/content/language.properties b/src/main/resources/content/language.properties
index 19a812766..8ec8b2b85 100644
--- a/src/main/resources/content/language.properties
+++ b/src/main/resources/content/language.properties
@@ -399,7 +399,7 @@ accession.page.data.intro=Explore the accession data with Genesys. To refine you
 accession.taxonomy-at-institute=View {0} at {1}
 accession.page.updates.title=Last updates of passport data
 accession.sort-data.lastUpdated=View updates of passport data for {0}
-accession.page.updates.intro=Updates of passport data by date of last update, institute and number of records updated
+accession.page.updates.intro=Updates of passport data by date of last update and institute
 accession.page.updated.never=Data provided before 2014
 
 accession.svalbard-data=Svalbard Global Seed Vault duplication data
diff --git a/src/main/webapp/WEB-INF/jsp/index.jsp b/src/main/webapp/WEB-INF/jsp/index.jsp
index 4b9279e8f..9ebee1ec9 100644
--- a/src/main/webapp/WEB-INF/jsp/index.jsp
+++ b/src/main/webapp/WEB-INF/jsp/index.jsp
@@ -165,9 +165,9 @@
 							<div class="post-inner clearfix post-left-pad">
 								<div class="post-content" dir="ltr" style="margin-bottom: 22px;">
 									<p><spring:message code='accession.page.updates.intro' /></p>
-									<c:forEach items="${accessionsUpdates}" var="update" begin="0" end="4">
+									<c:forEach items="${lastUpdatesOfInst}" var="update" begin="0" end="4">
 										<div class="row">
-											<div class="col-lg-3 col-md-3 col-sm-3 col-xs-4">
+											<div class="col-lg-3 col-md-3 col-sm-3 col-xs-5">
 												<b>
 													<c:if test="${update[0] ne null}">
 														${update[2]} ${monthNames[update[1]-1]} ${update[0]}
@@ -177,8 +177,7 @@
 													</c:if>
 												</b>
 											</div>
-											<div class="col-lg-6 col-md-6 col-sm-6 col-xs-4"><a href="<c:url value='/wiews/${update[3].code}' />">${update[3].fullName}</a></div>
-											<div class="col-lg-3 col-md-3 col-sm-3 col-xs-4 text-right"><b><fmt:formatNumber value="${update[4]}" /></b></div>
+											<div class="col-lg-9 col-md-9 col-sm-9 col-xs-7"><a href="<c:url value='/wiews/${update[3].code}' />">${update[3].fullName}</a></div>
 										</div>
 									</c:forEach>
 								</div>
-- 
GitLab