Commit 41b04a60 authored by Maxym Borodenko's avatar Maxym Borodenko Committed by Matija Obreza

Extend Recent activity.

- Updated query
- Fetch updates data using ES
parent 58331981
......@@ -16,16 +16,21 @@
package org.genesys2.server.mvc;
import java.util.Calendar;
import java.util.Locale;
import javax.servlet.http.HttpServletRequest;
import org.genesys.blocks.model.filters.DateFilter;
import org.genesys.catalog.service.DatasetService;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.StatisticsService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.servlet.filter.LocaleURLFilter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -61,10 +66,10 @@ public class HtmlController extends BaseController {
private OrganizationService organizationService;
@Autowired
private GenesysService genesysService;
private DatasetService datasetService;
@Autowired
private DatasetService datasetService;
private ElasticsearchService elasticsearchService;
@RequestMapping("/")
public String index(HttpServletRequest request) {
......@@ -90,16 +95,24 @@ public class HtmlController extends BaseController {
model.addAttribute("welcomeBlurp", contentService.getGlobalArticle(ContentService.WELCOME, getLocale()));
model.addAttribute("sideBlurp", contentService.getGlobalArticle(ContentService.SIDE_BLURP, getLocale()));
model.addAttribute("organizationsBlurb", contentService.getGlobalArticle(ContentService.WELCOME_ORGANIZATIONS, getLocale()));
model.addAttribute("numberOfCountries", statisticsService.numberOfCountries());
model.addAttribute("numberOfInstitutes", statisticsService.numberOfInstitutes());
model.addAttribute("numberOfAccessions", statisticsService.numberOfAccessions());
model.addAttribute("numberOfActiveAccessions", statisticsService.numberOfActiveAccessions());
model.addAttribute("numberOfHistoricAccessions", statisticsService.numberOfHistoricAccessions());
model.addAttribute("numberOfPublishedDatasets", statisticsService.numberOfPublishedDatasets());
model.addAttribute("accessionsUpdates", genesysService.getLastUpdatedStatistics());
model.addAttribute("lastDatasetsUpdates", datasetService.lastPublished());
{
final Calendar date = Calendar.getInstance();
//substract 6 months from current date
date.add(Calendar.MONTH, -6);
final AccessionFilter filter = new AccessionFilter();
filter.lastModifiedDate = new DateFilter();
filter.lastModifiedDate.gt = date.getTime();
model.addAttribute("lastUpdatesOfInst", elasticsearchService.aggregateDate(5, FaoInstitute.class, Accession.class, "lastModifiedDate", "institute.id", filter));
}
model.addAttribute("lastDatasetsUpdates", datasetService.lastPublished());
model.addAttribute("organizations", organizationService.list(new PageRequest(0, 5)));
return "/index";
......
......@@ -116,10 +116,18 @@ public interface ElasticsearchService {
long count(Class<? extends BasicModel> clazz, BasicModelFilter filter);
/**
* Aggregate by date
* @param size max size of results to be returned
* @param targetClass the target entity class
* @param indexClass the index class
* @param aggregatedDateField the name of date field to aggregate
* @param groupingByField the name of field to grouping
* @param filter the BasicModelFilter filter
* @return the result list
*/
List<Object[]> aggregateDate(int size, Class<? extends BasicModel> targetClass, Class<? extends BasicModel> indexClass, String aggregatedDateField, String groupingByField, BasicModelFilter filter);
public static class TermResult {
private List<Term> terms;
......
......@@ -18,6 +18,7 @@ package org.genesys2.server.service.filter;
import java.util.Set;
import java.util.UUID;
import org.genesys.blocks.model.filters.DateFilter;
import org.genesys.blocks.model.filters.NumberFilter;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter;
......@@ -29,13 +30,13 @@ import com.hazelcast.util.CollectionUtil;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate;
// TODO: Auto-generated Javadoc
/**
* Filters for {@link Accession}
* Filters for {@link Accession}.
*/
public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession> {
/** The historic. */
/// Include historic
public Boolean historic;
/** The mls status. */
......@@ -71,21 +72,34 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
/** The origin. */
public CountryFilter origin;
/** The geo. */
public AccessionGeoFilter geo;
/** The coll. */
public AccessionCollectFilter coll;
/** The storage. */
public Set<Integer> storage;
/** The sgsv. */
public Boolean sgsv;
/** The art 15. */
public Boolean art15;
/** The pdci. */
public NumberFilter<Double> pdci;
/** acce.lists uuid */
public Set<UUID> list;
/** The lastModifiedDate. */
public DateFilter lastModifiedDate;
/* (non-Javadoc)
* @see org.genesys.blocks.model.filters.BasicModelFilter#buildQuery()
*/
public Predicate buildQuery() {
return buildQuery(QAccession.accession);
}
......@@ -93,6 +107,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
/**
* Builds the query.
*
* @param accession the accession
* @return the predicate
*/
public Predicate buildQuery(QAccession accession) {
......@@ -123,6 +138,9 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
if (seqNo != null) {
and.and(seqNo.buildQuery(accession.seqNo));
}
if (lastModifiedDate != null) {
and.and(lastModifiedDate.buildQuery(accession.lastModifiedDate));
}
if (CollectionUtil.isNotEmpty(sampStat)) {
and.and(accession.sampStat.in(sampStat));
}
......@@ -164,6 +182,12 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
return and;
}
/**
* Convert.
*
* @param filters the filters
* @return the accession filter
*/
public static AccessionFilter convert(AppliedFilters filters) {
try {
return AppliedFiltersConverter.convert(filters);
......@@ -172,11 +196,22 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
}
}
/**
* Historic.
*
* @param historic the historic
* @return the accession filter
*/
public AccessionFilter historic(Boolean historic) {
this.historic = historic;
return this;
}
/**
* Holder.
*
* @return the institute filter
*/
public InstituteFilter holder() {
if (this.holder==null) {
return this.holder = new InstituteFilter();
......@@ -185,6 +220,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
}
}
/**
* Origin.
*
* @return the country filter
*/
public CountryFilter origin() {
if (this.origin == null) {
return this.origin = new CountryFilter();
......@@ -192,6 +232,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
return this.origin;
}
/**
* Geo.
*
* @return the accession geo filter
*/
public AccessionGeoFilter geo() {
if (this.geo == null) {
return this.geo = new AccessionGeoFilter();
......@@ -199,6 +244,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
return this.geo;
}
/**
* Coll.
*
* @return the accession collect filter
*/
public AccessionCollectFilter coll() {
if (this.coll == null) {
return this.coll = new AccessionCollectFilter();
......@@ -206,6 +256,11 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
return this.coll;
}
/**
* Taxa.
*
* @return the taxonomy filter
*/
public TaxonomyFilter taxa() {
if (this.taxa == null) {
return this.taxa = new TaxonomyFilter();
......
......@@ -18,10 +18,12 @@ package org.genesys2.server.service.impl;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -45,8 +47,14 @@ import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesNotFoundException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
......@@ -55,6 +63,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.search.sort.SortOrder;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.Publishable;
import org.genesys.blocks.model.VersionedModel;
......@@ -66,6 +76,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
......@@ -726,6 +737,59 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return hits.getHits().getTotalHits();
}
@Override
@Cacheable(value = "statistics", unless = "#result == null", key = "'stats.' + #root.methodName + '-' + #size + '-' + #targetClass.name + '-' + #indexClass.name")
public List<Object[]> aggregateDate(final int size, final Class<? extends BasicModel> targetClass, final Class<? extends BasicModel> indexClass,
final String aggregatedDateField, final String groupingByField, final BasicModelFilter filter) {
final String DATE_HISTOGRAM_AGG = "updates_over_time";
final String TERMS_AGG = "institute_terms";
final String TOP_HITS_AGG = "top_tag_hits";
final String indexName = toIndexName(indexClass) + INDEX_READ;
AggregationBuilder aggregation =
AggregationBuilders.dateHistogram(DATE_HISTOGRAM_AGG).field(aggregatedDateField).order(Histogram.Order.KEY_DESC).minDocCount(1).interval(DateHistogramInterval.DAY)
.subAggregation(AggregationBuilders.terms(TERMS_AGG).field(groupingByField).size(size)
.subAggregation(AggregationBuilders.topHits(TOP_HITS_AGG).addSort(aggregatedDateField, SortOrder.DESC).addFieldDataField(groupingByField).addFieldDataField(aggregatedDateField).setSize(1)));
SearchRequestBuilder esQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setSize(0).addAggregation(aggregation).setQuery(toEsQuery(indexClass, filter));
SearchResponse response = esQuery.get();
InternalHistogram internalHistogram = response.getAggregations().get(DATE_HISTOGRAM_AGG);
List<? extends InternalHistogram.Bucket> buckets = internalHistogram.getBuckets();
final Map<Long, Long> entities = new LinkedHashMap<>(size);
mainLoop:
for (InternalHistogram.Bucket bucket : buckets) {
Terms terms = bucket.getAggregations().get(TERMS_AGG);
for (Bucket instBucket: terms.getBuckets()) {
TopHits topHits = instBucket.getAggregations().get(TOP_HITS_AGG);
SearchHit searchHit = topHits.getHits().getHits()[0];
Map<String, SearchHitField> result = searchHit.getFields();
List<Object> datesList = result.get(aggregatedDateField).getValues();
Long maxDate = datesList.stream().mapToLong(v -> (Long)v).max().orElse((Long)datesList.get(0));
Long instId = Long.valueOf(result.get(groupingByField).getValues().get(0).toString());
entities.put(instId, maxDate);
if (entities.size() == size) {
break mainLoop;
}
}
}
List<Object[]> resultList = new ArrayList<>(size);
entities.keySet().forEach(key -> {
final Object itemInfo[] = new Object[4];
Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(entities.get(key));
itemInfo[0] = cal.get(Calendar.YEAR);
itemInfo[1] = cal.get(Calendar.MONTH) + 1;
itemInfo[2] = cal.get(Calendar.DAY_OF_MONTH);
itemInfo[3] = loadEntity(targetClass, key);
resultList.add(itemInfo);
});
return resultList;
}
@Override
public <T extends BasicModel> List<T> find(Class<T> clazz, BasicModelFilter filter) {
String indexName = toIndexName(clazz) + INDEX_READ;
......
......@@ -399,7 +399,7 @@ accession.page.data.intro=Explore the accession data with Genesys. To refine you
accession.taxonomy-at-institute=View {0} at {1}
accession.page.updates.title=Last updates of passport data
accession.sort-data.lastUpdated=View updates of passport data for {0}
accession.page.updates.intro=Updates of passport data by date of last update, institute and number of records updated
accession.page.updates.intro=Updates of passport data by date of last update and institute
accession.page.updated.never=Data provided before 2014
accession.svalbard-data=Svalbard Global Seed Vault duplication data
......
......@@ -165,9 +165,9 @@
<div class="post-inner clearfix post-left-pad">
<div class="post-content" dir="ltr" style="margin-bottom: 22px;">
<p><spring:message code='accession.page.updates.intro' /></p>
<c:forEach items="${accessionsUpdates}" var="update" begin="0" end="4">
<c:forEach items="${lastUpdatesOfInst}" var="update" begin="0" end="4">
<div class="row">
<div class="col-lg-3 col-md-3 col-sm-3 col-xs-4">
<div class="col-lg-3 col-md-3 col-sm-3 col-xs-5">
<b>
<c:if test="${update[0] ne null}">
${update[2]} ${monthNames[update[1]-1]} ${update[0]}
......@@ -177,8 +177,7 @@
</c:if>
</b>
</div>
<div class="col-lg-6 col-md-6 col-sm-6 col-xs-4"><a href="<c:url value='/wiews/${update[3].code}' />">${update[3].fullName}</a></div>
<div class="col-lg-3 col-md-3 col-sm-3 col-xs-4 text-right"><b><fmt:formatNumber value="${update[4]}" /></b></div>
<div class="col-lg-9 col-md-9 col-sm-9 col-xs-7"><a href="<c:url value='/wiews/${update[3].code}' />">${update[3].fullName}</a></div>
</div>
</c:forEach>
</div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment