Commit baf97745 authored by Matija Obreza's avatar Matija Obreza

Merge branch 'ui-582-wrong-counting-of-not-specify-datasets' into 'master'

Fixed: Wrong counting of "Not specify" datasets;

Closes genesys-ui#582

See merge request genesys-pgr/genesys-server!493
parents 6ae85973 e54dad96
......@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import javax.servlet.http.HttpServletResponse;
......@@ -41,6 +42,7 @@ import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.exception.InvalidApiUsageException;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.AccessionService.AccessionMapInfo;
......@@ -252,11 +254,15 @@ public class AccessionController {
@PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
@JsonView({ JsonViews.Public.class })
public AccessionOverview overview(@RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter)
throws IOException, SearchException {
throws IOException, SearchException, ExecutionException, InterruptedException {
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
Map<String, TermResult> overview = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, 10, terms.toArray(new String[] {}));
TermResult result = elasticsearchService.recountResult(QAccession.accession, QAccession.accession.accessionId.storage, filter, overview.get("storage"), "storage");
overview.put("storage", result);
long accessionCount = accessionService.countAccessions(filterInfo.filter);
Map<String, TermResult> suggestions = accessionService.getSuggestions(filterInfo.filter);
......
......@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletResponse;
......@@ -34,6 +35,7 @@ import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.dataset.DatasetAccessionRef;
import org.genesys.catalog.model.dataset.DatasetCreator;
import org.genesys.catalog.model.dataset.DatasetLocation;
import org.genesys.catalog.model.dataset.QDataset;
import org.genesys.catalog.model.filters.DatasetFilter;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.service.DatasetService;
......@@ -330,7 +332,7 @@ public class DatasetController extends ApiBaseController {
@PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
@JsonView({ JsonViews.Public.class })
public DatasetOverview overview(@RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) DatasetFilter filter)
throws IOException, SearchException {
throws IOException, SearchException, ExecutionException, InterruptedException {
if (filterCode != null) {
filter = shortFilterService.filterByCode(filterCode, DatasetFilter.class);
......@@ -343,6 +345,10 @@ public class DatasetController extends ApiBaseController {
overviewFilter.current = true;
Map<String, TermResult> overview = elasticsearchService.termStatisticsAuto(Dataset.class, overviewFilter, 10, terms.toArray(new String[] {}));
TermResult crops = elasticsearchService.recountResult(QDataset.dataset, QDataset.dataset.crops, overviewFilter, overview.get("crops"), "crops");
overview.put("crops", crops);
long datasetCount = datasetService.countDatasets(overviewFilter);
Map<String, ElasticsearchService.TermResult> suggestionRes = datasetService.getSuggestions(filter);
......
......@@ -20,7 +20,10 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import com.querydsl.core.types.dsl.EntityPathBase;
import com.querydsl.core.types.dsl.SetPath;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.filters.BasicModelFilter;
......@@ -123,6 +126,8 @@ public interface ElasticsearchService {
void deleteIndex(String indexName);
void reindexAll();
TermResult recountResult(EntityPathBase<?> qModel, SetPath<?, ?> setPath, BasicModelFilter<?, ?> filter, TermResult toRecount, String termName) throws ExecutionException, InterruptedException;
<T extends BasicModel> void reindex(Class<T> clazz, BasicModelFilter<?, ?> filter);
......
......@@ -28,21 +28,26 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
import com.querydsl.core.types.dsl.EntityPathBase;
import com.querydsl.core.types.dsl.SetPath;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
......@@ -75,6 +80,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.genesys.blocks.model.BasicModel;
......@@ -129,7 +135,7 @@ import com.querydsl.jpa.JPQLQuery;
/**
* Manage Elasticsearch indexing, indices.
*
*
* @author Matija Obreza
*/
// Not @Service
......@@ -254,7 +260,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
/**
* Set the size of IDs loaded in a single batch when reindexing an entity.
* {@link #batchSize}.
*
*
* @param batchSize
*/
public void setBatchSize(int batchSize) {
......@@ -278,6 +284,30 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
}
@Override
public TermResult recountResult(EntityPathBase<?> qModel, SetPath<?, ?> setPath, BasicModelFilter<?, ?> filter, TermResult toRecount, String termName) throws ExecutionException, InterruptedException {
String indexName = toIndexName(qModel.getType()) + INDEX_READ;
Predicate predicate = filter != null ? filter.buildPredicate() : new BooleanBuilder();
ElasticQueryBuilder setPropertyQueryBuilder = new ElasticQueryBuilder();
BooleanBuilder builder = new BooleanBuilder();
builder.and(setPath.size().gt(0)).and(predicate).accept(setPropertyQueryBuilder, null);
ValueCountBuilder aggregation = AggregationBuilders.count(termName).field(termName);
SearchRequestBuilder esQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setQuery(setPropertyQueryBuilder.getQuery()).setSize(0);
esQuery.addAggregation(aggregation);
long setCount = (new Double((double) esQuery.execute().get().getAggregations().get(termName).getProperty("value"))).longValue();
ElasticQueryBuilder entityCountQueryBuilder = new ElasticQueryBuilder();
builder = new BooleanBuilder();
builder.and(setPath.size().eq(0)).and(predicate).accept(entityCountQueryBuilder, null);
SearchRequestBuilder entityEsQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setQuery(entityCountQueryBuilder.getQuery()).setSize(0);
long entityCount = entityEsQuery.execute().get().getHits().getTotalHits();
return new TermResult(termName, setCount + entityCount, toRecount.getTerms(), toRecount.getOther());
}
@SuppressWarnings("unchecked")
@Override
public <R extends BasicModel> void indexEntity(Class<R> clazz) {
......@@ -429,7 +459,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
/**
* Schedule a parallel update.
*
*
* @param clazz Entity to reindex
* @param entityIds Entity identifiers to reindex
*/
......@@ -561,7 +591,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
/**
* Get index name for clazz
*
*
* @param clazz
* @return
*/
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment