Commit ac89e063 authored by Matija Obreza's avatar Matija Obreza

Merge branch '428-mcpd-coverage' into 'master'

Resolve "MCPD coverage"

Closes #428

See merge request genesys-pgr/genesys-server!369
parents 7ecce711 03f59281
......@@ -233,6 +233,12 @@
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jsonSchema</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-hibernate4</artifactId>
......
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -149,6 +149,27 @@ public class InstituteController {
return details;
}
/**
* Gets the passport data coverage for the institute.
*
* @param code the institute WIEWS code
* @return the coverage
*/
@JsonView({ JsonViews.Protected.class })
@GetMapping(value = "/{code:[A-Z]+[0-9]+}/coverage", produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("hasRole('USER')")
public Map<String, Long> getCoverage(@PathVariable(value = "code") String code) {
FaoInstitute faoInstitute = instituteService.getInstitute(code);
if (faoInstitute == null) {
throw new NotFoundElement();
}
AccessionFilter filter = new AccessionFilter();
filter.holder().id().add(faoInstitute.getId());
return elasticsearchService.countMissingValues(Accession.class, filter);
}
@RequestMapping(value = "/{wiewsCode}/download", method = RequestMethod.POST, params = { "dwca" })
public void downloadDwca(@PathVariable(value = "wiewsCode", required = true) String wiewsCode, HttpServletResponse response) throws Exception {
final FaoInstitute faoInstitute = instituteService.getInstitute(wiewsCode);
......
......@@ -116,6 +116,14 @@ public interface ElasticsearchService {
long count(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter);
/**
* Count missing values for all fields of specified class.
* @param clazz the index class
* @param filter the BasicModelFilter<?, ?> filter
* @return the map of all JSON paths with their missing value
*/
Map<String, Long> countMissingValues(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter);
/**
* Aggregate by date
* @param size max size of results to be returned
......
......@@ -36,7 +36,14 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
import com.google.common.collect.Sets;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.search.SearchRequestBuilder;
......@@ -56,6 +63,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
......@@ -73,6 +81,7 @@ import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys.custom.elasticsearch.CustomMapping;
import org.genesys2.server.component.elastic.ElasticQueryBuilder;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.spring.config.ElasticsearchConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -136,6 +145,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
private final Set<Class<? extends BasicModel>> indexedEntities = Collections.synchronizedSet(new HashSet<>());
private final Map<String, Class<BasicModel>> namesToClasses = Collections.synchronizedMap(new HashMap<>());
private final Map<Class<? extends BasicModel>, Set<String>> jsonSchemas = new HashMap<>();
/// Size of database batch scan for IDs
private int batchSize = 1000;
......@@ -160,6 +170,45 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
} else {
LOG.warn("Elasticsearch not accessible, not updating mappings");
}
ObjectMapper mapper = new ElasticsearchConfig.GenesysEntityMapper().getObjectMapper();
JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper);
for (Class<? extends BasicModel> clazz: indexedEntities) {
try {
JsonSchema schema = schemaGen.generateSchema(clazz);
jsonSchemas.put(clazz, buildJsonPaths(((ObjectSchema) schema).getProperties(), null));
} catch (Throwable e) {
LOG.error("The list of all {} fields is not created.", clazz.getSimpleName(), e);
}
}
}
/**
* Makes a list of all JSON paths for indexed entity and all related types.
*
* @param properties properties
* @param parentPath parentPath
*/
private Set<String> buildJsonPaths(Map<String, JsonSchema> properties, String parentPath) {
if (MapUtils.isEmpty(properties)) {
return Collections.emptySet();
}
Set<String> fieldList = new HashSet<>();
Set<String> keys = properties.keySet();
keys.removeAll(Sets.newHashSet("_class", "_permissions"));
for (String path: keys) {
JsonSchema schema = properties.get(path);
String fullPath = StringUtils.isBlank(parentPath) ? path : parentPath + "." + path;
if (schema instanceof ObjectSchema) {
fieldList.addAll(buildJsonPaths(((ObjectSchema)schema).getProperties(), fullPath));
} else {
fieldList.add(fullPath);
}
}
return fieldList;
}
/**
......@@ -740,6 +789,33 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return hits.getHits().getTotalHits();
}
@Override
public Map<String, Long> countMissingValues(final Class<? extends BasicModel> indexClass, final BasicModelFilter<?, ?> filter) {
final Map<String, Long> results = new HashMap<>();
final Set<String> fields = jsonSchemas.get(indexClass);
if (CollectionUtils.isEmpty(fields)) {
return results;
}
final String indexName = toIndexName(indexClass) + INDEX_READ;
final SearchRequestBuilder esQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setSize(0).setQuery(toEsQuery(indexClass, filter));
for (String fieldName: fields) {
esQuery.addAggregation(AggregationBuilders.missing(fieldName).field(fieldName));
}
final SearchResponse response = esQuery.get();
LOG.debug("Counting missing values took {}s", response.getTook().getSeconds());
for (Aggregation agg: response.getAggregations()) {
long missingCount = ((InternalMissing) agg).getDocCount();
if (missingCount > 0) {
results.put(agg.getName(), missingCount);
}
}
results.put("_totalCount", response.getHits().getTotalHits());
return results;
}
@Override
@Cacheable(value = "statistics", unless = "#result == null", key = "'stats.' + #root.methodName + '-' + #size + '-' + #targetClass.name + '-' + #indexClass.name")
public List<Object[]> aggregateDate(final int size, final Class<? extends BasicModel> targetClass, final Class<? extends BasicModel> indexClass,
......
......@@ -230,5 +230,9 @@ public class ElasticsearchConfig {
public <T> T mapToObject(final String source, final Class<T> clazz) throws IOException {
return mapper.readValue(source, clazz);
}
public ObjectMapper getObjectMapper() {
return mapper;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment