Commit 68cb2131 authored by Maxym Borodenko's avatar Maxym Borodenko Committed by Matija Obreza

MCPD coverage

Added ability to process any indexed model

Fix: Hardcoded index class

Added minor fix
parent 7ecce711
......@@ -233,6 +233,12 @@
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jsonSchema</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-hibernate4</artifactId>
......
......@@ -116,6 +116,14 @@ public interface ElasticsearchService {
long count(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter);
/**
* Count missing values for all fields of specified class.
* @param clazz the index class
* @param filter the BasicModelFilter<?, ?> filter
* @return the map of all JSON paths with their missing value
*/
Map<String, Long> countMissingValues(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter);
/**
* Aggregate by date
* @param size max size of results to be returned
......
......@@ -36,7 +36,14 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator;
import com.fasterxml.jackson.module.jsonSchema.types.ObjectSchema;
import com.google.common.collect.Sets;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.search.SearchRequestBuilder;
......@@ -56,6 +63,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
......@@ -72,7 +80,9 @@ import org.genesys.blocks.model.VersionedModel;
import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys.custom.elasticsearch.CustomMapping;
import org.genesys2.server.component.elastic.ElasticQueryBuilder;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.spring.config.ElasticsearchConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -136,6 +146,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
private final Set<Class<? extends BasicModel>> indexedEntities = Collections.synchronizedSet(new HashSet<>());
private final Map<String, Class<BasicModel>> namesToClasses = Collections.synchronizedMap(new HashMap<>());
private final Map<Class<? extends BasicModel>, Set<String>> jsonSchemas = new HashMap<>();
/// Size of database batch scan for IDs
private int batchSize = 1000;
......@@ -160,6 +171,45 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
} else {
LOG.warn("Elasticsearch not accessible, not updating mappings");
}
ObjectMapper mapper = new ElasticsearchConfig.GenesysEntityMapper().getObjectMapper();
JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper);
for (Class<? extends BasicModel> clazz: indexedEntities) {
try {
JsonSchema schema = schemaGen.generateSchema(clazz);
jsonSchemas.put(clazz, buildJsonPaths(((ObjectSchema) schema).getProperties(), null));
} catch (Throwable e) {
LOG.error("The list of all {} fields is not created.", clazz.getSimpleName(), e);
}
}
}
/**
* Makes a list of all JSON paths for indexed entity and all related types.
*
* @param properties properties
* @param parentPath parentPath
*/
private Set<String> buildJsonPaths(Map<String, JsonSchema> properties, String parentPath) {
if (MapUtils.isEmpty(properties)) {
return Collections.emptySet();
}
Set<String> fieldList = new HashSet<>();
Set<String> keys = properties.keySet();
keys.removeAll(Sets.newHashSet("_class", "_permissions"));
for (String path: keys) {
JsonSchema schema = properties.get(path);
String fullPath = StringUtils.isBlank(parentPath) ? path : parentPath + "." + path;
if (schema instanceof ObjectSchema) {
fieldList.addAll(buildJsonPaths(((ObjectSchema)schema).getProperties(), fullPath));
} else {
fieldList.add(fullPath);
}
}
return fieldList;
}
/**
......@@ -740,6 +790,33 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return hits.getHits().getTotalHits();
}
@Override
public Map<String, Long> countMissingValues(final Class<? extends BasicModel> indexClass, final BasicModelFilter<?, ?> filter) {
final Map<String, Long> results = new HashMap<>();
final Set<String> fields = jsonSchemas.get(indexClass);
if (CollectionUtils.isEmpty(fields)) {
return results;
}
final String indexName = toIndexName(indexClass) + INDEX_READ;
final SearchRequestBuilder esQuery = client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setSize(0).setQuery(toEsQuery(indexClass, filter));
for (String fieldName: fields) {
esQuery.addAggregation(AggregationBuilders.missing(fieldName).field(fieldName));
}
final SearchResponse response = esQuery.get();
LOG.debug("Counting missing values took {}s", response.getTook().getSeconds());
for (Aggregation agg: response.getAggregations()) {
long missingCount = ((InternalMissing) agg).getDocCount();
if (missingCount > 0) {
results.put(agg.getName(), missingCount);
}
}
results.put("totalCount", response.getHits().getTotalHits());
return results;
}
@Override
@Cacheable(value = "statistics", unless = "#result == null", key = "'stats.' + #root.methodName + '-' + #size + '-' + #targetClass.name + '-' + #indexClass.name")
public List<Object[]> aggregateDate(final int size, final Class<? extends BasicModel> targetClass, final Class<? extends BasicModel> indexClass,
......
......@@ -230,5 +230,9 @@ public class ElasticsearchConfig {
public <T> T mapToObject(final String source, final Class<T> clazz) throws IOException {
return mapper.readValue(source, clazz);
}
public ObjectMapper getObjectMapper() {
return mapper;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment