Commit b19ca6fa authored by Matija Obreza's avatar Matija Obreza

Upgrade Elasticsearch integration

- Mapping AppliedFilters to ModelFilters
- Mapping ModelFilters to ES queries
- updated /admin/elastic
parent 78e45a77
......@@ -560,6 +560,11 @@
<version>${spring.framework.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.5.2</version>
</dependency>
</dependencies>
<build>
......@@ -860,10 +865,6 @@
</includes>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>${project.build.directory}/generated-resources</directory>
<filtering>false</filtering>
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.api;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.catalog.model.filters.DatasetFilter;
import org.genesys.catalog.service.ElasticsearchFilter;
/**
* Wrapper for {@link DatasetFilter} to produce ES queries.
*
* @author Matija Obreza
*/
public class EsDatasetFilter extends DatasetFilter implements ElasticsearchFilter {
/*
* (non-Javadoc)
* @see org.genesys.catalog.server.service.ElasticsearchFilter#elasticQuery()
*/
@Override
public QueryBuilder elasticQuery() {
return null;
}
}
......@@ -16,34 +16,38 @@
package org.genesys.catalog.api.v0;
import com.fasterxml.jackson.annotation.JsonView;
import static com.google.common.collect.Sets.*;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.JsonViews;
import org.genesys.catalog.api.EsDatasetFilter;
import org.genesys.catalog.exceptions.InvalidApiUsageException;
import org.genesys.catalog.model.Partner;
import org.genesys.catalog.model.dataset.AccessionIdentifier;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.filters.DatasetFilter;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.service.ElasticsearchService;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.service.ElasticsearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
import com.fasterxml.jackson.annotation.JsonView;
/**
* API to search the Catalog.
......@@ -68,7 +72,7 @@ public class SearchController {
*/
@JsonView({ JsonViews.Minimal.class })
@PostMapping(value = "/dataset/suggest", produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, SearchResults> datasets(@RequestBody(required = false) final EsDatasetFilter filters, @RequestParam(value = "q", required = true) String searchQuery) {
public Map<String, SearchResults<?>> datasets(@RequestBody(required = false) final DatasetFilter filters, @RequestParam(value = "q", required = true) String searchQuery) {
LOG.trace("Incoming {}", searchQuery);
searchQuery = sanitizeQuery(searchQuery);
LOG.info("Suggestions for datasets for: {}", searchQuery);
......@@ -79,23 +83,23 @@ public class SearchController {
QueryBuilder extraFilters = filtersFromDataset(filters);
Map<Class<?>, List<BasicModel>> hitsByEntity = elasticsearch.search(extraFilters, searchQuery, newHashSet(Crop.class, Partner.class,
AccessionIdentifier.class, Descriptor.class));
Map<Class<BasicModel>, List<BasicModel>> hitsByEntity = elasticsearch.search(extraFilters, newHashSet(Crop.class, Partner.class,
AccessionIdentifier.class, Descriptor.class), searchQuery);
Map<String, SearchResults> suggestions = new HashMap<>();
Map<String, SearchResults<?>> suggestions = new HashMap<>();
suggestions.put("search.group.crop", SearchResults.from("code", Arrays.asList("crop"), hitsByEntity.get(Crop.class)));
suggestions.put("search.group.partner", SearchResults.from("uuid", Arrays.asList("owner.uuid"), hitsByEntity.get(Partner.class)));
suggestions.put("search.group.accession", SearchResults.from("doi", Arrays.asList("accessionIdentifier.doi"), hitsByEntity.get(AccessionIdentifier.class)));
suggestions.put("search.group.descriptor", SearchResults.from("uuid", Arrays.asList("descriptor.uuid"), hitsByEntity.get(Descriptor.class)));
// Search datasets
suggestions.put("search.matches", SearchResults.from("uuid", Arrays.asList("uuid"), elasticsearch.search(extraFilters, searchQuery, Dataset.class)));
suggestions.put("search.matches", SearchResults.from("uuid", Arrays.asList("uuid"), elasticsearch.search(extraFilters, Dataset.class, searchQuery)));
return suggestions;
}
/// Try to enhance the search by adding known Dataset filters
private QueryBuilder filtersFromDataset(EsDatasetFilter filters) {
private QueryBuilder filtersFromDataset(DatasetFilter filters) {
if (filters == null) {
return null;
}
......@@ -125,16 +129,16 @@ public class SearchController {
/**
* Wrapper for search results
*/
class SearchResults {
class SearchResults<T extends BasicModel> {
public List<String> filters;
public String key = "uuid";
public List<BasicModel> hits;
public List<T> hits;
public static SearchResults from(String key, List<String> filters, List<BasicModel> list) {
public static <T extends BasicModel> SearchResults<T> from(String key, List<String> filters, List<T> list) {
if (list == null || list.isEmpty())
return null;
SearchResults sr = new SearchResults();
SearchResults<T> sr = new SearchResults<T>();
sr.filters = filters;
sr.key = key;
sr.hits = list;
......
......@@ -15,6 +15,23 @@
*/
package org.genesys.catalog.custom.elasticsearch;
import static org.apache.commons.lang.StringUtils.*;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
import static org.springframework.util.StringUtils.*;
import java.io.IOException;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Lob;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
......@@ -22,22 +39,22 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.annotation.Transient;
import org.springframework.data.elasticsearch.annotations.*;
import org.springframework.data.elasticsearch.annotations.CompletionField;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
import org.springframework.data.elasticsearch.annotations.GeoPointField;
import org.springframework.data.elasticsearch.annotations.InnerField;
import org.springframework.data.elasticsearch.annotations.Mapping;
import org.springframework.data.elasticsearch.annotations.MultiField;
import org.springframework.data.elasticsearch.core.completion.Completion;
import org.springframework.data.elasticsearch.core.geo.GeoPoint;
import org.springframework.data.mapping.model.SimpleTypeHolder;
import org.springframework.data.util.ClassTypeInformation;
import org.springframework.data.util.TypeInformation;
import java.io.IOException;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.*;
import static org.apache.commons.lang.StringUtils.EMPTY;
import static org.apache.commons.lang.StringUtils.isNotBlank;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.springframework.util.StringUtils.hasText;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
/**
* Modified MappingBuilder that includes all simple (non-entity) types and forces index
......@@ -174,7 +191,9 @@ class CustomMappingBuilder {
for (java.lang.reflect.Field field : fields) {
if (field.isAnnotationPresent(Transient.class) || isInIgnoreFields(field, fieldAnnotation)) {
continue;
// Keep transient or ignored fields that have Field annotation
if (! field.isAnnotationPresent(Field.class))
continue;
}
if (field.isAnnotationPresent(Mapping.class)) {
......@@ -194,10 +213,16 @@ class CustomMappingBuilder {
Field singleField = field.getAnnotation(Field.class);
if (!isGeoPointField && !isCompletionField && isEntity(field) && isAnnotated(field)) {
if (singleField == null) {
LOG.info("Skipping {}#{}", field.getDeclaringClass(), field.getName());
continue;
}
boolean nestedOrObject = isNestedOrObjectField(field);
mapEntity(circularReferences, xContentBuilder, getFieldType(field), false, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(Field.class));
boolean includedInRoot = false;
{
JsonUnwrapped isUnwrapped = field.getAnnotation(JsonUnwrapped.class);
includedInRoot = isUnwrapped != null ? true : false;
}
mapEntity(circularReferences, xContentBuilder, getFieldType(field), includedInRoot, EMPTY, field.getName(), nestedOrObject, singleField.type(), field.getAnnotation(Field.class));
if (nestedOrObject) {
continue;
}
......@@ -316,6 +341,7 @@ class CustomMappingBuilder {
*/
private static void addSingleFieldMapping(XContentBuilder xContentBuilder, java.lang.reflect.Field field,
Field fieldAnnotation, boolean nestedOrObjectField) throws IOException {
LOG.trace("addSingleFieldMapping {}#{}", field.getDeclaringClass(), field.getName());
xContentBuilder.startObject(field.getName());
if(!nestedOrObjectField) {
xContentBuilder.field(FIELD_STORE, fieldAnnotation == null ? true : fieldAnnotation.store());
......@@ -343,6 +369,10 @@ class CustomMappingBuilder {
} else {
// Auto-detect
xContentBuilder.field(FIELD_TYPE, typeForField(field).name().toLowerCase());
FieldIndex indexType = indexForField(field);
if (indexType != null) {
xContentBuilder.field(FIELD_INDEX, indexType.name().toLowerCase());
}
}
xContentBuilder.endObject();
}
......@@ -351,13 +381,12 @@ class CustomMappingBuilder {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return typeForClass(paramClass);
return typeForClass(paramClass, field);
} else {
return typeForClass(field.getType());
return typeForClass(field.getType(), field);
}
}
private static FieldType typeForClass(Class<?> clazz) {
private static FieldType typeForClass(Class<?> clazz, java.lang.reflect.Field field) {
if (String.class.equals(clazz)) {
return FieldType.String;
} else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
......@@ -377,11 +406,59 @@ class CustomMappingBuilder {
} else if (clazz.isEnum()) {
return FieldType.String;
} else {
LOG.warn("Dummy mapping for class={}", clazz);
LOG.warn("Default mapping for class={} as {} for field={}#{}", clazz, FieldType.String, field.getDeclaringClass(), field.getName());
return FieldType.String;
}
}
private static FieldIndex indexForField(java.lang.reflect.Field field) {
if (isCollection(field)) {
ParameterizedType paramType = (ParameterizedType) field.getGenericType();
Class<?> paramClass = (Class<?>) paramType.getActualTypeArguments()[0];
return indexForClass(paramClass, field);
} else {
return indexForClass(field.getType(), field);
}
}
private static FieldIndex indexForClass(Class<?> clazz, java.lang.reflect.Field field) {
if (String.class.equals(clazz)) {
Lob jpaLob = field.getAnnotation(Lob.class);
Column jpaColumn = field.getAnnotation(Column.class);
boolean analyzed = false;
if (jpaLob != null) {
analyzed = true;
} else if (jpaColumn != null) {
if (jpaColumn.length() > 100) {
analyzed=true;
}
}
return analyzed ? FieldIndex.analyzed : FieldIndex.not_analyzed;
}
// else if (Boolean.TYPE.equals(clazz) || Boolean.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Long.TYPE.equals(clazz) || Long.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Double.TYPE.equals(clazz) || Double.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Float.TYPE.equals(clazz) || Float.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Integer.TYPE.equals(clazz) || Integer.class.equals(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Date.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// } else if (Number.class.isAssignableFrom(clazz)) {
// return FieldIndex.not_analyzed;
// }
else if (clazz.isEnum()) {
return FieldIndex.not_analyzed;
} else {
LOG.debug("Using default indexing for class={}", clazz);
return null;
}
}
/**
* Apply mapping for a single nested @Field annotation
*
......
......@@ -34,7 +34,7 @@ import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.json.AccessionJson;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysRESTService;
import org.genesys2.server.service.GenesysService;
......@@ -109,7 +109,7 @@ public class AccessionController extends ApiBaseController {
GenesysRESTService restService;
@Autowired
ElasticService elasticService;
ElasticsearchService elasticService;
/**
* Check if accessions exists in the system
......@@ -335,20 +335,21 @@ public class AccessionController extends ApiBaseController {
}
@RequestMapping(value = "/search", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody Page<AccessionDetails> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(StringUtils.defaultIfBlank(query, "*"), new PageRequest(page - 1, 20));
public @ResponseBody List<Accession> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(null, Accession.class, StringUtils.defaultIfBlank(query, "*")); //, new PageRequest(page - 1, 20));
}
// FIXME Not using institute...
@RequestMapping(value = "/{instCode}/list", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody Page<AccessionDetails> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
public @ResponseBody List<Accession> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
throws SearchException {
FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
throw new ResourceNotFoundException();
}
query = StringUtils.defaultIfBlank(query, "*");
return elasticService.search(query, new PageRequest(page - 1, 20));
// TODO filer by inst
return elasticService.search(null, Accession.class, query); // , new PageRequest(page - 1, 20));
}
@RequestMapping(value = "/{id}", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
......
......@@ -14,9 +14,15 @@
* limitations under the License.
*/
package org.genesys.catalog.component;
package org.genesys2.server.component.elastic;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import javax.annotation.Resource;
import com.hazelcast.core.IQueue;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
......@@ -27,11 +33,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
* AspectJ powered listener on repository save and delete operations
* adds indexed entities to re-index queue, handled by {@link ElasticReindexProcessor}.
......@@ -45,7 +46,7 @@ public class ElasticJPAListener {
private Set<Object> ignoredClasses;
@Resource
private IQueue<ElasticReindex> elasticReindexQueue;
private BlockingQueue<ElasticReindex> elasticReindexQueue;
/**
* Instantiates a new elastic JPA listener.
......@@ -109,7 +110,7 @@ public class ElasticJPAListener {
} else {
Class<?> clazz = toReindex.getClass();
if (isIndexed(clazz)) {
LOG.warn("Reindexing {} {}", clazz.getName(), toReindex);
LOG.debug("Reindexing {} {}", clazz.getName(), toReindex);
if (toReindex instanceof BasicModel) {
BasicModel entity = (BasicModel) toReindex;
elasticReindexQueue.add(new ElasticReindex(clazz.getName(), entity.getId()));
......
package org.genesys2.server.component.elastic;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.List;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
import com.querydsl.core.QueryMetadata;
import com.querydsl.core.types.Constant;
import com.querydsl.core.types.Expression;
import com.querydsl.core.types.FactoryExpression;
import com.querydsl.core.types.Operation;
import com.querydsl.core.types.Operator;
import com.querydsl.core.types.Ops;
import com.querydsl.core.types.ParamExpression;
import com.querydsl.core.types.Path;
import com.querydsl.core.types.PathImpl;
import com.querydsl.core.types.PathMetadata;
import com.querydsl.core.types.PathType;
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.SubQueryExpression;
import com.querydsl.core.types.TemplateExpression;
import com.querydsl.core.types.Visitor;
public class ElasticQueryBuilder implements Visitor<Void, Void> {
private static Logger LOG = LoggerFactory.getLogger(ElasticQueryBuilder.class);
BoolQueryBuilder root = QueryBuilders.boolQuery();
private final ElasticQueryBuilder self = this;
public BoolQueryBuilder getQuery() {
return QueryBuilders.boolQuery().filter(root);
}
private String customizedPath(String path) {
// Just remove the entity name from the path -- hopefully that's fine
if (path.contains(".")) {
int lastDot = path.indexOf('.');
// String base = path.substring(0, lastDot);
path = path.substring(lastDot + 1);
}
if (path.contains("accessionId.")) {
return path.replace("accessionId.", "");
}
return path;
}
@Override
public Void visit(Constant<?> c, Void context) {
LOG.debug("+Constant: {}", c.getConstant());
return null;
}
@Override
public Void visit(FactoryExpression<?> expr, Void context) {
LOG.debug("+FactoryExpression: {}", expr.getArgs());
return null;
}
@Override
public Void visit(Operation<?> expr, Void context) {
LOG.debug("+Operation: {} {} {}", expr.getType(), expr.getOperator(), expr.getArgs());
visitOperation(expr.getType(), expr.getOperator(), expr.getArgs());
return null;
}
private void visitOperation(Class<?> type, Operator operator, List<Expression<?>> args) {
if (operator == Ops.AND) {
for (Expression<?> expr : args) {
printExpression(".. " + operator, expr);
expr.accept(self, null);
}
} else if (operator == Ops.EQ || operator == Ops.IN) {
LOG.debug("EQUALS: {}", args);
for (Expression<?> expr : args) {
printExpression("EQUALS.. ", expr);
}
Path<?> a0 = (Path<?>) args.get(0);
Expression<?> a1 = args.get(1);
handleEquals(a0, a1);
} else if (operator == Ops.LOE || operator == Ops.GOE || operator == Ops.BETWEEN || operator == Ops.LT || operator == Ops.GT) {
LOG.debug("Range: {}", args);
for (Expression<?> expr : args) {
printExpression("LOE.. ", expr);
}
Path<?> a0 = (Path<?>) args.get(0);
handleRange(operator, a0, args.get(1), args.size() > 2 ? args.get(2) : null);
} else if (operator == Ops.STRING_CONTAINS || operator == Ops.STARTS_WITH) {
LOG.debug("{}: {}", operator, args);
for (Expression<?> expr : args) {
printExpression(operator + ".. ", expr);
}
Path<?> a0 = (Path<?>) args.get(0);
Expression<?> a1 = args.get(1);
handleLike(operator, a0, a1);
} else {
LOG.error("Op {}: {}", operator, args);
}
// Expression<?> a0 = args.get(0);
// Expression<?> a1 = args.get(1);
// printExpression("a1: " + type.getName() + " " + operator, a1);
}
private void handleLike(Operator operator, Path<?> path, Expression<?> val) {
PathMetadata pmd = path.getMetadata();
// SimpleQueryStringBuilder qsq = simpleQueryStringQuery( +":" + toValue(val));
if (operator == Ops.STARTS_WITH) {
MatchQueryBuilder matchPrefixQuery = matchPhrasePrefixQuery(customizedPath(pmd.getParent().toString() + "." + pmd.getName()), toValue(val));
root.must(matchPrefixQuery);
} else if (operator == Ops.STRING_CONTAINS) {
MatchQueryBuilder matchPrefixQuery = matchPhraseQuery(customizedPath(pmd.getParent().toString() + "." + pmd.getName()), toValue(val));
root.must(matchPrefixQuery);
} else {
throw new RuntimeException("Unsupported ES handleLike operator: " + operator);
}
}