Commit 28b20edb authored by Matija Obreza's avatar Matija Obreza

ElasticsearchService using AppliedFilters

parent 64e3c84c
......@@ -16,22 +16,37 @@
package org.genesys2.server.model.filters;
public class BasicFilter implements GenesysFilter {
private final String name;
private final DataType dataType;
private FilterType filterType;
private final Integer maxLength;
private boolean analyzed = false;
@Override
public boolean isCore() {
return true;
}
/**
* Is the field analyzed by indexer?
*/
@Override
public boolean isAnalyzed() {
return this.analyzed;
};
public BasicFilter setAnalyzed(boolean analyzed) {
this.analyzed = analyzed;
return this;
}
public BasicFilter(String name, DataType type) {
this.name = name;
this.dataType = type;
if (this.dataType == DataType.NUMERIC) {
this.analyzed = false;
this.filterType = FilterType.RANGE;
} else {
this.filterType = FilterType.EXACT;
......@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter {
return name;
}
public String getName() {
return name;
}
@Override
public DataType getDataType() {
return dataType;
}
@Override
public FilterType getFilterType() {
return filterType;
}
......
......@@ -16,15 +16,24 @@
package org.genesys2.server.model.filters;
public interface GenesysFilter {
public String getKey();
public enum DataType {
FIXEDSTRING, STRING, NUMERIC, BOOLEAN
STRING, NUMERIC, BOOLEAN
}
public enum FilterType {
EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST
}
boolean isCore();
DataType getDataType();
FilterType getFilterType();
public boolean isAnalyzed();
}
\ No newline at end of file
......@@ -38,7 +38,7 @@ public interface ElasticService {
void refreshIndex(String className);
Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException;
Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException;
}
package org.genesys2.server.service.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
......@@ -9,19 +8,31 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.filters.GenesysFilter;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.impl.FilterHandler.FilterValue;
import org.genesys2.server.service.impl.FilterHandler.LiteralValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Service
public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
......@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired
private GenesysService genesysService;
@Autowired
private OrganizationService organizationService;
@Autowired
private GenesysFilterService filterService;
@Autowired
private FilterHandler filterHandler;
@Autowired
private ObjectMapper objectMapper;
......@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
@Override
public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException {
Map<String, List<String>> filters;
try {
filters = objectMapper.readValue(jsonFilter, Map.class);
} catch (IOException e) {
throw new SearchException(e.getMessage(), e);
}
public Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException {
AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
for (String key : filters.keySet()) {
for (AppliedFilter appliedFilter : appliedFilters) {
String key = appliedFilter.getFilterName();
GenesysFilter genesysFilter = filterHandler.getFilter(key);
List<String> filterValues = filters.get(key);
if (filterValues == null || filterValues.isEmpty())
if (genesysFilter == null) {
LOG.warn("No such filter " + key);
continue;
}
System.err.println("key=" + key);
if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
// indexed field
filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
} else if (FilterConstants.ALIAS.equals(key)) {
// Nested object
filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
} else if (FilterConstants.SGSV.equals(key)) {
// Check for existance nested object
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
// Filter-level OR
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (filterValues.contains(true) || filterValues.contains(false))
filterBuilder.add(orFilter);
} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) {
OrFilterBuilder orFilter = FilterBuilders.orFilter();
// range filters
for (Object val : filterValues) {
System.err.println(val.getClass() + " " + val);
Map<String, Object> v = (Map<String, Object>) val;
List<Number> range = (List<Number>) v.get("range");
Number max = (Number) v.get("max");
Number min = (Number) v.get("min");
if (range != null) {
System.err.println("Range " + range.getClass() + " " + range);
orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1)));
} else if (max != null) {
System.err.println("Max " + max);
orFilter.add(FilterBuilders.rangeFilter(key).to(max));
} else if (min != null) {
System.err.println("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min));
// null
if (appliedFilter.getWithNull()) {
orFilter.add(FilterBuilders.missingFilter(key));
}
Set<FilterValue> filterValues = appliedFilter.getValues();
if (filterValues != null && !filterValues.isEmpty()) {
{
// Handle literals
Set<Object> literals = new HashSet<Object>();
for (FilterValue filterValue : filterValues) {
if (filterValue instanceof FilterHandler.LiteralValueFilter) {
FilterHandler.LiteralValueFilter literal = (LiteralValueFilter) filterValue;
literals.add(literal.getValue());
}
}
}
filterBuilder.add(orFilter);
} else {
Set<Object> nonNull = new HashSet<Object>(filters.size());
boolean hasNull = false;
for (Object val : filterValues) {
if (val != null) {
nonNull.add(val);
} else {
hasNull = true;
if (!literals.isEmpty()) {
if (genesysFilter.isAnalyzed()) {
// query
StringBuilder sb = new StringBuilder();
for (Object val : literals) {
if (sb.length() > 0)
sb.append(",");
if (val instanceof String)
sb.append("\"" + val + "\"");
else
sb.append(val);
}
if (FilterConstants.ALIAS.equals(key)) {
// Nested
orFilter.add(FilterBuilders.nestedFilter("aliases", QueryBuilders.queryString("aliases.name" + ":(" + sb.toString() + ")")));
} else if (FilterConstants.SGSV.equals(key)) {
// Check if exists
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":(" + sb.toString() + ")")));
}
} else {
// terms
if (FilterConstants.ORGANIZATION.equals(key)) {
Set<String> instCodes = new HashSet<String>();
for (Object literal : literals) {
if (literal instanceof String) {
Organization organization = organizationService.getOrganization((String) literal);
if (organization != null)
for (FaoInstitute inst : organizationService.getMembers(organization)) {
instCodes.add(inst.getCode());
}
}
}
if (!instCodes.isEmpty()) {
orFilter.add(FilterBuilders.termsFilter(FilterConstants.INSTCODE, instCodes).execution("or"));
}
} else {
orFilter.add(FilterBuilders.termsFilter(key, literals).execution("or"));
}
}
}
}
if (hasNull) {
filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or")));
} else {
filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or"));
{
// Handle operations
for (FilterValue filterValue : filterValues) {
if (filterValue instanceof ValueRangeFilter) {
ValueRangeFilter range = (ValueRangeFilter) filterValue;
LOG.debug("Range " + range.getClass() + " " + range);
orFilter.add(FilterBuilders.rangeFilter(key).from(range.getFrom()).to(range.getTo()));
} else if (filterValue instanceof MaxValueFilter) {
MaxValueFilter max = (MaxValueFilter) filterValue;
LOG.debug("Max " + max);
orFilter.add(FilterBuilders.rangeFilter(key).to(max.getTo()));
} else if (filterValue instanceof MinValueFilter) {
MinValueFilter min = (MinValueFilter) filterValue;
LOG.debug("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min.getFrom()));
} else if (filterValue instanceof StartsWithFilter) {
StartsWithFilter startsWith = (StartsWithFilter) filterValue;
LOG.debug("startsWith " + startsWith);
if (genesysFilter.isAnalyzed()) {
if (FilterConstants.ALIAS.equals(key)) {
orFilter.add(FilterBuilders.nestedFilter("aliases",
QueryBuilders.queryString("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":" + startsWith.getStartsWith() + "*")));
}
} else {
orFilter.add(FilterBuilders.prefixFilter(key, startsWith.getStartsWith()));
}
}
}
}
}
filterBuilder.add(orFilter);
}
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build();
// System.err.println("Filter query: " + searchQuery.toString());
SortBuilder sortBuilder = SortBuilders.fieldSort("acceNumb").order(SortOrder.ASC);
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
......@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
}
private QueryBuilder makeQuery(String key, List<String> list) {
// Indexed fields
StringBuilder sb = new StringBuilder();
for (String val : list) {
if (sb.length() > 0)
sb.append(",");
sb.append("\"" + val + "\"");
}
return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
}
@Override
public void update(String className, long id) {
if (!clazzMap.containsKey(className)) {
......@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Initializing index");
elasticsearchTemplate.createIndex("genesysarchive");
}
Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object key : indexMapping.keySet()) {
System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key));
}
Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
// for (Object key : indexMapping.keySet()) {
// System.err.println("Mapping ke=" + key + " val=" +
// indexMapping.get(key));
// }
LOG.info("Copying mapping");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
}
......
......@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
......@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType;
import org.genesys2.server.model.filters.GenesysFilter.FilterType;
import org.genesys2.server.model.filters.I18nListFilter;
import org.genesys2.server.model.filters.MethodFilter;
import org.genesys2.server.model.filters.NoSuchFilterException;
import org.genesys2.server.model.filters.NoSuchFilterValueException;
import org.genesys2.server.model.filters.UnsupportedFilterOperation;
import org.genesys2.server.model.filters.ValueName;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.TraitCode;
......@@ -96,13 +94,13 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING));
this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
13, 20, 30, 40, 50, 99 }));
}
......@@ -111,37 +109,39 @@ public class FilterHandler {
return Collections.unmodifiableList(this.availableFilters);
}
public GenesysFilter getFilter(String key) {
if (key.startsWith("gm:")) {
return getMethodFilter(key);
}
for (GenesysFilter f : this.availableFilters) {
if (f.getKey().equals(key))
return f;
}
return null;
}
public List<GenesysFilter> selectFilters(String[] selectedFilters) {
LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters));
final List<GenesysFilter> filters = new ArrayList<GenesysFilter>();
for (final String selectedFilter : selectedFilters) {
if (selectedFilter.startsWith("gm:")) {
try {
final GenesysFilter methodFilter = getMethodFilter(Long.parseLong(selectedFilter.substring(3)));
if (methodFilter != null) {
filters.add(methodFilter);
}
} catch (NumberFormatException | NullPointerException e) {
LOG.warn(e);
}
} else {
final GenesysFilter coreFilter = CollectionUtils.find(this.availableFilters, new Predicate<GenesysFilter>() {
@Override
public boolean evaluate(GenesysFilter object) {
return object.getKey().equals(selectedFilter);
}
});
if (coreFilter != null) {
filters.add(coreFilter);
try {
final GenesysFilter filter = getFilter(selectedFilter);
if (filter != null) {
filters.add(filter);
}
} catch (NumberFormatException | NullPointerException e) {
LOG.warn(e);
}
}
return filters;
}
GenesysFilter getMethodFilter(long methodId) {
GenesysFilter getMethodFilter(String methodFilterName) {
long methodId = Long.parseLong(methodFilterName.substring(3));
final Method method = traitService.getMethod(methodId);
return method == null ? null : toFilter(method);
}
......@@ -172,75 +172,6 @@ public class FilterHandler {
return filter;
}
public List<AppliedFilter> fromJSON(String jsonFilter) throws IOException, NoSuchFilterException, UnsupportedFilterOperation, NoSuchFilterValueException {
@SuppressWarnings("unchecked")
Map<String, List<Object>> filters = objectMapper.readValue(jsonFilter, Map.class);
AppliedFilters appliedFilters = new AppliedFilters();
for (String key : filters.keySet()) {
List<Object> filterValues = filters.get(key);
if (filterValues == null || filterValues.isEmpty())
continue;
LOG.info("key=" + key);
AppliedFilter appliedFilter = new AppliedFilter();
appliedFilter.setFilter(getFilterByName(key));
for (Object filterValue : filterValues) {
appliedFilter.addFilterValue(toFilterValue(filterValue));
}
}
return appliedFilters;
}
private FilterValue toFilterValue(Object filterValue) throws UnsupportedFilterOperation, NoSuchFilterValueException {
if (filterValue == null)
return null;
if (filterValue instanceof Map) {
return toFilterOperation((Map<?, ?>) filterValue);
} else if (filterValue instanceof Number || filterValue instanceof String) {
return new LiteralValueFilter(filterValue);
} else {
throw new NoSuchFilterValueException(filterValue);
}
}
private FilterValue toFilterOperation(Map<?, ?> filterValue) throws UnsupportedFilterOperation {
@SuppressWarnings("unchecked")
List<Number> range = (List<Number>) filterValue.get("range");
if (range != null) {
return new ValueRangeFilter(range.get(0), range.get(1));
}
Number max = (Number) filterValue.get("max");
if (max != null) {
return new MaxValueFilter(max);
}
Number min = (Number) filterValue.get("min");
if (min != null) {
return new MinValueFilter(min);
}
String like = (String) filterValue.get("like");
if (like != null) {
return new StartsWithFilter(like);
}
throw new UnsupportedFilterOperation(filterValue.toString());
}
private GenesysFilter getFilterByName(String key) throws NoSuchFilterException {
for (GenesysFilter filter : this.availableFilters) {
if (filter.getKey().equals(key)) {
return filter;
}
}
throw new NoSuchFilterException(key);
}
public static interface FilterValue {
String getType();
}
......@@ -316,7 +247,11 @@ public class FilterHandler {
jp.nextToken();
String op = jp.getCurrentName();
if ("like".equals(op)) {
af.addFilterValue(new StartsWithFilter(jp.nextTextValue()));
String startsWith = jp.nextTextValue();
if (startsWith == null || StringUtils.isBlank(startsWith))
throw new JsonParseException("StartsWithFilter expects a non-blank string", jp.getCurrentLocation());
af.addFilterValue(new StartsWithFilter(startsWith));
} else if ("min".equals(op)) {
Number number1 = null;
jp.nextToken();
......@@ -436,6 +371,7 @@ public class FilterHandler {