Commit 28b20edb authored by Matija Obreza's avatar Matija Obreza

ElasticsearchService using AppliedFilters

parent 64e3c84c
...@@ -16,22 +16,37 @@ ...@@ -16,22 +16,37 @@
package org.genesys2.server.model.filters; package org.genesys2.server.model.filters;
public class BasicFilter implements GenesysFilter { public class BasicFilter implements GenesysFilter {
private final String name; private final String name;
private final DataType dataType; private final DataType dataType;
private FilterType filterType; private FilterType filterType;
private final Integer maxLength; private final Integer maxLength;
private boolean analyzed = false;
@Override
public boolean isCore() { public boolean isCore() {
return true; return true;
} }
/**
* Is the field analyzed by indexer?
*/
@Override
public boolean isAnalyzed() {
return this.analyzed;
};
public BasicFilter setAnalyzed(boolean analyzed) {
this.analyzed = analyzed;
return this;
}
public BasicFilter(String name, DataType type) { public BasicFilter(String name, DataType type) {
this.name = name; this.name = name;
this.dataType = type; this.dataType = type;
if (this.dataType == DataType.NUMERIC) { if (this.dataType == DataType.NUMERIC) {
this.analyzed = false;
this.filterType = FilterType.RANGE; this.filterType = FilterType.RANGE;
} else { } else {
this.filterType = FilterType.EXACT; this.filterType = FilterType.EXACT;
...@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter { ...@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter {
return name; return name;
} }
public String getName() { @Override
return name;
}
public DataType getDataType() { public DataType getDataType() {
return dataType; return dataType;
} }
@Override
public FilterType getFilterType() { public FilterType getFilterType() {
return filterType; return filterType;
} }
......
...@@ -16,15 +16,24 @@ ...@@ -16,15 +16,24 @@
package org.genesys2.server.model.filters; package org.genesys2.server.model.filters;
public interface GenesysFilter { public interface GenesysFilter {
public String getKey(); public String getKey();
public enum DataType { public enum DataType {
FIXEDSTRING, STRING, NUMERIC, BOOLEAN STRING, NUMERIC, BOOLEAN
} }
public enum FilterType { public enum FilterType {
EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST
} }
boolean isCore();
DataType getDataType();
FilterType getFilterType();
public boolean isAnalyzed();
} }
\ No newline at end of file
...@@ -38,7 +38,7 @@ public interface ElasticService { ...@@ -38,7 +38,7 @@ public interface ElasticService {
void refreshIndex(String className); void refreshIndex(String className);
Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException; Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException;
} }
package org.genesys2.server.service.impl; package org.genesys2.server.service.impl;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
...@@ -9,19 +8,31 @@ import java.util.List; ...@@ -9,19 +8,31 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.AndFilterBuilder; import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder; import org.elasticsearch.index.query.OrFilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.genesys2.server.model.elastic.AccessionDetails; import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.filters.GenesysFilter;
import org.genesys2.server.model.genesys.Accession; import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.ElasticService; import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants; import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService; import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters; import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.impl.FilterHandler.FilterValue;
import org.genesys2.server.service.impl.FilterHandler.LiteralValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
...@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service; ...@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch; import org.springframework.util.StopWatch;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Service @Service
public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean { public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
...@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired @Autowired
private GenesysService genesysService; private GenesysService genesysService;
@Autowired
private OrganizationService organizationService;
@Autowired @Autowired
private GenesysFilterService filterService; private GenesysFilterService filterService;
@Autowired
private FilterHandler filterHandler;
@Autowired @Autowired
private ObjectMapper objectMapper; private ObjectMapper objectMapper;
...@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} }
@Override @Override
public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException { public Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException {
Map<String, List<String>> filters;
try {
filters = objectMapper.readValue(jsonFilter, Map.class);
} catch (IOException e) {
throw new SearchException(e.getMessage(), e);
}
AndFilterBuilder filterBuilder = FilterBuilders.andFilter(); AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
for (String key : filters.keySet()) { for (AppliedFilter appliedFilter : appliedFilters) {
String key = appliedFilter.getFilterName();
GenesysFilter genesysFilter = filterHandler.getFilter(key);
List<String> filterValues = filters.get(key); if (genesysFilter == null) {
if (filterValues == null || filterValues.isEmpty()) LOG.warn("No such filter " + key);
continue; continue;
}
System.err.println("key=" + key); // Filter-level OR
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
// indexed field
filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
} else if (FilterConstants.ALIAS.equals(key)) {
// Nested object
filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
} else if (FilterConstants.SGSV.equals(key)) {
// Check for existance nested object
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
if (filterValues.contains(true) || filterValues.contains(false)) // null
filterBuilder.add(orFilter); if (appliedFilter.getWithNull()) {
} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) { orFilter.add(FilterBuilders.missingFilter(key));
OrFilterBuilder orFilter = FilterBuilders.orFilter(); }
// range filters
for (Object val : filterValues) { Set<FilterValue> filterValues = appliedFilter.getValues();
System.err.println(val.getClass() + " " + val); if (filterValues != null && !filterValues.isEmpty()) {
Map<String, Object> v = (Map<String, Object>) val;
List<Number> range = (List<Number>) v.get("range"); {
Number max = (Number) v.get("max"); // Handle literals
Number min = (Number) v.get("min"); Set<Object> literals = new HashSet<Object>();
if (range != null) { for (FilterValue filterValue : filterValues) {
System.err.println("Range " + range.getClass() + " " + range); if (filterValue instanceof FilterHandler.LiteralValueFilter) {
orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1))); FilterHandler.LiteralValueFilter literal = (LiteralValueFilter) filterValue;
} else if (max != null) { literals.add(literal.getValue());
System.err.println("Max " + max); }
orFilter.add(FilterBuilders.rangeFilter(key).to(max));
} else if (min != null) {
System.err.println("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min));
} }
} if (!literals.isEmpty()) {
filterBuilder.add(orFilter);
if (genesysFilter.isAnalyzed()) {
} else { // query
Set<Object> nonNull = new HashSet<Object>(filters.size()); StringBuilder sb = new StringBuilder();
boolean hasNull = false; for (Object val : literals) {
for (Object val : filterValues) { if (sb.length() > 0)
if (val != null) { sb.append(",");
nonNull.add(val); if (val instanceof String)
} else { sb.append("\"" + val + "\"");
hasNull = true; else
sb.append(val);
}
if (FilterConstants.ALIAS.equals(key)) {
// Nested
orFilter.add(FilterBuilders.nestedFilter("aliases", QueryBuilders.queryString("aliases.name" + ":(" + sb.toString() + ")")));
} else if (FilterConstants.SGSV.equals(key)) {
// Check if exists
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":(" + sb.toString() + ")")));
}
} else {
// terms
if (FilterConstants.ORGANIZATION.equals(key)) {
Set<String> instCodes = new HashSet<String>();
for (Object literal : literals) {
if (literal instanceof String) {
Organization organization = organizationService.getOrganization((String) literal);
if (organization != null)
for (FaoInstitute inst : organizationService.getMembers(organization)) {
instCodes.add(inst.getCode());
}
}
}
if (!instCodes.isEmpty()) {
orFilter.add(FilterBuilders.termsFilter(FilterConstants.INSTCODE, instCodes).execution("or"));
}
} else {
orFilter.add(FilterBuilders.termsFilter(key, literals).execution("or"));
}
}
} }
} }
if (hasNull) {
filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or"))); {
} else { // Handle operations
filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or")); for (FilterValue filterValue : filterValues) {
if (filterValue instanceof ValueRangeFilter) {
ValueRangeFilter range = (ValueRangeFilter) filterValue;
LOG.debug("Range " + range.getClass() + " " + range);
orFilter.add(FilterBuilders.rangeFilter(key).from(range.getFrom()).to(range.getTo()));
} else if (filterValue instanceof MaxValueFilter) {
MaxValueFilter max = (MaxValueFilter) filterValue;
LOG.debug("Max " + max);
orFilter.add(FilterBuilders.rangeFilter(key).to(max.getTo()));
} else if (filterValue instanceof MinValueFilter) {
MinValueFilter min = (MinValueFilter) filterValue;
LOG.debug("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min.getFrom()));
} else if (filterValue instanceof StartsWithFilter) {
StartsWithFilter startsWith = (StartsWithFilter) filterValue;
LOG.debug("startsWith " + startsWith);
if (genesysFilter.isAnalyzed()) {
if (FilterConstants.ALIAS.equals(key)) {
orFilter.add(FilterBuilders.nestedFilter("aliases",
QueryBuilders.queryString("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":" + startsWith.getStartsWith() + "*")));
}
} else {
orFilter.add(FilterBuilders.prefixFilter(key, startsWith.getStartsWith()));
}
}
}
} }
} }
filterBuilder.add(orFilter);
} }
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build(); SortBuilder sortBuilder = SortBuilders.fieldSort("acceNumb").order(SortOrder.ASC);
// System.err.println("Filter query: " + searchQuery.toString()); SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
try { try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class); Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
...@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} }
} }
private QueryBuilder makeQuery(String key, List<String> list) {
// Indexed fields
StringBuilder sb = new StringBuilder();
for (String val : list) {
if (sb.length() > 0)
sb.append(",");
sb.append("\"" + val + "\"");
}
return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
}
@Override @Override
public void update(String className, long id) { public void update(String className, long id) {
if (!clazzMap.containsKey(className)) { if (!clazzMap.containsKey(className)) {
...@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Initializing index"); LOG.info("Initializing index");
elasticsearchTemplate.createIndex("genesysarchive"); elasticsearchTemplate.createIndex("genesysarchive");
} }
Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class); Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object key : indexMapping.keySet()) { // for (Object key : indexMapping.keySet()) {
System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key)); // System.err.println("Mapping ke=" + key + " val=" +
} // indexMapping.get(key));
// }
LOG.info("Copying mapping"); LOG.info("Copying mapping");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping); elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
} }
......
...@@ -27,6 +27,7 @@ import java.util.Set; ...@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate; import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
...@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType; ...@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType;
import org.genesys2.server.model.filters.GenesysFilter.FilterType; import org.genesys2.server.model.filters.GenesysFilter.FilterType;
import org.genesys2.server.model.filters.I18nListFilter; import org.genesys2.server.model.filters.I18nListFilter;
import org.genesys2.server.model.filters.MethodFilter; import org.genesys2.server.model.filters.MethodFilter;
import org.genesys2.server.model.filters.NoSuchFilterException;
import org.genesys2.server.model.filters.NoSuchFilterValueException;
import org.genesys2.server.model.filters.UnsupportedFilterOperation;
import org.genesys2.server.model.filters.ValueName; import org.genesys2.server.model.filters.ValueName;
import org.genesys2.server.model.genesys.Method; import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.TraitCode; import org.genesys2.server.model.genesys.TraitCode;
...@@ -96,13 +94,13 @@ public class FilterHandler { ...@@ -96,13 +94,13 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING)); this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode")); this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE)); this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE)); this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING)); this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12, this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
13, 20, 30, 40, 50, 99 })); 13, 20, 30, 40, 50, 99 }));
} }
...@@ -111,37 +109,39 @@ public class FilterHandler { ...@@ -111,37 +109,39 @@ public class FilterHandler {
return Collections.unmodifiableList(this.availableFilters); return Collections.unmodifiableList(this.availableFilters);
} }
public GenesysFilter getFilter(String key) {
if (key.startsWith("gm:")) {
return getMethodFilter(key);
}
for (GenesysFilter f : this.availableFilters) {
if (f.getKey().equals(key))
return f;
}
return null;
}
public List<GenesysFilter> selectFilters(String[] selectedFilters) { public List<GenesysFilter> selectFilters(String[] selectedFilters) {
LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters)); LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters));
final List<GenesysFilter> filters = new ArrayList<GenesysFilter>(); final List<GenesysFilter> filters = new ArrayList<GenesysFilter>();
for (final String selectedFilter : selectedFilters) { for (final String selectedFilter : selectedFilters) {
if (selectedFilter.startsWith("gm:")) { try {
try { final GenesysFilter filter = getFilter(selectedFilter);
final GenesysFilter methodFilter = getMethodFilter(Long.parseLong(selectedFilter.substring(3))); if (filter != null) {
if (methodFilter != null) { filters.add(filter);
filters.add(methodFilter);
}
} catch (NumberFormatException | NullPointerException e) {
LOG.warn(e);
}