Skip to content

GitLab

Genesys Backend
Genesys Backend
Commit 28b20edb authored by Matija Obreza's avatar Matija Obreza
Browse files
Options

ElasticsearchService using AppliedFilters

parent 64e3c84c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 839 additions and 301 deletions
src/main/java/org/genesys2/server/model/filters/BasicFilter.java
View file @ 28b20edb
...@@ -16,22 +16,37 @@ ...@@ -16,22 +16,37 @@
package org.genesys2.server.model.filters; package org.genesys2.server.model.filters;
public class BasicFilter implements GenesysFilter { public class BasicFilter implements GenesysFilter {
private final String name; private final String name;
private final DataType dataType; private final DataType dataType;
private FilterType filterType; private FilterType filterType;
private final Integer maxLength; private final Integer maxLength;
private boolean analyzed = false;
@Override
public boolean isCore() { public boolean isCore() {
return true; return true;
} }
/**
* Is the field analyzed by indexer?
*/
@Override
public boolean isAnalyzed() {
return this.analyzed;
};
public BasicFilter setAnalyzed(boolean analyzed) {
this.analyzed = analyzed;
return this;
}
public BasicFilter(String name, DataType type) { public BasicFilter(String name, DataType type) {
this.name = name; this.name = name;
this.dataType = type; this.dataType = type;
if (this.dataType == DataType.NUMERIC) { if (this.dataType == DataType.NUMERIC) {
this.analyzed = false;
this.filterType = FilterType.RANGE; this.filterType = FilterType.RANGE;
} else { } else {
this.filterType = FilterType.EXACT; this.filterType = FilterType.EXACT;
...@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter { ...@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter {
return name; return name;
} }
public String getName() { @Override
return name;
}
public DataType getDataType() { public DataType getDataType() {
return dataType; return dataType;
} }
@Override
public FilterType getFilterType() { public FilterType getFilterType() {
return filterType; return filterType;
} }
......
src/main/java/org/genesys2/server/model/filters/GenesysFilter.java
View file @ 28b20edb
...@@ -16,15 +16,24 @@ ...@@ -16,15 +16,24 @@
package org.genesys2.server.model.filters; package org.genesys2.server.model.filters;
public interface GenesysFilter { public interface GenesysFilter {
public String getKey(); public String getKey();
public enum DataType { public enum DataType {
FIXEDSTRING, STRING, NUMERIC, BOOLEAN STRING, NUMERIC, BOOLEAN
} }
public enum FilterType { public enum FilterType {
EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST
} }
boolean isCore();
DataType getDataType();
FilterType getFilterType();
public boolean isAnalyzed();
} }
\ No newline at end of file
src/main/java/org/genesys2/server/service/ElasticService.java
View file @ 28b20edb
...@@ -38,7 +38,7 @@ public interface ElasticService { ...@@ -38,7 +38,7 @@ public interface ElasticService {
void refreshIndex(String className); void refreshIndex(String className);
Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException; Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException;
} }
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @ 28b20edb
package org.genesys2.server.service.impl; package org.genesys2.server.service.impl;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
...@@ -9,19 +8,31 @@ import java.util.List; ...@@ -9,19 +8,31 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.AndFilterBuilder; import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder; import org.elasticsearch.index.query.OrFilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.genesys2.server.model.elastic.AccessionDetails; import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.filters.GenesysFilter;
import org.genesys2.server.model.genesys.Accession; import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.ElasticService; import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants; import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService; import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters; import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.impl.FilterHandler.FilterValue;
import org.genesys2.server.service.impl.FilterHandler.LiteralValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
...@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service; ...@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch; import org.springframework.util.StopWatch;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Service @Service
public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean { public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
...@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired @Autowired
private GenesysService genesysService; private GenesysService genesysService;
@Autowired
private OrganizationService organizationService;
@Autowired @Autowired
private GenesysFilterService filterService; private GenesysFilterService filterService;
@Autowired
private FilterHandler filterHandler;
@Autowired @Autowired
private ObjectMapper objectMapper; private ObjectMapper objectMapper;
...@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} }
@Override @Override
public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException { public Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException {
Map<String, List<String>> filters;
try {
filters = objectMapper.readValue(jsonFilter, Map.class);
} catch (IOException e) {
throw new SearchException(e.getMessage(), e);
}
AndFilterBuilder filterBuilder = FilterBuilders.andFilter(); AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
for (String key : filters.keySet()) { for (AppliedFilter appliedFilter : appliedFilters) {
String key = appliedFilter.getFilterName();
GenesysFilter genesysFilter = filterHandler.getFilter(key);
List<String> filterValues = filters.get(key); if (genesysFilter == null) {
if (filterValues == null || filterValues.isEmpty()) LOG.warn("No such filter " + key);
continue; continue;
}
System.err.println("key=" + key); // Filter-level OR
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
// indexed field
filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
} else if (FilterConstants.ALIAS.equals(key)) {
// Nested object
filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
} else if (FilterConstants.SGSV.equals(key)) {
// Check for existance nested object
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
if (filterValues.contains(true) || filterValues.contains(false)) // null
filterBuilder.add(orFilter); if (appliedFilter.getWithNull()) {
} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) { orFilter.add(FilterBuilders.missingFilter(key));
OrFilterBuilder orFilter = FilterBuilders.orFilter(); }
// range filters
for (Object val : filterValues) { Set<FilterValue> filterValues = appliedFilter.getValues();
System.err.println(val.getClass() + " " + val); if (filterValues != null && !filterValues.isEmpty()) {
Map<String, Object> v = (Map<String, Object>) val;
List<Number> range = (List<Number>) v.get("range"); {
Number max = (Number) v.get("max"); // Handle literals
Number min = (Number) v.get("min"); Set<Object> literals = new HashSet<Object>();
if (range != null) { for (FilterValue filterValue : filterValues) {
System.err.println("Range " + range.getClass() + " " + range); if (filterValue instanceof FilterHandler.LiteralValueFilter) {
orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1))); FilterHandler.LiteralValueFilter literal = (LiteralValueFilter) filterValue;
} else if (max != null) { literals.add(literal.getValue());
System.err.println("Max " + max); }
orFilter.add(FilterBuilders.rangeFilter(key).to(max));
} else if (min != null) {
System.err.println("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min));
} }
} if (!literals.isEmpty()) {
filterBuilder.add(orFilter);
if (genesysFilter.isAnalyzed()) {
} else { // query
Set<Object> nonNull = new HashSet<Object>(filters.size()); StringBuilder sb = new StringBuilder();
boolean hasNull = false; for (Object val : literals) {
for (Object val : filterValues) { if (sb.length() > 0)
if (val != null) { sb.append(",");
nonNull.add(val); if (val instanceof String)
} else { sb.append("\"" + val + "\"");
hasNull = true; else
sb.append(val);
}
if (FilterConstants.ALIAS.equals(key)) {
// Nested
orFilter.add(FilterBuilders.nestedFilter("aliases", QueryBuilders.queryString("aliases.name" + ":(" + sb.toString() + ")")));
} else if (FilterConstants.SGSV.equals(key)) {
// Check if exists
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":(" + sb.toString() + ")")));
}
} else {
// terms
if (FilterConstants.ORGANIZATION.equals(key)) {
Set<String> instCodes = new HashSet<String>();
for (Object literal : literals) {
if (literal instanceof String) {
Organization organization = organizationService.getOrganization((String) literal);
if (organization != null)
for (FaoInstitute inst : organizationService.getMembers(organization)) {
instCodes.add(inst.getCode());
}
}
}
if (!instCodes.isEmpty()) {
orFilter.add(FilterBuilders.termsFilter(FilterConstants.INSTCODE, instCodes).execution("or"));
}
} else {
orFilter.add(FilterBuilders.termsFilter(key, literals).execution("or"));
}
}
} }
} }
if (hasNull) {
filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or"))); {
} else { // Handle operations
filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or")); for (FilterValue filterValue : filterValues) {
if (filterValue instanceof ValueRangeFilter) {
ValueRangeFilter range = (ValueRangeFilter) filterValue;
LOG.debug("Range " + range.getClass() + " " + range);
orFilter.add(FilterBuilders.rangeFilter(key).from(range.getFrom()).to(range.getTo()));
} else if (filterValue instanceof MaxValueFilter) {
MaxValueFilter max = (MaxValueFilter) filterValue;
LOG.debug("Max " + max);
orFilter.add(FilterBuilders.rangeFilter(key).to(max.getTo()));
} else if (filterValue instanceof MinValueFilter) {
MinValueFilter min = (MinValueFilter) filterValue;
LOG.debug("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min.getFrom()));
} else if (filterValue instanceof StartsWithFilter) {
StartsWithFilter startsWith = (StartsWithFilter) filterValue;
LOG.debug("startsWith " + startsWith);
if (genesysFilter.isAnalyzed()) {
if (FilterConstants.ALIAS.equals(key)) {
orFilter.add(FilterBuilders.nestedFilter("aliases",
QueryBuilders.queryString("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":" + startsWith.getStartsWith() + "*")));
}
} else {
orFilter.add(FilterBuilders.prefixFilter(key, startsWith.getStartsWith()));
}
}
}
} }
} }
filterBuilder.add(orFilter);
} }
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build(); SortBuilder sortBuilder = SortBuilders.fieldSort("acceNumb").order(SortOrder.ASC);
// System.err.println("Filter query: " + searchQuery.toString()); SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
try { try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class); Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
...@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} }
} }
private QueryBuilder makeQuery(String key, List<String> list) {
// Indexed fields
StringBuilder sb = new StringBuilder();
for (String val : list) {
if (sb.length() > 0)
sb.append(",");
sb.append("\"" + val + "\"");
}
return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
}
@Override @Override
public void update(String className, long id) { public void update(String className, long id) {
if (!clazzMap.containsKey(className)) { if (!clazzMap.containsKey(className)) {
...@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Initializing index"); LOG.info("Initializing index");
elasticsearchTemplate.createIndex("genesysarchive"); elasticsearchTemplate.createIndex("genesysarchive");
} }
Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class); Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object key : indexMapping.keySet()) { // for (Object key : indexMapping.keySet()) {
System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key)); // System.err.println("Mapping ke=" + key + " val=" +
} // indexMapping.get(key));
// }
LOG.info("Copying mapping"); LOG.info("Copying mapping");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping); elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
} }
......
src/main/java/org/genesys2/server/service/impl/FilterHandler.java
View file @ 28b20edb
...@@ -27,6 +27,7 @@ import java.util.Set; ...@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate; import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
...@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType; ...@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType;
import org.genesys2.server.model.filters.GenesysFilter.FilterType; import org.genesys2.server.model.filters.GenesysFilter.FilterType;
import org.genesys2.server.model.filters.I18nListFilter; import org.genesys2.server.model.filters.I18nListFilter;
import org.genesys2.server.model.filters.MethodFilter; import org.genesys2.server.model.filters.MethodFilter;
import org.genesys2.server.model.filters.NoSuchFilterException;
import org.genesys2.server.model.filters.NoSuchFilterValueException;
import org.genesys2.server.model.filters.UnsupportedFilterOperation;
import org.genesys2.server.model.filters.ValueName; import org.genesys2.server.model.filters.ValueName;
import org.genesys2.server.model.genesys.Method; import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.TraitCode; import org.genesys2.server.model.genesys.TraitCode;
...@@ -96,13 +94,13 @@ public class FilterHandler { ...@@ -96,13 +94,13 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING)); this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode")); this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE)); this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE)); this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN)); this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING)); this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12, this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
13, 20, 30, 40, 50, 99 })); 13, 20, 30, 40, 50, 99 }));
} }
...@@ -111,37 +109,39 @@ public class FilterHandler { ...@@ -111,37 +109,39 @@ public class FilterHandler {
return Collections.unmodifiableList(this.availableFilters); return Collections.unmodifiableList(this.availableFilters);
} }
public GenesysFilter getFilter(String key) {
if (key.startsWith("gm:")) {
return getMethodFilter(key);
}
for (GenesysFilter f : this.availableFilters) {
if (f.getKey().equals(key))
return f;
}
return null;
}
public List<GenesysFilter> selectFilters(String[] selectedFilters) { public List<GenesysFilter> selectFilters(String[] selectedFilters) {
LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters)); LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters));
final List<GenesysFilter> filters = new ArrayList<GenesysFilter>(); final List<GenesysFilter> filters = new ArrayList<GenesysFilter>();
for (final String selectedFilter : selectedFilters) { for (final String selectedFilter : selectedFilters) {
if (selectedFilter.startsWith("gm:")) { try {
try { final GenesysFilter filter = getFilter(selectedFilter);
final GenesysFilter methodFilter = getMethodFilter(Long.parseLong(selectedFilter.substring(3))); if (filter != null) {
if (methodFilter != null) { filters.add(filter);
filters.add(methodFilter);
}
} catch (NumberFormatException | NullPointerException e) {
LOG.warn(e);
}