ElasticsearchService using AppliedFilters

src/main/java/org/genesys2/server/model/filters/BasicFilter.java

@@ -16,22 +16,37 @@
 
 package org.genesys2.server.model.filters;
 
-
 public class BasicFilter implements GenesysFilter {
 
 	private final String name;
 	private final DataType dataType;
 	private FilterType filterType;
 	private final Integer maxLength;
+	private boolean analyzed = false;
 
+	@Override
 	public boolean isCore() {
 		return true;
 	}
 
+	/**
+	 * Is the field analyzed by indexer?
+	 */
+	@Override
+	public boolean isAnalyzed() {
+		return this.analyzed;
+	};
+
+	public BasicFilter setAnalyzed(boolean analyzed) {
+		this.analyzed = analyzed;
+		return this;
+	}
+
 	public BasicFilter(String name, DataType type) {
 		this.name = name;
 		this.dataType = type;
 		if (this.dataType == DataType.NUMERIC) {
+			this.analyzed = false;
 			this.filterType = FilterType.RANGE;
 		} else {
 			this.filterType = FilterType.EXACT;
@@ -62,14 +77,12 @@ public class BasicFilter implements GenesysFilter {
 		return name;
 	}
 
-	public String getName() {
-		return name;
-	}
-
+	@Override
 	public DataType getDataType() {
 		return dataType;
 	}
 
+	@Override
 	public FilterType getFilterType() {
 		return filterType;
 	}

src/main/java/org/genesys2/server/model/filters/GenesysFilter.java

@@ -16,15 +16,24 @@
 
 package org.genesys2.server.model.filters;
 
+
 public interface GenesysFilter {
 
 	public String getKey();
 
 	public enum DataType {
-		FIXEDSTRING, STRING, NUMERIC, BOOLEAN
+		STRING, NUMERIC, BOOLEAN
 	}
 
 	public enum FilterType {
 		EXACT, RANGE, LIST, AUTOCOMPLETE, I18NLIST
 	}
+
+	boolean isCore();
+
+	DataType getDataType();
+
+	FilterType getFilterType();
+
+	public boolean isAnalyzed();
 }
\ No newline at end of file

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -38,7 +38,7 @@ public interface ElasticService {
 
 	void refreshIndex(String className);
 
-	Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException;
+	Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException;
 
 
 }

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

 package org.genesys2.server.service.impl;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -9,19 +8,31 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.index.query.AndFilterBuilder;
 import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.OrFilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
 import org.genesys2.server.model.elastic.AccessionDetails;
+import org.genesys2.server.model.filters.GenesysFilter;
 import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.model.impl.Organization;
 import org.genesys2.server.service.ElasticService;
 import org.genesys2.server.service.FilterConstants;
 import org.genesys2.server.service.GenesysFilterService;
 import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.OrganizationService;
+import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
 import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
+import org.genesys2.server.service.impl.FilterHandler.FilterValue;
+import org.genesys2.server.service.impl.FilterHandler.LiteralValueFilter;
+import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
+import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
+import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
+import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -37,7 +48,6 @@ import org.springframework.stereotype.Service;
 import org.springframework.util.StopWatch;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 
 @Service
 public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
@@ -49,9 +59,15 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 	@Autowired
 	private GenesysService genesysService;
 
+	@Autowired
+	private OrganizationService organizationService;
+
 	@Autowired
 	private GenesysFilterService filterService;
 
+	@Autowired
+	private FilterHandler filterHandler;
+
 	@Autowired
 	private ObjectMapper objectMapper;
 
@@ -76,86 +92,134 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 	}
 
 	@Override
-	public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException {
-		Map<String, List<String>> filters;
-		try {
-			filters = objectMapper.readValue(jsonFilter, Map.class);
-		} catch (IOException e) {
-			throw new SearchException(e.getMessage(), e);
-		}
+	public Page<AccessionDetails> filter(AppliedFilters appliedFilters, Pageable pageable) throws SearchException {
 
 		AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
-		for (String key : filters.keySet()) {
+		for (AppliedFilter appliedFilter : appliedFilters) {
+
+			String key = appliedFilter.getFilterName();
+
+			GenesysFilter genesysFilter = filterHandler.getFilter(key);
 
-			List<String> filterValues = filters.get(key);
-			if (filterValues == null || filterValues.isEmpty())
+			if (genesysFilter == null) {
+				LOG.warn("No such filter " + key);
 				continue;
+			}
 
-			System.err.println("key=" + key);
-
-			if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
-				// indexed field
-				filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
-			} else if (FilterConstants.ALIAS.equals(key)) {
-				// Nested object
-				filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
-			} else if (FilterConstants.SGSV.equals(key)) {
-				// Check for existance nested object
-				OrFilterBuilder orFilter = FilterBuilders.orFilter();
-				if (filterValues.contains(true)) {
-					// true:
-					orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
-				}
-				if (filterValues.contains(false)) {
-					// false:
-					orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
-				}
+			// Filter-level OR
+			OrFilterBuilder orFilter = FilterBuilders.orFilter();
 
-				if (filterValues.contains(true) || filterValues.contains(false))
-					filterBuilder.add(orFilter);
-			} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) {
-				OrFilterBuilder orFilter = FilterBuilders.orFilter();
-				// range filters
-				for (Object val : filterValues) {
-					System.err.println(val.getClass() + " " + val);
-					Map<String, Object> v = (Map<String, Object>) val;
-					List<Number> range = (List<Number>) v.get("range");
-					Number max = (Number) v.get("max");
-					Number min = (Number) v.get("min");
-					if (range != null) {
-						System.err.println("Range " + range.getClass() + " " + range);
-						orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1)));
-					} else if (max != null) {
-						System.err.println("Max " + max);
-						orFilter.add(FilterBuilders.rangeFilter(key).to(max));
-					} else if (min != null) {
-						System.err.println("Min " + min);
-						orFilter.add(FilterBuilders.rangeFilter(key).from(min));
+			// null
+			if (appliedFilter.getWithNull()) {
+				orFilter.add(FilterBuilders.missingFilter(key));
+			}
+
+			Set<FilterValue> filterValues = appliedFilter.getValues();
+			if (filterValues != null && !filterValues.isEmpty()) {
+
+				{
+					// Handle literals
+					Set<Object> literals = new HashSet<Object>();
+					for (FilterValue filterValue : filterValues) {
+						if (filterValue instanceof FilterHandler.LiteralValueFilter) {
+							FilterHandler.LiteralValueFilter literal = (LiteralValueFilter) filterValue;
+							literals.add(literal.getValue());
+						}
 					}
 
-				}
-				filterBuilder.add(orFilter);
-
-			} else {
-				Set<Object> nonNull = new HashSet<Object>(filters.size());
-				boolean hasNull = false;
-				for (Object val : filterValues) {
-					if (val != null) {
-						nonNull.add(val);
-					} else {
-						hasNull = true;
+					if (!literals.isEmpty()) {
+
+						if (genesysFilter.isAnalyzed()) {
+							// query
+							StringBuilder sb = new StringBuilder();
+							for (Object val : literals) {
+								if (sb.length() > 0)
+									sb.append(",");
+								if (val instanceof String)
+									sb.append("\"" + val + "\"");
+								else
+									sb.append(val);
+							}
+
+							if (FilterConstants.ALIAS.equals(key)) {
+								// Nested
+								orFilter.add(FilterBuilders.nestedFilter("aliases", QueryBuilders.queryString("aliases.name" + ":(" + sb.toString() + ")")));
+							} else if (FilterConstants.SGSV.equals(key)) {
+								// Check if exists
+								if (filterValues.contains(true)) {
+									// true:
+									orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
+								}
+								if (filterValues.contains(false)) {
+									// false:
+									orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
+								}
+							} else {
+								orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":(" + sb.toString() + ")")));
+							}
+						} else {
+							// terms
+
+							if (FilterConstants.ORGANIZATION.equals(key)) {
+								Set<String> instCodes = new HashSet<String>();
+								for (Object literal : literals) {
+									if (literal instanceof String) {
+										Organization organization = organizationService.getOrganization((String) literal);
+										if (organization != null)
+											for (FaoInstitute inst : organizationService.getMembers(organization)) {
+												instCodes.add(inst.getCode());
+											}
+									}
+								}
+								if (!instCodes.isEmpty()) {
+									orFilter.add(FilterBuilders.termsFilter(FilterConstants.INSTCODE, instCodes).execution("or"));
+								}
+							} else {
+								orFilter.add(FilterBuilders.termsFilter(key, literals).execution("or"));
+							}
+						}
 					}
 				}
-				if (hasNull) {
-					filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or")));
-				} else {
-					filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or"));
+
+				{
+					// Handle operations
+					for (FilterValue filterValue : filterValues) {
+						if (filterValue instanceof ValueRangeFilter) {
+							ValueRangeFilter range = (ValueRangeFilter) filterValue;
+							LOG.debug("Range " + range.getClass() + " " + range);
+							orFilter.add(FilterBuilders.rangeFilter(key).from(range.getFrom()).to(range.getTo()));
+						} else if (filterValue instanceof MaxValueFilter) {
+							MaxValueFilter max = (MaxValueFilter) filterValue;
+							LOG.debug("Max " + max);
+							orFilter.add(FilterBuilders.rangeFilter(key).to(max.getTo()));
+						} else if (filterValue instanceof MinValueFilter) {
+							MinValueFilter min = (MinValueFilter) filterValue;
+							LOG.debug("Min " + min);
+							orFilter.add(FilterBuilders.rangeFilter(key).from(min.getFrom()));
+						} else if (filterValue instanceof StartsWithFilter) {
+							StartsWithFilter startsWith = (StartsWithFilter) filterValue;
+							LOG.debug("startsWith " + startsWith);
+							if (genesysFilter.isAnalyzed()) {
+								if (FilterConstants.ALIAS.equals(key)) {
+									orFilter.add(FilterBuilders.nestedFilter("aliases",
+											QueryBuilders.queryString("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
+								} else {
+									orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryString(key + ":" + startsWith.getStartsWith() + "*")));
+								}
+							} else {
+								orFilter.add(FilterBuilders.prefixFilter(key, startsWith.getStartsWith()));
+							}
+						}
+					}
 				}
 			}
+
+			filterBuilder.add(orFilter);
+
 		}
 
-		SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build();
-		// System.err.println("Filter query: " + searchQuery.toString());
+		SortBuilder sortBuilder = SortBuilders.fieldSort("acceNumb").order(SortOrder.ASC);
+		SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
 
 		try {
 			Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
@@ -165,17 +229,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 		}
 	}
 
-	private QueryBuilder makeQuery(String key, List<String> list) {
-		// Indexed fields
-		StringBuilder sb = new StringBuilder();
-		for (String val : list) {
-			if (sb.length() > 0)
-				sb.append(",");
-			sb.append("\"" + val + "\"");
-		}
-		return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
-	}
-
 	@Override
 	public void update(String className, long id) {
 		if (!clazzMap.containsKey(className)) {
@@ -343,10 +396,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 			LOG.info("Initializing index");
 			elasticsearchTemplate.createIndex("genesysarchive");
 		}
-		Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
-		for (Object key : indexMapping.keySet()) {
-			System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key));
-		}
+		Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
+		// for (Object key : indexMapping.keySet()) {
+		// System.err.println("Mapping ke=" + key + " val=" +
+		// indexMapping.get(key));
+		// }
 		LOG.info("Copying mapping");
 		elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
 	}

src/main/java/org/genesys2/server/service/impl/FilterHandler.java

@@ -27,6 +27,7 @@ import java.util.Set;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.Predicate;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -38,9 +39,6 @@ import org.genesys2.server.model.filters.GenesysFilter.DataType;
 import org.genesys2.server.model.filters.GenesysFilter.FilterType;
 import org.genesys2.server.model.filters.I18nListFilter;
 import org.genesys2.server.model.filters.MethodFilter;
-import org.genesys2.server.model.filters.NoSuchFilterException;
-import org.genesys2.server.model.filters.NoSuchFilterValueException;
-import org.genesys2.server.model.filters.UnsupportedFilterOperation;
 import org.genesys2.server.model.filters.ValueName;
 import org.genesys2.server.model.genesys.Method;
 import org.genesys2.server.model.genesys.TraitCode;
@@ -96,13 +94,13 @@ public class FilterHandler {
 
 		this.availableFilters.add(new BasicFilter(FilterConstants.ORGANIZATION, DataType.STRING));
 		this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTCODE, "/explore/ac/instCode"));
-		this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE));
-		this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE));
+		this.availableFilters.add(new BasicFilter(FilterConstants.ACCENUMB, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
+		this.availableFilters.add(new BasicFilter(FilterConstants.ALIAS, DataType.STRING, FilterType.RANGE).setAnalyzed(true));
 		this.availableFilters.add(new BasicFilter(FilterConstants.SGSV, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.MLSSTATUS, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.ART15, DataType.BOOLEAN));
 		this.availableFilters.add(new BasicFilter(FilterConstants.AVAILABLE, DataType.BOOLEAN));
-		this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING));
+		this.availableFilters.add(new BasicFilter(FilterConstants.COLLMISSID, DataType.STRING).setAnalyzed(true));
 		this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.STORAGE, DataType.NUMERIC).build("accession.storage", new Integer[] { 10, 11, 12,
 				13, 20, 30, 40, 50, 99 }));
 	}
@@ -111,37 +109,39 @@ public class FilterHandler {
 		return Collections.unmodifiableList(this.availableFilters);
 	}
 
+	public GenesysFilter getFilter(String key) {
+		if (key.startsWith("gm:")) {
+			return getMethodFilter(key);
+		}
+
+		for (GenesysFilter f : this.availableFilters) {
+			if (f.getKey().equals(key))
+				return f;
+		}
+
+		return null;
+	}
+
 	public List<GenesysFilter> selectFilters(String[] selectedFilters) {
 		LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters));
 		final List<GenesysFilter> filters = new ArrayList<GenesysFilter>();
 
 		for (final String selectedFilter : selectedFilters) {
-			if (selectedFilter.startsWith("gm:")) {
-				try {
-					final GenesysFilter methodFilter = getMethodFilter(Long.parseLong(selectedFilter.substring(3)));
-					if (methodFilter != null) {
-						filters.add(methodFilter);
-					}
-				} catch (NumberFormatException | NullPointerException e) {
-					LOG.warn(e);
-				}
-			} else {
-				final GenesysFilter coreFilter = CollectionUtils.find(this.availableFilters, new Predicate<GenesysFilter>() {
-					@Override
-					public boolean evaluate(GenesysFilter object) {
-						return object.getKey().equals(selectedFilter);
-					}
-				});
-				if (coreFilter != null) {
-					filters.add(coreFilter);
+			try {
+				final GenesysFilter filter = getFilter(selectedFilter);
+				if (filter != null) {
+					filters.add(filter);
 				}
+			} catch (NumberFormatException | NullPointerException e) {
+				LOG.warn(e);
 			}
 		}
 
 		return filters;
 	}
 
-	GenesysFilter getMethodFilter(long methodId) {
+	GenesysFilter getMethodFilter(String methodFilterName) {
+		long methodId = Long.parseLong(methodFilterName.substring(3));
 		final Method method = traitService.getMethod(methodId);
 		return method == null ? null : toFilter(method);
 	}
@@ -172,75 +172,6 @@ public class FilterHandler {
 		return filter;
 	}
 
-	public List<AppliedFilter> fromJSON(String jsonFilter) throws IOException, NoSuchFilterException, UnsupportedFilterOperation, NoSuchFilterValueException {
-		@SuppressWarnings("unchecked")
-		Map<String, List<Object>> filters = objectMapper.readValue(jsonFilter, Map.class);
-		AppliedFilters appliedFilters = new AppliedFilters();
-
-		for (String key : filters.keySet()) {
-
-			List<Object> filterValues = filters.get(key);
-			if (filterValues == null || filterValues.isEmpty())
-				continue;
-
-			LOG.info("key=" + key);
-
-			AppliedFilter appliedFilter = new AppliedFilter();
-			appliedFilter.setFilter(getFilterByName(key));
-			for (Object filterValue : filterValues) {
-				appliedFilter.addFilterValue(toFilterValue(filterValue));
-			}
-		}
-
-		return appliedFilters;
-	}
-
-	private FilterValue toFilterValue(Object filterValue) throws UnsupportedFilterOperation, NoSuchFilterValueException {
-		if (filterValue == null)
-			return null;
-
-		if (filterValue instanceof Map) {
-			return toFilterOperation((Map<?, ?>) filterValue);
-		} else if (filterValue instanceof Number || filterValue instanceof String) {
-			return new LiteralValueFilter(filterValue);
-		} else {
-			throw new NoSuchFilterValueException(filterValue);
-		}
-	}
-
-	private FilterValue toFilterOperation(Map<?, ?> filterValue) throws UnsupportedFilterOperation {
-		@SuppressWarnings("unchecked")
-		List<Number> range = (List<Number>) filterValue.get("range");
-
-		if (range != null) {
-			return new ValueRangeFilter(range.get(0), range.get(1));
-		}
-		Number max = (Number) filterValue.get("max");
-		if (max != null) {
-			return new MaxValueFilter(max);
-		}
-		Number min = (Number) filterValue.get("min");
-		if (min != null) {
-			return new MinValueFilter(min);
-		}
-
-		String like = (String) filterValue.get("like");
-		if (like != null) {
-			return new StartsWithFilter(like);
-		}
-
-		throw new UnsupportedFilterOperation(filterValue.toString());
-	}
-
-	private GenesysFilter getFilterByName(String key) throws NoSuchFilterException {
-		for (GenesysFilter filter : this.availableFilters) {
-			if (filter.getKey().equals(key)) {
-				return filter;
-			}
-		}
-		throw new NoSuchFilterException(key);
-	}
-
 	public static interface FilterValue {
 		String getType();
 	}
@@ -316,7 +247,11 @@ public class FilterHandler {
 								jp.nextToken();
 								String op = jp.getCurrentName();
 								if ("like".equals(op)) {
-									af.addFilterValue(new StartsWithFilter(jp.nextTextValue()));
+									String startsWith = jp.nextTextValue();
+									if (startsWith == null || StringUtils.isBlank(startsWith))
+										throw new JsonParseException("StartsWithFilter expects a non-blank string", jp.getCurrentLocation());
+
+									af.addFilterValue(new StartsWithFilter(startsWith));
 								} else if ("min".equals(op)) {
 									Number number1 = null;
 									jp.nextToken();
@@ -436,6 +371,7 @@ public class FilterHandler {
 		 * @param class1