Filtering with elasticsearch

src/main/java/org/genesys2/server/service/ContentService.java

@@ -16,6 +16,7 @@
 
 package org.genesys2.server.service;
 
+import java.io.Writer;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -102,4 +103,6 @@ public interface ContentService {
 	Locale getDefaultLocale();
 
 	String processTemplate(String body, Map<String, Object> root);
+
+	void processTemplate(String templateStr, Map<String, Object> root, Writer writer);
 }

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -37,4 +37,6 @@ public interface ElasticService {
 
 	void refreshIndex(String className);
 
+	Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException;
+
 }

src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java

@@ -17,6 +17,7 @@
 package org.genesys2.server.service.impl;
 
 import java.io.StringWriter;
+import java.io.Writer;
 import java.util.Calendar;
 import java.util.List;
 import java.util.Locale;
@@ -169,7 +170,7 @@ public class ContentServiceImpl implements ContentService {
 
 	/**
 	 * Creates or updates an article
-	 *
+	 * 
 	 * @param entity
 	 * @param body
 	 * @param locale
@@ -289,7 +290,22 @@ public class ContentServiceImpl implements ContentService {
 		 * Merge data and template
 		 */
 		velocityEngine.evaluate(context, swOut, "log tag name", templateStr);
-		System.out.println(swOut);
+		if (LOG.isTraceEnabled()) {
+			LOG.trace(swOut);
+		}
 		return swOut.toString();
 	}
+
+	@Override
+	public void processTemplate(String templateStr, Map<String, Object> root, Writer writer) {
+		final VelocityContext context = new VelocityContext();
+		for (final String key : root.keySet()) {
+			context.put(key, root.get(key));
+		}
+
+		/**
+		 * Merge data and template
+		 */
+		velocityEngine.evaluate(context, writer, "log tag name", templateStr);
+	}
 }

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

@@ -4,17 +4,23 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.index.query.AndFilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.OrFilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.genesys2.server.model.elastic.AccessionDetails;
 import org.genesys2.server.model.genesys.Accession;
-import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.ElasticService;
+import org.genesys2.server.service.FilterConstants;
 import org.genesys2.server.service.GenesysFilterService;
 import org.genesys2.server.service.GenesysService;
-import org.genesys2.server.service.OrganizationService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -45,15 +51,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 	@Autowired
 	private GenesysFilterService filterService;
 
-	@Autowired
-	private OrganizationService organizationService;
-
 	@Autowired
 	private ObjectMapper objectMapper;
 
-	@Autowired
-	private CropService cropService;
-
 	private final Map<String, Class<?>> clazzMap;
 
 	public ElasticsearchSearchServiceImpl() {
@@ -74,6 +74,107 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 		}
 	}
 
+	@Override
+	public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException {
+		Map<String, List<String>> filters;
+		try {
+			filters = objectMapper.readValue(jsonFilter, Map.class);
+		} catch (IOException e) {
+			throw new SearchException(e.getMessage(), e);
+		}
+
+		AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
+		for (String key : filters.keySet()) {
+
+			List<String> filterValues = filters.get(key);
+			if (filterValues == null || filterValues.isEmpty())
+				continue;
+
+			System.err.println("key=" + key);
+
+			if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
+				// indexed field
+				filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
+			} else if (FilterConstants.ALIAS.equals(key)) {
+				// Nested object
+				filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
+			} else if (FilterConstants.SGSV.equals(key)) {
+				// Check for existance nested object
+				OrFilterBuilder orFilter = FilterBuilders.orFilter();
+				if (filterValues.contains(true)) {
+					// true:
+					orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
+				}
+				if (filterValues.contains(false)) {
+					// false:
+					orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
+				}
+
+				if (filterValues.contains(true) || filterValues.contains(false))
+					filterBuilder.add(orFilter);
+			} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) {
+				OrFilterBuilder orFilter = FilterBuilders.orFilter();
+				// range filters
+				for (Object val : filterValues) {
+					System.err.println(val.getClass() + " " + val);
+					Map<String, Object> v = (Map<String, Object>) val;
+					List<Number> range = (List<Number>) v.get("range");
+					Number max = (Number) v.get("max");
+					Number min = (Number) v.get("min");
+					if (range != null) {
+						System.err.println("Range " + range.getClass() + " " + range);
+						orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1)));
+					} else if (max != null) {
+						System.err.println("Max " + max);
+						orFilter.add(FilterBuilders.rangeFilter(key).to(max));
+					} else if (min != null) {
+						System.err.println("Min " + min);
+						orFilter.add(FilterBuilders.rangeFilter(key).from(min));
+					}
+
+				}
+				filterBuilder.add(orFilter);
+
+			} else {
+				Set<Object> nonNull = new HashSet<Object>(filters.size());
+				boolean hasNull = false;
+				for (Object val : filterValues) {
+					if (val != null) {
+						nonNull.add(val);
+					} else {
+						hasNull = true;
+					}
+				}
+				if (hasNull) {
+					filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or")));
+				} else {
+					filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or"));
+				}
+			}
+		}
+
+		SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build();
+		// System.err.println("Filter query: " + searchQuery.toString());
+
+		try {
+			Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
+			return sampleEntities;
+		} catch (Throwable e) {
+			throw new SearchException(e.getMessage(), e);
+		}
+	}
+
+	private QueryBuilder makeQuery(String key, List<String> list) {
+		// Indexed fields
+		StringBuilder sb = new StringBuilder();
+		for (String val : list) {
+			if (sb.length() > 0)
+				sb.append(",");
+			sb.append("\"" + val + "\"");
+		}
+		return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
+	}
+
 	@Override
 	public void update(String className, long id) {
 		if (!clazzMap.containsKey(className)) {
@@ -255,6 +356,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 			elasticsearchTemplate.createIndex("genesysarchive");
 		}
 		Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
+		for (Object key : indexMapping.keySet()) {
+			System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key));
+		}
 		LOG.info("Copying mapping");
 		elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
 	}

src/main/java/org/genesys2/server/servlet/controller/rdf/AccessionControllerRdf.java

@@ -16,6 +16,13 @@
 
 package org.genesys2.server.servlet.controller.rdf;
 
+import java.io.IOException;
+import java.io.OutputStream;
+
+import javax.servlet.http.HttpServletResponse;
+
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.GenesysService;
@@ -28,6 +35,7 @@ import org.springframework.ui.ModelMap;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
 
 @Controller
 @RequestMapping(value = "/acn", headers = "accept=text/turtle", produces = "text/turtle")
@@ -67,4 +75,16 @@ public class AccessionControllerRdf {
 
 		return "/accession/details-turtle";
 	}
+
+	@RequestMapping(value = "/search", method = RequestMethod.POST)
+	public void downloadFiltered(@RequestParam(value = "filter", required = false, defaultValue = "{}") String jsonFilter, HttpServletResponse response)
+			throws IOException {
+		final OutputStream outputStream = response.getOutputStream();
+		
+		// TODO
+		
+		
+		response.flushBuffer();
+	}
+
 }

src/main/webapp/WEB-INF/jsp/accession/details-turtle.jsp

@@ -13,7 +13,7 @@
 <http://www.genesys-pgr.org/acn/id/${accession.id}>
   a germplasm:GermplasmAccession;
   germplasm:germplasmID "<c:out value="${accession.id}" />";
-  dwc:cataogNumber "<c:out value="${accession.accessionName}" />";
+  dwc:catalogNumber "<c:out value="${accession.accessionName}" />";
   dwc:institutionCode "<c:out value="${accession.instituteCode}" />";
   dwc:instituteId "<c:out value="${accession.institute.id}" />";
   dwc:countryCode "<c:out value="${accession.institute.country.code3}" />";

src/test/java/org/genesys2/server/service/impl/ElasticsearchTest.java

@@ -16,46 +16,146 @@
 
 package org.genesys2.server.service.impl;
 
+import java.io.IOException;
+
 import org.apache.log4j.Logger;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.exception.VelocityException;
+import org.genesys2.server.aspect.AsAdminAspect;
 import org.genesys2.server.model.elastic.AccessionDetails;
+import org.genesys2.server.service.AclService;
+import org.genesys2.server.service.ContentService;
+import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.ElasticService;
+import org.genesys2.server.service.FilterConstants;
+import org.genesys2.server.service.GenesysFilterService;
+import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.GeoService;
+import org.genesys2.server.service.HtmlSanitizer;
+import org.genesys2.server.service.InstituteService;
+import org.genesys2.server.service.OrganizationService;
+import org.genesys2.server.service.TaxonomyService;
+import org.genesys2.server.service.UserService;
+import org.genesys2.server.test.JpaDataConfig;
 import org.genesys2.server.test.PropertyPlacholderInitializer;
 import org.genesys2.spring.config.ElasticsearchConfig;
+import org.genesys2.spring.config.HazelcastConfig;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.support.NoOpCacheManager;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.ui.velocity.VelocityEngineFactoryBean;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes = ElasticsearchTest.Config.class, initializers = PropertyPlacholderInitializer.class)
+@ActiveProfiles("cacher")
 public class ElasticsearchTest {
 	private static final Logger LOG = Logger.getLogger(ElasticsearchTest.class);
 
-	@Import(ElasticsearchConfig.class)
+	@Import({ ElasticsearchConfig.class, JpaDataConfig.class, HazelcastConfig.class })
+	@ComponentScan(basePackages = { "org.genesys2.server.persistence.domain" })
 	public static class Config {
 
 		@Bean
 		public ElasticService searchService() {
 			return new ElasticsearchSearchServiceImpl();
 		}
-		
+
 		@Bean
 		public ObjectMapper objectMapper() {
 			return new ObjectMapper();
 		}
+
+		@Bean
+		public AsAdminAspect asAdminAspect() {
+			return new AsAdminAspect();
+		}
+
+		@Bean
+		public UserService userService() {
+			return new UserServiceImpl();
+		}
+
+		@Bean
+		public AclService aclService() {
+			return new AclServiceImpl();
+		}
+
+		@Bean
+		public TaxonomyService taxonomyService() {
+			return new TaxonomyServiceImpl();
+		}
+
+		@Bean
+		public GenesysService genesysService() {
+			return new GenesysServiceImpl();
+		}
+
+		@Bean
+		public CacheManager cacheManager() {
+			return new NoOpCacheManager();
+		}
+
+		@Bean
+		public HtmlSanitizer htmlSanitizer() {
+			return new OWASPSanitizer();
+		}
+
+		@Bean
+		public GeoService geoService() {
+			return new GeoServiceImpl();
+		}
+
+		@Bean
+		public ContentService contentService() {
+			return new ContentServiceImpl();
+		}
+
+		@Bean
+		public VelocityEngine velocityEngine() throws VelocityException, IOException {
+			final VelocityEngineFactoryBean vf = new VelocityEngineFactoryBean();
+			return vf.createVelocityEngine();
+		}
+
+		@Bean
+		public OrganizationService organizationService() {
+			return new OrganizationServiceImpl();
+		}
+
+		@Bean
+		public InstituteService instituteService() {
+			return new InstituteServiceImpl();
+		}
+
+		@Bean
+		public CropService cropService() {
+			return new CropServiceImpl();
+		}
+
+		@Bean
+		public GenesysFilterService genesysFilterService() {
+			return new GenesysFilterServiceImpl();
+		}
 	}
 
 	@Autowired
 	private ObjectMapper mapper;
-	
+
 	@Autowired
 	private ElasticService elasticService;
 
@@ -67,4 +167,202 @@ public class ElasticsearchTest {
 			LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
 		}
 	}
+
+	@Test
+	public void testInstituteCode() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.INSTCODE);
+		arr.add("NGA039");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testGenus() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.TAXONOMY_GENUS);
+		arr.add("Vigna");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+
+	}
+
+	@Test
+	public void testGenusAndSpecies() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.TAXONOMY_GENUS);
+		arr.add("Vigna");
+		arr = jsonFilter.putArray(FilterConstants.TAXONOMY_SPECIES);
+		arr.add("unguiculata");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+
+	}
+
+	@Test
+	public void testAcceNumb() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
+		arr.add("TVu-7268");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testAccenumb() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
+		arr.add("VI039981");
+		arr.add("VI039982");
+		// ALIAS is analyzed (not a term)
+		// arr = jsonFilter.putArray(FilterConstants.ALIAS);
+		// arr.add("TOT0956");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testAliases() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
+		arr.add("VI039981");
+		arr.add("VI039982"); // should not be found!
+		// ALIAS is analyzed (not a term)
+		arr = jsonFilter.putArray(FilterConstants.ALIAS);
+		arr.add("TOT0956");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testSampStat() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.SAMPSTAT);
+		arr.add(110);
+		arr.add(120);
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testStorage() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.STORAGE);
+		arr.add(30);
+		arr.add(40);
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testAvailable() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.AVAILABLE);
+		arr.add(NullNode.getInstance());
+		arr.add(true);
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements() + " total=" + a.getTotalElements());
+		for (AccessionDetails aa : a.getContent()) {
+			// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
+			LOG.info(mapper.writeValueAsString(aa));
+		}
+	}
+
+	@Test
+	public void testCollMissId() throws JsonProcessingException, SearchException {
+		ObjectNode jsonFilter = mapper.createObjectNode();
+		// ACCENUMB is analyzed (not a term)
+		ArrayNode arr = jsonFilter.putArray(FilterConstants.COLLMISSID);
+		arr.add("SDC-CLVNET GERMPLASM COLLECTING PROJECT");
+
+		Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
+		LOG.info("Size: " + a.getNumberOfElements() + " total=" + a.getTotalElements());
+		for (AccessionDetails aa :