Commit 98e0f20f authored by Matija Obreza's avatar Matija Obreza

Filtering with elasticsearch

parent 5dd0a047
......@@ -16,6 +16,7 @@
package org.genesys2.server.service;
import java.io.Writer;
import java.util.List;
import java.util.Locale;
import java.util.Map;
......@@ -102,4 +103,6 @@ public interface ContentService {
Locale getDefaultLocale();
String processTemplate(String body, Map<String, Object> root);
void processTemplate(String templateStr, Map<String, Object> root, Writer writer);
}
......@@ -37,4 +37,6 @@ public interface ElasticService {
void refreshIndex(String className);
Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException;
}
......@@ -17,6 +17,7 @@
package org.genesys2.server.service.impl;
import java.io.StringWriter;
import java.io.Writer;
import java.util.Calendar;
import java.util.List;
import java.util.Locale;
......@@ -169,7 +170,7 @@ public class ContentServiceImpl implements ContentService {
/**
* Creates or updates an article
*
*
* @param entity
* @param body
* @param locale
......@@ -289,7 +290,22 @@ public class ContentServiceImpl implements ContentService {
* Merge data and template
*/
velocityEngine.evaluate(context, swOut, "log tag name", templateStr);
System.out.println(swOut);
if (LOG.isTraceEnabled()) {
LOG.trace(swOut);
}
return swOut.toString();
}
@Override
public void processTemplate(String templateStr, Map<String, Object> root, Writer writer) {
final VelocityContext context = new VelocityContext();
for (final String key : root.keySet()) {
context.put(key, root.get(key));
}
/**
* Merge data and template
*/
velocityEngine.evaluate(context, writer, "log tag name", templateStr);
}
}
......@@ -4,17 +4,23 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -45,15 +51,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired
private GenesysFilterService filterService;
@Autowired
private OrganizationService organizationService;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private CropService cropService;
private final Map<String, Class<?>> clazzMap;
public ElasticsearchSearchServiceImpl() {
......@@ -74,6 +74,107 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
}
@Override
public Page<AccessionDetails> filter(String jsonFilter, Pageable pageable) throws SearchException {
Map<String, List<String>> filters;
try {
filters = objectMapper.readValue(jsonFilter, Map.class);
} catch (IOException e) {
throw new SearchException(e.getMessage(), e);
}
AndFilterBuilder filterBuilder = FilterBuilders.andFilter();
for (String key : filters.keySet()) {
List<String> filterValues = filters.get(key);
if (filterValues == null || filterValues.isEmpty())
continue;
System.err.println("key=" + key);
if (FilterConstants.ACCENUMB.equals(key) || FilterConstants.COLLMISSID.equals(key)) {
// indexed field
filterBuilder.add(FilterBuilders.queryFilter(makeQuery(key, filterValues)));
} else if (FilterConstants.ALIAS.equals(key)) {
// Nested object
filterBuilder.add(FilterBuilders.nestedFilter("aliases", makeQuery("aliases.name", filterValues)));
} else if (FilterConstants.SGSV.equals(key)) {
// Check for existance nested object
OrFilterBuilder orFilter = FilterBuilders.orFilter();
if (filterValues.contains(true)) {
// true:
orFilter.add(FilterBuilders.notFilter(FilterBuilders.missingFilter(FilterConstants.SGSV)));
}
if (filterValues.contains(false)) {
// false:
orFilter.add(FilterBuilders.missingFilter(FilterConstants.SGSV));
}
if (filterValues.contains(true) || filterValues.contains(false))
filterBuilder.add(orFilter);
} else if (FilterConstants.GEO_ELEVATION.equals(key) || FilterConstants.GEO_LONGITUDE.equals(key) || FilterConstants.GEO_LATITUDE.equals(key)) {
OrFilterBuilder orFilter = FilterBuilders.orFilter();
// range filters
for (Object val : filterValues) {
System.err.println(val.getClass() + " " + val);
Map<String, Object> v = (Map<String, Object>) val;
List<Number> range = (List<Number>) v.get("range");
Number max = (Number) v.get("max");
Number min = (Number) v.get("min");
if (range != null) {
System.err.println("Range " + range.getClass() + " " + range);
orFilter.add(FilterBuilders.rangeFilter(key).from(range.get(0)).to(range.get(1)));
} else if (max != null) {
System.err.println("Max " + max);
orFilter.add(FilterBuilders.rangeFilter(key).to(max));
} else if (min != null) {
System.err.println("Min " + min);
orFilter.add(FilterBuilders.rangeFilter(key).from(min));
}
}
filterBuilder.add(orFilter);
} else {
Set<Object> nonNull = new HashSet<Object>(filters.size());
boolean hasNull = false;
for (Object val : filterValues) {
if (val != null) {
nonNull.add(val);
} else {
hasNull = true;
}
}
if (hasNull) {
filterBuilder.add(FilterBuilders.orFilter(FilterBuilders.missingFilter(key), FilterBuilders.termsFilter(key, nonNull).execution("or")));
} else {
filterBuilder.add(FilterBuilders.termsFilter(key, filterValues).execution("or"));
}
}
}
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withPageable(pageable).build();
// System.err.println("Filter query: " + searchQuery.toString());
try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
return sampleEntities;
} catch (Throwable e) {
throw new SearchException(e.getMessage(), e);
}
}
private QueryBuilder makeQuery(String key, List<String> list) {
// Indexed fields
StringBuilder sb = new StringBuilder();
for (String val : list) {
if (sb.length() > 0)
sb.append(",");
sb.append("\"" + val + "\"");
}
return QueryBuilders.queryString(key + ":(" + sb.toString() + ")");
}
@Override
public void update(String className, long id) {
if (!clazzMap.containsKey(className)) {
......@@ -255,6 +356,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate.createIndex("genesysarchive");
}
Map indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object key : indexMapping.keySet()) {
System.err.println("Mapping ke=" + key + " val=" + indexMapping.get(key));
}
LOG.info("Copying mapping");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
}
......
......@@ -16,6 +16,13 @@
package org.genesys2.server.servlet.controller.rdf;
import java.io.IOException;
import java.io.OutputStream;
import javax.servlet.http.HttpServletResponse;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GenesysService;
......@@ -28,6 +35,7 @@ import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
@Controller
@RequestMapping(value = "/acn", headers = "accept=text/turtle", produces = "text/turtle")
......@@ -67,4 +75,16 @@ public class AccessionControllerRdf {
return "/accession/details-turtle";
}
@RequestMapping(value = "/search", method = RequestMethod.POST)
public void downloadFiltered(@RequestParam(value = "filter", required = false, defaultValue = "{}") String jsonFilter, HttpServletResponse response)
throws IOException {
final OutputStream outputStream = response.getOutputStream();
// TODO
response.flushBuffer();
}
}
......@@ -13,7 +13,7 @@
<http://www.genesys-pgr.org/acn/id/${accession.id}>
a germplasm:GermplasmAccession;
germplasm:germplasmID "<c:out value="${accession.id}" />";
dwc:cataogNumber "<c:out value="${accession.accessionName}" />";
dwc:catalogNumber "<c:out value="${accession.accessionName}" />";
dwc:institutionCode "<c:out value="${accession.instituteCode}" />";
dwc:instituteId "<c:out value="${accession.institute.id}" />";
dwc:countryCode "<c:out value="${accession.institute.country.code3}" />";
......
......@@ -16,46 +16,146 @@
package org.genesys2.server.service.impl;
import java.io.IOException;
import org.apache.log4j.Logger;
import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.exception.VelocityException;
import org.genesys2.server.aspect.AsAdminAspect;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.service.AclService;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.UserService;
import org.genesys2.server.test.JpaDataConfig;
import org.genesys2.server.test.PropertyPlacholderInitializer;
import org.genesys2.spring.config.ElasticsearchConfig;
import org.genesys2.spring.config.HazelcastConfig;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.CacheManager;
import org.springframework.cache.support.NoOpCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Import;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.ui.velocity.VelocityEngineFactoryBean;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.NullNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = ElasticsearchTest.Config.class, initializers = PropertyPlacholderInitializer.class)
@ActiveProfiles("cacher")
public class ElasticsearchTest {
private static final Logger LOG = Logger.getLogger(ElasticsearchTest.class);
@Import(ElasticsearchConfig.class)
@Import({ ElasticsearchConfig.class, JpaDataConfig.class, HazelcastConfig.class })
@ComponentScan(basePackages = { "org.genesys2.server.persistence.domain" })
public static class Config {
@Bean
public ElasticService searchService() {
return new ElasticsearchSearchServiceImpl();
}
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
}
@Bean
public AsAdminAspect asAdminAspect() {
return new AsAdminAspect();
}
@Bean
public UserService userService() {
return new UserServiceImpl();
}
@Bean
public AclService aclService() {
return new AclServiceImpl();
}
@Bean
public TaxonomyService taxonomyService() {
return new TaxonomyServiceImpl();
}
@Bean
public GenesysService genesysService() {
return new GenesysServiceImpl();
}
@Bean
public CacheManager cacheManager() {
return new NoOpCacheManager();
}
@Bean
public HtmlSanitizer htmlSanitizer() {
return new OWASPSanitizer();
}
@Bean
public GeoService geoService() {
return new GeoServiceImpl();
}
@Bean
public ContentService contentService() {
return new ContentServiceImpl();
}
@Bean
public VelocityEngine velocityEngine() throws VelocityException, IOException {
final VelocityEngineFactoryBean vf = new VelocityEngineFactoryBean();
return vf.createVelocityEngine();
}
@Bean
public OrganizationService organizationService() {
return new OrganizationServiceImpl();
}
@Bean
public InstituteService instituteService() {
return new InstituteServiceImpl();
}
@Bean
public CropService cropService() {
return new CropServiceImpl();
}
@Bean
public GenesysFilterService genesysFilterService() {
return new GenesysFilterServiceImpl();
}
}
@Autowired
private ObjectMapper mapper;
@Autowired
private ElasticService elasticService;
......@@ -67,4 +167,202 @@ public class ElasticsearchTest {
LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
}
}
@Test
public void testInstituteCode() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
ArrayNode arr = jsonFilter.putArray(FilterConstants.INSTCODE);
arr.add("NGA039");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testGenus() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
ArrayNode arr = jsonFilter.putArray(FilterConstants.TAXONOMY_GENUS);
arr.add("Vigna");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testGenusAndSpecies() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
ArrayNode arr = jsonFilter.putArray(FilterConstants.TAXONOMY_GENUS);
arr.add("Vigna");
arr = jsonFilter.putArray(FilterConstants.TAXONOMY_SPECIES);
arr.add("unguiculata");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testAcceNumb() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
arr.add("TVu-7268");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testAccenumb() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
arr.add("VI039981");
arr.add("VI039982");
// ALIAS is analyzed (not a term)
// arr = jsonFilter.putArray(FilterConstants.ALIAS);
// arr.add("TOT0956");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testAliases() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.ACCENUMB);
arr.add("VI039981");
arr.add("VI039982"); // should not be found!
// ALIAS is analyzed (not a term)
arr = jsonFilter.putArray(FilterConstants.ALIAS);
arr.add("TOT0956");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testSampStat() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.SAMPSTAT);
arr.add(110);
arr.add(120);
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testStorage() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.STORAGE);
arr.add(30);
arr.add(40);
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testAvailable() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.AVAILABLE);
arr.add(NullNode.getInstance());
arr.add(true);
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements() + " total=" + a.getTotalElements());
for (AccessionDetails aa : a.getContent()) {
// LOG.info(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(aa));
LOG.info(mapper.writeValueAsString(aa));
}
}
@Test
public void testCollMissId() throws JsonProcessingException, SearchException {
ObjectNode jsonFilter = mapper.createObjectNode();
// ACCENUMB is analyzed (not a term)
ArrayNode arr = jsonFilter.putArray(FilterConstants.COLLMISSID);
arr.add("SDC-CLVNET GERMPLASM COLLECTING PROJECT");
Page<AccessionDetails> a = elasticService.filter(jsonFilter.toString(), new PageRequest(0, 100));
LOG.info("Size: " + a.getNumberOfElements() + " total=" + a.getTotalElements());
for (AccessionDetails aa :