Commit 74b20ca1 authored by Matija Obreza's avatar Matija Obreza

Merge branch '308-es-query-error-on-storage'

parents 642fceb4 bc0ff540
......@@ -106,7 +106,7 @@ public class ElasticJPAListener {
*
* @param joinPoint the join point
*/
@After(value = "execution(* org.springframework.data.jpa.repository.JpaRepository.deleteAll())")
@After(value = "execution(* org.springframework.data.jpa.repository.JpaRepository.deleteAll()) || execution(* org.springframework.data.jpa.repository.JpaRepository.deleteAllInBatch())")
public void afterDeleteAll(final JoinPoint joinPoint) {
try {
LOG.trace("JPA afterDeleteAll: {} {}", joinPoint.toLongString(), joinPoint.getTarget());
......
package org.genesys2.server.component.elastic;
import static com.google.common.collect.Lists.*;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.Collection;
import java.util.List;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.slf4j.Logger;
......@@ -32,11 +35,12 @@ import com.querydsl.core.types.Visitor;
public class ElasticQueryBuilder implements Visitor<Void, Void> {
private static Logger LOG = LoggerFactory.getLogger(ElasticQueryBuilder.class);
BoolQueryBuilder root = QueryBuilders.boolQuery();
private BoolQueryBuilder root = QueryBuilders.boolQuery();
private final ElasticQueryBuilder self = this;
public BoolQueryBuilder getQuery() {
return QueryBuilders.boolQuery().filter(root);
public QueryBuilder getQuery() {
return root;
}
private String customizedPath(String path) {
......@@ -48,7 +52,8 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> {
path = path.substring(firstDot + 1);
}
// FIXME We should try to build this from annotations on startup. This is very hacky.
// FIXME We should try to build this from annotations on startup. This is very
// hacky.
if ("accession".equals(root)) {
if (path.equals("accessionId.lists.uuid")) {
// Example of @JsonIdentityReference(alwaysAsId = true)
......@@ -121,13 +126,13 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> {
private void handleLike(Operator operator, Path<?> path, Expression<?> val) {
PathMetadata pmd = path.getMetadata();
// SimpleQueryStringBuilder qsq = simpleQueryStringQuery( +":" + toValue(val));
// SimpleQueryStringBuilder qsq = simpleQueryStringQuery( +":" + toValue(val));
if (operator == Ops.STARTS_WITH) {
MatchQueryBuilder matchPrefixQuery = matchPhrasePrefixQuery(customizedPath(pmd.getParent().toString() + "." + pmd.getName()), toValue(val));
root.must(matchPrefixQuery);
root.filter(matchPrefixQuery);
} else if (operator == Ops.STRING_CONTAINS) {
MatchQueryBuilder matchPrefixQuery = matchPhraseQuery(customizedPath(pmd.getParent().toString() + "." + pmd.getName()), toValue(val));
root.must(matchPrefixQuery);
root.filter(matchPrefixQuery);
} else {
throw new RuntimeException("Unsupported ES handleLike operator: " + operator);
}
......@@ -150,16 +155,16 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> {
} else if (operator == Ops.LOE) {
rq.lte(toValue(val1));
}
root.must(rq);
root.filter(rq);
}
private void handleEquals(Path<?> path, Expression<?> value) {
PathMetadata pmd = path.getMetadata();
if (pmd.getPathType() == PathType.COLLECTION_ANY) {
LOG.error("Path ANY for {}={}", pmd.getParent(), value);
root.must(termsQuery(customizedPath(pmd.getParent().toString()), toValue(value)));
root.filter(termsQuery(customizedPath(pmd.getParent().toString()), toValues(value)));
} else {
root.must(termsQuery(customizedPath(getParentPath(pmd.getParent()) + "." + pmd.getName()), toValue(value)));
root.filter(termsQuery(customizedPath(getParentPath(pmd.getParent()) + "." + pmd.getName()), toValues(value)));
}
}
......@@ -178,16 +183,28 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> {
Constant<?> cons = (Constant<?>) value;
Object obj = cons.getConstant();
LOG.debug("toValue of {}: c={}", obj, obj.getClass());
// if (obj instanceof Collection<?>) {
// Collection<?> c = (Collection<?>) obj;
// return c.toArray();
// }
return obj;
}
throw new RuntimeException("Unhandled value " + value);
}
private Collection<?> toValues(Expression<?> value) {
if (value instanceof Constant<?>) {
Constant<?> cons = (Constant<?>) value;
Object obj = cons.getConstant();
LOG.debug("toValues of {}: c={}", obj, obj.getClass());
if (obj instanceof Collection<?>) {
Collection<?> c = (Collection<?>) obj;
return c;
} else {
return newArrayList(obj);
}
}
throw new RuntimeException("Unhandled value " + value);
}
private void printExpression(String prefix, Expression<?> expr) {
if (expr instanceof Path<?>) {
PathImpl<?> path = (PathImpl<?>) expr;
......
......@@ -98,6 +98,7 @@ public class AccessionId extends AuditedVersionedModel implements IdUUID {
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "accession_storage", joinColumns = @JoinColumn(name = "accessionId", referencedColumnName = "id"))
@OrderBy("storage")
@Field(type = FieldType.Auto, index = FieldIndex.not_analyzed)
private Set<Integer> storage = null;
@Column(name = "duplSite", nullable = false, length = 9)
......
......@@ -653,6 +653,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
esQuery.addAggregation(aggregation);
}
LOG.debug("ES query {}", esQuery);
SearchResponse response = esQuery.execute().actionGet();
Map<String, Aggregation> results = response.getAggregations().asMap();
......@@ -690,7 +691,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
if (filters != null) {
filters.buildQuery().accept(esQb, null);
}
BoolQueryBuilder esQuery = esQb.getQuery();
QueryBuilder esQuery = esQb.getQuery();
try {
LOG.trace("Converted {} to\ncurl -XGET 'localhost:9200/accession_read/_search?pretty' -d '{ \"query\": \n{} }'", new ObjectMapper().writeValueAsString(filters),
esQuery);
......@@ -732,13 +733,13 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
client.prepareSearch(indexName).setTypes(COMMON_TYPE_NAME).setQuery(toEsQuery(clazz, filter)).get()
// map
.getHits().forEach(hit -> {
System.err.println("Mapping " + clazz + " id=" + hit.getId());
LOG.trace("Mapping {} id={}", clazz, hit.getId());
T x = loadEntity(clazz, Long.parseLong(hit.getId()));
if (x != null) {
System.err.println("Adding to results: " + x);
LOG.trace("Adding to results: {}", x);
results.add(x);
} else {
System.err.println("Got null");
LOG.trace("Got null");
}
});
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.test.base;
import org.genesys.test.config.ApplicationConfig;
import org.genesys2.server.component.elastic.ElasticJPAListener;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.ContextHierarchy;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.annotation.Transactional;
@RunWith(SpringJUnit4ClassRunner.class)
@ActiveProfiles("dev")
@ContextHierarchy(@ContextConfiguration(name = "services", classes = { AbstractElasticServiceTest.Config.class }))
public abstract class AbstractElasticServiceTest extends AbstractTest {
@Configuration
@EnableAspectJAutoProxy
public static class Config extends ApplicationConfig.BaseConfig {
@Bean
public ElasticJPAListener listener() {
return new ElasticJPAListener();
}
@Bean
public ElasticReindexProcessor reindexProcessor() {
return new ElasticReindexProcessor();
}
}
@Before
@Transactional
@Override
public void beforeTest() throws Exception {
super.beforeTest();
}
@After
@Transactional
@Override
public void cleanup() throws Exception {
super.cleanup();
}
}
\ No newline at end of file
......@@ -69,16 +69,6 @@ public class TestElasticsearchConfig implements InitializingBean, DisposableBean
System.err.println("Deleting " + dataPath.getAbsolutePath());
FileUtils.deleteQuietly(dataPath);
}
// @Bean
// public ElasticJPAListener listener() {
// return new ElasticJPAListener();
// }
//
// @Bean
// public ElasticReindexProcessor reindexProcessor() {
// return new ElasticReindexProcessor();
// }
/**
* Node client.
......
......@@ -15,12 +15,16 @@
*/
package org.genesys.test.server.services;
import static com.google.common.collect.Sets.*;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.test.base.AbstractElasticServiceTest;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionIdRepository;
......@@ -31,22 +35,26 @@ import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.TermResult;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.filter.InstituteFilter;
import org.genesys2.server.service.impl.SearchException;
import org.genesys2.server.service.worker.AccessionUploader;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
@Ignore
public class ElasticQueryBuilderTest extends AbstractServicesTest {
public class ElasticQueryBuilderTest extends AbstractElasticServiceTest {
private static final int MCPD_STORAGE_FIELD = 20;
private static final int MCPD_STORAGE_INVITRO = 30;
private static final int MCPD_STORAGE_CRYO = 40;
@Autowired
private ElasticsearchService elasticService;
......@@ -85,28 +93,37 @@ public class ElasticQueryBuilderTest extends AbstractServicesTest {
institute = instituteService.findInstitute(institute.getCode());
ArrayNode batch = objectMapper.createArrayNode();
ObjectNode a = objectMapper.createObjectNode();
a.put("instituteCode", "INS001");
a.put("accessionNumber", "ACCE 1000");
ObjectNode t = objectMapper.createObjectNode();
t.put("genus", "Manihot");
a.set("taxonomy", t);
batch.add(a);
batch.add(makeAccessionJson(institute, "ACCE 1000", "Manihot", newHashSet(MCPD_STORAGE_INVITRO, MCPD_STORAGE_FIELD)));
batch.add(makeAccessionJson(institute, "M 1001", "Musa", newHashSet(MCPD_STORAGE_INVITRO)));
batch.add(makeAccessionJson(institute, "M 1002", "Musa", newHashSet(MCPD_STORAGE_CRYO, MCPD_STORAGE_FIELD)));
batch.add(makeAccessionJson(institute, "ACCE 1003", "Dioscorea", newHashSet(MCPD_STORAGE_FIELD)));
accessionUploader.upsertAccessions(institute, batch);
assertThat(accessionRepository.count(), greaterThan(0l));
assertThat(accessionRepository.count(), is((long) batch.size()));
long count = 0;
do {
count = elasticsearchService.count(Accession.class, null);
if (count != 1) {
LOG.warn("ES count of {} is {}!=1", Accession.class, count);
if (count != batch.size()) {
LOG.warn("ES count of {} is {}!={}", Accession.class, count, batch.size());
Thread.sleep(1000);
} else {
List<Accession> search = elasticsearchService.find(Accession.class, null);
LOG.warn("Got: ", search);
}
} while (count != 1);
} while (count != batch.size());
}
private JsonNode makeAccessionJson(FaoInstitute institute, String acceNumb, String genus, Set<Integer> storage) {
ObjectNode a = objectMapper.createObjectNode();
a.put("instituteCode", institute.getCode());
a.put("accessionNumber", acceNumb);
ObjectNode taxon = a.putObject("taxonomy");
taxon.put("genus", genus);
ArrayNode s = a.putArray("storage");
storage.forEach(st -> s.add(st));
return a;
}
@Override
......@@ -114,7 +131,7 @@ public class ElasticQueryBuilderTest extends AbstractServicesTest {
accessionRepository.deleteAll();
accessionIdRepository.deleteAll();
taxonomyRepository.deleteAll();
instituteRepository.deleteAll();
instituteRepository.deleteAllInBatch();
super.cleanup();
long count = 0;
......@@ -129,16 +146,28 @@ public class ElasticQueryBuilderTest extends AbstractServicesTest {
}
@Test
public void testEq() throws SearchException, JsonProcessingException {
public void testHolderCode() throws SearchException, JsonProcessingException {
AccessionFilter af = new AccessionFilter();
af.acceNumb = new StringFilter();
af.acceNumb.eq = "ACCE 1000";
af.holder = new InstituteFilter();
af.holder.code = newHashSet("INS001");
TermResult terms = elasticService.termStatistics(Accession.class, af, 10, "institute.code");
LOG.warn("WTF {}", objectMapper.writeValueAsString(terms));
assertThat(terms.getTotal(), is(1));
LOG.debug("institute.code {}", objectMapper.writeValueAsString(terms));
assertThat(terms.getTotal(), is(4l));
assertThat(terms.getTerms().get(0).getTerm(), is("INS001"));
assertThat(terms.getTerms().get(0).getCount(), is(1l));
assertThat(terms.getTerms().get(0).getCount(), is(4l));
terms = elasticService.termStatisticsAuto(Accession.class, af, 10, "storage");
LOG.debug("Storage {}", objectMapper.writeValueAsString(terms));
terms.getTerms().forEach(t -> {
if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_FIELD))) {
assertThat(t.getCount(), is(3l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_INVITRO))) {
assertThat(t.getCount(), is(2l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_CRYO))) {
assertThat(t.getCount(), is(1l));
}
});
}
@Test
......@@ -147,13 +176,79 @@ public class ElasticQueryBuilderTest extends AbstractServicesTest {
af.acceNumb = new StringFilter();
af.acceNumb.contains = "ACCE";
List<Accession> search = elasticsearchService.find(Accession.class, null);
assertThat(search.size(), is(1));
List<Accession> search = elasticsearchService.find(Accession.class, af);
assertThat(search.size(), is(2));
assertThat(search.get(0).getInstitute().getCode(), is("INS001"));
TermResult terms = elasticService.termStatistics(Accession.class, af, 10, "institute.code");
assertThat(terms.getTotal(), is(1l));
assertThat(terms.getTotal(), is(2l));
assertThat(terms.getTerms().get(0).getTerm(), is("INS001"));
assertThat(terms.getTerms().get(0).getCount(), is(1l));
assertThat(terms.getTerms().get(0).getCount(), is(2l));
terms = elasticService.termStatisticsAuto(Accession.class, af, 10, "storage");
LOG.debug("Storage {}", objectMapper.writeValueAsString(terms));
assertThat(terms.getTotal(), is(3l));
terms.getTerms().forEach(t -> {
if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_FIELD))) {
assertThat(t.getCount(), is(2l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_INVITRO))) {
assertThat(t.getCount(), is(1l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_CRYO))) {
assertThat(t.getCount(), is(0l));
}
});
}
@Test
public void testStorageFilter() throws SearchException, JsonProcessingException {
AccessionFilter af = new AccessionFilter();
af.storage = newHashSet(MCPD_STORAGE_INVITRO);
Map<String, TermResult> stats = elasticService.termStatistics(Accession.class, af, 10, "institute.code", "storage");
TermResult terms = stats.get("institute.code");
LOG.debug("Terms {}", objectMapper.writeValueAsString(terms));
assertThat(terms.getTotal(), is(2l));
assertThat(terms.getTerms().get(0).getTerm(), is("INS001"));
assertThat(terms.getTerms().get(0).getCount(), is(2l));
terms = stats.get("storage");
assertThat(terms.getTotal(), is(3l));
terms.getTerms().forEach(t -> {
if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_FIELD))) {
assertThat(t.getCount(), is(1l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_INVITRO))) {
assertThat(t.getCount(), is(2l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_CRYO))) {
assertThat(t.getCount(), is(0l));
}
});
af.storage = newHashSet(MCPD_STORAGE_FIELD);
stats = elasticService.termStatistics(Accession.class, af, 10, "institute.code", "storage");
terms = stats.get("storage");
assertThat(terms.getTotal(), is(5l));
terms.getTerms().forEach(t -> {
if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_FIELD))) {
assertThat(t.getCount(), is(3l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_INVITRO))) {
assertThat(t.getCount(), is(1l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_CRYO))) {
assertThat(t.getCount(), is(1l));
}
});
af.storage = newHashSet(MCPD_STORAGE_INVITRO, MCPD_STORAGE_CRYO);
terms = elasticService.termStatistics(Accession.class, af, 10, "storage");
assertThat(terms.getTotal(), is(5l));
terms.getTerms().forEach(t -> {
if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_FIELD))) {
assertThat(t.getCount(), is(2l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_INVITRO))) {
assertThat(t.getCount(), is(2l));
} else if (t.getTerm().equals(Integer.toString(MCPD_STORAGE_CRYO))) {
assertThat(t.getCount(), is(1l));
}
});
}
}
package org.genesys.test.simpletest;
import static com.google.common.collect.Sets.*;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.List;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.filters.NumberFilter;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys2.server.component.elastic.ElasticQueryBuilder;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.filter.AccessionGeoFilter;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jayway.jsonpath.JsonPath;
import com.querydsl.core.types.Predicate;
public class EsQueryTest {
// private static Logger LOG = LoggerFactory.getLogger(EsQueryTest.class);
private static Logger LOG = LoggerFactory.getLogger(EsQueryTest.class);
@Test
public void test1() {
......@@ -19,13 +30,50 @@ public class EsQueryTest {
af.acceNumb = new StringFilter();
af.acceNumb.eq = "IRGC 1";
af.acceNumb.sw = "FOO";
af.geo=new AccessionGeoFilter();
af.geo.latitude=new NumberFilter<>();
af.geo = new AccessionGeoFilter();
af.geo.latitude = new NumberFilter<>();
af.geo.latitude.between = new Double[] { 10d, 20d };
Predicate predicate = af.buildQuery();
ElasticQueryBuilder visitor = new ElasticQueryBuilder();
predicate.accept(visitor, null);
LOG.debug("ES query: {}", visitor.getQuery());
// Object active = JsonPath.read(esQuery.toString(),
// "bool.filter.bool.must.terms.active");
// assertThat(active, notNullValue());
// assertThat(((List<?>) active), hasSize(1));
}
@Test
public void testStorage() throws IOException {
AccessionFilter af = new AccessionFilter();
af.storage = newHashSet(30);
Predicate predicate = af.buildQuery();
ElasticQueryBuilder visitor = new ElasticQueryBuilder();
predicate.accept(visitor, null);
QueryBuilder esQuery = visitor.getQuery();
LOG.debug("ES query: {}", visitor.getQuery());
Object storage = JsonPath.read(esQuery.toString(), "bool.filter.terms.storage");
assertThat(storage, notNullValue());
assertThat(((List<?>) storage), hasSize(1));
assertThat(((List<?>) storage), containsInAnyOrder(30));
}
@Test
public void testStorage2() throws IOException {
AccessionFilter af = new AccessionFilter();
af.storage = newHashSet(10, 20);
Predicate predicate = af.buildQuery();
ElasticQueryBuilder visitor = new ElasticQueryBuilder();
predicate.accept(visitor, null);
QueryBuilder esQuery = visitor.getQuery();
LOG.debug("ES query: {}", visitor.getQuery());
Object storage = JsonPath.read(esQuery.toString(), "bool.filter.terms.storage");
assertThat(storage, notNullValue());
assertThat(((List<?>) storage), hasSize(2));
assertThat(((List<?>) storage), containsInAnyOrder(20, 10));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment