Commit 6339acaa authored by Matija Obreza's avatar Matija Obreza

WIP: Facilitated search

- JSON ignore pdciStatistics
- Search
parent d3f85821
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.model.filters;
import static org.genesys.catalog.model.vocab.QVocabularyTerm.vocabularyTerm;
import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.catalog.model.vocab.VocabularyTerm;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate;
/**
* The Class VocabularyTermFilter.
*/
public class VocabularyTermFilter extends BasicModelFilter<VocabularyTermFilter, VocabularyTerm> {
/** The descriptor. */
public DescriptorFilter descriptor;
/** The title. */
public StringFilter title;
/** The description. */
public StringFilter description;
/**
* Builds the query.
*
* @return the predicate
*/
public Predicate buildPredicate() {
final BooleanBuilder and = new BooleanBuilder();
super.buildPredicate(vocabularyTerm, vocabularyTerm._super, and);
if (title != null) {
and.and(title.buildQuery(vocabularyTerm.title));
}
if (description != null) {
and.and(description.buildQuery(vocabularyTerm.description));
}
if (descriptor != null) {
and.and(descriptor.buildQuery(vocabularyTerm.descriptor));
}
return and;
}
}
......@@ -120,4 +120,12 @@ public interface VocabularyService {
// 'read')")
Page<VocabularyTerm> listTerms(ControlledVocabulary vocabulary, Pageable page);
/**
* Reload and lazy load provided terms
*
* @param terms
* @return
*/
List<VocabularyTerm> lazyLoad(List<VocabularyTerm> terms);
}
......@@ -263,4 +263,18 @@ public class VocabularyServiceImpl implements VocabularyService {
public Page<VocabularyTerm> listTerms(final ControlledVocabulary vocabulary, final Pageable page) {
return vocabRepository.listVocabularyTerms(vocabulary, page);
}
@Override
public List<VocabularyTerm> lazyLoad(List<VocabularyTerm> terms) {
List<VocabularyTerm> loaded = termRepository.findAll(terms.stream().map(term -> term.getId()).collect(Collectors.toList()));
loaded.forEach(term -> {
if (term.getDescriptor() != null) {
term.getDescriptor().getId();
}
if (term.getVocabulary() != null) {
term.getVocabulary().getId();
}
});
return loaded;
}
}
......@@ -40,6 +40,7 @@ import org.genesys2.server.model.json.AccessionJson;
import org.genesys2.server.model.json.Api0Constants;
import org.genesys2.server.model.json.Api1Constants;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.Hit;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysRESTService;
import org.genesys2.server.service.GenesysService;
......@@ -353,13 +354,13 @@ public class AccessionController extends ApiBaseController {
}
@RequestMapping(value = "/search", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody List<Accession> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
public @ResponseBody List<Hit<Accession>> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(null, StringUtils.defaultIfBlank(query, "*"), Accession.class); //, new PageRequest(page - 1, 20));
}
// FIXME Not using institute...
@RequestMapping(value = "/{instCode}/list", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody List<Accession> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
public @ResponseBody List<Hit<Accession>> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
throws SearchException {
FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
......
......@@ -91,7 +91,7 @@ public abstract class AccessionData extends AuditedVersionedModel implements IdU
@JoinColumn(name = "instituteId")
@JsonView({ JsonViews.Minimal.class })
@Field(type = FieldType.Object)
@JsonIgnoreProperties({"settings"})
@JsonIgnoreProperties({ "settings", "pdciHistogram", "statisticsPDCI" })
@QueryInit({ "country.*", "owner.*"})
private FaoInstitute institute;
......
......@@ -35,6 +35,7 @@ import org.genesys2.server.model.GlobalVersionedAuditedModel;
import org.genesys2.server.model.impl.CropTaxonomy;
import org.genesys2.server.model.json.Api1Constants;
import org.hibernate.annotations.Type;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
......@@ -48,6 +49,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
// Add index on all fields
@Table(name = "taxonomy2", uniqueConstraints = { @UniqueConstraint(name="UK_taxonomy2", columnNames = { "genus", "species", "spAuthor", "subtaxa", "subtAuthor" }) })
@JsonIgnoreProperties(ignoreUnknown = true)
@Document(indexName = "genesys")
public class Taxonomy2 extends GlobalVersionedAuditedModel {
private static final long serialVersionUID = 8881324404490162933L;
......
......@@ -31,6 +31,7 @@ import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.Hit;
import org.genesys2.server.service.impl.SearchException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -76,7 +77,7 @@ public class SearchController {
if (!StringUtils.isBlank(searchQuery)) {
try {
List<Accession> x = elasticService.search(null, searchQuery, Accession.class);
List<Hit<Accession>> x = elasticService.search(null, searchQuery, Accession.class);
model.addAttribute("pagedData", x);
LOG.info("Searching for: {} returns {}", searchQuery, x.size());
} catch (Throwable e) {
......@@ -185,7 +186,7 @@ public class SearchController {
return "/search/index2";
}
private <T extends BasicModel> Page<T> convertToPage(List<T> results, int page) {
private <T extends BasicModel> Page<Hit<T>> convertToPage(List<Hit<T>> results, int page) {
int total = results == null ? 0 : results.size();
return new PageImpl<>(results, new PageRequest(page - 1, Math.max(20, total)), total);
......
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -19,12 +19,13 @@ import java.beans.Transient;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys2.server.service.impl.SearchException;
import org.springframework.data.domain.Pageable;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
/**
* The Interface ElasticsearchService.
......@@ -68,29 +69,21 @@ public interface ElasticsearchService {
/**
* Search single entity.
*
* @param shouldMatch the should match
* @param <T> the generic type
* @param filter the filter
* @param searchQuery the search query
* @param clazz the clazz
* @return the list
*/
<T extends BasicModel> List<T> search(QueryBuilder shouldMatch, String searchQuery, Class<T> clazz);
/**
* Search using query for selected entity types.
*
* @param shouldMatch Any additional filters that should match
* @param searchQuery the search query
* @param clazzes the clazzes
* @return the map
*/
<T extends BasicModel> Map<Class<? extends BasicModel>, List<? extends BasicModel>> search(QueryBuilder shouldMatch, String searchQuery, Set<Class<? extends BasicModel>> clazzes);
<T extends BasicModel> List<Hit<T>> search(BasicModelFilter<?, T> filter, String searchQuery, Class<T> clazz);
/**
* Term statistics auto.
* @param filters the filters
* @param instcode the instcode
* @param i the i
*
* @param clazz the clazz
* @param filters the filters
* @param size the size
* @param term the term
* @return the term result
* @throws SearchException the search exception
*/
......@@ -125,7 +118,8 @@ public interface ElasticsearchService {
Map<String, Long> countMissingValues(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter);
/**
* Aggregate by date
* Aggregate by date.
*
* @param size max size of results to be returned
* @param targetClass the target entity class
* @param indexClass the index class
......@@ -199,15 +193,21 @@ public interface ElasticsearchService {
<T extends BasicModel> List<T> find(Class<T> clazz, BasicModelFilter<?, ?> filter);
public static class Hit<T> {
public double score;
@JsonUnwrapped
public T hit;
}
/**
* Wrapper for search results
*/
public static class SearchResults<T extends BasicModel> {
public List<String> filters;
public String key = "uuid";
public List<T> hits;
public List<Hit<T>> hits;
public static <T extends BasicModel> SearchResults<T> from(String key, List<String> filters, List<T> list) {
public static <T extends BasicModel> SearchResults<T> from(String key, List<String> filters, List<Hit<T>> list) {
if (list == null || list.isEmpty())
return null;
......@@ -218,4 +218,6 @@ public interface ElasticsearchService {
return sr;
}
}
<T extends BasicModel> List<Hit<String>> searchFields(Class<T> clazz, Collection<String> fields, BasicModelFilter<?, T> filter, String searchQuery, Pageable pageable);
}
......@@ -51,9 +51,13 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.SimpleQueryStringFlag;
import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesNotFoundException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
......@@ -581,49 +585,50 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
@Override
public <T extends BasicModel> List<T> search(QueryBuilder shouldMatch, String searchQuery, Class<T> clazz) {
public <T extends BasicModel> List<Hit<T>> search(BasicModelFilter<?, T> filter, String searchQuery, Class<T> clazz) {
if (!indexedEntities.contains(clazz)) {
throw new RuntimeException("Class is not indexed " + clazz);
}
Pageable pageable = new PageRequest(0, 20);
return searchIndex(clazz, shouldMatch, searchQuery, pageable);
return searchIndex(clazz, filter, searchQuery, pageable);
}
@Override
public <T extends BasicModel> Map<Class<? extends BasicModel>, List<? extends BasicModel>> search(QueryBuilder shouldMatch, String searchQuery, Set<Class<? extends BasicModel>> clazzes) {
final Set<String> indexNames = clazzes.stream().filter(clazz -> indexedEntities.contains(clazz)).map(clazz -> toIndexName(clazz) + INDEX_READ).collect(Collectors.toSet());
if (indexNames.size() == 0) {
LOG.warn("No clazzes to search");
return null;
}
LOG.debug("Searching {} in indices {}", clazzes, indexNames);
Map<Class<? extends BasicModel>, List<? extends BasicModel>> hits = new HashMap<>();
// @Override
// public Map<Class<? extends BasicModel>, List<Hit<? extends BasicModel>>> search(BasicModelFilter<?, ?> filter, String searchQuery, Set<Class<? extends BasicModel>> clazzes) {
//
// final Set<String> indexNames = clazzes.stream().filter(clazz -> indexedEntities.contains(clazz)).map(clazz -> toIndexName(clazz) + INDEX_READ).collect(Collectors.toSet());
// if (indexNames.size() == 0) {
// LOG.warn("No clazzes to search");
// return null;
// }
// LOG.debug("Searching {} in indices {}", clazzes, indexNames);
//
// Map<Class<?>, List<Hit<?>>> hits = new HashMap<>();
//
// Pageable pageable = new PageRequest(0, 20);
//
// for (Class<?> clazz : clazzes) {
// hits = searchIndex(clazz, filter, searchQuery, pageable);
// hits.put(clazz, hits);
// }
//
// return hits;
// }
Pageable pageable = new PageRequest(0, 20);
private <T extends BasicModel> List<Hit<T>> searchIndex(Class<T> clazz, BasicModelFilter<?, T> filter, String searchQuery, Pageable pageable) {
for (Class<? extends BasicModel> clazz : clazzes) {
hits.put(clazz, searchIndex(clazz, shouldMatch, searchQuery, pageable));
if (!indexedEntities.contains(clazz)) {
throw new RuntimeException("Class is not indexed: " + clazz);
}
return hits;
}
private <T extends BasicModel> List<T> searchIndex(Class<T> clazz, QueryBuilder shouldMatch, String searchQuery, Pageable pageable) {
String indexName = toIndexName(clazz) + INDEX_READ;
LOG.debug("Searching {} in index {}", clazz, indexName);
LOG.debug("Searching {} in index {}", searchQuery, indexName);
BoolQueryBuilder theQuery = boolQuery()
/*@formatter:off*/
.should(multiMatchQuery(searchQuery, "_all")
.field("crop", 0.2f)
.field("code", 1.5f)
.field("name", 1.5f)
.field("title", 1.4f)
.fuzziness(Fuzziness.ONE)
.should(commonTermsQuery("_all", searchQuery)
.boost(1.2f)
)
.should(multiMatchQuery(searchQuery, "code", "name", "title", "description")
......@@ -633,17 +638,17 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
.field("title", 1.4f)
.field("description", 0.8f)
.fuzziness(Fuzziness.AUTO)
.boost(2.0f)
.boost(1.0f)
)
.should(queryStringQuery(searchQuery)
.useDisMax(true)
.fuzziness(Fuzziness.TWO)
.boost(0.9f)
.should(simpleQueryStringQuery(searchQuery)
.lenient(true)
.flags(SimpleQueryStringFlag.FUZZY, SimpleQueryStringFlag.OR)
.boost(1.5f)
);
/*@formatter:on*/
if (shouldMatch != null) {
theQuery.should(shouldMatch);
if (filter != null) {
theQuery.must(toEsQuery(clazz, filter));
}
SearchQuery query = new NativeSearchQueryBuilder()
......@@ -651,50 +656,129 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
.withIndices(indexName)
.withTypes(COMMON_TYPE_NAME)
.withFields("id", "_class")
.withMinScore(0.2f)
.withQuery(theQuery)
.withPageable(pageable)
.build();
/*@formatter:on*/
if (!indexedEntities.contains(clazz)) {
throw new RuntimeException("Class is not indexed: " + clazz);
}
return elasticsearchTemplate.query(query, (response) -> {
LOG.debug("Search response: {}", response.status());
LOG.debug("Search response: {} totalHits={}", response.status(), response.getHits().getTotalHits());
List<T> entities = new ArrayList<>();
List<Hit<T>> hits = new ArrayList<>();
response.getHits().forEach(searchHit -> {
LOG.debug("Hit score={} id={} _class={} _source={}", searchHit.getScore(), searchHit.getId(), clazz, searchHit.getSourceAsString());
T entity = loadEntity(clazz, Long.parseLong(searchHit.getId()));
if (entity != null) {
Hit<T> hit = new Hit<>();
hit.score = searchHit.getScore();
hit.hit = entity;
// Filter things
if (entity instanceof Publishable) {
if (((Publishable) entity).isPublished()) {
entities.add(entity);
hits.add(hit);
}
} else if (entity instanceof VersionedModel) {
if (((VersionedModel) entity).isActive()) {
entities.add(entity);
hits.add(hit);
}
} else {
entities.add(entity);
hits.add(hit);
}
}
});
return entities;
return hits;
});
}
private <T extends BasicModel> T loadEntity(Class<T> clazz, Long id) {
return (T) em.find(clazz, id);
}
@Override
public <T extends BasicModel> List<Hit<String>> searchFields(Class<T> clazz, Collection<String> fields, BasicModelFilter<?, T> filter, String searchQuery, Pageable pageable) {
if (!indexedEntities.contains(clazz)) {
throw new RuntimeException("Class is not indexed: " + clazz);
}
String indexName = toIndexName(clazz) + INDEX_READ;
LOG.debug("Searching {} in index {}", clazz, indexName);
BoolQueryBuilder theQuery = boolQuery();
final MultiMatchQueryBuilder mmq = multiMatchQuery(searchQuery, "_all")
.fuzziness(Fuzziness.ONE)
.boost(1.2f);
fields.forEach(field -> mmq.field(field, 1.5f));
theQuery.should(mmq);
final MultiMatchQueryBuilder mmq2 = multiMatchQuery(searchQuery, fields.toArray(Strings.EMPTY_ARRAY))
.fuzziness(Fuzziness.AUTO)
.boost(2.0f);
fields.forEach(field -> mmq2.field(field, 2.0f));
theQuery.should(mmq2);
// QueryStringQueryBuilder qsq = queryStringQuery(searchQuery)
// .useDisMax(true)
// .fuzziness(Fuzziness.ONE)
// .boost(1.5f);
// fields.forEach(field -> qsq.field(field, 5.0f));
// theQuery.should(qsq);
if (filter != null) {
theQuery.must(toEsQuery(clazz, filter));
}
SearchQuery query = new NativeSearchQueryBuilder()
/*@formatter:off*/
.withIndices(indexName)
.withTypes(COMMON_TYPE_NAME)
.withFields(fields.toArray(Strings.EMPTY_ARRAY))
.withMinScore(0.2f)
.withQuery(theQuery)
.withPageable(pageable)
.build();
/*@formatter:on*/
return elasticsearchTemplate.query(query, (response) -> {
LOG.debug("Search response: {} totalHits={}", response.status(), response.getHits().getTotalHits());
List<Hit<String>> hits = new ArrayList<>();
ObjectMapper objectMapper=new ObjectMapper();
response.getHits().forEach(searchHit -> {
LOG.debug("Hit score={} id={} _class={} _source={}", searchHit.getScore(), searchHit.getId(), clazz, searchHit.getSourceAsString());
System.out.println("\nsearchHit " + searchHit.getScore());
searchHit.fields().forEach((fld, fldHit) -> {
try {
System.out.println(fld + " :" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(fldHit));
} catch (JsonProcessingException e) {
System.out.println(e.getMessage());
}
});
// searchHit.highlightFields().forEach((fld, fldHit) -> {
// try {
// System.out.println("Highlight " + fld + " :" + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(fldHit));
// } catch (JsonProcessingException e) {
// System.out.println(e.getMessage());
// }
// });
});
return hits;
});
}
@Override
public TermResult termStatistics(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filters, int size, String term) throws SearchException {
......@@ -755,10 +839,8 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
filters.buildPredicate().accept(esQb, null);
}
QueryBuilder esQuery = esQb.getQuery();
try {
LOG.trace("Converted {} to\ncurl -XGET 'localhost:9200/accession_read/_search?pretty' -d '{ \"query\": \n{} }'", new ObjectMapper().writeValueAsString(filters),
esQuery);
} catch (JsonProcessingException e) {
if (LOG.isTraceEnabled()) {
LOG.trace("Test using\ncurl -XGET 'localhost:9200/{}_read/_search?pretty&q=' -d '{ \"query\": \n{} }'", toIndexName(clazz), esQuery);
}
return esQuery;
}
......
......@@ -34,11 +34,13 @@ import org.genesys2.server.component.elastic.ElasticJacksonAnnotationIntrospecto
import org.genesys2.server.component.elastic.ElasticReindex;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.GeoRegion;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.impl.ElasticsearchServiceImpl;
import org.slf4j.Logger;
......@@ -142,8 +144,10 @@ public class ElasticsearchConfig {
es.indexEntity(Dataset.class);
es.indexEntity(Accession.class);
es.indexEntity(Taxonomy2.class);
es.indexEntity(FaoInstitute.class);
es.indexEntity(Country.class);
es.indexEntity(GeoRegion.class);
es.indexEntity(Article.class);
es.indexEntity(ActivityPost.class);
......@@ -222,7 +226,7 @@ public class ElasticsearchConfig {
@Override
public String mapToString(final Object object) throws IOException {
String x = mapper.writeValueAsString(object);
// System.err.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object));
System.err.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object));
return x;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment