From c38258c545cad7201ef87e331a1aac6821991128 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Tue, 3 May 2016 22:24:17 +0200 Subject: [PATCH] Added score and score-based search, added FullTextDocument is used in Elasticsearch mapping */ @Document(indexName = "fulltext", refreshInterval = "60s") public class FullTextDocument { - // FIXME Revisit! - private static final String ENGLISH_LOCALE = "en"; - @Id private Long id; @@ -61,6 +53,10 @@ public class FullTextDocument { @Field(index = FieldIndex.not_analyzed, type = FieldType.Date) private Date lastModifiedDate; + /** The score here will boost result relevance */ + @Field(index = FieldIndex.not_analyzed) + private float score = 0.5f; + public Long getId() { return this.id; } @@ -101,21 +97,6 @@ public class FullTextDocument { this.summary = summary; } - public String getTextByLocale(final Locale locale) { - try { - final JsonObject json = JsonObject.readFrom(this.body); - JsonValue value = json.get(locale.getLanguage()); - if (value == null) { - value = json.get(ENGLISH_LOCALE); - } - - return value.toString().replace("\"", ""); - } catch (final ParseException ignored) { - } - - return getBody(); - } - public String getLanguage() { return this.language; } @@ -139,4 +120,12 @@ public class FullTextDocument { public void setLastModifiedDate(final Date lastModifiedDate) { this.lastModifiedDate = lastModifiedDate; } + + public float getScore() { + return score; + } + + public void setScore(float score) { + this.score = score; + } } diff --git a/src/main/java/org/genesys2/server/service/ContentService.java b/src/main/java/org/genesys2/server/service/ContentService.java index 06c500c02..52219ad18 100644 --- a/src/main/java/org/genesys2/server/service/ContentService.java +++ b/src/main/java/org/genesys2/server/service/ContentService.java @@ -40,6 +40,8 @@ public interface ContentService { ClassPK getClassPk(String shortName); + ClassPK getClassPk(Class clazz); + /** * Load article with {@link ClassPK} of clazz with specified id and the * "slug" for the locale diff --git a/src/main/java/org/genesys2/server/service/FullTextSearchService.java b/src/main/java/org/genesys2/server/service/FullTextSearchService.java index d87ed034e..d4178e237 100644 --- a/src/main/java/org/genesys2/server/service/FullTextSearchService.java +++ b/src/main/java/org/genesys2/server/service/FullTextSearchService.java @@ -43,4 +43,6 @@ public interface FullTextSearchService { void reindex(String type); void deleteIndex(String indexName); + + Page search(String query, Pageable pageable, Class type) throws SearchException; } diff --git a/src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java index b1431dbc9..0cd8ae962 100644 --- a/src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java @@ -247,6 +247,7 @@ public class ContentServiceImpl implements ContentService { return article; } + @Override public ClassPK getClassPk(Class clazz) { return classPkRepository.findByClassName(clazz.getName()); } diff --git a/src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java index 0c58d9f92..494c2a107 100644 --- a/src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java @@ -16,8 +16,10 @@ package org.genesys2.server.service.impl; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.queryString; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; import java.sql.ResultSet; import java.sql.SQLException; @@ -40,10 +42,10 @@ import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.genesys2.server.model.BusinessModel; -import org.genesys2.server.model.elastic.AccessionDetails; import org.genesys2.server.model.elastic.FullTextDocument; import org.genesys2.server.model.impl.ActivityPost; import org.genesys2.server.model.impl.Article; +import org.genesys2.server.model.impl.ClassPK; import org.genesys2.server.model.impl.Country; import org.genesys2.server.model.impl.FaoInstitute; import org.genesys2.server.persistence.domain.ActivityPostRepository; @@ -59,9 +61,11 @@ import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.elasticsearch.ElasticsearchException; import org.springframework.data.elasticsearch.core.ElasticsearchTemplate; +import org.springframework.data.elasticsearch.core.FacetedPage; import org.springframework.data.elasticsearch.core.query.AliasQuery; import org.springframework.data.elasticsearch.core.query.IndexQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; @@ -69,6 +73,7 @@ import org.springframework.data.elasticsearch.core.query.SearchQuery; import org.springframework.jdbc.core.RowCallbackHandler; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import com.hazelcast.core.ILock; @@ -82,7 +87,7 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial private static final String INDEXALIAS_FULLTEXT_READ = "fulltextRead"; private static final String INDEXALIAS_FULLTEXT_WRITE = "fulltextWrite"; - private static final String ACCESSION_SECTION = "Accession"; + private static final String ACCESSION_SECTION = "accession"; private static final String CLASSPK_SHORTNAME = "classPK.shortName"; private static final String REINDEX_TYPE_ALL = "All"; @@ -151,19 +156,68 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial } } + @Override + @Transactional(readOnly = true) + public Page search(final String query, final Pageable pageable, final Class clazz) throws SearchException { + ClassPK classPK = contentService.getClassPk(clazz); + if (classPK == null) { + throw new SearchException("No such fulltext type"); + } + + SearchQuery searchQuery = new NativeSearchQueryBuilder() + .withIndices(INDEXALIAS_FULLTEXT_READ) + .withQuery( + functionScoreQuery( + boolQuery().must(matchQuery(CLASSPK_SHORTNAME, classPK.getShortName())).must( + queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add( + fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build(); + + try { + FacetedPage fulltextResults = this.elasticsearchTemplate.queryForPage(searchQuery, FullTextDocument.class); + return toEntities(fulltextResults, pageable); + } catch (final Throwable e) { + throw new SearchException(e.getMessage(), e); + } + } + + private Page toEntities(FacetedPage fulltextResults, Pageable pageable) { + List content = new ArrayList(fulltextResults.getSize()); + for (FullTextDocument document : fulltextResults.getContent()) { + content.add(toEntity(document.getClassPK().getClassName(), document.getId())); + } + return new PageImpl<>(content, pageable, fulltextResults.getTotalElements()); + } + + private Object toEntity(String className, Long id) { + if (FaoInstitute.class.getName().equals(className)) { + return instituteRepository.findOne(id); + } else if (Country.class.getName().equals(className)) { + return countryRepository.findOne(id); + } else { + throw new UnsupportedOperationException("Missing toEntity implementation for class=" + className); + } + } + @Override public Page search(final String query, final Pageable pageable, final String section) throws SearchException { + ClassPK classPK = contentService.getClassPk(section); + if (classPK == null) { + throw new SearchException("No such fulltext type"); + } + SearchQuery searchQuery; Class clazz; if (section.equals(ACCESSION_SECTION)) { - searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes(ElasticsearchSearchServiceImpl.PASSPORT_TYPE) - .withQuery(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).withPageable(pageable).build(); - clazz = AccessionDetails.class; + throw new UnsupportedOperationException("Use ElasticsearchServiceImpl for accession search"); } else { - searchQuery = new NativeSearchQueryBuilder().withIndices(INDEXALIAS_FULLTEXT_READ).withQuery( - boolQuery().must(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).must(matchQuery(CLASSPK_SHORTNAME, section))) - .withPageable(pageable).build(); + searchQuery = new NativeSearchQueryBuilder() + .withIndices(INDEXALIAS_FULLTEXT_READ) + .withQuery( + functionScoreQuery( + boolQuery().must(matchQuery(CLASSPK_SHORTNAME, section)).must( + queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add( + fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build(); clazz = FullTextDocument.class; } @@ -436,18 +490,29 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial } private void updateDocumentForCountry(Country country, FullTextDocument document) { - document.setBody(country.getNameL()); - document.setSummary(country.getCode3()); document.setUrlToContent("/geo/" + country.getCode3()); + document.setLanguage(contentService.getDefaultLocale().getLanguage()); + document.setSummary(country.getCode3() + " " + country.getNameL()); + + Article blurb = contentService.getArticle(country, "blurp", contentService.getDefaultLocale()); + if (blurb != null) { + document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody()); + } } private void updateDocumentForFaoInstitute(FaoInstitute institute, FullTextDocument document) { + document.setUrlToContent("/wiews/" + institute.getCode()); + document.setLanguage(contentService.getDefaultLocale().getLanguage()); document.setSummary(institute.getCode() + " " + institute.getAcronym() + " " + institute.getFullName()); + Article blurb = contentService.getArticle(institute, "blurp", contentService.getDefaultLocale()); if (blurb != null) { document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody()); } - document.setUrlToContent("/wiews/" + institute.getCode()); + // If the institute has accessions, we boost it + if (institute.getAccessionCount() > 0) { + document.setScore(institute.getAccessionCount()); + } } class CustomRowCallbackHandler implements RowCallbackHandler { diff --git a/src/main/java/org/genesys2/server/servlet/controller/SearchController.java b/src/main/java/org/genesys2/server/servlet/controller/SearchController.java index 0b7a80237..2445941d7 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/SearchController.java +++ b/src/main/java/org/genesys2/server/servlet/controller/SearchController.java @@ -24,6 +24,10 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.genesys2.server.model.impl.ClassPK; +import org.genesys2.server.model.impl.Country; +import org.genesys2.server.model.impl.FaoInstitute; +import org.genesys2.server.service.ContentService; import org.genesys2.server.service.ElasticService; import org.genesys2.server.service.FullTextSearchService; import org.genesys2.server.service.impl.SearchException; @@ -47,9 +51,15 @@ public class SearchController { @Autowired ElasticService searchService; + @Autowired + private ContentService contentService; + @Autowired FullTextSearchService fullTextSearchService; + @Autowired + ElasticService accessionSearchService; + @Value("${base.url}") private String baseUrl; @@ -89,8 +99,8 @@ public class SearchController { * @throws IOException */ @RequestMapping(value = "/acn/opensearch-gossip", produces = MediaType.APPLICATION_JSON_VALUE) - public @ResponseBody - List openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery) throws IOException { + public @ResponseBody List openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery) + throws IOException { List res = new ArrayList(); res.add(searchQuery); @@ -131,16 +141,28 @@ public class SearchController { @RequestMapping("/acn/search2") public String findFullTextDocument(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery, - @RequestParam(value = "page", required = false, defaultValue = "1") int page, - @RequestParam(value = "section", required = false, defaultValue = "Accession") String section) { + @RequestParam(value = "page", required = true, defaultValue = "1") int page, + @RequestParam(value = "section", required = true, defaultValue = "accession") String section) throws SearchException { + + ClassPK classPK = contentService.getClassPk(section); + if (classPK == null) { + LOG.warn("No search section " + section); + throw new SearchException("No such search section"); + } model.addAttribute("q", searchQuery); + model.addAttribute("section", section); if (!StringUtils.isBlank(searchQuery)) { try { - final Page x = fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section); - model.addAttribute("pagedData", x); - model.addAttribute("section", section); - LOG.info("Searching for: " + searchQuery + " returns " + x.getNumberOfElements()); + if ("accession".equals(section)) { + model.addAttribute("pagedData", accessionSearchService.search(searchQuery, new PageRequest(page - 1, 50))); + } else if ("institute".equals(section)) { + model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), FaoInstitute.class)); + } else if ("country".equals(section)) { + model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), Country.class)); + } else { + model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section)); + } } catch (SearchException e) { LOG.info("Searching for: " + searchQuery + " failed with error " + e.getMessage()); LOG.error(e.getMessage(), e); diff --git a/src/main/resources/content/language.properties b/src/main/resources/content/language.properties index cedc7da47..860f27707 100644 --- a/src/main/resources/content/language.properties +++ b/src/main/resources/content/language.properties @@ -463,6 +463,11 @@ search.search-query-missing=Please enter your search query. search.search-query-failed=Sorry, search failed with error {0} search.button.label=Search search.add-genesys-opensearch=Register Genesys Search with your browser +search.section.accession=Accessions +search.section.article=Content +search.section.activitypost=News Items +search.section.institute=Genebanks +search.section.country=Countries admin.page.title=Genesys 2 Administration metadata.page.title=Datasets diff --git a/src/main/webapp/WEB-INF/jsp/search2/index.jsp b/src/main/webapp/WEB-INF/jsp/search2/index.jsp index 7fd41d8ab..db345c359 100644 --- a/src/main/webapp/WEB-INF/jsp/search2/index.jsp +++ b/src/main/webapp/WEB-INF/jsp/search2/index.jsp @@ -2,6 +2,7 @@ <%@include file="/WEB-INF/jsp/init.jsp"%> <%@ taglib prefix="sec" uri="http://www.springframework.org/security/tags"%> +<%@ taglib prefix="mini" tagdir="/WEB-INF/tags/mini"%> @@ -38,6 +39,7 @@
"> + " />
" name="q" class="form-control" value="" />
" class="btn" />
@@ -45,12 +47,12 @@
-Accession,Article,ActivityPost,Country,FaoInstitute +accession,article,activitypost,country,institute @@ -60,8 +62,7 @@ - - + @@ -69,30 +70,25 @@ - + - + - - - + - - - + - - + + + + + + +
Body Summary Body Title NameCodeFull NameAcronym
${status.count + pagedData.size * pagedData.number} "> @@ -102,12 +98,19 @@ "> + " ${section} " "> - + - + @@ -121,7 +124,7 @@
-
No results.
+
diff --git a/src/main/webapp/WEB-INF/tags/mini/country.tag b/src/main/webapp/WEB-INF/tags/mini/country.tag new file mode 100644 index 000000000..7934d1141 --- /dev/null +++ b/src/main/webapp/WEB-INF/tags/mini/country.tag @@ -0,0 +1,20 @@ +<%@ tag description="Display country data" pageEncoding="UTF-8"%> +<%@ tag body-content="empty"%> +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%> +<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%> +<%@ taglib prefix="security" + uri="http://www.springframework.org/security/tags"%> +<%@ taglib prefix="local" tagdir="/WEB-INF/tags"%> +<%@ taglib prefix="cms" tagdir="/WEB-INF/tags/cms"%> +<%@ attribute name="country" required="true" + type="org.genesys2.server.model.impl.Country"%> + +
+
+ "> +
+
+ +
+
diff --git a/src/main/webapp/WEB-INF/tags/mini/institute.tag b/src/main/webapp/WEB-INF/tags/mini/institute.tag new file mode 100644 index 000000000..b70ac5b56 --- /dev/null +++ b/src/main/webapp/WEB-INF/tags/mini/institute.tag @@ -0,0 +1,33 @@ +<%@ tag description="Display FAO institute data" pageEncoding="UTF-8"%> +<%@ tag body-content="empty"%> +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%> +<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%> +<%@ taglib prefix="security" + uri="http://www.springframework.org/security/tags"%> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt"%> +<%@ attribute name="institute" required="true" + type="org.genesys2.server.model.impl.FaoInstitute"%> + +
+
+ "> +
+
+ +
+
+ +
+
+ + + + "> + + + + + +
+
-- GitLab