Skip to content
GitLab
Commit c38258c5 authored by Matija Obreza's avatar Matija Obreza
Browse files

Added score and score-based search, added <mini:* tags for display, returning entities from search

parent 5e6aa0f7
Hide whitespace changes
Inline Side-by-side
src/main/java/org/genesys2/server/model/elastic/FullTextDocument.java
View file @ c38258c5
......@@ -16,7 +16,6 @@
package org.genesys2.server.model.elastic;
import java.util.Date;
import java.util.Locale;
import org.genesys2.server.model.impl.ClassPK;
import org.springframework.data.annotation.Id;
......@@ -25,18 +24,11 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
import com.eclipsesource.json.JsonObject;
import com.eclipsesource.json.JsonValue;
import com.eclipsesource.json.ParseException;
/**
* <code>FullTextDocument</code> is used in Elasticsearch mapping
*/
@Document(indexName = "fulltext", refreshInterval = "60s")
public class FullTextDocument {
// FIXME Revisit!
private static final String ENGLISH_LOCALE = "en";
@Id
private Long id;
......@@ -61,6 +53,10 @@ public class FullTextDocument {
@Field(index = FieldIndex.not_analyzed, type = FieldType.Date)
private Date lastModifiedDate;
/** The score here will boost result relevance */
@Field(index = FieldIndex.not_analyzed)
private float score = 0.5f;
public Long getId() {
return this.id;
}
......@@ -101,21 +97,6 @@ public class FullTextDocument {
this.summary = summary;
}
public String getTextByLocale(final Locale locale) {
try {
final JsonObject json = JsonObject.readFrom(this.body);
JsonValue value = json.get(locale.getLanguage());
if (value == null) {
value = json.get(ENGLISH_LOCALE);
}
return value.toString().replace("\"", "");
} catch (final ParseException ignored) {
}
return getBody();
}
public String getLanguage() {
return this.language;
}
......@@ -139,4 +120,12 @@ public class FullTextDocument {
public void setLastModifiedDate(final Date lastModifiedDate) {
this.lastModifiedDate = lastModifiedDate;
}
public float getScore() {
return score;
}
public void setScore(float score) {
this.score = score;
}
}
src/main/java/org/genesys2/server/service/ContentService.java
View file @ c38258c5
......@@ -40,6 +40,8 @@ public interface ContentService {
ClassPK getClassPk(String shortName);
ClassPK getClassPk(Class<?> clazz);
/**
* Load article with {@link ClassPK} of clazz with specified id and the
* "slug" for the locale
......
src/main/java/org/genesys2/server/service/FullTextSearchService.java
View file @ c38258c5
......@@ -43,4 +43,6 @@ public interface FullTextSearchService {
void reindex(String type);
void deleteIndex(String indexName);
Page<?> search(String query, Pageable pageable, Class<?> type) throws SearchException;
}
src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java
View file @ c38258c5
......@@ -247,6 +247,7 @@ public class ContentServiceImpl implements ContentService {
return article;
}
@Override
public ClassPK getClassPk(Class<?> clazz) {
return classPkRepository.findByClassName(clazz.getName());
}
......
src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java
View file @ c38258c5
......@@ -16,8 +16,10 @@
package org.genesys2.server.service.impl;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.queryString;
import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
import java.sql.ResultSet;
import java.sql.SQLException;
......@@ -40,10 +42,10 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.genesys2.server.model.BusinessModel;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.elastic.FullTextDocument;
import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
import org.genesys2.server.model.impl.ClassPK;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.domain.ActivityPostRepository;
......@@ -59,9 +61,11 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.FacetedPage;
import org.springframework.data.elasticsearch.core.query.AliasQuery;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
......@@ -69,6 +73,7 @@ import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.hazelcast.core.ILock;
......@@ -82,7 +87,7 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
private static final String INDEXALIAS_FULLTEXT_READ = "fulltextRead";
private static final String INDEXALIAS_FULLTEXT_WRITE = "fulltextWrite";
private static final String ACCESSION_SECTION = "Accession";
private static final String ACCESSION_SECTION = "accession";
private static final String CLASSPK_SHORTNAME = "classPK.shortName";
private static final String REINDEX_TYPE_ALL = "All";
......@@ -151,19 +156,68 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
}
}
@Override
@Transactional(readOnly = true)
public Page<?> search(final String query, final Pageable pageable, final Class<?> clazz) throws SearchException {
ClassPK classPK = contentService.getClassPk(clazz);
if (classPK == null) {
throw new SearchException("No such fulltext type");
}
SearchQuery searchQuery = new NativeSearchQueryBuilder()
.withIndices(INDEXALIAS_FULLTEXT_READ)
.withQuery(
functionScoreQuery(
boolQuery().must(matchQuery(CLASSPK_SHORTNAME, classPK.getShortName())).must(
queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add(
fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build();
try {
FacetedPage<FullTextDocument> fulltextResults = this.elasticsearchTemplate.queryForPage(searchQuery, FullTextDocument.class);
return toEntities(fulltextResults, pageable);
} catch (final Throwable e) {
throw new SearchException(e.getMessage(), e);
}
}
private Page<?> toEntities(FacetedPage<FullTextDocument> fulltextResults, Pageable pageable) {
List content = new ArrayList(fulltextResults.getSize());
for (FullTextDocument document : fulltextResults.getContent()) {
content.add(toEntity(document.getClassPK().getClassName(), document.getId()));
}
return new PageImpl<>(content, pageable, fulltextResults.getTotalElements());
}
private Object toEntity(String className, Long id) {
if (FaoInstitute.class.getName().equals(className)) {
return instituteRepository.findOne(id);
} else if (Country.class.getName().equals(className)) {
return countryRepository.findOne(id);
} else {
throw new UnsupportedOperationException("Missing toEntity implementation for class=" + className);
}
}
@Override
public Page<?> search(final String query, final Pageable pageable, final String section) throws SearchException {
ClassPK classPK = contentService.getClassPk(section);
if (classPK == null) {
throw new SearchException("No such fulltext type");
}
SearchQuery searchQuery;
Class clazz;
if (section.equals(ACCESSION_SECTION)) {
searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes(ElasticsearchSearchServiceImpl.PASSPORT_TYPE)
.withQuery(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).withPageable(pageable).build();
clazz = AccessionDetails.class;
throw new UnsupportedOperationException("Use ElasticsearchServiceImpl for accession search");
} else {
searchQuery = new NativeSearchQueryBuilder().withIndices(INDEXALIAS_FULLTEXT_READ).withQuery(
boolQuery().must(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).must(matchQuery(CLASSPK_SHORTNAME, section)))
.withPageable(pageable).build();
searchQuery = new NativeSearchQueryBuilder()
.withIndices(INDEXALIAS_FULLTEXT_READ)
.withQuery(
functionScoreQuery(
boolQuery().must(matchQuery(CLASSPK_SHORTNAME, section)).must(
queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add(
fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build();
clazz = FullTextDocument.class;
}
......@@ -436,18 +490,29 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
}
private void updateDocumentForCountry(Country country, FullTextDocument document) {
document.setBody(country.getNameL());
document.setSummary(country.getCode3());
document.setUrlToContent("/geo/" + country.getCode3());
document.setLanguage(contentService.getDefaultLocale().getLanguage());
document.setSummary(country.getCode3() + " " + country.getNameL());
Article blurb = contentService.getArticle(country, "blurp", contentService.getDefaultLocale());
if (blurb != null) {
document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody());
}
}
private void updateDocumentForFaoInstitute(FaoInstitute institute, FullTextDocument document) {
document.setUrlToContent("/wiews/" + institute.getCode());
document.setLanguage(contentService.getDefaultLocale().getLanguage());
document.setSummary(institute.getCode() + " " + institute.getAcronym() + " " + institute.getFullName());
Article blurb = contentService.getArticle(institute, "blurp", contentService.getDefaultLocale());
if (blurb != null) {
document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody());
}
document.setUrlToContent("/wiews/" + institute.getCode());
// If the institute has accessions, we boost it
if (institute.getAccessionCount() > 0) {
document.setScore(institute.getAccessionCount());
}
}
class CustomRowCallbackHandler implements RowCallbackHandler {
......
src/main/java/org/genesys2/server/servlet/controller/SearchController.java
View file @ c38258c5
......@@ -24,6 +24,10 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.impl.ClassPK;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FullTextSearchService;
import org.genesys2.server.service.impl.SearchException;
......@@ -47,9 +51,15 @@ public class SearchController {
@Autowired
ElasticService searchService;
@Autowired
private ContentService contentService;
@Autowired
FullTextSearchService fullTextSearchService;
@Autowired
ElasticService accessionSearchService;
@Value("${base.url}")
private String baseUrl;
......@@ -89,8 +99,8 @@ public class SearchController {
* @throws IOException
*/
@RequestMapping(value = "/acn/opensearch-gossip", produces = MediaType.APPLICATION_JSON_VALUE)
public @ResponseBody
List<Object> openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery) throws IOException {
public @ResponseBody List<Object> openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery)
throws IOException {
List<Object> res = new ArrayList<Object>();
res.add(searchQuery);
......@@ -131,16 +141,28 @@ public class SearchController {
@RequestMapping("/acn/search2")
public String findFullTextDocument(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery,
@RequestParam(value = "page", required = false, defaultValue = "1") int page,
@RequestParam(value = "section", required = false, defaultValue = "Accession") String section) {
@RequestParam(value = "page", required = true, defaultValue = "1") int page,
@RequestParam(value = "section", required = true, defaultValue = "accession") String section) throws SearchException {
ClassPK classPK = contentService.getClassPk(section);
if (classPK == null) {
LOG.warn("No search section " + section);
throw new SearchException("No such search section");
}
model.addAttribute("q", searchQuery);
model.addAttribute("section", section);
if (!StringUtils.isBlank(searchQuery)) {
try {
final Page<?> x = fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section);
model.addAttribute("pagedData", x);
model.addAttribute("section", section);
LOG.info("Searching for: " + searchQuery + " returns " + x.getNumberOfElements());
if ("accession".equals(section)) {
model.addAttribute("pagedData", accessionSearchService.search(searchQuery, new PageRequest(page - 1, 50)));
} else if ("institute".equals(section)) {
model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), FaoInstitute.class));
} else if ("country".equals(section)) {
model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), Country.class));
} else {
model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section));
}
} catch (SearchException e) {
LOG.info("Searching for: " + searchQuery + " failed with error " + e.getMessage());
LOG.error(e.getMessage(), e);
......
src/main/resources/content/language.properties
View file @ c38258c5
......@@ -463,6 +463,11 @@ search.search-query-missing=Please enter your search query.
search.search-query-failed=Sorry, search failed with error {0}
search.button.label=Search
search.add-genesys-opensearch=Register Genesys Search with your browser
search.section.accession=Accessions
search.section.article=Content
search.section.activitypost=News Items
search.section.institute=Genebanks
search.section.country=Countries
admin.page.title=Genesys 2 Administration
metadata.page.title=Datasets
......
src/main/webapp/WEB-INF/jsp/search2/index.jsp
View file @ c38258c5
......@@ -2,6 +2,7 @@
<%@include file="/WEB-INF/jsp/init.jsp"%>
<%@ taglib prefix="sec" uri="http://www.springframework.org/security/tags"%>
<%@ taglib prefix="mini" tagdir="/WEB-INF/tags/mini"%>
<html>
<head>
......@@ -38,6 +39,7 @@
<div class="applied-filters">
<form class="" method="get" action="<c:url value="/acn/search2" />">
<input type="hidden" name="section" value="<c:out value="${section}" />" />
<div class="row">
<div class="col-md-4"><input type="text" placeholder="<spring:message code="search.input.placeholder" />" name="q" class="form-control" value="<c:out value="${q}" />" /></div>
<div class="col-md-2"><input type="submit" value="<spring:message code="search.button.label" />" class="btn" /></div>
......@@ -45,12 +47,12 @@
</form>
</div>
<c:set var="sectionList">Accession,Article,ActivityPost,Country,FaoInstitute</c:set>
<c:set var="sectionList">accession,article,activitypost,country,institute</c:set>
<ul class="nav nav-tabs">
<c:forTokens items="${sectionList}" delims="," var="sect">
<li class=${sect eq section ? "active" : ""}>
<a href="<c:url value="/acn/search2" />?q=${q}&amp;section=${sect}">${sect}</a>
<a href="<c:url value="/acn/search2" />?q=${q}&amp;section=${sect}"><spring:message code="search.section.${sect}" /></a>
</li>
</c:forTokens>
</ul>
......@@ -60,8 +62,7 @@
<table class="accessions">
<thead>
<tr>
<td class="idx-col"></td>
<c:if test="${section eq 'Accession'}">
<c:if test="${section eq 'accession'}">
<td />
<td><spring:message code="accession.accessionName" /></td>
<td><spring:message code="accession.taxonomy" /></td>
......@@ -69,30 +70,25 @@
<td class="notimportant"><spring:message code="accession.sampleStatus" /></td>
<td class="notimportant"><spring:message code="accession.holdingInstitute" /></td>
</c:if>
<c:if test="${section eq 'Article'}">
<c:if test="${section eq 'article'}">
<td>Body</td>
<td>Summary</td>
</c:if>
<c:if test="${section eq 'ActivityPost'}">
<c:if test="${section eq 'activitypost'}">
<td>Body</td>
<td>Title</td>
</c:if>
<c:if test="${section eq 'Country'}">
<td>Name</td>
<td>Code</td>
<c:if test="${section eq 'country'}">
</c:if>
<c:if test="${section eq 'FaoInstitute'}">
<td>Full Name</td>
<td>Acronym</td>
<c:if test="${section eq 'institute'}">
</c:if>
</tr>
</thead>
<tbody>
<c:forEach items="${pagedData.content}" var="unit" varStatus="status">
<tr class="acn">
<td class="idx-col">${status.count + pagedData.size * pagedData.number}</td>
<c:choose>
<c:when test="${section eq 'Accession'}">
<c:when test="${section eq 'accession'}">
<td class="sel" x-aid="${unit.id}"></td>
<td><a href="<c:url value="/acn/id/${unit.id}" />"><b>
<c:out value="${unit.acceNumb}" />
......@@ -102,12 +98,19 @@
<td class="notimportant"><spring:message code="accession.sampleStatus.${unit.sampStat}" /></td>
<td class="notimportant"><a href="<c:url value="/wiews/${unit.institute.code}" />"><c:out value="${unit.institute.code}" /></a></td>
</c:when>
<c:when test="${section eq 'country'}">
<td><mini:country country="${unit}" /></td>
</c:when>
<c:when test="${section eq 'institute'}">
<td><mini:institute institute="${unit}" /></td>
</c:when>
<c:otherwise>
<td>
" ${section} "
<a href="<c:url value="${unit.urlToContent}" />">
<c:set var="body" value="${unit.getTextByLocale(pageContext.response.locale)}" />
<c:set var="body" value="${unit.body}" />
<c:if test="${fn:length(body) > 100}">
<b><c:set var="body" value="${fn:substring(body, 0, 100)}..." /></b>
<c:set var="body" value="${fn:substring(body, 0, 100)}..." />
</c:if>
<b><c:out value="${body}" escapeXml="false" /></b>
</a>
......@@ -121,7 +124,7 @@
</table>
</c:when>
<c:otherwise>
<div class="alert alert-info">No results.</div>
<div class="alert alert-info"><spring:message code="search.no-results" /></div>
</c:otherwise>
</c:choose>
</body>
......
src/main/webapp/WEB-INF/tags/mini/country.tag 0 → 100644
View file @ c38258c5
<%@ tag description="Display country data" pageEncoding="UTF-8"%>
<%@ tag body-content="empty"%>
<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%>
<%@ taglib prefix="security"
uri="http://www.springframework.org/security/tags"%>
<%@ taglib prefix="local" tagdir="/WEB-INF/tags"%>
<%@ taglib prefix="cms" tagdir="/WEB-INF/tags/cms"%>
<%@ attribute name="country" required="true"
type="org.genesys2.server.model.impl.Country"%>
<div class="genesys-mini genesys-country">
<div class="code3">
<a href="<c:url value="/geo/${country.code3}" />"><c:out
value="${country.code3}" /></a>
</div>
<div class="name">
<c:out value="${country.getName(pageContext.response.locale)}" />
</div>
</div>
src/main/webapp/WEB-INF/tags/mini/institute.tag 0 → 100644
View file @ c38258c5
<%@ tag description="Display FAO institute data" pageEncoding="UTF-8"%>
<%@ tag body-content="empty"%>
<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%>
<%@ taglib prefix="security"
uri="http://www.springframework.org/security/tags"%>
<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt"%>
<%@ attribute name="institute" required="true"
type="org.genesys2.server.model.impl.FaoInstitute"%>
<div class="genesys-mini genesys-institute">
<div class="code">
<a href="<c:url value="/wiews/${institute.code}" />"><c:out
value="${institute.code}" /></a>
</div>
<div class="fullName">
<c:out value="${institute.fullName}" />
</div>
<div class="acronym">
<c:out value="${institute.acronym}" />
</div>
<div class="accessionCount">
<c:choose>
<c:when test="${institute.accessionCount gt 0}">
<spring:message code="faoInstitutes.stat.accessionCount" />
<a href="<c:url value="/wiews/${institute.code}/data" />"><fmt:formatNumber value="${institute.accessionCount}" /></a>
</c:when>
<c:otherwise>
</c:otherwise>
</c:choose>
</div>
</div>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment