Added score and score-based search, added <mini:* tags for display, returning entities from search

src/main/java/org/genesys2/server/model/elastic/FullTextDocument.java

@@ -16,7 +16,6 @@
 package org.genesys2.server.model.elastic;
 
 import java.util.Date;
-import java.util.Locale;
 
 import org.genesys2.server.model.impl.ClassPK;
 import org.springframework.data.annotation.Id;
@@ -25,18 +24,11 @@ import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldIndex;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
-import com.eclipsesource.json.JsonObject;
-import com.eclipsesource.json.JsonValue;
-import com.eclipsesource.json.ParseException;
-
 /**
  * <code>FullTextDocument</code> is used in Elasticsearch mapping
  */
 @Document(indexName = "fulltext", refreshInterval = "60s")
 public class FullTextDocument {
-	// FIXME Revisit!
-	private static final String ENGLISH_LOCALE = "en";
-
 	@Id
 	private Long id;
 
@@ -61,6 +53,10 @@ public class FullTextDocument {
 	@Field(index = FieldIndex.not_analyzed, type = FieldType.Date)
 	private Date lastModifiedDate;
 
+	/** The score here will boost result relevance */
+	@Field(index = FieldIndex.not_analyzed)
+	private float score = 0.5f;
+
 	public Long getId() {
 		return this.id;
 	}
@@ -101,21 +97,6 @@ public class FullTextDocument {
 		this.summary = summary;
 	}
 
-	public String getTextByLocale(final Locale locale) {
-		try {
-			final JsonObject json = JsonObject.readFrom(this.body);
-			JsonValue value = json.get(locale.getLanguage());
-			if (value == null) {
-				value = json.get(ENGLISH_LOCALE);
-			}
-
-			return value.toString().replace("\"", "");
-		} catch (final ParseException ignored) {
-		}
-
-		return getBody();
-	}
-
 	public String getLanguage() {
 		return this.language;
 	}
@@ -139,4 +120,12 @@ public class FullTextDocument {
 	public void setLastModifiedDate(final Date lastModifiedDate) {
 		this.lastModifiedDate = lastModifiedDate;
 	}
+
+	public float getScore() {
+		return score;
+	}
+
+	public void setScore(float score) {
+		this.score = score;
+	}
 }

src/main/java/org/genesys2/server/service/ContentService.java

@@ -40,6 +40,8 @@ public interface ContentService {
 	
 	ClassPK getClassPk(String shortName);
 
+	ClassPK getClassPk(Class<?> clazz);
+
 	/**
 	 * Load article with {@link ClassPK} of clazz with specified id and the
 	 * "slug" for the locale

src/main/java/org/genesys2/server/service/FullTextSearchService.java

@@ -43,4 +43,6 @@ public interface FullTextSearchService {
 	void reindex(String type);
 
 	void deleteIndex(String indexName);
+
+	Page<?> search(String query, Pageable pageable, Class<?> type) throws SearchException;
 }

src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java

@@ -247,6 +247,7 @@ public class ContentServiceImpl implements ContentService {
 		return article;
 	}
 
+	@Override
 	public ClassPK getClassPk(Class<?> clazz) {
 		return classPkRepository.findByClassName(clazz.getName());
 	}

src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java

@@ -16,8 +16,10 @@
 package org.genesys2.server.service.impl;
 
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.queryString;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -40,10 +42,10 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.genesys2.server.model.BusinessModel;
-import org.genesys2.server.model.elastic.AccessionDetails;
 import org.genesys2.server.model.elastic.FullTextDocument;
 import org.genesys2.server.model.impl.ActivityPost;
 import org.genesys2.server.model.impl.Article;
+import org.genesys2.server.model.impl.ClassPK;
 import org.genesys2.server.model.impl.Country;
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.persistence.domain.ActivityPostRepository;
@@ -59,9 +61,11 @@ import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.ElasticsearchException;
 import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
+import org.springframework.data.elasticsearch.core.FacetedPage;
 import org.springframework.data.elasticsearch.core.query.AliasQuery;
 import org.springframework.data.elasticsearch.core.query.IndexQuery;
 import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
@@ -69,6 +73,7 @@ import org.springframework.data.elasticsearch.core.query.SearchQuery;
 import org.springframework.jdbc.core.RowCallbackHandler;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import com.hazelcast.core.ILock;
 
@@ -82,7 +87,7 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
 	private static final String INDEXALIAS_FULLTEXT_READ = "fulltextRead";
 	private static final String INDEXALIAS_FULLTEXT_WRITE = "fulltextWrite";
 
-	private static final String ACCESSION_SECTION = "Accession";
+	private static final String ACCESSION_SECTION = "accession";
 	private static final String CLASSPK_SHORTNAME = "classPK.shortName";
 	private static final String REINDEX_TYPE_ALL = "All";
 
@@ -151,19 +156,68 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
 		}
 	}
 
+	@Override
+	@Transactional(readOnly = true)
+	public Page<?> search(final String query, final Pageable pageable, final Class<?> clazz) throws SearchException {
+		ClassPK classPK = contentService.getClassPk(clazz);
+		if (classPK == null) {
+			throw new SearchException("No such fulltext type");
+		}
+
+		SearchQuery searchQuery = new NativeSearchQueryBuilder()
+				.withIndices(INDEXALIAS_FULLTEXT_READ)
+				.withQuery(
+						functionScoreQuery(
+								boolQuery().must(matchQuery(CLASSPK_SHORTNAME, classPK.getShortName())).must(
+										queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add(
+								fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build();
+
+		try {
+			FacetedPage<FullTextDocument> fulltextResults = this.elasticsearchTemplate.queryForPage(searchQuery, FullTextDocument.class);
+			return toEntities(fulltextResults, pageable);
+		} catch (final Throwable e) {
+			throw new SearchException(e.getMessage(), e);
+		}
+	}
+
+	private Page<?> toEntities(FacetedPage<FullTextDocument> fulltextResults, Pageable pageable) {
+		List content = new ArrayList(fulltextResults.getSize());
+		for (FullTextDocument document : fulltextResults.getContent()) {
+			content.add(toEntity(document.getClassPK().getClassName(), document.getId()));
+		}
+		return new PageImpl<>(content, pageable, fulltextResults.getTotalElements());
+	}
+
+	private Object toEntity(String className, Long id) {
+		if (FaoInstitute.class.getName().equals(className)) {
+			return instituteRepository.findOne(id);
+		} else if (Country.class.getName().equals(className)) {
+			return countryRepository.findOne(id);
+		} else {
+			throw new UnsupportedOperationException("Missing toEntity implementation for class=" + className);
+		}
+	}
+
 	@Override
 	public Page<?> search(final String query, final Pageable pageable, final String section) throws SearchException {
+		ClassPK classPK = contentService.getClassPk(section);
+		if (classPK == null) {
+			throw new SearchException("No such fulltext type");
+		}
+
 		SearchQuery searchQuery;
 		Class clazz;
 
 		if (section.equals(ACCESSION_SECTION)) {
-			searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes(ElasticsearchSearchServiceImpl.PASSPORT_TYPE)
-					.withQuery(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).withPageable(pageable).build();
-			clazz = AccessionDetails.class;
+			throw new UnsupportedOperationException("Use ElasticsearchServiceImpl for accession search");
 		} else {
-			searchQuery = new NativeSearchQueryBuilder().withIndices(INDEXALIAS_FULLTEXT_READ).withQuery(
-					boolQuery().must(queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND)).must(matchQuery(CLASSPK_SHORTNAME, section)))
-					.withPageable(pageable).build();
+			searchQuery = new NativeSearchQueryBuilder()
+					.withIndices(INDEXALIAS_FULLTEXT_READ)
+					.withQuery(
+							functionScoreQuery(
+									boolQuery().must(matchQuery(CLASSPK_SHORTNAME, section)).must(
+											queryString(query).defaultOperator(QueryStringQueryBuilder.Operator.AND))).add(
+									fieldValueFactorFunction("score").factor(1.0f))).withPageable(pageable).build();
 			clazz = FullTextDocument.class;
 		}
 
@@ -436,18 +490,29 @@ public class FullTextSearchServiceImpl implements FullTextSearchService, Initial
 	}
 
 	private void updateDocumentForCountry(Country country, FullTextDocument document) {
-		document.setBody(country.getNameL());
-		document.setSummary(country.getCode3());
 		document.setUrlToContent("/geo/" + country.getCode3());
+		document.setLanguage(contentService.getDefaultLocale().getLanguage());
+		document.setSummary(country.getCode3() + " " + country.getNameL());
+
+		Article blurb = contentService.getArticle(country, "blurp", contentService.getDefaultLocale());
+		if (blurb != null) {
+			document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody());
+		}
 	}
 
 	private void updateDocumentForFaoInstitute(FaoInstitute institute, FullTextDocument document) {
+		document.setUrlToContent("/wiews/" + institute.getCode());
+		document.setLanguage(contentService.getDefaultLocale().getLanguage());
 		document.setSummary(institute.getCode() + " " + institute.getAcronym() + " " + institute.getFullName());
+
 		Article blurb = contentService.getArticle(institute, "blurp", contentService.getDefaultLocale());
 		if (blurb != null) {
 			document.setBody(blurb.getSummary() + "\n\n" + blurb.getBody());
 		}
-		document.setUrlToContent("/wiews/" + institute.getCode());
+		// If the institute has accessions, we boost it
+		if (institute.getAccessionCount() > 0) {
+			document.setScore(institute.getAccessionCount());
+		}
 	}
 
 	class CustomRowCallbackHandler implements RowCallbackHandler {

src/main/java/org/genesys2/server/servlet/controller/SearchController.java

@@ -24,6 +24,10 @@ import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.genesys2.server.model.impl.ClassPK;
+import org.genesys2.server.model.impl.Country;
+import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.service.ContentService;
 import org.genesys2.server.service.ElasticService;
 import org.genesys2.server.service.FullTextSearchService;
 import org.genesys2.server.service.impl.SearchException;
@@ -47,9 +51,15 @@ public class SearchController {
 	@Autowired
 	ElasticService searchService;
 
+	@Autowired
+	private ContentService contentService;
+
 	@Autowired
 	FullTextSearchService fullTextSearchService;
 
+	@Autowired
+	ElasticService accessionSearchService;
+
 	@Value("${base.url}")
 	private String baseUrl;
 
@@ -89,8 +99,8 @@ public class SearchController {
 	 * @throws IOException
 	 */
 	@RequestMapping(value = "/acn/opensearch-gossip", produces = MediaType.APPLICATION_JSON_VALUE)
-	public @ResponseBody
-	List<Object> openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery) throws IOException {
+	public @ResponseBody List<Object> openSearchAutocomplete(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery)
+			throws IOException {
 		List<Object> res = new ArrayList<Object>();
 		res.add(searchQuery);
 
@@ -131,16 +141,28 @@ public class SearchController {
 
 	@RequestMapping("/acn/search2")
 	public String findFullTextDocument(ModelMap model, @RequestParam(required = false, value = "q") String searchQuery,
-								@RequestParam(value = "page", required = false, defaultValue = "1") int page,
-								@RequestParam(value = "section", required = false, defaultValue = "Accession") String section) {
+			@RequestParam(value = "page", required = true, defaultValue = "1") int page,
+			@RequestParam(value = "section", required = true, defaultValue = "accession") String section) throws SearchException {
+
+		ClassPK classPK = contentService.getClassPk(section);
+		if (classPK == null) {
+			LOG.warn("No search section " + section);
+			throw new SearchException("No such search section");
+		}
 
 		model.addAttribute("q", searchQuery);
+		model.addAttribute("section", section);
 		if (!StringUtils.isBlank(searchQuery)) {
 			try {
-				final Page<?> x = fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section);
-				model.addAttribute("pagedData", x);
-				model.addAttribute("section", section);
-				LOG.info("Searching for: " + searchQuery + " returns " + x.getNumberOfElements());
+				if ("accession".equals(section)) {
+					model.addAttribute("pagedData", accessionSearchService.search(searchQuery, new PageRequest(page - 1, 50)));
+				} else if ("institute".equals(section)) {
+					model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), FaoInstitute.class));
+				} else if ("country".equals(section)) {
+					model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), Country.class));
+				} else {
+					model.addAttribute("pagedData", fullTextSearchService.search(searchQuery, new PageRequest(page - 1, 50), section));
+				}
 			} catch (SearchException e) {
 				LOG.info("Searching for: " + searchQuery + " failed with error " + e.getMessage());
 				LOG.error(e.getMessage(), e);

src/main/resources/content/language.properties

@@ -463,6 +463,11 @@ search.search-query-missing=Please enter your search query.
 search.search-query-failed=Sorry, search failed with error {0}
 search.button.label=Search
 search.add-genesys-opensearch=Register Genesys Search with your browser
+search.section.accession=Accessions
+search.section.article=Content
+search.section.activitypost=News Items
+search.section.institute=Genebanks
+search.section.country=Countries
 
 admin.page.title=Genesys 2 Administration
 metadata.page.title=Datasets

src/main/webapp/WEB-INF/jsp/search2/index.jsp

@@ -2,6 +2,7 @@
 
 <%@include file="/WEB-INF/jsp/init.jsp"%>
 <%@ taglib prefix="sec" uri="http://www.springframework.org/security/tags"%>
+<%@ taglib prefix="mini" tagdir="/WEB-INF/tags/mini"%>
 
 <html>
 <head>
@@ -38,6 +39,7 @@
 
 <div class="applied-filters">
   <form class="" method="get" action="<c:url value="/acn/search2" />">
+  	<input type="hidden" name="section" value="<c:out value="${section}" />" />
     <div class="row">
       <div class="col-md-4"><input type="text" placeholder="<spring:message code="search.input.placeholder" />" name="q" class="form-control" value="<c:out value="${q}" />" /></div>
       <div class="col-md-2"><input type="submit" value="<spring:message code="search.button.label" />" class="btn" /></div>
@@ -45,12 +47,12 @@
   </form>
 </div>
 
-<c:set var="sectionList">Accession,Article,ActivityPost,Country,FaoInstitute</c:set>
+<c:set var="sectionList">accession,article,activitypost,country,institute</c:set>
 
 <ul class="nav nav-tabs">
   <c:forTokens items="${sectionList}" delims="," var="sect">
     <li class=${sect eq section ? "active" : ""}>
-      <a href="<c:url value="/acn/search2" />?q=${q}&amp;section=${sect}">${sect}</a>
+      <a href="<c:url value="/acn/search2" />?q=${q}&amp;section=${sect}"><spring:message code="search.section.${sect}" /></a>
     </li>
   </c:forTokens>
 </ul>
@@ -60,8 +62,7 @@
     <table class="accessions">
       <thead>
       <tr>
-        <td class="idx-col"></td>
-        <c:if test="${section eq 'Accession'}">
+        <c:if test="${section eq 'accession'}">
           <td />
           <td><spring:message code="accession.accessionName" /></td>
           <td><spring:message code="accession.taxonomy" /></td>
@@ -69,30 +70,25 @@
           <td class="notimportant"><spring:message code="accession.sampleStatus" /></td>
           <td class="notimportant"><spring:message code="accession.holdingInstitute" /></td>
         </c:if>
-        <c:if test="${section eq 'Article'}">
+        <c:if test="${section eq 'article'}">
           <td>Body</td>
           <td>Summary</td>
         </c:if>
-        <c:if test="${section eq 'ActivityPost'}">
+        <c:if test="${section eq 'activitypost'}">
           <td>Body</td>
           <td>Title</td>
         </c:if>
-        <c:if test="${section eq 'Country'}">
-          <td>Name</td>
-          <td>Code</td>
+        <c:if test="${section eq 'country'}">
         </c:if>
-        <c:if test="${section eq 'FaoInstitute'}">
-          <td>Full Name</td>
-          <td>Acronym</td>
+        <c:if test="${section eq 'institute'}">
         </c:if>
       </tr>
       </thead>
       <tbody>
       <c:forEach items="${pagedData.content}" var="unit" varStatus="status">
         <tr class="acn">
-          <td class="idx-col">${status.count + pagedData.size * pagedData.number}</td>
           <c:choose>
-            <c:when test="${section eq 'Accession'}">
+            <c:when test="${section eq 'accession'}">
               <td class="sel" x-aid="${unit.id}"></td>
               <td><a href="<c:url value="/acn/id/${unit.id}" />"><b>
                 <c:out value="${unit.acceNumb}" />
@@ -102,12 +98,19 @@
               <td class="notimportant"><spring:message code="accession.sampleStatus.${unit.sampStat}" /></td>
               <td class="notimportant"><a href="<c:url value="/wiews/${unit.institute.code}" />"><c:out value="${unit.institute.code}" /></a></td>
             </c:when>
+            <c:when test="${section eq 'country'}">
+            	<td><mini:country country="${unit}" /></td>
+            </c:when>
+            <c:when test="${section eq 'institute'}">
+            	<td><mini:institute institute="${unit}" /></td>
+            </c:when>
             <c:otherwise>
               <td>
+              	" ${section} "
                 <a href="<c:url value="${unit.urlToContent}" />">
-                  <c:set var="body" value="${unit.getTextByLocale(pageContext.response.locale)}" />
+                  <c:set var="body" value="${unit.body}" />
                   <c:if test="${fn:length(body) > 100}">
-                    <b><c:set var="body" value="${fn:substring(body, 0, 100)}..." /></b>
+                    <c:set var="body" value="${fn:substring(body, 0, 100)}..." />
                   </c:if>
                   <b><c:out value="${body}" escapeXml="false" /></b>
                 </a>
@@ -121,7 +124,7 @@
     </table>
   </c:when>
   <c:otherwise>
-    <div class="alert alert-info">No results.</div>
+    <div class="alert alert-info"><spring:message code="search.no-results" /></div>
   </c:otherwise>
 </c:choose>
 </body>

src/main/webapp/WEB-INF/tags/mini/country.tag

+<%@ tag description="Display country data" pageEncoding="UTF-8"%>
+<%@ tag body-content="empty"%>
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%>
+<%@ taglib prefix="security"
+	uri="http://www.springframework.org/security/tags"%>
+<%@ taglib prefix="local" tagdir="/WEB-INF/tags"%>
+<%@ taglib prefix="cms" tagdir="/WEB-INF/tags/cms"%>
+<%@ attribute name="country" required="true"
+	type="org.genesys2.server.model.impl.Country"%>
+
+<div class="genesys-mini genesys-country">
+	<div class="code3">
+		<a href="<c:url value="/geo/${country.code3}" />"><c:out
+				value="${country.code3}" /></a>
+	</div>
+	<div class="name">
+		<c:out value="${country.getName(pageContext.response.locale)}" />
+	</div>
+</div>

src/main/webapp/WEB-INF/tags/mini/institute.tag

+<%@ tag description="Display FAO institute data" pageEncoding="UTF-8"%>
+<%@ tag body-content="empty"%>
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+<%@ taglib prefix="spring" uri="http://www.springframework.org/tags"%>
+<%@ taglib prefix="security"
+	uri="http://www.springframework.org/security/tags"%>
+<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt"%>
+<%@ attribute name="institute" required="true"
+	type="org.genesys2.server.model.impl.FaoInstitute"%>
+
+<div class="genesys-mini genesys-institute">
+	<div class="code">
+		<a href="<c:url value="/wiews/${institute.code}" />"><c:out
+				value="${institute.code}" /></a>
+	</div>
+	<div class="fullName">
+		<c:out value="${institute.fullName}" />
+	</div>
+	<div class="acronym">
+		<c:out value="${institute.acronym}" />
+	</div>
+	<div class="accessionCount">
+		<c:choose>
+			<c:when test="${institute.accessionCount gt 0}">
+				<spring:message code="faoInstitutes.stat.accessionCount" />
+				<a href="<c:url value="/wiews/${institute.code}/data" />"><fmt:formatNumber value="${institute.accessionCount}" /></a>
+			</c:when>
+			<c:otherwise>
+			
+			</c:otherwise>
+		</c:choose>
+	</div>
+</div>