Commit 2b17d953 authored by Matija Obreza's avatar Matija Obreza

ES startup fixed and organized

parent b1c13105
......@@ -22,9 +22,7 @@ public interface IndexAliasConstants {
String INDEX_FULLTEXT = "fulltext";
String INDEX_PASSPORT = "passport";
String INDEX_GENESYS = "genesys";
String INDEX_PASSPORT = "genesys";
String INDEX_GENESYS_ARCHIVE = "genesysarchive";
......
......@@ -16,7 +16,18 @@
package org.genesys2.server.service.impl;
import com.hazelcast.core.ILock;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Resource;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
......@@ -31,7 +42,7 @@ import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.domain.*;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.ElasticSearchManagementService;
import org.genesys2.server.service.IndexAliasConstants;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
......@@ -45,11 +56,7 @@ import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import com.hazelcast.core.ILock;
@Service
public class ElasticSearchManagementServiceImpl implements ElasticSearchManagementService {
......
......@@ -135,8 +135,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
AndFilterBuilder filterBuilder = getFilterBuilder(appliedFilters);
SortBuilder sortBuilder = SortBuilders.fieldSort(FilterConstants.ACCENUMB).order(SortOrder.ASC);
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(IndexAliasConstants.INDEXALIAS_PASSPORT_READ).withTypes(IndexAliasConstants.PASSPORT_TYPE)
.withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(IndexAliasConstants.INDEXALIAS_PASSPORT_READ).withTypes(IndexAliasConstants.PASSPORT_TYPE).withFilter(filterBuilder)
.withSort(sortBuilder).withPageable(pageable).build();
try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
......@@ -159,8 +159,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} else {
termFacetRequest = new TermFacetRequestBuilder("f").applyQueryFilter().fields(term).size(size).build();
}
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(IndexAliasConstants.INDEXALIAS_PASSPORT_READ).withTypes(IndexAliasConstants.PASSPORT_TYPE)
.withFilter(filterBuilder).withFacet(termFacetRequest).build();
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(IndexAliasConstants.INDEXALIAS_PASSPORT_READ).withTypes(IndexAliasConstants.PASSPORT_TYPE).withFilter(filterBuilder)
.withFacet(termFacetRequest).build();
try {
FacetedPage<AccessionDetails> page = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
......@@ -172,8 +172,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
/**
* Runs TermFacet, but will automatically increase size if #otherCount is
* more than 10%
* Runs TermFacet, but will automatically increase size if #otherCount is more than 10%
*/
@Override
public TermResult termStatisticsAuto(AppliedFilters appliedFilters, String term, int size) throws SearchException {
......@@ -295,8 +294,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.debug("startsWith " + startsWith);
if (genesysFilter.isAnalyzed()) {
if (FilterConstants.ALIAS.equals(key)) {
orFilter.add(FilterBuilders.nestedFilter("aliases",
QueryBuilders.queryStringQuery("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
orFilter.add(FilterBuilders.nestedFilter("aliases", QueryBuilders.queryStringQuery("aliases.name" + ":" + startsWith.getStartsWith() + "*")));
} else {
orFilter.add(FilterBuilders.queryFilter(QueryBuilders.queryStringQuery(key + ":" + startsWith.getStartsWith() + "*")));
}
......@@ -365,8 +363,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Removing from index " + clazz2 + " id=" + id);
if (clazz2.equals(AccessionDetails.class)) {
elasticsearchTemplate.delete(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE,
IndexAliasConstants.PASSPORT_TYPE, String.valueOf(id));
elasticsearchTemplate.delete(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE, IndexAliasConstants.PASSPORT_TYPE, String.valueOf(id));
} else {
// Default
elasticsearchTemplate.delete(clazz2, String.valueOf(id));
......@@ -376,12 +373,16 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public void afterPropertiesSet() throws Exception {
LOG.info("Initializing index");
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
try {
elasticsearchTemplate.putMapping(AccessionDetails.class);
} catch (Throwable e) {
LOG.error("Mapping mismatch. Need to reindex.", e);
for (int i = 0; i < 5; i++) {
try {
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
elasticsearchTemplate.putMapping(AccessionDetails.class);
break;
} catch (Throwable e) {
LOG.error("Mapping mismatch. Deleting template index " + IndexAliasConstants.INDEX_PASSPORT, e);
elasticsearchTemplate.deleteIndex(IndexAliasConstants.INDEX_PASSPORT);
}
}
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
......@@ -396,7 +397,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate.putMapping(IndexAliasConstants.INDEX_GENESYS_ARCHIVE, IndexAliasConstants.PASSPORT_TYPE, indexMapping);
}
String newIndex = IndexAliasConstants.INDEX_GENESYS + System.currentTimeMillis();
String newIndex = IndexAliasConstants.INDEX_PASSPORT + System.currentTimeMillis();
if (!elasticSearchManagementService.aliasExists(IndexAliasConstants.INDEXALIAS_PASSPORT_READ)) {
elasticsearchTemplate.createIndex(newIndex);
elasticsearchTemplate.putMapping(newIndex, IndexAliasConstants.PASSPORT_TYPE, indexMapping);
......@@ -458,10 +459,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
IndexRequest indexRequest = new IndexRequest();
indexRequest.index(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE).type(IndexAliasConstants.PASSPORT_TYPE);
indexRequest.source("seqNo", o[2]);
UpdateQuery updateQuery = new UpdateQueryBuilder().withClass(AccessionDetails.class)
.withIndexName(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE).withType(IndexAliasConstants.PASSPORT_TYPE)
.withId(o[0].toString())
.withIndexRequest(indexRequest).build();
UpdateQuery updateQuery = new UpdateQueryBuilder().withClass(AccessionDetails.class).withIndexName(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE)
.withType(IndexAliasConstants.PASSPORT_TYPE).withId(o[0].toString()).withIndexRequest(indexRequest).build();
queries.add(updateQuery);
// LOG.debug("ES added seqNo to " + o[0].toString());
......
......@@ -22,6 +22,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.client.Client;
......@@ -37,6 +39,7 @@ import org.genesys2.server.service.ElasticSearchManagementService;
import org.genesys2.server.service.IndexAliasConstants;
import org.genesys2.server.service.impl.FilterHandler;
import org.genesys2.server.service.worker.ElasticUpdater;
import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
......@@ -46,6 +49,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.hazelcast.core.IQueue;
@Controller
@RequestMapping("/admin/elastic")
......@@ -59,6 +63,12 @@ public class ElasticSearchController {
@Autowired
private ElasticSearchManagementService elasticSearchManagementService;
@Resource
private IQueue<ElasticNode> elasticRemoveQueue;
@Resource
private IQueue<ElasticNode> elasticUpdateQueue;
@Autowired
private Client client;
......@@ -89,6 +99,9 @@ public class ElasticSearchController {
model.addAttribute("indexes", indexMap);
model.addAttribute("reindexTypes", reindexTypes);
model.addAttribute("removeQueueSize", elasticRemoveQueue.size());
model.addAttribute("updateQueueSize", elasticUpdateQueue.size());
return "/admin/elastic/index";
}
......
......@@ -8,6 +8,10 @@
</head>
<body>
<%@ include file="/WEB-INF/jsp/admin/menu.jsp" %>
<h3>Queues</h3>
<div>Remove queue size: <c:out value="${removeQueueSize}" /></div>
<div>Update queue size: <c:out value="${updateQueueSize}" /></div>
<h3>Accession data</h3>
<form method="post" action="<c:url value="/admin/elastic/action" />">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment