Skip to content

GitLab

Genesys Backend
Genesys Backend
Commit 9d0edef9 authored by Matija Obreza's avatar Matija Obreza
Browse files
Options

Using aliases for ES indexes

parent 90f566ea
Hide whitespace changes
Inline Side-by-side
Showing with 239 additions and 168 deletions
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
View file @ 9d0edef9
......@@ -28,6 +28,9 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
/**
* <code>AccessionDetails</code> is used in Elasticsearch mapping
*/
@Document(indexName = "genesys", type = "mcpd", refreshInterval = "60s")
public class AccessionDetails {
private static final Logger LOG = Logger.getLogger(AccessionDetails.class);
......
src/main/java/org/genesys2/server/service/ElasticService.java
View file @ 9d0edef9
......@@ -46,4 +46,16 @@ public interface ElasticService {
void regenerateAccessionSequentialNumber();
/**
* Reindex part of accession database
*
* @param filters
*/
void reindex(AppliedFilters filters);
/**
* Create new indexes and fill them with data
*/
void regenerateIndexes();
}
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @ 9d0edef9
......@@ -9,10 +9,16 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Resource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder;
......@@ -27,9 +33,7 @@ import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilter;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.impl.FilterHandler.FilterValue;
......@@ -38,6 +42,7 @@ import org.genesys2.server.service.impl.FilterHandler.MaxValueFilter;
import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.genesys2.server.service.worker.ElasticUpdater;
import org.genesys2.util.NumberUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -45,11 +50,13 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.FacetedPage;
import org.springframework.data.elasticsearch.core.facet.FacetRequest;
import org.springframework.data.elasticsearch.core.facet.request.TermFacetRequestBuilder;
import org.springframework.data.elasticsearch.core.facet.result.TermResult;
import org.springframework.data.elasticsearch.core.query.AliasQuery;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
......@@ -58,29 +65,36 @@ import org.springframework.data.elasticsearch.core.query.UpdateQueryBuilder;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.hazelcast.core.ILock;
@Service
public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
private static final Log LOG = LogFactory.getLog(ElasticsearchSearchServiceImpl.class);
private static final String INDEXALIAS_PASSPORT_READ = "passport";
private static final String INDEXALIAS_PASSPORT_WRITE = "passportWrite";
private static final String INDEXALIAS_ARCHIVE_READ = "archive";
private static final String INDEXALIAS_ARCHIVE_WRITE = "archiveWrite";
@Autowired
private Client client;
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Autowired
private GenesysService genesysService;
// @Autowired
// private OrganizationService organizationService;
// @Autowired
// private GenesysFilterService filterService;
@Autowired
private FilterHandler filterHandler;
// @Autowired
// private ObjectMapper objectMapper;
@Autowired
private ElasticUpdater elasticUpdater;
@Resource
private ILock elasticsearchAdminLock;
private final Map<String, Class<?>> clazzMap;
......@@ -95,7 +109,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public Page<AccessionDetails> search(String query, Pageable pageable) throws SearchException {
SearchQuery searchQuery = new NativeSearchQueryBuilder()
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes("mcpd")
.withQuery(org.elasticsearch.index.query.QueryBuilders.queryString(query).defaultOperator(Operator.AND)).withPageable(pageable).build();
try {
......@@ -108,7 +122,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public List<String> autocompleteSearch(String query) throws SearchException {
SearchQuery searchQuery = new NativeSearchQueryBuilder()
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes("mcpd")
.withQuery(org.elasticsearch.index.query.QueryBuilders.queryString("acceNumb:(" + query + "*)").defaultOperator(Operator.AND))
.withSort(SortBuilders.fieldSort(FilterConstants.ACCENUMB).order(SortOrder.ASC)).withPageable(new PageRequest(0, 10)).build();
......@@ -130,7 +144,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
AndFilterBuilder filterBuilder = getFilterBuilder(appliedFilters);
SortBuilder sortBuilder = SortBuilders.fieldSort(FilterConstants.ACCENUMB).order(SortOrder.ASC);
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes("mcpd")
.withFilter(filterBuilder).withSort(sortBuilder).withPageable(pageable).build();
try {
Page<AccessionDetails> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
......@@ -153,7 +168,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} else {
termFacetRequest = new TermFacetRequestBuilder("f").applyQueryFilter().fields(term).size(size).build();
}
SearchQuery searchQuery = new NativeSearchQueryBuilder().withFilter(filterBuilder).withFacet(termFacetRequest).build();
SearchQuery searchQuery = new NativeSearchQueryBuilder().withIndices(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ).withTypes("mcpd")
.withFilter(filterBuilder).withFacet(termFacetRequest).build();
try {
FacetedPage<AccessionDetails> page = elasticsearchTemplate.queryForPage(searchQuery, AccessionDetails.class);
......@@ -315,11 +331,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
return;
}
// TODO Isn't that handled above?
if (!Accession.class.getName().equals(className)) {
LOG.warn("Unsupported class " + className);
return;
}
if (ids.isEmpty()) {
LOG.info("Skipping empty updateAll.");
return;
......@@ -331,14 +342,14 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
List<IndexQuery> queries = new ArrayList<IndexQuery>();
List<AccessionDetails> ads = genesysService.getAccessionDetails(ids);
// If Accession, update PDCI
genesysService.updatePDCI(ads);
for (AccessionDetails ad : ads) {
if (ad == null)
continue; // Skip null id
IndexQuery iq = new IndexQuery();
iq.setIndexName(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE);
iq.setType("mcpd");
iq.setId(String.valueOf(ad.getId()));
iq.setObject(ad);
......@@ -359,7 +370,13 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
return;
}
LOG.info("Removing from index " + clazz2 + " id=" + id);
elasticsearchTemplate.delete(clazz2, String.valueOf(id));
if (clazz2.equals(AccessionDetails.class)) {
elasticsearchTemplate.delete(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE, "mcpd", String.valueOf(id));
} else {
// Default
elasticsearchTemplate.delete(clazz2, String.valueOf(id));
}
}
@Override
......@@ -382,7 +399,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate.putMapping(AccessionDetails.class);
} catch (Throwable e) {
LOG.error("Mapping mismatch. Need to reindex.");
reindexExistingData(AccessionDetails.class);
}
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
......@@ -396,21 +412,129 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG.info("Copying mapping to genesysarchive");
elasticsearchTemplate.putMapping("genesysarchive", "mcpd", indexMapping);
}
if (!aliasExists(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ)) {
realias(INDEXALIAS_PASSPORT_READ, "genesys");
}
if (!aliasExists(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE)) {
realias(INDEXALIAS_PASSPORT_WRITE, "genesys");
}
if (!aliasExists(INDEXALIAS_ARCHIVE_READ)) {
realias(INDEXALIAS_ARCHIVE_READ, "genesysarchive");
}
if (!aliasExists(INDEXALIAS_ARCHIVE_WRITE)) {
realias(INDEXALIAS_ARCHIVE_WRITE, "genesysarchive");
}
}
/**
* Check if ES alias exists and points to some index
*
* @param aliasName
* @return true if alias exists
*/
private boolean aliasExists(String aliasName) {
if (LOG.isDebugEnabled()) {
LOG.debug("Checking for ES alias " + aliasName + " on client=" + client);
}
ImmutableOpenMap<String, AliasMetaData> x = client.admin().cluster().prepareState().execute().actionGet().getState().getMetaData().getAliases()
.get(aliasName);
if (x == null) {
LOG.debug("Alias does not exist");
return false;
} else {
LOG.debug("Got something back " + x);
return x.keysIt().hasNext();
}
}
/**
* Create a new index based on current timestamp, reindex it, change alias
* to point to index
*/
@Override
public void regenerateIndexes() throws ElasticsearchException {
try {
if (elasticsearchAdminLock.tryLock(10, TimeUnit.SECONDS)) {
long time = System.currentTimeMillis();
String passportIndexName = ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ + time;
// String archiveIndexName = "archive" + time;
Map<?, ?> indexMapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
Map<?, ?> settings = elasticsearchTemplate.getSetting(AccessionDetails.class);
if (elasticsearchTemplate.indexExists(passportIndexName)) {
throw new ElasticsearchException("Index already exists with name " + passportIndexName);
}
// if (elasticsearchTemplate.indexExists(archiveIndexName)) {
// throw new ElasticsearchException("Index already exists with
// name " + passportIndexName);
// }
createIndex(passportIndexName, indexMapping, settings);
realias(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE, passportIndexName);
reindex(new AppliedFilters());
realias(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_READ, passportIndexName);
// createIndex(archiveIndexName, indexMapping, settings);
// realias(ElasticsearchSearchServiceImpl.INDEXALIAS_ARCHIVE_WRITE,
// passportIndexName);
// reindexArchive();
// realias(INDEXALIAS_ARCHIVE_READ, archiveIndexName);
} else {
throw new ElasticsearchException("Could not acquire elasticsearchAdminLock lock");
}
} catch (InterruptedException e) {
} finally {
elasticsearchAdminLock.unlock();
}
}
// TODO FIXME Don't delete index; use aliases!
private void reindexExistingData(Class<?> clazz) {
LOG.warn("Deleting index for class=" + clazz);
elasticsearchTemplate.deleteIndex(clazz);
// Make new index
LOG.warn("Creating new index for class=" + clazz);
elasticsearchTemplate.createIndex(clazz);
elasticsearchTemplate.putMapping(clazz);
/**
* Make the alias point exclusively to the specified index
*
* @param aliasName
* The alias name
* @param indexName
* The index the alias points to
*/
private void realias(String aliasName, String indexName) {
if (LOG.isDebugEnabled())
LOG.debug("Loading alias definition for " + aliasName);
ImmutableOpenMap<String, AliasMetaData> x = client.admin().cluster().prepareState().execute().actionGet().getState().getMetaData().getAliases()
.get(aliasName);
if (x != null) {
final Set<String> allIndices = new HashSet<>();
x.keysIt().forEachRemaining(allIndices::add);
// TODO Scan & Scroll
// TODO Re-alias
for (String aliasIndex : allIndices) {
AliasQuery query = new AliasQuery();
query.setAliasName(aliasName);
query.setIndexName(aliasIndex);
// But... For now, don't do anything
LOG.info("Removing alias " + aliasName + " from index " + aliasIndex);
elasticsearchTemplate.removeAlias(query);
}
}
AliasQuery query = new AliasQuery();
query.setAliasName(aliasName);
query.setIndexName(indexName);
LOG.info("Adding alias " + aliasName + " to index " + indexName);
elasticsearchTemplate.addAlias(query);
}
private void createIndex(String indexName, Map<?, ?> indexMapping, Map<?, ?> settings) {
LOG.info("Creating index " + indexName);
elasticsearchTemplate.createIndex(indexName, settings);
LOG.info("Copying mapping to genesysarchive");
elasticsearchTemplate.putMapping(indexName, "mcpd", indexMapping);
}
@Override
......@@ -458,9 +582,11 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
try {
IndexRequest indexRequest = new IndexRequest();
indexRequest.index(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE).type("mcpd");
indexRequest.source("seqNo", o[2]);
UpdateQuery updateQuery = new UpdateQueryBuilder().withClass(AccessionDetails.class).withId(o[0].toString()).withIndexRequest(indexRequest)
.build();
UpdateQuery updateQuery = new UpdateQueryBuilder().withClass(AccessionDetails.class)
.withIndexName(ElasticsearchSearchServiceImpl.INDEXALIAS_PASSPORT_WRITE).withType("mcpd").withId(o[0].toString())
.withIndexRequest(indexRequest).build();
queries.add(updateQuery);
// LOG.debug("ES added seqNo to " + o[0].toString());
......@@ -482,4 +608,20 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
}
@Override
public void reindex(AppliedFilters filters) {
genesysLowlevelRepository.listAccessionIds(filters, null, new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
long accessionId = rs.getLong(1);
elasticUpdater.update(Accession.class, accessionId);
}
});
LOG.info("Done.");
}
}
src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java
View file @ 9d0edef9
......@@ -25,14 +25,10 @@ import java.util.Set;
import javax.annotation.Resource;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.GenesysService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
......@@ -53,9 +49,6 @@ public class ElasticUpdater {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Autowired
private GenesysService genesysService;
@Resource
private IQueue<ElasticNode> elasticRemoveQueue;
@Resource
......@@ -137,63 +130,6 @@ public class ElasticUpdater {
return set;
}
public void fullReindex() {
clearQueues();
try {
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread.sleep(1000);
} catch (InterruptedException e) {
}
// Delete index manually if you have to with: $ curl -XDELETE
// http://localhost:9200/genesys
// LOG.info("Deleting index");
// // This fails if an update happens in the meantime
// elasticsearchTemplate.deleteIndex(AccessionDetails.class);
LOG.info("Creating index");
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
elasticsearchTemplate.putMapping(AccessionDetails.class);
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
Map<?, ?> setting = elasticsearchTemplate.getSetting(AccessionDetails.class);
for (Object k : setting.keySet()) {
LOG.info("es setting " + k + " = " + setting.get(k));
}
Map<?, ?> mapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object k : mapping.keySet()) {
LOG.info("es mapping " + k + " = " + mapping.get(k));
}
int page = 0, size = 100;
do {
if ((page * size) % 10000 == 0) {
LOG.info("Full reindex enqueue progress " + (page * size));
try {
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
List<Long> accessions = genesysService.listAccessionsIds(new PageRequest(page, size));
if (accessions.isEmpty()) {
LOG.info("No more content");
break;
}
page++;
updateAll(Accession.class, accessions.toArray(ArrayUtils.EMPTY_LONG_OBJECT_ARRAY));
} while (true);
elasticsearchTemplate.refresh(AccessionDetails.class, true);
LOG.info("Done.");
}
public void clearQueues() {
elasticRemoveQueue.clear();
elasticUpdateQueue.clear();
......
src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java
View file @ 9d0edef9
......@@ -9,7 +9,6 @@ import javax.annotation.Resource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
......@@ -42,21 +41,8 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
@Resource
private IQueue<ElasticNode> elasticUpdateQueue;
@Autowired
private GenesysService genesysService;
private long indexDelay = 5000;
private HashMap<String, Set<Long>> buckets = new HashMap<String, Set<Long>>();
public void setIndexDelay(long indexDelay) {
this.indexDelay = indexDelay;
}
public long getIndexDelay() {
return indexDelay;
}
@Override
public void run() {
LOG.info("Started.");
......
src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java
View file @ 9d0edef9
......@@ -30,11 +30,9 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
......@@ -47,7 +45,6 @@ import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
import org.genesys2.server.service.worker.WorldClimUpdater;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
......@@ -72,6 +69,7 @@ public class AdminController {
@Autowired
ElasticUpdater elasticUpdater;
@Autowired
ElasticService elasticService;
......@@ -96,18 +94,14 @@ public class AdminController {
@Autowired
private GenesysLowlevelRepository genesysLowlevelRepository;
@Autowired
private TaskExecutor taskExecutor;
@Autowired
private GenesysFilterService filterService;
@Autowired
private WorldClimUpdater worldClimUpdater;
@Autowired
private GeoRegionService geoRegionService;
ObjectMapper mapper = new ObjectMapper();
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -133,37 +127,38 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic")
public String reindexElastic(@RequestParam(value = "startAt", required = false) Long startAt,
@RequestParam(value = "slow", required = true, defaultValue = "true") boolean slow) {
// LOG.info("Json filter: " + jsonFilter);
elasticUpdater.fullReindex();
/**
* Completely recreate Elasticsearch indexes: create, index, re-alias.
*
* @return
*/
@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "regenerate" })
public String regenerateElastic() {
elasticService.regenerateIndexes();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic", params = { "filter" })
public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter,
@RequestParam(value = "slow", required = false, defaultValue = "true") boolean slow) throws IOException {
ObjectMapper mapper = new ObjectMapper();
/**
* This method refreshes data in the currently active index. It is very
* handy when having to refresh part of ES after direct database update.
*
* @param jsonFilter
* @throws IOException
*/
@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "reindex", "filter" })
public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter) throws IOException {
AppliedFilters filters = mapper.