Commit 9d0edef9 authored by Matija Obreza's avatar Matija Obreza

Using aliases for ES indexes

parent 90f566ea
......@@ -28,6 +28,9 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
/**
* <code>AccessionDetails</code> is used in Elasticsearch mapping
*/
@Document(indexName = "genesys", type = "mcpd", refreshInterval = "60s")
public class AccessionDetails {
private static final Logger LOG = Logger.getLogger(AccessionDetails.class);
......
......@@ -46,4 +46,16 @@ public interface ElasticService {
void regenerateAccessionSequentialNumber();
/**
* Reindex part of accession database
*
* @param filters
*/
void reindex(AppliedFilters filters);
/**
* Create new indexes and fill them with data
*/
void regenerateIndexes();
}
......@@ -25,14 +25,10 @@ import java.util.Set;
import javax.annotation.Resource;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.service.GenesysService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
......@@ -53,9 +49,6 @@ public class ElasticUpdater {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Autowired
private GenesysService genesysService;
@Resource
private IQueue<ElasticNode> elasticRemoveQueue;
@Resource
......@@ -137,63 +130,6 @@ public class ElasticUpdater {
return set;
}
public void fullReindex() {
clearQueues();
try {
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread.sleep(1000);
} catch (InterruptedException e) {
}
// Delete index manually if you have to with: $ curl -XDELETE
// http://localhost:9200/genesys
// LOG.info("Deleting index");
// // This fails if an update happens in the meantime
// elasticsearchTemplate.deleteIndex(AccessionDetails.class);
LOG.info("Creating index");
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
elasticsearchTemplate.putMapping(AccessionDetails.class);
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
Map<?, ?> setting = elasticsearchTemplate.getSetting(AccessionDetails.class);
for (Object k : setting.keySet()) {
LOG.info("es setting " + k + " = " + setting.get(k));
}
Map<?, ?> mapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
for (Object k : mapping.keySet()) {
LOG.info("es mapping " + k + " = " + mapping.get(k));
}
int page = 0, size = 100;
do {
if ((page * size) % 10000 == 0) {
LOG.info("Full reindex enqueue progress " + (page * size));
try {
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
List<Long> accessions = genesysService.listAccessionsIds(new PageRequest(page, size));
if (accessions.isEmpty()) {
LOG.info("No more content");
break;
}
page++;
updateAll(Accession.class, accessions.toArray(ArrayUtils.EMPTY_LONG_OBJECT_ARRAY));
} while (true);
elasticsearchTemplate.refresh(AccessionDetails.class, true);
LOG.info("Done.");
}
public void clearQueues() {
elasticRemoveQueue.clear();
elasticUpdateQueue.clear();
......
......@@ -9,7 +9,6 @@ import javax.annotation.Resource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
......@@ -42,21 +41,8 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
@Resource
private IQueue<ElasticNode> elasticUpdateQueue;
@Autowired
private GenesysService genesysService;
private long indexDelay = 5000;
private HashMap<String, Set<Long>> buckets = new HashMap<String, Set<Long>>();
public void setIndexDelay(long indexDelay) {
this.indexDelay = indexDelay;
}
public long getIndexDelay() {
return indexDelay;
}
@Override
public void run() {
LOG.info("Started.");
......
......@@ -30,11 +30,9 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
......@@ -47,7 +45,6 @@ import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
import org.genesys2.server.service.worker.WorldClimUpdater;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
......@@ -72,6 +69,7 @@ public class AdminController {
@Autowired
ElasticUpdater elasticUpdater;
@Autowired
ElasticService elasticService;
......@@ -96,18 +94,14 @@ public class AdminController {
@Autowired
private GenesysLowlevelRepository genesysLowlevelRepository;
@Autowired
private TaskExecutor taskExecutor;
@Autowired
private GenesysFilterService filterService;
@Autowired
private WorldClimUpdater worldClimUpdater;
@Autowired
private GeoRegionService geoRegionService;
ObjectMapper mapper = new ObjectMapper();
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -133,37 +127,38 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic")
public String reindexElastic(@RequestParam(value = "startAt", required = false) Long startAt,
@RequestParam(value = "slow", required = true, defaultValue = "true") boolean slow) {
// LOG.info("Json filter: " + jsonFilter);
elasticUpdater.fullReindex();
/**
* Completely recreate Elasticsearch indexes: create, index, re-alias.
*
* @return
*/
@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "regenerate" })
public String regenerateElastic() {
elasticService.regenerateIndexes();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic", params = { "filter" })
public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter,
@RequestParam(value = "slow", required = false, defaultValue = "true") boolean slow) throws IOException {
ObjectMapper mapper = new ObjectMapper();
/**
* This method refreshes data in the currently active index. It is very
* handy when having to refresh part of ES after direct database update.
*
* @param jsonFilter
* @throws IOException
*/
@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "reindex", "filter" })
public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter) throws IOException {
AppliedFilters filters = mapper.readValue(jsonFilter, AppliedFilters.class);
genesysLowlevelRepository.listAccessionIds(filters, null, new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
long accessionId = rs.getLong(1);
elasticUpdater.update(Accession.class, accessionId);
}
});
LOG.info("Done.");
elasticService.reindex(filters);
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/clear-queues")
/**
* Clear ES queue
*
* @return
*/
@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "clear-queues" })
public String clearElasticQueues() {
elasticUpdater.clearQueues();
return "redirect:/admin/";
......
......@@ -66,6 +66,13 @@ public abstract class HazelcastConfig {
return lock;
}
@Bean
public ILock elasticsearchAdminLock(HazelcastInstance hazelcast) {
ILock lock = hazelcast.getLock("elasticsearchAdminLock");
return lock;
}
/**
* Loads an instance and sets the static field
* @param hazelcastCacheRegionFactory
......
......@@ -25,9 +25,9 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %t %5p %c{1}:%L - %m
### set log levels - for more verbose logging change 'info' to 'debug' ###
log4j.rootLogger=warn, stdout
#log4j.category.org.genesys2=info
log4j.category.org.genesys2.server.service=debug
#log4j.category.org.genesys2.server.servlet.controller=debug
log4j.category.org.hibernate.cfg.Configuration=debug
#log4j.category.org.hibernate.cfg.Configuration=debug
#log4j.category.org.hibernate.search=debug
#log4j.category.org.apache.tomcat.jdbc.pool=debug
#log4j.category.org.springframework.jdbc.core.JdbcTemplate=debug
......@@ -35,7 +35,7 @@ log4j.category.org.hibernate.cfg.Configuration=debug
#log4j.category.org.springframework.security.access=trace
#log4j.category.org.springframework.security.acl=trace
#log4j.category.org.springframework.expression=trace
log4j.category.com.hazelcast=info
#log4j.category.com.hazelcast=info
#log4j.category.com.hazelcast.web=debug
#log4j.category.com.hazelcast.aws=trace
#log4j.category.org.eclipse.jetty.servlets=trace
......
......@@ -24,7 +24,7 @@ base.cookie-http-only=false
# robots.txt
robots.allow=false
db.url=jdbc:mysql://localhost/genesys20150630?useUnicode=true&characterEncoding=UTF-8&useFastDateParsing=false
db.url=jdbc:mysql://localhost/genesys20160331?useUnicode=true&characterEncoding=UTF-8&useFastDateParsing=false
db.driverClassName=com.mysql.jdbc.Driver
db.username=root
db.password=
......
......@@ -35,19 +35,16 @@
<h3>Full-text Search</h3>
<form method="post" action="<c:url value="/admin/reindex-elastic" />">
<input type="text" name="startAt" disabled="disabled" /> <input type="text" name="filter" /> <label> <input
type="checkbox" name="slow" value="false"
/> No sleep
</label> <input type="submit" class="btn btn-default" value="ES reindex" />
<!-- CSRF protection -->
<form method="post" action="<c:url value="/admin/elastic" />">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/clear-queues" />">
<input type="submit" class="btn btn-default" value="Clear ES update queues" />
<!-- CSRF protection -->
<input type="text" name="filter" placeholder="Genesys filter {}" value="{}" />
<button type="submit" class="btn btn-default" name="reindex">Reindex</button>
<button type="submit" class="btn btn-default" name="regenerate">Regenerate</button>
</form>
<form method="post" action="<c:url value="/admin/elastic" />">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
<button type="submit" class="btn btn-default" name="clear-queues">Clear ES update queues</button>
</form>
<h3>Country data</h3>
......@@ -123,13 +120,6 @@
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>Full-text Search</h3>
<form method="post" action="<c:url value="/admin/reindexEverything" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Reindex search indexes" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/reindexEntity" />">
<select name="entity">
<option value="org.genesys2.server.model.impl.Country">Countries</option>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment