Using aliases for ES indexes

src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java

@@ -28,6 +28,9 @@ import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldIndex;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
+/**
+ * <code>AccessionDetails</code> is used in Elasticsearch mapping
+ */
 @Document(indexName = "genesys", type = "mcpd", refreshInterval = "60s")
 public class AccessionDetails {
 	private static final Logger LOG = Logger.getLogger(AccessionDetails.class);

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -46,4 +46,16 @@ public interface ElasticService {
 
 	void regenerateAccessionSequentialNumber();
 
+	/**
+	 * Reindex part of accession database
+	 * 
+	 * @param filters
+	 */
+	void reindex(AppliedFilters filters);
+
+	/**
+	 * Create new indexes and fill them with data
+	 */
+	void regenerateIndexes();
+
 }

src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java

@@ -25,14 +25,10 @@ import java.util.Set;
 
 import javax.annotation.Resource;
 
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.model.elastic.AccessionDetails;
-import org.genesys2.server.model.genesys.Accession;
-import org.genesys2.server.service.GenesysService;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.domain.PageRequest;
 import org.springframework.data.elasticsearch.ElasticsearchException;
 import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
 import org.springframework.data.elasticsearch.core.query.IndexQuery;
@@ -53,9 +49,6 @@ public class ElasticUpdater {
 	@Autowired
 	private ElasticsearchTemplate elasticsearchTemplate;
 
-	@Autowired
-	private GenesysService genesysService;
-
 	@Resource
 	private IQueue<ElasticNode> elasticRemoveQueue;
 	@Resource
@@ -137,63 +130,6 @@ public class ElasticUpdater {
 		return set;
 	}
 
-	public void fullReindex() {
-		clearQueues();
-
-		try {
-			// Sleep to allow {@link ElasticUpdaterProcessor} to finish
-			Thread.sleep(1000);
-		} catch (InterruptedException e) {
-		}
-
-		// Delete index manually if you have to with: $ curl -XDELETE
-		// http://localhost:9200/genesys
-		// LOG.info("Deleting index");
-		// // This fails if an update happens in the meantime
-		// elasticsearchTemplate.deleteIndex(AccessionDetails.class);
-
-		LOG.info("Creating index");
-		elasticsearchTemplate.createIndex(AccessionDetails.class);
-		LOG.info("Putting mapping");
-		elasticsearchTemplate.putMapping(AccessionDetails.class);
-		LOG.info("Refreshing");
-		elasticsearchTemplate.refresh(AccessionDetails.class, true);
-
-		Map<?, ?> setting = elasticsearchTemplate.getSetting(AccessionDetails.class);
-		for (Object k : setting.keySet()) {
-			LOG.info("es setting " + k + " = " + setting.get(k));
-		}
-		Map<?, ?> mapping = elasticsearchTemplate.getMapping(AccessionDetails.class);
-		for (Object k : mapping.keySet()) {
-			LOG.info("es mapping " + k + " = " + mapping.get(k));
-		}
-
-		int page = 0, size = 100;
-		do {
-			if ((page * size) % 10000 == 0) {
-				LOG.info("Full reindex enqueue progress " + (page * size));
-				try {
-					// Sleep to allow {@link ElasticUpdaterProcessor} to finish
-					Thread.sleep(500);
-				} catch (InterruptedException e) {
-				}
-			}
-			List<Long> accessions = genesysService.listAccessionsIds(new PageRequest(page, size));
-			if (accessions.isEmpty()) {
-				LOG.info("No more content");
-				break;
-			}
-
-			page++;
-
-			updateAll(Accession.class, accessions.toArray(ArrayUtils.EMPTY_LONG_OBJECT_ARRAY));
-
-		} while (true);
-
-		elasticsearchTemplate.refresh(AccessionDetails.class, true);
-		LOG.info("Done.");
-	}
-
 	public void clearQueues() {
 		elasticRemoveQueue.clear();
 		elasticUpdateQueue.clear();

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java

@@ -9,7 +9,6 @@ import javax.annotation.Resource;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.service.ElasticService;
-import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.worker.ElasticUpdater.ElasticNode;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.InitializingBean;
@@ -42,21 +41,8 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
 	@Resource
 	private IQueue<ElasticNode> elasticUpdateQueue;
 
-	@Autowired
-	private GenesysService genesysService;
-
-	private long indexDelay = 5000;
-
 	private HashMap<String, Set<Long>> buckets = new HashMap<String, Set<Long>>();
 
-	public void setIndexDelay(long indexDelay) {
-		this.indexDelay = indexDelay;
-	}
-
-	public long getIndexDelay() {
-		return indexDelay;
-	}
-
 	@Override
 	public void run() {
 		LOG.info("Started.");

src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java

@@ -30,11 +30,9 @@ import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
 import org.genesys2.server.service.CountryNamesUpdater;
 import org.genesys2.server.service.ElasticService;
-import org.genesys2.server.service.GenesysFilterService;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.GeoRegionService;
 import org.genesys2.server.service.GeoService;
@@ -47,7 +45,6 @@ import org.genesys2.server.service.worker.InstituteUpdater;
 import org.genesys2.server.service.worker.SGSVUpdate;
 import org.genesys2.server.service.worker.WorldClimUpdater;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.task.TaskExecutor;
 import org.springframework.jdbc.core.RowCallbackHandler;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
@@ -72,6 +69,7 @@ public class AdminController {
 
 	@Autowired
 	ElasticUpdater elasticUpdater;
+
 	@Autowired
 	ElasticService elasticService;
 
@@ -96,18 +94,14 @@ public class AdminController {
 	@Autowired
 	private GenesysLowlevelRepository genesysLowlevelRepository;
 
-	@Autowired
-	private TaskExecutor taskExecutor;
-
-	@Autowired
-	private GenesysFilterService filterService;
-
 	@Autowired
 	private WorldClimUpdater worldClimUpdater;
 
 	@Autowired
 	private GeoRegionService geoRegionService;
 
+	ObjectMapper mapper = new ObjectMapper();
+
 	@RequestMapping("/")
 	public String root(Model model) {
 		return "/admin/index";
@@ -133,37 +127,38 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
-	@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic")
-	public String reindexElastic(@RequestParam(value = "startAt", required = false) Long startAt,
-			@RequestParam(value = "slow", required = true, defaultValue = "true") boolean slow) {
-		// LOG.info("Json filter: " + jsonFilter);
-		elasticUpdater.fullReindex();
+	/**
+	 * Completely recreate Elasticsearch indexes: create, index, re-alias.
+	 * 
+	 * @return
+	 */
+	@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "regenerate" })
+	public String regenerateElastic() {
+		elasticService.regenerateIndexes();
 		return "redirect:/admin/";
 	}
 
-	@RequestMapping(method = RequestMethod.POST, value = "/reindex-elastic", params = { "filter" })
-	public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter,
-			@RequestParam(value = "slow", required = false, defaultValue = "true") boolean slow) throws IOException {
-
-		ObjectMapper mapper = new ObjectMapper();
+	/**
+	 * This method refreshes data in the currently active index. It is very
+	 * handy when having to refresh part of ES after direct database update.
+	 * 
+	 * @param jsonFilter
+	 * @throws IOException
+	 */
+	@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "reindex", "filter" })
+	public String reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter) throws IOException {
 
 		AppliedFilters filters = mapper.readValue(jsonFilter, AppliedFilters.class);
-
-		genesysLowlevelRepository.listAccessionIds(filters, null, new RowCallbackHandler() {
-			@Override
-			public void processRow(ResultSet rs) throws SQLException {
-				long accessionId = rs.getLong(1);
-
-				elasticUpdater.update(Accession.class, accessionId);
-			}
-		});
-
-		LOG.info("Done.");
-
+		elasticService.reindex(filters);
 		return "redirect:/admin/";
 	}
 
-	@RequestMapping(method = RequestMethod.POST, value = "/clear-queues")
+	/**
+	 * Clear ES queue
+	 * 
+	 * @return
+	 */
+	@RequestMapping(method = RequestMethod.POST, value = "/elastic", params = { "clear-queues" })
 	public String clearElasticQueues() {
 		elasticUpdater.clearQueues();
 		return "redirect:/admin/";

src/main/java/org/genesys2/spring/config/HazelcastConfig.java

@@ -66,6 +66,13 @@ public abstract class HazelcastConfig {
 		return lock;
 	}
 
+	@Bean
+	public ILock elasticsearchAdminLock(HazelcastInstance hazelcast) {
+		ILock lock = hazelcast.getLock("elasticsearchAdminLock");
+		return lock;
+	}
+
+
 	/**
 	 * Loads an instance and sets the static field
 	 * @param hazelcastCacheRegionFactory 

src/main/resources/log4j.properties

@@ -25,9 +25,9 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %t %5p %c{1}:%L - %m
 ### set log levels - for more verbose logging change 'info' to 'debug' ###
 log4j.rootLogger=warn, stdout
 
-#log4j.category.org.genesys2=info
+log4j.category.org.genesys2.server.service=debug
 #log4j.category.org.genesys2.server.servlet.controller=debug
-log4j.category.org.hibernate.cfg.Configuration=debug
+#log4j.category.org.hibernate.cfg.Configuration=debug
 #log4j.category.org.hibernate.search=debug
 #log4j.category.org.apache.tomcat.jdbc.pool=debug
 #log4j.category.org.springframework.jdbc.core.JdbcTemplate=debug
@@ -35,7 +35,7 @@ log4j.category.org.hibernate.cfg.Configuration=debug
 #log4j.category.org.springframework.security.access=trace
 #log4j.category.org.springframework.security.acl=trace
 #log4j.category.org.springframework.expression=trace
-log4j.category.com.hazelcast=info
+#log4j.category.com.hazelcast=info
 #log4j.category.com.hazelcast.web=debug
 #log4j.category.com.hazelcast.aws=trace
 #log4j.category.org.eclipse.jetty.servlets=trace

src/main/resources/spring/spring.properties

@@ -24,7 +24,7 @@ base.cookie-http-only=false
 # robots.txt
 robots.allow=false
 
-db.url=jdbc:mysql://localhost/genesys20150630?useUnicode=true&characterEncoding=UTF-8&useFastDateParsing=false
+db.url=jdbc:mysql://localhost/genesys20160331?useUnicode=true&characterEncoding=UTF-8&useFastDateParsing=false
 db.driverClassName=com.mysql.jdbc.Driver
 db.username=root
 db.password=

src/main/webapp/WEB-INF/jsp/admin/index.jsp

@@ -35,19 +35,16 @@
 
 
   <h3>Full-text Search</h3>
-  <form method="post" action="<c:url value="/admin/reindex-elastic" />">
-    <input type="text" name="startAt" disabled="disabled" /> <input type="text" name="filter" /> <label> <input
-      type="checkbox" name="slow" value="false"
-    /> No sleep
-    </label> <input type="submit" class="btn btn-default" value="ES reindex" />
-    <!-- CSRF protection -->
+  <form method="post" action="<c:url value="/admin/elastic" />">
     <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
-  </form>
 
-  <form method="post" action="<c:url value="/admin/clear-queues" />">
-    <input type="submit" class="btn btn-default" value="Clear ES update queues" />
-    <!-- CSRF protection -->
+		<input type="text" name="filter" placeholder="Genesys filter {}" value="{}" />
+    <button type="submit" class="btn btn-default" name="reindex">Reindex</button>
+    <button type="submit" class="btn btn-default" name="regenerate">Regenerate</button>
+  </form>
+  <form method="post" action="<c:url value="/admin/elastic" />">
     <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
+    <button type="submit" class="btn btn-default" name="clear-queues">Clear ES update queues</button>
   </form>
 
   <h3>Country data</h3>
@@ -123,13 +120,6 @@
     <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
   </form>
 
-  <h3>Full-text Search</h3>
-  <form method="post" action="<c:url value="/admin/reindexEverything" />">
-    <input type="submit" class="btn btn-default" class="btn btn-default" value="Reindex search indexes" />
-    <!-- CSRF protection -->
-    <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
-  </form>
-
   <form method="post" action="<c:url value="/admin/reindexEntity" />">
     <select name="entity">
       <option value="org.genesys2.server.model.impl.Country">Countries</option>