Commit 5627ea21 authored by Matija Obreza's avatar Matija Obreza
Browse files

Use scrollable result to reindex ES instead of paginated queries

parent 2d846a15
...@@ -46,9 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -46,9 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.Cache; import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager; import org.springframework.cache.CacheManager;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.jdbc.core.RowCallbackHandler; import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.ui.Model; import org.springframework.ui.Model;
...@@ -151,34 +148,14 @@ public class AdminController { ...@@ -151,34 +148,14 @@ public class AdminController {
AppliedFilters filters = mapper.readValue(jsonFilter, AppliedFilters.class); AppliedFilters filters = mapper.readValue(jsonFilter, AppliedFilters.class);
int page = 0, size = 100; genesysLowlevelRepository.listAccessionIds(filters, null, new RowCallbackHandler() {
List<IndexQuery> queries = new ArrayList<IndexQuery>(); @Override
do { public void processRow(ResultSet rs) throws SQLException {
LOG.info("Reindexing progress " + (page * size)); long accessionId = rs.getLong(1);
queries.clear();
Page<Accession> accessions = filterService.listAccessions(filters, new PageRequest(page, size));
if (!accessions.hasContent()) {
LOG.info("No more content");
break;
}
page++;
for (Accession a : accessions) {
elasticUpdater.update(Accession.class, a.getId());
}
if (slow) { elasticUpdater.update(Accession.class, accessionId);
try {
Thread.sleep(50);
} catch (InterruptedException e) {
break;
}
} }
});
} while (true);
LOG.info("Done."); LOG.info("Done.");
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment