From 05ab953c7eda9a3188b621b795f310902310e804 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Thu, 22 Sep 2016 08:20:17 +0200 Subject: [PATCH] Resource conscious accession#crop update --- .../domain/AccessionRepository.java | 5 ++-- .../server/service/GenesysService.java | 5 ++-- .../service/impl/GenesysServiceImpl.java | 15 +++++------ .../worker/ElasticUpdaterProcessor.java | 8 +++--- .../controller/admin/AdminController.java | 26 ++++++++++++------- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java b/src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java index c334588a6..e828d76ee 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java @@ -128,8 +128,9 @@ public interface AccessionRepository extends JpaRepository { public long countByHistoric(boolean historic); - @Query("select a from Accession a where a.cropName != null") - Stream streamWithCropname(); + + @Query("select a.id from Accession a where a.cropName != null") + List listAccessionIdsWithCropname(); @Query("update Accession a set a.crop = ?2 where a = ?1") @Modifying diff --git a/src/main/java/org/genesys2/server/service/GenesysService.java b/src/main/java/org/genesys2/server/service/GenesysService.java index 79da9f7dd..49fb3b944 100644 --- a/src/main/java/org/genesys2/server/service/GenesysService.java +++ b/src/main/java/org/genesys2/server/service/GenesysService.java @@ -268,8 +268,7 @@ public interface GenesysService { void regenerateAccessionSequentialNumber(); - Stream streamWithCropname(); - - List saveAccessionCrops(ArrayList copy); + + List saveAccessionCrops(ArrayList accessionIds); } diff --git a/src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java index db8fd9218..ff25f92c9 100644 --- a/src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.stream.Stream; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -775,7 +775,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { if (LOG.isDebugEnabled()) LOG.debug("Updating " + accession); - System.err.println("Saving " + accession + " ST=" + accession.getStorage()); + LOG.debug("Saving " + accession + " ST=" + accession.getStorage()); AccessionData res = null; if (accession instanceof AccessionHistoric) { res = accessionHistoricRepository.save((AccessionHistoric) accession); @@ -1790,17 +1790,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { } - @Override - public Stream streamWithCropname() { - return accessionRepository.streamWithCropname(); - } - @Override @Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW) // @PreAuthorize("hasRole('ADMINISTRATOR')") - public List saveAccessionCrops(ArrayList accessions) { - for (Accession accession : accessions) + public List saveAccessionCrops(ArrayList accessionIds) { + List accessions = new ArrayList<>(accessionRepository.findAll(accessionIds)); + for (Accession accession : accessions) { accessionRepository.updateCrop(accession, cropService.getCrop(accession.getCropName())); + } return accessions; } } diff --git a/src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java b/src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java index 5bcf915cf..9f29dcfde 100644 --- a/src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java +++ b/src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java @@ -19,9 +19,8 @@ import org.springframework.stereotype.Component; import com.hazelcast.core.IQueue; /** - * ES Processor component uses Spring's @Scheduled annotation to scan queues - * with 2000ms delay measured from the completion time of each preceding - * invocation. + * ES Processor component uses Spring's @Scheduled annotation to scan queues with 2000ms delay measured from the + * completion time of each preceding invocation. */ @Component class ElasticUpdaterProcessor { @@ -88,8 +87,7 @@ class ElasticUpdaterProcessor { } if (LOG.isInfoEnabled() && (i % 500 == 0)) { - LOG.info("Queue size update=" + elasticUpdateQueue.size() + " indexed=" + i + " rate=" + (1000.0 * i / (stopWatch.getTime())) - + " records/s"); + LOG.info("Queue size update=" + elasticUpdateQueue.size() + " indexed=" + i + " rate=" + Math.round(1000.0 * i / (stopWatch.getTime())) + " records/s"); } } while (toUpdate != null); diff --git a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java index 3f94f76f3..cf01c187e 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java +++ b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java @@ -32,9 +32,11 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.genesys2.server.model.genesys.Accession; +import org.genesys2.server.persistence.domain.AccessionRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepository; import org.genesys2.server.service.CountryNamesUpdater; import org.genesys2.server.service.CropService; @@ -118,6 +120,9 @@ public class AdminController { ObjectMapper mapper = new ObjectMapper(); + @Autowired + private AccessionRepository accessionRepository; + @RequestMapping("/") public String root(Model model) { return "/admin/index"; @@ -362,8 +367,7 @@ public class AdminController { } /** - * Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract - * the number from the ACCNUMB) + * Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract the number from the ACCNUMB) * * @return */ @@ -387,26 +391,28 @@ public class AdminController { return "redirect:/admin/"; } - @Transactional(readOnly=true, propagation=Propagation.REQUIRES_NEW) @RequestMapping(value = "/cropname-crop", method = RequestMethod.POST) public String assignCropWithCropname() { LOG.info("Assigning crops to accessions with CROPNAME."); AtomicLong counter = new AtomicLong(0); - List batch = Collections.synchronizedList(new ArrayList<>()); - genesysService.streamWithCropname().parallel().forEach(accession -> { - batch.add(accession); - ArrayList copy = null; + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + final List batch = Collections.synchronizedList(new ArrayList<>()); + List list = accessionRepository.listAccessionIdsWithCropname(); + LOG.info("The list has " + list.size() + " elements"); + list.stream().parallel().forEach(accessionId -> { + batch.add(accessionId); + ArrayList copy = null; synchronized (batch) { if (batch.size() > 100) { copy = new ArrayList<>(batch); batch.clear(); } } - if (copy != null) { + if (copy != null) genesysService.saveAccessionCrops(copy); - } if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) { - LOG.info("Updated " + counter.get() + " records"); + LOG.info("Updated " + counter.get() + " records overall_rate=" + Math.round(1000.0 * counter.get() / (stopWatch.getTime())) + " records/s"); } }); LOG.info("Done assigning crops to accessions with CROPNAME."); -- GitLab