Resource conscious accession#crop update

src/main/java/org/genesys2/server/persistence/domain/AccessionRepository.java

@@ -128,8 +128,9 @@ public interface AccessionRepository extends JpaRepository<Accession, Long> {
 
 	public long countByHistoric(boolean historic);
 
-	@Query("select a from Accession a where a.cropName != null")
-	Stream<Accession> streamWithCropname();
+
+	@Query("select a.id from Accession a where a.cropName != null")
+	List<Long> listAccessionIdsWithCropname();
 
 	@Query("update Accession a set a.crop = ?2 where a = ?1")
 	@Modifying

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -268,8 +268,7 @@ public interface GenesysService {
 
 	void regenerateAccessionSequentialNumber();
 
-	Stream<Accession> streamWithCropname();
-
-	List<Accession> saveAccessionCrops(ArrayList<Accession> copy);
+	
+	List<Accession> saveAccessionCrops(ArrayList<Long> accessionIds);
 
 }

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -32,7 +32,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -775,7 +775,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		if (LOG.isDebugEnabled())
 			LOG.debug("Updating " + accession);
 
-		System.err.println("Saving " + accession + " ST=" + accession.getStorage());
+		LOG.debug("Saving " + accession + " ST=" + accession.getStorage());
 		AccessionData res = null;
 		if (accession instanceof AccessionHistoric) {
 			res = accessionHistoricRepository.save((AccessionHistoric) accession);
@@ -1790,17 +1790,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	}
 
 	
-	@Override
-	public Stream<Accession> streamWithCropname() {
-		return accessionRepository.streamWithCropname();
-	}
-
 	@Override
 	@Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
 	// @PreAuthorize("hasRole('ADMINISTRATOR')")
-	public List<Accession> saveAccessionCrops(ArrayList<Accession> accessions) {
-		for (Accession accession : accessions) 
+	public List<Accession> saveAccessionCrops(ArrayList<Long> accessionIds) {
+		List<Accession> accessions = new ArrayList<>(accessionRepository.findAll(accessionIds));
+		for (Accession accession : accessions) {
 			accessionRepository.updateCrop(accession, cropService.getCrop(accession.getCropName()));
+		}
 		return accessions;
 	}
 }

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java

@@ -19,9 +19,8 @@ import org.springframework.stereotype.Component;
 import com.hazelcast.core.IQueue;
 
 /**
- * ES Processor component uses Spring's @Scheduled annotation to scan queues
- * with 2000ms delay measured from the completion time of each preceding
- * invocation.
+ * ES Processor component uses Spring's @Scheduled annotation to scan queues with 2000ms delay measured from the
+ * completion time of each preceding invocation.
  */
 @Component
 class ElasticUpdaterProcessor {
@@ -88,8 +87,7 @@ class ElasticUpdaterProcessor {
 				}
 
 				if (LOG.isInfoEnabled() && (i % 500 == 0)) {
-					LOG.info("Queue size update=" + elasticUpdateQueue.size() + " indexed=" + i + " rate=" + (1000.0 * i / (stopWatch.getTime()))
-							+ " records/s");
+					LOG.info("Queue size update=" + elasticUpdateQueue.size() + " indexed=" + i + " rate=" + Math.round(1000.0 * i / (stopWatch.getTime())) + " records/s");
 				}
 
 			} while (toUpdate != null);

src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java

@@ -32,9 +32,11 @@ import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.StopWatch;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.persistence.domain.AccessionRepository;
 import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
 import org.genesys2.server.service.CountryNamesUpdater;
 import org.genesys2.server.service.CropService;
@@ -118,6 +120,9 @@ public class AdminController {
 
 	ObjectMapper mapper = new ObjectMapper();
 
+	@Autowired
+	private AccessionRepository accessionRepository;
+
 	@RequestMapping("/")
 	public String root(Model model) {
 		return "/admin/index";
@@ -362,8 +367,7 @@ public class AdminController {
 	}
 
 	/**
-	 * Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract
-	 * the number from the ACCNUMB)
+	 * Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract the number from the ACCNUMB)
 	 * 
 	 * @return
 	 */
@@ -387,26 +391,28 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
-	@Transactional(readOnly=true, propagation=Propagation.REQUIRES_NEW)
 	@RequestMapping(value = "/cropname-crop", method = RequestMethod.POST)
 	public String assignCropWithCropname() {
 		LOG.info("Assigning crops to accessions with CROPNAME.");
 		AtomicLong counter = new AtomicLong(0);
-		List<Accession> batch = Collections.synchronizedList(new ArrayList<>());
-		genesysService.streamWithCropname().parallel().forEach(accession -> {
-			batch.add(accession);
-			ArrayList<Accession> copy = null;
+		StopWatch stopWatch = new StopWatch();
+		stopWatch.start();
+		final List<Long> batch = Collections.synchronizedList(new ArrayList<>());
+		List<Long> list = accessionRepository.listAccessionIdsWithCropname();
+		LOG.info("The list has " + list.size() + " elements");
+		list.stream().parallel().forEach(accessionId -> {
+			batch.add(accessionId);
+			ArrayList<Long> copy = null;
 			synchronized (batch) {
 				if (batch.size() > 100) {
 					copy = new ArrayList<>(batch);
 					batch.clear();
 				}
 			}
-			if (copy != null) {
+			if (copy != null)
 				genesysService.saveAccessionCrops(copy);
-			}
 			if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) {
-				LOG.info("Updated " + counter.get() + " records");
+				LOG.info("Updated " + counter.get() + " records overall_rate=" + Math.round(1000.0 * counter.get() / (stopWatch.getTime())) + " records/s");
 			}
 		});
 		LOG.info("Done assigning crops to accessions with CROPNAME.");