Load data in batches for MCPD export

src/main/java/org/genesys2/server/persistence/domain/AccessionAliasRepository.java

@@ -38,4 +38,7 @@ public interface AccessionAliasRepository extends JpaRepository<AccessionAlias,
 
 	@Query("select distinct aa.accession from AccessionAlias aa where aa.usedBy=?1 and aa.name=?2 and aa.aliasType=?3 and aa.accession.institute.code=?1")
 	Accession findAccession(String instCode, String name, int aliasType);
+
+	@Query("from AccessionAlias aa where aa.accession.id in ?1")
+	List<AccessionAlias> findAllFor(List<Long> accessionIds);
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionBreedingRepository.java

@@ -17,6 +17,7 @@
 package org.genesys2.server.persistence.domain;
 
 import java.util.Collection;
+import java.util.List;
 
 import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.model.genesys.AccessionBreeding;
@@ -31,4 +32,7 @@ public interface AccessionBreedingRepository extends JpaRepository<AccessionBree
 	@Modifying
 	@Query("delete from AccessionBreeding ab where ab.accession.id in ?1")
 	void deleteForAccessions(Collection<Long> accessionIds);
+
+	@Query("from AccessionBreeding ab where ab.accession.id in ?1")
+	List<AccessionBreeding> findAllFor(List<Long> accessionIds);
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionCollectRepository.java

@@ -17,6 +17,7 @@
 package org.genesys2.server.persistence.domain;
 
 import java.util.Collection;
+import java.util.List;
 
 import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.model.genesys.AccessionCollect;
@@ -31,4 +32,7 @@ public interface AccessionCollectRepository extends JpaRepository<AccessionColle
 	@Modifying
 	@Query("delete from AccessionCollect ac where ac.accession.id in ?1")
 	void deleteForAccessions(Collection<Long> accessionIds);
+
+	@Query("from AccessionCollect ac where ac.accession.id in ?1")
+	List<AccessionCollect> findAllFor(List<Long> accessionId);
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionExchangeRepository.java

@@ -17,6 +17,7 @@
 package org.genesys2.server.persistence.domain;
 
 import java.util.Collection;
+import java.util.List;
 
 import org.genesys2.server.model.genesys.Accession;
 import org.genesys2.server.model.genesys.AccessionExchange;
@@ -31,4 +32,7 @@ public interface AccessionExchangeRepository extends JpaRepository<AccessionExch
 	@Modifying
 	@Query("delete from AccessionExchange ae where ae.accession.id in ?1")
 	void deleteForAccessions(Collection<Long> accessionIds);
+
+	@Query("from AccessionExchange ae where ae.accession.id in ?1")
+	List<AccessionExchange> findAllFor(List<Long> accessionIds);
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java

@@ -37,4 +37,7 @@ public interface AccessionGeoRepository extends JpaRepository<AccessionGeo, Long
 	@Query("delete from AccessionGeo ag where ag.accession.id in ?1")
 	void deleteForAccessions(Collection<Long> accessionIds);
 
+	@Query("from AccessionGeo ag where ag.accession.id in ?1")
+	List<AccessionGeo> findAllFor(List<Long> accessionIds);
+
 }

src/main/java/org/genesys2/server/persistence/domain/AccessionRemarkRepository.java

@@ -36,4 +36,7 @@ public interface AccessionRemarkRepository extends JpaRepository<AccessionRemark
 	@Modifying
 	@Query("delete from AccessionRemark ar where ar.accession in (from Accession a where a.id in ( :ids ))")
 	void deleteForAccessions(@Param("ids") Collection<Long> accessionIds);
+
+	@Query("from AccessionRemark ar where ar.accession.id in ?1")
+	List<AccessionRemark> findAllFor(List<Long> accessionIds);
 }

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -103,7 +103,7 @@ public interface GenesysService {
 
 	Page<Object[]> statisticsGenusByInstitute(FaoInstitute faoInstitute, Pageable pageable);
 
-    Page<Object[]> statisticsSpeciesByInstitute(FaoInstitute faoInstitute, Pageable pageable);
+	Page<Object[]> statisticsSpeciesByInstitute(FaoInstitute faoInstitute, Pageable pageable);
 
 	// Page<Object[]> statisticsCropByInstitute(FaoInstitute faoInstitute,
 	// Pageable pageable);
@@ -204,4 +204,20 @@ public interface GenesysService {
 
 	List<Long> listAccessionsIds(Taxonomy2 taxonomy);
 
+	public static class AllStuff {
+		public AllStuff(Long id) {
+			this.id = id;
+		}
+
+		public Long id;
+		public Accession accession = null;
+		public AccessionGeo geo = null;
+		public AccessionCollect collect = null;
+		public AccessionBreeding bred = null;
+		public List<AccessionAlias> names = null;
+		public AccessionExchange exch = null;
+		public List<AccessionRemark> remarks = null;
+	}
+
+	List<AllStuff> loadAllStuff(List<Long> accessionIds);
 }

src/main/java/org/genesys2/server/service/impl/DownloadServiceImpl.java

@@ -20,7 +20,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
@@ -49,6 +49,7 @@ import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
 import org.genesys2.server.persistence.domain.MethodRepository;
 import org.genesys2.server.service.DownloadService;
 import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.GenesysService.AllStuff;
 import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -91,7 +92,6 @@ public class DownloadServiceImpl implements DownloadService {
 		dateStyle.setAlignment(CellStyle.ALIGN_LEFT);
 
 		Sheet legal = wb.getSheet("Legal information");
-		System.err.println("Rows: " + legal.getLastRowNum());
 		Row r;
 		Cell c;
 		r = legal.createRow(0);
@@ -117,65 +117,23 @@ public class DownloadServiceImpl implements DownloadService {
 
 		final Sheet sheet = wb.getSheet("MCPD");
 
+		final ArrayList<Long> batch = new ArrayList<Long>(100);
+		final Writer writer=new Writer();
+		
 		// Write accession information
 		genesysLowlevelRepository.listAccessionIds(filters, new Sort("acceNumb"), new RowCallbackHandler() {
-			int i = 0;
-
 			@Override
 			public void processRow(ResultSet rs) throws SQLException {
-				if (i > XLSX_ROWS_LIMIT) {
-					LOG.info("Row limit exceeded");
-					return;
-				}
-
 				long accessionId = rs.getLong(1);
-				Accession accession = entityManager.find(Accession.class, accessionId);
-				Row row = sheet.createRow(i + 1);
-
-				AccessionGeo geo = genesysService.listAccessionGeo(accession);
-				AccessionCollect collect = genesysService.listAccessionCollect(accession);
-				AccessionBreeding bred = genesysService.listAccessionBreeding(accession);
-				List<AccessionAlias> names = genesysService.listAccessionAliases(accession);
-				AccessionExchange exch = genesysService.listAccessionExchange(accession);
-				List<AccessionRemark> remarks = genesysService.listAccessionRemarks(accession);
-
-				writeMCPDRow(sheet, row, accession, geo, collect, bred, names, exch, remarks);
-
-				detach(accession);
-				detach(geo);
-				detach(collect);
-				detach(bred);
-				detach(names);
-				detach(exch);
-				detach(remarks);
-
-				i++;
-
-				if (i % 100 == 0 && LOG.isDebugEnabled()) {
-					LOG.debug("Writing MCPD row " + i);
-				}
-
-				if (i % 1000 == 0) {
-					try {
-						((SXSSFSheet) sheet).flushRows();
-					} catch (IOException e) {
-						LOG.error(e.getMessage(), e);
-					}
-				}
-
-			}
 
-			private void detach(Object obj) {
-				if (obj != null) {
-					if (obj instanceof Collection) {
-						for (Object o : (Collection<?>) obj)
-							detach(o);
-					} else {
-						entityManager.detach(obj);
-					}
+				if (batch.size() == 500) {
+					writer.processBatch(sheet, batch);
+					batch.clear();
 				}
+				batch.add(accessionId);
 			}
 		});
+		writer.processBatch(sheet, batch);
 
 		((SXSSFSheet) sheet).flushRows();
 
@@ -185,6 +143,33 @@ public class DownloadServiceImpl implements DownloadService {
 		LOG.info("Done");
 	}
 
+	private class Writer {
+		int i = 0;
+
+		private void processBatch(Sheet sheet, List<Long> batch) {
+			List<AllStuff> alls = genesysService.loadAllStuff(batch);
+
+			for (AllStuff all : alls) {
+				if (i > XLSX_ROWS_LIMIT) {
+					LOG.info("Row limit exceeded");
+					return;
+				}
+
+				Row row = sheet.createRow(i + 1);
+				writeMCPDRow(sheet, row, all.accession, all.geo, all.collect, all.bred, all.names, all.exch, all.remarks);
+				i++;
+
+			}
+
+			LOG.info("Writing MCPD row " + i);
+			try {
+				((SXSSFSheet) sheet).flushRows();
+			} catch (IOException e) {
+				LOG.error(e.getMessage(), e);
+			}
+		}
+	}
+
 	private void writeMCPDRow(Sheet sheet, Row row, Accession accession, AccessionGeo geo, AccessionCollect collect, AccessionBreeding bred,
 			List<AccessionAlias> names, AccessionExchange exch, List<AccessionRemark> remarks) {
 

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -33,6 +34,8 @@ import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
+import javax.persistence.EntityManager;
+
 import org.apache.commons.collections.ListUtils;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.Predicate;
@@ -112,6 +115,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 
 	public static final Log LOG = LogFactory.getLog(GenesysServiceImpl.class);
 
+	@Autowired
+	private EntityManager entityManager;
+
 	// Services
 	@Autowired
 	private TaxonomyService taxonomyService;
@@ -336,7 +342,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		for (Accession accn : accessions) {
 			if (accn == null)
 				continue;
-			
+
 			set.add(getAccessionDetails(accn.getId()));
 		}
 		return set;
@@ -413,6 +419,87 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		return accessionAliasRepository.findByAccession(accession);
 	}
 
+	public AllStuff loadAllStuff(Accession accession) {
+		if (accession == null)
+			return null;
+		AllStuff all = new AllStuff(1l);
+		all.names = listAccessionAliases(accession);
+		all.remarks = listAccessionRemarks(accession);
+		return all;
+	}
+
+	@Override
+	public List<AllStuff> loadAllStuff(List<Long> accessionIds) {
+		List<AllStuff> alls = new ArrayList<AllStuff>(accessionIds.size());
+		HashMap<Long, AllStuff> map = new HashMap<Long, AllStuff>();
+
+		for (Long accessionId : accessionIds) {
+			AllStuff all = new AllStuff(accessionId);
+			alls.add(all);
+			map.put(accessionId, all);
+		}
+		for (Accession a : accessionRepository.findAll(accessionIds)) {
+			AllStuff all = map.get(a.getId());
+			all.accession = a;
+			a.getStoRage().size();
+			all.names = new ArrayList<AccessionAlias>();
+			all.remarks = new ArrayList<AccessionRemark>();
+		}
+
+		for (AccessionCollect c : accessionCollectRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(c.getAccession().getId());
+			all.collect = c;
+		}
+
+		for (AccessionGeo g : accessionGeoRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(g.getAccession().getId());
+			all.geo = g;
+		}
+
+		for (AccessionBreeding b : accessionBreedingRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(b.getAccession().getId());
+			all.bred = b;
+		}
+
+		for (AccessionExchange e : accessionExchangeRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(e.getAccession().getId());
+			all.exch = e;
+		}
+
+		for (AccessionAlias aa : accessionAliasRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(aa.getAccession().getId());
+			all.names.add(aa);
+		}
+
+		for (AccessionRemark ar : accessionRemarkRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(ar.getAccession().getId());
+			all.remarks.add(ar);
+		}
+
+		for (AllStuff all : alls) {
+			detach(all.accession);
+			detach(all.geo);
+			detach(all.collect);
+			detach(all.bred);
+			detach(all.names);
+			detach(all.exch);
+			detach(all.remarks);
+		}
+
+		return alls;
+	}
+
+	private void detach(Object obj) {
+		if (obj != null) {
+			if (obj instanceof Collection) {
+				for (Object o : (Collection<?>) obj)
+					detach(o);
+			} else {
+				entityManager.detach(obj);
+			}
+		}
+	}
+
 	@Override
 	public List<Metadata> listMetadata(Accession accession) {
 		final List<Long> x = accessionTraitRepository.listMetadataIds(accession);

src/main/webapp/WEB-INF/jsp/selection/index.jsp

@@ -21,21 +21,26 @@
 		
 		<div class="main-col-header clearfix">
 		<div class="nav-header">
-			<form class="pull-right form-horizontal" method="post" action="<c:url value="/sel/dwca" />">
-				<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
-				<div class="row" style="margin-top: 2em;">
-					<div class="col-sm-4">
-						<button class="btn btn-default" type="submit"><spring:message code="filter.download-dwca" /></button>
+			<div class="results"><spring:message code="accessions.number" arguments="${pagedData.totalElements}" /></div>
+
+			<div class="row">
+				<div class="col-sm-12 col-md-6">
+					<div class="pagination">
+						<spring:message code="paged.pageOfPages" arguments="${pagedData.number+1},${pagedData.totalPages}" />
+						<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number eq 0 ? 1 : pagedData.number}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.previous-page" /></a>
+						<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number+2}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.next-page" /></a>
 					</div>
 				</div>
-			</form>
-
-			<div class="results"><spring:message code="accessions.number" arguments="${pagedData.totalElements}" /></div>
-			
-			<div class="pagination">
-				<spring:message code="paged.pageOfPages" arguments="${pagedData.number+1},${pagedData.totalPages}" />
-				<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number eq 0 ? 1 : pagedData.number}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.previous-page" /></a>
-				<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number+2}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.next-page" /></a>
+				<div class="col-sm-12 col-md-6" style="text-align: right; padding-top: 12px">
+					<form style="display: inline-block" method="post" action="<c:url value="/sel/dwca" />">
+						<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
+						<button class="btn btn-default" type="submit"><spring:message code="filter.download-dwca" /></button>
+					</form>
+					<form style="display: inline-block" method="post" action="<c:url value="/sel/download/mcpd" />">
+						<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
+						<button class="btn btn-default" type="submit"><spring:message code="filter.download-mcpd" /></button>
+					</form>
+				</div>
 			</div>
 		</div>
 		</div>