Commit 0abdbf0c authored by Matija Obreza's avatar Matija Obreza
Browse files

Load data in batches for MCPD export

parent c054dac5
......@@ -38,4 +38,7 @@ public interface AccessionAliasRepository extends JpaRepository<AccessionAlias,
@Query("select distinct aa.accession from AccessionAlias aa where aa.usedBy=?1 and aa.name=?2 and aa.aliasType=?3 and aa.accession.institute.code=?1")
Accession findAccession(String instCode, String name, int aliasType);
@Query("from AccessionAlias aa where aa.accession.id in ?1")
List<AccessionAlias> findAllFor(List<Long> accessionIds);
}
......@@ -17,6 +17,7 @@
package org.genesys2.server.persistence.domain;
import java.util.Collection;
import java.util.List;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionBreeding;
......@@ -31,4 +32,7 @@ public interface AccessionBreedingRepository extends JpaRepository<AccessionBree
@Modifying
@Query("delete from AccessionBreeding ab where ab.accession.id in ?1")
void deleteForAccessions(Collection<Long> accessionIds);
@Query("from AccessionBreeding ab where ab.accession.id in ?1")
List<AccessionBreeding> findAllFor(List<Long> accessionIds);
}
......@@ -17,6 +17,7 @@
package org.genesys2.server.persistence.domain;
import java.util.Collection;
import java.util.List;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionCollect;
......@@ -31,4 +32,7 @@ public interface AccessionCollectRepository extends JpaRepository<AccessionColle
@Modifying
@Query("delete from AccessionCollect ac where ac.accession.id in ?1")
void deleteForAccessions(Collection<Long> accessionIds);
@Query("from AccessionCollect ac where ac.accession.id in ?1")
List<AccessionCollect> findAllFor(List<Long> accessionId);
}
......@@ -17,6 +17,7 @@
package org.genesys2.server.persistence.domain;
import java.util.Collection;
import java.util.List;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionExchange;
......@@ -31,4 +32,7 @@ public interface AccessionExchangeRepository extends JpaRepository<AccessionExch
@Modifying
@Query("delete from AccessionExchange ae where ae.accession.id in ?1")
void deleteForAccessions(Collection<Long> accessionIds);
@Query("from AccessionExchange ae where ae.accession.id in ?1")
List<AccessionExchange> findAllFor(List<Long> accessionIds);
}
......@@ -37,4 +37,7 @@ public interface AccessionGeoRepository extends JpaRepository<AccessionGeo, Long
@Query("delete from AccessionGeo ag where ag.accession.id in ?1")
void deleteForAccessions(Collection<Long> accessionIds);
@Query("from AccessionGeo ag where ag.accession.id in ?1")
List<AccessionGeo> findAllFor(List<Long> accessionIds);
}
......@@ -36,4 +36,7 @@ public interface AccessionRemarkRepository extends JpaRepository<AccessionRemark
@Modifying
@Query("delete from AccessionRemark ar where ar.accession in (from Accession a where a.id in ( :ids ))")
void deleteForAccessions(@Param("ids") Collection<Long> accessionIds);
@Query("from AccessionRemark ar where ar.accession.id in ?1")
List<AccessionRemark> findAllFor(List<Long> accessionIds);
}
......@@ -103,7 +103,7 @@ public interface GenesysService {
Page<Object[]> statisticsGenusByInstitute(FaoInstitute faoInstitute, Pageable pageable);
Page<Object[]> statisticsSpeciesByInstitute(FaoInstitute faoInstitute, Pageable pageable);
Page<Object[]> statisticsSpeciesByInstitute(FaoInstitute faoInstitute, Pageable pageable);
// Page<Object[]> statisticsCropByInstitute(FaoInstitute faoInstitute,
// Pageable pageable);
......@@ -204,4 +204,20 @@ public interface GenesysService {
List<Long> listAccessionsIds(Taxonomy2 taxonomy);
public static class AllStuff {
public AllStuff(Long id) {
this.id = id;
}
public Long id;
public Accession accession = null;
public AccessionGeo geo = null;
public AccessionCollect collect = null;
public AccessionBreeding bred = null;
public List<AccessionAlias> names = null;
public AccessionExchange exch = null;
public List<AccessionRemark> remarks = null;
}
List<AllStuff> loadAllStuff(List<Long> accessionIds);
}
......@@ -20,7 +20,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
......@@ -49,6 +49,7 @@ import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.MethodRepository;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GenesysService.AllStuff;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -91,7 +92,6 @@ public class DownloadServiceImpl implements DownloadService {
dateStyle.setAlignment(CellStyle.ALIGN_LEFT);
Sheet legal = wb.getSheet("Legal information");
System.err.println("Rows: " + legal.getLastRowNum());
Row r;
Cell c;
r = legal.createRow(0);
......@@ -117,65 +117,23 @@ public class DownloadServiceImpl implements DownloadService {
final Sheet sheet = wb.getSheet("MCPD");
final ArrayList<Long> batch = new ArrayList<Long>(100);
final Writer writer=new Writer();
// Write accession information
genesysLowlevelRepository.listAccessionIds(filters, new Sort("acceNumb"), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
if (i > XLSX_ROWS_LIMIT) {
LOG.info("Row limit exceeded");
return;
}
long accessionId = rs.getLong(1);
Accession accession = entityManager.find(Accession.class, accessionId);
Row row = sheet.createRow(i + 1);
AccessionGeo geo = genesysService.listAccessionGeo(accession);
AccessionCollect collect = genesysService.listAccessionCollect(accession);
AccessionBreeding bred = genesysService.listAccessionBreeding(accession);
List<AccessionAlias> names = genesysService.listAccessionAliases(accession);
AccessionExchange exch = genesysService.listAccessionExchange(accession);
List<AccessionRemark> remarks = genesysService.listAccessionRemarks(accession);
writeMCPDRow(sheet, row, accession, geo, collect, bred, names, exch, remarks);
detach(accession);
detach(geo);
detach(collect);
detach(bred);
detach(names);
detach(exch);
detach(remarks);
i++;
if (i % 100 == 0 && LOG.isDebugEnabled()) {
LOG.debug("Writing MCPD row " + i);
}
if (i % 1000 == 0) {
try {
((SXSSFSheet) sheet).flushRows();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
}
private void detach(Object obj) {
if (obj != null) {
if (obj instanceof Collection) {
for (Object o : (Collection<?>) obj)
detach(o);
} else {
entityManager.detach(obj);
}
if (batch.size() == 500) {
writer.processBatch(sheet, batch);
batch.clear();
}
batch.add(accessionId);
}
});
writer.processBatch(sheet, batch);
((SXSSFSheet) sheet).flushRows();
......@@ -185,6 +143,33 @@ public class DownloadServiceImpl implements DownloadService {
LOG.info("Done");
}
private class Writer {
int i = 0;
private void processBatch(Sheet sheet, List<Long> batch) {
List<AllStuff> alls = genesysService.loadAllStuff(batch);
for (AllStuff all : alls) {
if (i > XLSX_ROWS_LIMIT) {
LOG.info("Row limit exceeded");
return;
}
Row row = sheet.createRow(i + 1);
writeMCPDRow(sheet, row, all.accession, all.geo, all.collect, all.bred, all.names, all.exch, all.remarks);
i++;
}
LOG.info("Writing MCPD row " + i);
try {
((SXSSFSheet) sheet).flushRows();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
}
private void writeMCPDRow(Sheet sheet, Row row, Accession accession, AccessionGeo geo, AccessionCollect collect, AccessionBreeding bred,
List<AccessionAlias> names, AccessionExchange exch, List<AccessionRemark> remarks) {
......
......@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
......@@ -33,6 +34,8 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.persistence.EntityManager;
import org.apache.commons.collections.ListUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
......@@ -112,6 +115,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
public static final Log LOG = LogFactory.getLog(GenesysServiceImpl.class);
@Autowired
private EntityManager entityManager;
// Services
@Autowired
private TaxonomyService taxonomyService;
......@@ -336,7 +342,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
for (Accession accn : accessions) {
if (accn == null)
continue;
set.add(getAccessionDetails(accn.getId()));
}
return set;
......@@ -413,6 +419,87 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return accessionAliasRepository.findByAccession(accession);
}
public AllStuff loadAllStuff(Accession accession) {
if (accession == null)
return null;
AllStuff all = new AllStuff(1l);
all.names = listAccessionAliases(accession);
all.remarks = listAccessionRemarks(accession);
return all;
}
@Override
public List<AllStuff> loadAllStuff(List<Long> accessionIds) {
List<AllStuff> alls = new ArrayList<AllStuff>(accessionIds.size());
HashMap<Long, AllStuff> map = new HashMap<Long, AllStuff>();
for (Long accessionId : accessionIds) {
AllStuff all = new AllStuff(accessionId);
alls.add(all);
map.put(accessionId, all);
}
for (Accession a : accessionRepository.findAll(accessionIds)) {
AllStuff all = map.get(a.getId());
all.accession = a;
a.getStoRage().size();
all.names = new ArrayList<AccessionAlias>();
all.remarks = new ArrayList<AccessionRemark>();
}
for (AccessionCollect c : accessionCollectRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(c.getAccession().getId());
all.collect = c;
}
for (AccessionGeo g : accessionGeoRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(g.getAccession().getId());
all.geo = g;
}
for (AccessionBreeding b : accessionBreedingRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(b.getAccession().getId());
all.bred = b;
}
for (AccessionExchange e : accessionExchangeRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(e.getAccession().getId());
all.exch = e;
}
for (AccessionAlias aa : accessionAliasRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(aa.getAccession().getId());
all.names.add(aa);
}
for (AccessionRemark ar : accessionRemarkRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(ar.getAccession().getId());
all.remarks.add(ar);
}
for (AllStuff all : alls) {
detach(all.accession);
detach(all.geo);
detach(all.collect);
detach(all.bred);
detach(all.names);
detach(all.exch);
detach(all.remarks);
}
return alls;
}
private void detach(Object obj) {
if (obj != null) {
if (obj instanceof Collection) {
for (Object o : (Collection<?>) obj)
detach(o);
} else {
entityManager.detach(obj);
}
}
}
@Override
public List<Metadata> listMetadata(Accession accession) {
final List<Long> x = accessionTraitRepository.listMetadataIds(accession);
......
......@@ -21,21 +21,26 @@
<div class="main-col-header clearfix">
<div class="nav-header">
<form class="pull-right form-horizontal" method="post" action="<c:url value="/sel/dwca" />">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
<div class="row" style="margin-top: 2em;">
<div class="col-sm-4">
<button class="btn btn-default" type="submit"><spring:message code="filter.download-dwca" /></button>
<div class="results"><spring:message code="accessions.number" arguments="${pagedData.totalElements}" /></div>
<div class="row">
<div class="col-sm-12 col-md-6">
<div class="pagination">
<spring:message code="paged.pageOfPages" arguments="${pagedData.number+1},${pagedData.totalPages}" />
<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number eq 0 ? 1 : pagedData.number}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.previous-page" /></a>
<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number+2}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.next-page" /></a>
</div>
</div>
</form>
<div class="results"><spring:message code="accessions.number" arguments="${pagedData.totalElements}" /></div>
<div class="pagination">
<spring:message code="paged.pageOfPages" arguments="${pagedData.number+1},${pagedData.totalPages}" />
<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number eq 0 ? 1 : pagedData.number}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.previous-page" /></a>
<a href="<spring:url value=""><spring:param name="page" value="${pagedData.number+2}" /><spring:param name="filter" value="${jsonFilter}" /></spring:url>"><spring:message code="pagination.next-page" /></a>
<div class="col-sm-12 col-md-6" style="text-align: right; padding-top: 12px">
<form style="display: inline-block" method="post" action="<c:url value="/sel/dwca" />">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
<button class="btn btn-default" type="submit"><spring:message code="filter.download-dwca" /></button>
</form>
<form style="display: inline-block" method="post" action="<c:url value="/sel/download/mcpd" />">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
<button class="btn btn-default" type="submit"><spring:message code="filter.download-mcpd" /></button>
</form>
</div>
</div>
</div>
</div>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment