Download worldclim.org dataset as Excel

src/main/java/org/genesys2/server/mvc/admin/DS2Controller.java

@@ -16,7 +16,11 @@
 
 package org.genesys2.server.mvc.admin;
 
+import java.io.EOFException;
 import java.io.IOException;
+import java.io.OutputStream;
+
+import javax.servlet.http.HttpServletResponse;
 
 import org.genesys2.server.model.dataset.DS;
 import org.genesys2.server.model.dataset.DSColumn;
@@ -68,4 +72,25 @@ public class DS2Controller {
 		}
 		return "redirect:/admin/ds2/";
 	}
+	
+
+	@RequestMapping(value = "/worldclim/download", method = RequestMethod.POST)
+	public void worldclimDownload(HttpServletResponse response) throws IOException {
+
+		DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
+		
+		// Write MCPD to the stream.
+		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+		response.addHeader("Content-Disposition", String.format("attachment; filename=\"worldclim-%1s.xlsx\"", System.currentTimeMillis()));
+		// response.flushBuffer();
+
+		final OutputStream outputStream = response.getOutputStream();
+		try {
+			dsService.download(ds, outputStream);
+			response.flushBuffer();
+		} catch (EOFException e) {
+			LOG.warn("Download was aborted: {}", e.getMessage());
+		}
+	}
+
 }

src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java

@@ -18,8 +18,10 @@ import java.util.regex.Pattern;
 
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Row;
-import org.apache.poi.xssf.streaming.SXSSFSheet;
-import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.streaming.SXSSFRow;
+import org.apache.poi.xssf.streaming.SuperSXSSFSheet;
+import org.apache.poi.xssf.streaming.SuperSXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.genesys2.server.model.dataset.DS;
 import org.genesys2.server.model.dataset.DSColumn;
@@ -413,29 +415,32 @@ public class DSServiceImpl implements DSService {
 
 	@Override
 	public void download(DS ds, List<DSColumn> dsds, OutputStream outputStream) throws IOException {
-		XSSFWorkbook template = new XSSFWorkbook();
-
-		// keep 1000 rows in memory, exceeding rows will be flushed to disk
-		SXSSFWorkbook wb = new SXSSFWorkbook(template, 100);
-
-		SXSSFSheet sheet = (SXSSFSheet) wb.createSheet("worldclim.org");
-
-		int rowIndex = 0, cellIndex = 0;
-		Row r = sheet.createRow(rowIndex++);
-		r.createCell(cellIndex++).setCellValue("row_id");
-		for (DSQualifier dsq : ds.getQualifiers()) {
-			r.createCell(cellIndex++).setCellValue(dsq.getDescriptor().getCode());
-		}
-		// Prepare descriptor mapping
 		Long[] columnDescriptors = new Long[dsds.size()];
+
+		XSSFWorkbook template = new XSSFWorkbook();
 		{
-			int columnIndex = 0;
-			for (DSColumn dsd : dsds) {
-				r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode());
-				columnDescriptors[columnIndex++] = dsd.getId();
+			XSSFSheet sheet = template.createSheet("worldclim.org");
+			int rowIndex = 0, cellIndex = 0;
+			Row r = sheet.createRow(rowIndex++);
+			r.createCell(cellIndex++).setCellValue("row_id");
+			for (DSQualifier dsq : ds.getQualifiers()) {
+				r.createCell(cellIndex++).setCellValue(dsq.getDescriptor().getCode());
+			}
+			// Prepare descriptor mapping
+			{
+				int columnIndex = 0;
+				for (DSColumn dsd : dsds) {
+					r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode());
+					columnDescriptors[columnIndex++] = dsd.getId();
+				}
 			}
 		}
 
+		// keep 1000 rows in memory, exceeding rows will be flushed to disk
+		SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(template, 50);
+
+		SuperSXSSFSheet sheet = wb.getStreamingSheet("worldclim.org");
+		
 		// List<DSRow> rows = dsRowRepo.findByDataset(ds);
 		// LOG.info("Retrieved DS rows " +rows.size());
 
@@ -443,43 +448,47 @@ public class DSServiceImpl implements DSService {
 		List<Object[]> allDsrq = dsRowRepo.getQualifiers(ds);
 		LOG.info("Got {} row qualifiers", allDsrq.size());
 
-		int batchSize = 100;
-
-		for (int fromIndex = 0; fromIndex < allDsrq.size(); fromIndex += batchSize) {
-			List<Object[]> batch = allDsrq.subList(fromIndex, Math.min(fromIndex + batchSize, allDsrq.size()));
-			LOG.info("Processing position: {} of {}", fromIndex, allDsrq.size());
-
-			List<Long> rowIds = new ArrayList<Long>(batchSize);
-			for (Object[] x : batch) {
-				rowIds.add((Long) x[0]);
-			}
-			List<Object[]> allValues = dsRowRepo.getRowValues(rowIds, columnDescriptors);
-			int batchPos = 0;
-			for (Object[] x : batch) {
-				r = sheet.createRow(rowIndex++);
-				cellIndex = 0;
-				for (Object v : x) {
-					addCell(r, cellIndex++, v);
+		sheet.setRowGenerator((sheet2) -> {
+			int batchSize = 100;
+			int rowIndex = 1, cellIndex = 0;
+
+			for (int fromIndex = 0; fromIndex < allDsrq.size(); fromIndex += batchSize) {
+				List<Object[]> batch = allDsrq.subList(fromIndex, Math.min(fromIndex + batchSize, allDsrq.size()));
+				LOG.info("Processing position: {} of {}", fromIndex, allDsrq.size());
+	
+				List<Long> rowIds = new ArrayList<Long>(batchSize);
+				for (Object[] x : batch) {
+					rowIds.add((Long) x[0]);
 				}
-
-				Object[] values = allValues.get(batchPos++);
-				if (values != null) {
-					for (Object v : values) {
+				List<Object[]> allValues = dsRowRepo.getRowValues(rowIds, columnDescriptors);
+				int batchPos = 0;
+				for (Object[] x : batch) {
+					SXSSFRow r = sheet.createRow(rowIndex++);
+					cellIndex = 0;
+					for (Object v : x) {
 						addCell(r, cellIndex++, v);
 					}
+	
+					Object[] values = allValues.get(batchPos++);
+					if (values != null) {
+						for (Object v : values) {
+							addCell(r, cellIndex++, v);
+						}
+					}
 				}
+	
+				// if (rowIndex > 10000) {
+				// LOG.warn("Breaking");
+				// break;
+				// }
 			}
-
-			// if (rowIndex > 10000) {
-			// LOG.warn("Breaking");
-			// break;
-			// }
-		}
-		sheet.flushRows();
-
+		});
+		
 		LOG.info("Writing to output stream");
 		wb.write(outputStream);
+		wb.close();
 		wb.dispose();
+		outputStream.flush();
 		LOG.info("Done");
 	}
 

src/main/webapp/WEB-INF/jsp/admin/ds2/index.jsp

@@ -15,12 +15,19 @@
         <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
 	</form>
 
+	<form method="post" action="<c:url value="/admin/ds2/worldclim/download" />">
+		<input type="submit" class="btn btn-default" value="Download worldclim.org" />
+        <!-- CSRF protection -->
+        <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
+	</form>
+
 	<form method="post" action="<c:url value="/admin/ds2/worldclim/delete" />">
 		<input type="submit" class="btn btn-default" value="Delete worldclim.org" />
         <!-- CSRF protection -->
         <input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
 	</form>
 
+
 	<content tag="javascript">
 		<script type="text/javascript">
             $(document).ready(function() {