Downloading + Hibernate Stateless test

src/main/java/org/genesys2/server/model/dataset/DSValue.java

@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset;
 import javax.persistence.DiscriminatorColumn;
 import javax.persistence.DiscriminatorType;
 import javax.persistence.Entity;
+import javax.persistence.FetchType;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -22,11 +23,11 @@ public abstract class DSValue<T> {
 	@GeneratedValue(strategy = GenerationType.TABLE)
 	private Long id;
 
-	@ManyToOne(optional = false)
+	@ManyToOne(fetch=FetchType.LAZY, optional = false)
 	@JoinColumn(name = "r")
 	private DSRow row;
 
-	@ManyToOne(optional = false)
+	@ManyToOne(fetch=FetchType.LAZY, optional = false)
 	@JoinColumn(name = "dsd")
 	private DSDescriptor datasetDescriptor;
 

src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java

 package org.genesys2.server.persistence.domain.dataset;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
 import org.genesys2.server.model.dataset.DS;
@@ -13,4 +14,25 @@ public interface DSRowCustomRepository {
 	DSRow findRow(DS ds, Object[] qualifiers);
 
 	Map<Long, DSValue<?>> rowValueMap(Collection<DSRow> dsrs, DSDescriptor dsd);
+
+	List<Object[]> getQualifiers(DS ds);
+
+	/**
+	 * Array of dsv.values as per DSDescriptor order
+	 * 
+	 * @param rowId
+	 * @param columnDescriptors
+	 * @return
+	 */
+	Object[] getRowValues(long rowId, Long[] columnDescriptors);
+
+	/**
+	 * List of arrays of dsv.values as per rowId order and DSDescriptor order
+	 * 
+	 * @param rowId
+	 * @param columnDescriptors
+	 * @return
+	 */
+	List<Object[]> getRowValues(List<Long> rowId, Long[] columnDescriptors);
+
 }

src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepository.java

@@ -20,4 +20,7 @@ public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustom
 	@Query("select dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value in ( ?2 )")
 	List<DSRow> findByLongQualifier(DS ds, Collection<?> qualifiers);
 
+	@Query("select distinct dsrq.row from DSRowQualifier dsrq where dsrq.datasetQualifier.dataset = ?1")
+	List<DSRow> findByDataset(DS ds);
+
 }

src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java

@@ -16,8 +16,14 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.genesys2.server.model.dataset.DS;
 import org.genesys2.server.model.dataset.DSDescriptor;
+import org.genesys2.server.model.dataset.DSQualifier;
 import org.genesys2.server.model.dataset.DSRow;
 import org.genesys2.server.model.dataset.DSValue;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
+import org.hibernate.SessionFactory;
+import org.hibernate.StatelessSession;
+import org.hibernate.Transaction;
 import org.springframework.stereotype.Repository;
 
 @Repository
@@ -95,7 +101,121 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
 				map.put((Long) r[0], (DSValue<?>) r[1]);
 			}
 		}
-		
+
 		return map;
 	}
+
+	/**
+	 * Return row.id, dsdq1.value, dsdq2.value...
+	 */
+	@SuppressWarnings("unchecked")
+	@Override
+	public List<Object[]> getQualifiers(DS ds) {
+		StringBuilder select = new StringBuilder();
+		StringBuilder where = new StringBuilder();
+		StringBuilder from = new StringBuilder();
+
+		List<DSQualifier> dsq = ds.getQualifiers();
+		Class<?> clazz;
+		final int OFFSET = 1;
+		int paramIdx = OFFSET;
+		for (int i = 0; i < dsq.size(); i++) {
+			if (i > 0)
+				from.append(", ");
+
+			// TODO Get from descriptor
+			clazz = Long.class;
+
+			if (clazz.equals(Long.class)) {
+				from.append("DSRowQualifierLong dsrq").append(i);
+			} else if (clazz.equals(Double.class)) {
+				from.append("DSRowQualifierDouble dsrq").append(i);
+			} else if (clazz.equals(String.class)) {
+				from.append("DSRowQualifierString dsrq").append(i);
+			}
+
+			select.append(", ").append("dsrq").append(i).append(".value");
+
+			if (i > 0) {
+				where.append(" and dsrq" + (i - 1) + ".row = dsrq" + i + ".row and ");
+			}
+			where.append(" dsrq" + i + ".datasetQualifier = ?").append(paramIdx++);
+		}
+
+		LOG.info("select dsrq0.row from " + from + " where " + where.toString());
+		Query q = entityManager.createQuery("select dsrq0.row.id " + select + " from " + from + " where " + where.toString() + "");
+
+		paramIdx = OFFSET;
+		for (int i = 0; i < dsq.size(); i++) {
+			q.setParameter(paramIdx++, dsq);
+		}
+
+		return q.getResultList();
+	}
+
+	@Override
+	public Object[] getRowValues(long rowId, Long[] columnDescriptors) {
+		Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv from DSValue dsrv where dsrv.row.id = ?1");
+		q.setParameter(1, rowId);
+
+		// List of [dsd.id, dsrv]
+		@SuppressWarnings("unchecked")
+		List<Object[]> values = q.getResultList();
+
+		Object[] result = new Object[columnDescriptors.length];
+		for (Object[] v : values) {
+			Long dsdId = (Long) v[0];
+
+			for (int i = 0; i < columnDescriptors.length; i++) {
+				if (columnDescriptors[i].equals(dsdId)) {
+					result[i] = ((DSValue<?>) v[1]).getValue();
+				}
+			}
+		}
+
+		return result;
+	}
+
+	@Override
+	public List<Object[]> getRowValues(List<Long> rowIds, Long[] columnDescriptors) {
+//		LOG.info("Making stateless");
+		SessionFactory sessionFactory = entityManager.getEntityManagerFactory().unwrap(SessionFactory.class);
+		StatelessSession session = sessionFactory.openStatelessSession();
+		Transaction tx = session.beginTransaction();
+		org.hibernate.Query q = session.createQuery("select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids)");
+		q.setParameterList("ids", rowIds);
+//		LOG.info("Query ready");
+
+		// System.out.println("-");
+		// Initialize results list with blank arrays
+		List<Object[]> results = new ArrayList<Object[]>(rowIds.size());
+		for (int i = 0; i < rowIds.size(); i++) {
+			results.add(new Object[columnDescriptors.length]);
+		}
+		// List of [dsd.id, dsrv]
+		ScrollableResults scroll = q.scroll(ScrollMode.FORWARD_ONLY);
+//		LOG.info("Scrolling");
+		int entries = 0;
+		while (scroll.next()) {
+			entries++;
+			Object[] v = scroll.get();
+			// LOG.info("Got next");
+			Long dsdId = (Long) v[0];
+			Long rowId = (Long) v[1];
+			DSValue<?> dsv = (DSValue<?>) v[2];
+
+			Object[] result = results.get(rowIds.indexOf(rowId));
+			for (int i = 0; i < columnDescriptors.length; i++) {
+				if (columnDescriptors[i].equals(dsdId)) {
+					result[i] = dsv.getValue();
+				}
+			}
+		}
+//		LOG.info("Done scrolling entries=" + entries);
+		tx.commit();
+		session.close();
+//		LOG.info("Closing session");
+
+		return results;
+	}
 }

src/main/java/org/genesys2/server/service/DSService.java

 package org.genesys2.server.service;
 
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.MappedByteBuffer;
 import java.util.List;
 import java.util.Set;
@@ -31,4 +34,6 @@ public interface DSService {
 	void tempStore(List<AccessionGeo> toSave);
 
 	WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex);
+
+	void download(DS ds, OutputStream fos) throws IOException;
 }

src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java

 package org.genesys2.server.service.impl;
 
+import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -18,6 +21,11 @@ import java.util.regex.Pattern;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.xssf.streaming.SXSSFSheet;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.genesys2.server.model.dataset.DS;
 import org.genesys2.server.model.dataset.DSDescriptor;
 import org.genesys2.server.model.dataset.DSQualifier;
@@ -264,7 +272,7 @@ public class DSServiceImpl implements DSService {
 				missing.remove(dsr.getRowQualifiers().get(0).getValue());
 			}
 			for (Long miss : missing) {
-//				LOG.info("Adding missing row for " + miss);
+				// LOG.info("Adding missing row for " + miss);
 				dsrs.add(makeRow(ds, miss));
 			}
 		}
@@ -421,4 +429,90 @@ public class DSServiceImpl implements DSService {
 		}
 		return wc;
 	}
+
+	@Override
+	public void download(DS ds, OutputStream outputStream) throws IOException {
+		XSSFWorkbook template = new XSSFWorkbook();
+
+		// keep 1000 rows in memory, exceeding rows will be flushed to disk
+		SXSSFWorkbook wb = new SXSSFWorkbook(template, 1000);
+
+		SXSSFSheet sheet = (SXSSFSheet) wb.createSheet("worldclim.org");
+
+		int rowIndex = 0, cellIndex = 0;
+		Row r = sheet.createRow(rowIndex++);
+		r.createCell(cellIndex++).setCellValue("row_id");
+		for (DSQualifier dsq : ds.getQualifiers()) {
+			r.createCell(cellIndex++).setCellValue(dsq.getDescriptor().getCode());
+		}
+		// Prepare descriptor mapping
+		Long[] columnDescriptors = new Long[ds.getDescriptors().size()];
+		{
+			int columnIndex = 0;
+			for (DSDescriptor dsd : ds.getDescriptors()) {
+				r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode());
+				columnDescriptors[columnIndex++] = dsd.getId();
+			}
+		}
+
+		// List<DSRow> rows = dsRowRepo.findByDataset(ds);
+		// LOG.info("Retrieved DS rows " +rows.size());
+
+		// rowId, dsdq1.value, dsdq2.value...
+		List<Object[]> allDsrq = dsRowRepo.getQualifiers(ds);
+		LOG.info("Got " + allDsrq.size() + " row qualifiers");
+
+		int batchSize = 100;
+
+		for (int fromIndex = 0; fromIndex < allDsrq.size(); fromIndex += batchSize) {
+			List<Object[]> batch = allDsrq.subList(fromIndex, Math.min(fromIndex + batchSize, allDsrq.size()));
+			LOG.info("Processing position: " + fromIndex + " of " + allDsrq.size());
+
+			List<Long> rowIds = new ArrayList<Long>(batchSize);
+			for (Object[] x : batch) {
+				rowIds.add((Long) x[0]);
+			}
+			List<Object[]> allValues = dsRowRepo.getRowValues(rowIds, columnDescriptors);
+			int batchPos = 0;
+			for (Object[] x : batch) {
+				r = sheet.createRow(rowIndex++);
+				cellIndex = 0;
+				for (Object v : x) {
+					addCell(r, cellIndex++, v);
+				}
+
+				Object[] values = allValues.get(batchPos++);
+				if (values != null) {
+					for (Object v : values) {
+						addCell(r, cellIndex++, v);
+					}
+				}
+			}
+			
+
+			if (rowIndex > 10000) {
+				LOG.warn("Breaking");
+				break;
+			}
+		}
+		sheet.flushRows();
+
+		LOG.info("Writing to output stream");
+		wb.write(outputStream);
+		wb.dispose();
+		LOG.info("Done");
+	}
+
+	private void addCell(Row r, int cellIndex, Object v) {
+		if (v == null)
+			return;
+		Cell c = r.createCell(cellIndex);
+		if (v instanceof String) {
+			c.setCellValue((String) v);
+		} else if (v instanceof Number) {
+			c.setCellValue(((Number) v).doubleValue());
+		} else if (v instanceof Date) {
+			c.setCellValue((Date) v);
+		}
+	}
 }

src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java

@@ -77,7 +77,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 public class WorldClimUpdater implements InitializingBean {
 	public static final Log LOG = LogFactory.getLog(WorldClimUpdater.class);
 
-	private static final UUID WORLDCLIM_DATASET = UUID.fromString("BC84433B-A626-4BDF-97D3-DB36D79499C6");
+	public static final UUID WORLDCLIM_DATASET = UUID.fromString("BC84433B-A626-4BDF-97D3-DB36D79499C6");
 
 	public static String WORLDCLIM_ORG = "http://www.worldclim.org";
 	public static String WORLDCLIM_ORG_FILEBASE = "http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/";

src/test/java/org/genesys2/server/service/impl/DSTest.java

@@ -35,7 +35,7 @@ import com.hazelcast.spring.cache.HazelcastCacheManager;
 @ActiveProfiles("dev")
 public class DSTest {
 
-	@Import({ HazelcastConfig.class, JpaRealDataConfig.class })
+	@Import({ HazelcastConfig.class, JpaDataConfig.class })
 	public static class Config {
 
 		@Bean

src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java

 package org.genesys2.server.service.impl;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.genesys2.server.model.dataset.DS;
 import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
 import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl;
 import org.genesys2.server.service.DSService;
@@ -67,6 +71,9 @@ public class WorldClimUpdaterTest {
 	@Autowired
 	private WorldClimUpdater worldClimUpdater;
 
+	@Autowired
+	private DSService dsService;
+
 	@Test
 	public void testUpdateAlt() throws IOException {
 		worldClimUpdater.update("alt");
@@ -83,4 +90,14 @@ public class WorldClimUpdaterTest {
 		}
 	}
 
+	@Test
+	public void testDownload() throws IOException {
+		DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
+		File file = new File("worldclim " + System.currentTimeMillis() + ".xls");
+		BufferedOutputStream fos=new BufferedOutputStream(new FileOutputStream(file));
+		dsService.download(ds, fos);
+		fos.flush();
+		fos.close();
+		//		file.delete();
+	}
 }