Commit 2509c2d8 authored by Matija Obreza's avatar Matija Obreza

Downloading + Hibernate Stateless test

parent 62df87bd
......@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset;
import javax.persistence.DiscriminatorColumn;
import javax.persistence.DiscriminatorType;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
......@@ -22,11 +23,11 @@ public abstract class DSValue<T> {
@GeneratedValue(strategy = GenerationType.TABLE)
private Long id;
@ManyToOne(optional = false)
@ManyToOne(fetch=FetchType.LAZY, optional = false)
@JoinColumn(name = "r")
private DSRow row;
@ManyToOne(optional = false)
@ManyToOne(fetch=FetchType.LAZY, optional = false)
@JoinColumn(name = "dsd")
private DSDescriptor datasetDescriptor;
......
package org.genesys2.server.persistence.domain.dataset;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.genesys2.server.model.dataset.DS;
......@@ -13,4 +14,25 @@ public interface DSRowCustomRepository {
DSRow findRow(DS ds, Object[] qualifiers);
Map<Long, DSValue<?>> rowValueMap(Collection<DSRow> dsrs, DSDescriptor dsd);
List<Object[]> getQualifiers(DS ds);
/**
* Array of dsv.values as per DSDescriptor order
*
* @param rowId
* @param columnDescriptors
* @return
*/
Object[] getRowValues(long rowId, Long[] columnDescriptors);
/**
* List of arrays of dsv.values as per rowId order and DSDescriptor order
*
* @param rowId
* @param columnDescriptors
* @return
*/
List<Object[]> getRowValues(List<Long> rowId, Long[] columnDescriptors);
}
......@@ -20,4 +20,7 @@ public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustom
@Query("select dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value in ( ?2 )")
List<DSRow> findByLongQualifier(DS ds, Collection<?> qualifiers);
@Query("select distinct dsrq.row from DSRowQualifier dsrq where dsrq.datasetQualifier.dataset = ?1")
List<DSRow> findByDataset(DS ds);
}
......@@ -16,8 +16,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.dataset.DSRow;
import org.genesys2.server.model.dataset.DSValue;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.SessionFactory;
import org.hibernate.StatelessSession;
import org.hibernate.Transaction;
import org.springframework.stereotype.Repository;
@Repository
......@@ -95,7 +101,121 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
map.put((Long) r[0], (DSValue<?>) r[1]);
}
}
return map;
}
/**
* Return row.id, dsdq1.value, dsdq2.value...
*/
@SuppressWarnings("unchecked")
@Override
public List<Object[]> getQualifiers(DS ds) {
StringBuilder select = new StringBuilder();
StringBuilder where = new StringBuilder();
StringBuilder from = new StringBuilder();
List<DSQualifier> dsq = ds.getQualifiers();
Class<?> clazz;
final int OFFSET = 1;
int paramIdx = OFFSET;
for (int i = 0; i < dsq.size(); i++) {
if (i > 0)
from.append(", ");
// TODO Get from descriptor
clazz = Long.class;
if (clazz.equals(Long.class)) {
from.append("DSRowQualifierLong dsrq").append(i);
} else if (clazz.equals(Double.class)) {
from.append("DSRowQualifierDouble dsrq").append(i);
} else if (clazz.equals(String.class)) {
from.append("DSRowQualifierString dsrq").append(i);
}
select.append(", ").append("dsrq").append(i).append(".value");
if (i > 0) {
where.append(" and dsrq" + (i - 1) + ".row = dsrq" + i + ".row and ");
}
where.append(" dsrq" + i + ".datasetQualifier = ?").append(paramIdx++);
}
LOG.info("select dsrq0.row from " + from + " where " + where.toString());
Query q = entityManager.createQuery("select dsrq0.row.id " + select + " from " + from + " where " + where.toString() + "");
paramIdx = OFFSET;
for (int i = 0; i < dsq.size(); i++) {
q.setParameter(paramIdx++, dsq);
}
return q.getResultList();
}
@Override
public Object[] getRowValues(long rowId, Long[] columnDescriptors) {
Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv from DSValue dsrv where dsrv.row.id = ?1");
q.setParameter(1, rowId);
// List of [dsd.id, dsrv]
@SuppressWarnings("unchecked")
List<Object[]> values = q.getResultList();
Object[] result = new Object[columnDescriptors.length];
for (Object[] v : values) {
Long dsdId = (Long) v[0];
for (int i = 0; i < columnDescriptors.length; i++) {
if (columnDescriptors[i].equals(dsdId)) {
result[i] = ((DSValue<?>) v[1]).getValue();
}
}
}
return result;
}
@Override
public List<Object[]> getRowValues(List<Long> rowIds, Long[] columnDescriptors) {
// LOG.info("Making stateless");
SessionFactory sessionFactory = entityManager.getEntityManagerFactory().unwrap(SessionFactory.class);
StatelessSession session = sessionFactory.openStatelessSession();
Transaction tx = session.beginTransaction();
org.hibernate.Query q = session.createQuery("select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids)");
q.setParameterList("ids", rowIds);
// LOG.info("Query ready");
// System.out.println("-");
// Initialize results list with blank arrays
List<Object[]> results = new ArrayList<Object[]>(rowIds.size());
for (int i = 0; i < rowIds.size(); i++) {
results.add(new Object[columnDescriptors.length]);
}
// List of [dsd.id, dsrv]
ScrollableResults scroll = q.scroll(ScrollMode.FORWARD_ONLY);
// LOG.info("Scrolling");
int entries = 0;
while (scroll.next()) {
entries++;
Object[] v = scroll.get();
// LOG.info("Got next");
Long dsdId = (Long) v[0];
Long rowId = (Long) v[1];
DSValue<?> dsv = (DSValue<?>) v[2];
Object[] result = results.get(rowIds.indexOf(rowId));
for (int i = 0; i < columnDescriptors.length; i++) {
if (columnDescriptors[i].equals(dsdId)) {
result[i] = dsv.getValue();
}
}
}
// LOG.info("Done scrolling entries=" + entries);
tx.commit();
session.close();
// LOG.info("Closing session");
return results;
}
}
package org.genesys2.server.service;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.MappedByteBuffer;
import java.util.List;
import java.util.Set;
......@@ -31,4 +34,6 @@ public interface DSService {
void tempStore(List<AccessionGeo> toSave);
WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex);
void download(DS ds, OutputStream fos) throws IOException;
}
package org.genesys2.server.service.impl;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
......@@ -18,6 +21,11 @@ import java.util.regex.Pattern;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier;
......@@ -264,7 +272,7 @@ public class DSServiceImpl implements DSService {
missing.remove(dsr.getRowQualifiers().get(0).getValue());
}
for (Long miss : missing) {
// LOG.info("Adding missing row for " + miss);
// LOG.info("Adding missing row for " + miss);
dsrs.add(makeRow(ds, miss));
}
}
......@@ -421,4 +429,90 @@ public class DSServiceImpl implements DSService {
}
return wc;
}
@Override
public void download(DS ds, OutputStream outputStream) throws IOException {
XSSFWorkbook template = new XSSFWorkbook();
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SXSSFWorkbook wb = new SXSSFWorkbook(template, 1000);
SXSSFSheet sheet = (SXSSFSheet) wb.createSheet("worldclim.org");
int rowIndex = 0, cellIndex = 0;
Row r = sheet.createRow(rowIndex++);
r.createCell(cellIndex++).setCellValue("row_id");
for (DSQualifier dsq : ds.getQualifiers()) {
r.createCell(cellIndex++).setCellValue(dsq.getDescriptor().getCode());
}
// Prepare descriptor mapping
Long[] columnDescriptors = new Long[ds.getDescriptors().size()];
{
int columnIndex = 0;
for (DSDescriptor dsd : ds.getDescriptors()) {
r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode());
columnDescriptors[columnIndex++] = dsd.getId();
}
}
// List<DSRow> rows = dsRowRepo.findByDataset(ds);
// LOG.info("Retrieved DS rows " +rows.size());
// rowId, dsdq1.value, dsdq2.value...
List<Object[]> allDsrq = dsRowRepo.getQualifiers(ds);
LOG.info("Got " + allDsrq.size() + " row qualifiers");
int batchSize = 100;
for (int fromIndex = 0; fromIndex < allDsrq.size(); fromIndex += batchSize) {
List<Object[]> batch = allDsrq.subList(fromIndex, Math.min(fromIndex + batchSize, allDsrq.size()));
LOG.info("Processing position: " + fromIndex + " of " + allDsrq.size());
List<Long> rowIds = new ArrayList<Long>(batchSize);
for (Object[] x : batch) {
rowIds.add((Long) x[0]);
}
List<Object[]> allValues = dsRowRepo.getRowValues(rowIds, columnDescriptors);
int batchPos = 0;
for (Object[] x : batch) {
r = sheet.createRow(rowIndex++);
cellIndex = 0;
for (Object v : x) {
addCell(r, cellIndex++, v);
}
Object[] values = allValues.get(batchPos++);
if (values != null) {
for (Object v : values) {
addCell(r, cellIndex++, v);
}
}
}
if (rowIndex > 10000) {
LOG.warn("Breaking");
break;
}
}
sheet.flushRows();
LOG.info("Writing to output stream");
wb.write(outputStream);
wb.dispose();
LOG.info("Done");
}
private void addCell(Row r, int cellIndex, Object v) {
if (v == null)
return;
Cell c = r.createCell(cellIndex);
if (v instanceof String) {
c.setCellValue((String) v);
} else if (v instanceof Number) {
c.setCellValue(((Number) v).doubleValue());
} else if (v instanceof Date) {
c.setCellValue((Date) v);
}
}
}
......@@ -77,7 +77,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
public class WorldClimUpdater implements InitializingBean {
public static final Log LOG = LogFactory.getLog(WorldClimUpdater.class);
private static final UUID WORLDCLIM_DATASET = UUID.fromString("BC84433B-A626-4BDF-97D3-DB36D79499C6");
public static final UUID WORLDCLIM_DATASET = UUID.fromString("BC84433B-A626-4BDF-97D3-DB36D79499C6");
public static String WORLDCLIM_ORG = "http://www.worldclim.org";
public static String WORLDCLIM_ORG_FILEBASE = "http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/";
......
......@@ -35,7 +35,7 @@ import com.hazelcast.spring.cache.HazelcastCacheManager;
@ActiveProfiles("dev")
public class DSTest {
@Import({ HazelcastConfig.class, JpaRealDataConfig.class })
@Import({ HazelcastConfig.class, JpaDataConfig.class })
public static class Config {
@Bean
......
package org.genesys2.server.service.impl;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.http.impl.client.HttpClientBuilder;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl;
import org.genesys2.server.service.DSService;
......@@ -67,6 +71,9 @@ public class WorldClimUpdaterTest {
@Autowired
private WorldClimUpdater worldClimUpdater;
@Autowired
private DSService dsService;
@Test
public void testUpdateAlt() throws IOException {
worldClimUpdater.update("alt");
......@@ -83,4 +90,14 @@ public class WorldClimUpdaterTest {
}
}
@Test
public void testDownload() throws IOException {
DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
File file = new File("worldclim " + System.currentTimeMillis() + ".xls");
BufferedOutputStream fos=new BufferedOutputStream(new FileOutputStream(file));
dsService.download(ds, fos);
fos.flush();
fos.close();
// file.delete();
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment