Commit 6df3434c authored by Matija Obreza's avatar Matija Obreza
Browse files

Using lazy fetching where possible. Added DSService download() methods

parent 957accc3
......@@ -4,6 +4,7 @@ import java.util.List;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
......@@ -16,20 +17,20 @@ import org.genesys2.server.model.EntityId;
@Entity
@Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) })
public class DSRow implements EntityId {
@OneToMany(mappedBy = "row")
@OneToMany(fetch=FetchType.LAZY, mappedBy = "row")
private List<DSRowQualifier<?>> rowQualifiers;
@OneToMany(mappedBy = "row")
@OneToMany(fetch=FetchType.LAZY, mappedBy = "row")
private List<DSValue<?>> values;
@Id
@GeneratedValue(strategy = GenerationType.TABLE)
private Long id;
@Column(columnDefinition="binary(16)")
@Column(columnDefinition="binary(16)", updatable=false)
private byte[] md5;
@Column(columnDefinition="binary(20)")
@Column(columnDefinition="binary(20)", updatable=false)
private byte[] sha1;
@Override
......
......@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset;
import javax.persistence.DiscriminatorColumn;
import javax.persistence.DiscriminatorType;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
......@@ -22,12 +23,12 @@ public abstract class DSRowQualifier<T> {
@GeneratedValue(strategy = GenerationType.TABLE)
private Long id;
@ManyToOne(optional = false)
@JoinColumn(name = "r")
@ManyToOne(fetch=FetchType.LAZY, optional = false)
@JoinColumn(name = "r", updatable=false)
private DSRow row;
@ManyToOne(optional = false)
@JoinColumn(name = "dsq")
@ManyToOne(fetch=FetchType.LAZY, optional = false)
@JoinColumn(name = "dsq", updatable=false)
private DSQualifier datasetQualifier;
public abstract T getValue();
......
......@@ -8,7 +8,7 @@ import javax.persistence.Entity;
@DiscriminatorValue("1")
public class DSRowQualifierLong extends DSRowQualifier<Long> {
@Column(name = "vall")
@Column(name = "vall", updatable=false)
private Long value;
@Override
......
......@@ -17,8 +17,8 @@ public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustom
@Query("select dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value = ?2")
DSRow findByLongQualifier(DS dataset, Long value);
@Query("select dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value in ( ?2 )")
List<DSRow> findByLongQualifier(DS ds, Collection<?> qualifiers);
@Query("select dsrq.value, dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value in ( ?2 )")
List<Object[]> findByLongQualifier(DS ds, Collection<?> qualifiers);
@Query("select distinct dsrq.row from DSRowQualifier dsrq where dsrq.datasetQualifier.dataset = ?1")
List<DSRow> findByDataset(DS ds);
......
......@@ -19,11 +19,6 @@ import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.dataset.DSRow;
import org.genesys2.server.model.dataset.DSValue;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.SessionFactory;
import org.hibernate.StatelessSession;
import org.hibernate.Transaction;
import org.springframework.stereotype.Repository;
@Repository
......@@ -90,6 +85,7 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
Map<Long, DSValue<?>> map = new HashMap<Long, DSValue<?>>();
if (!validRows.isEmpty()) {
// LOG.info("getting row value map");
Query q = entityManager.createQuery("select dsv.row.id, dsv from DSValue dsv where dsv.row in (?1) and dsv.datasetDescriptor=?2");
q.setParameter(1, validRows);
q.setParameter(2, dsd);
......@@ -177,16 +173,16 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
}
@Override
public List<Object[]> getRowValues(List<Long> rowIds, Long[] columnDescriptors) {
Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids)");
public List<Object[]> getRowValues(List<Long> rowIds, Long[] datasetDescriptors) {
Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids) and dsrv.datasetDescriptor.id in (:dsds)");
q.setParameter("ids", rowIds);
q.setParameter("dsds", Arrays.asList(datasetDescriptors));
// LOG.info("Query ready");
// System.out.println("-");
// Initialize results list with blank arrays
List<Object[]> results = new ArrayList<Object[]>(rowIds.size());
for (int i = 0; i < rowIds.size(); i++) {
results.add(new Object[columnDescriptors.length]);
results.add(new Object[datasetDescriptors.length]);
}
List<Object[]> values = q.getResultList();
......@@ -199,8 +195,8 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
entityManager.detach(dsv);
Object[] result = results.get(rowIds.indexOf(rowId));
for (int i = 0; i < columnDescriptors.length; i++) {
if (columnDescriptors[i].equals(dsdId)) {
for (int i = 0; i < datasetDescriptors.length; i++) {
if (datasetDescriptors[i].equals(dsdId)) {
result[i] = dsv.getValue();
}
}
......
package org.genesys2.server.service;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.MappedByteBuffer;
......@@ -35,5 +34,7 @@ public interface DSService {
WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex);
void download(DS ds, OutputStream fos) throws IOException;
void download(DS ds, OutputStream outputStream) throws IOException;
void download(DS ds, List<DSDescriptor> dsds, OutputStream outputStream) throws IOException;
}
......@@ -242,8 +242,8 @@ public class DSServiceImpl implements DSService {
return dsr;
}
private List<DSRow> findOrMakeRows(DS ds, Collection<?> qualifiers) {
List<DSRow> dsrs = new ArrayList<DSRow>();
private Map<Object, DSRow> findOrMakeRows(DS ds, Collection<?> qualifiers) {
Map<Object, DSRow> dsrs = new HashMap<Object, DSRow>();
Iterator<?> it = qualifiers.iterator();
if (!it.hasNext()) {
......@@ -256,24 +256,29 @@ public class DSServiceImpl implements DSService {
LOG.info("Qualifiers are arrays!");
// TODO Sort it out
for (Object qs : qualifiers) {
dsrs.add(findOrMakeRow(ds, (Object[]) qs));
dsrs.put(qs, findOrMakeRow(ds, (Object[]) qs));
}
} else if (sampleQualifier instanceof Long) {
LOG.info("Qualifiers are Longs");
List<DSRow> existing = dsRowRepo.findByLongQualifier(ds, qualifiers);
dsrs.addAll(existing);
List<Object[]> existing = dsRowRepo.findByLongQualifier(ds, qualifiers);
Map<Long, DSRow> qualifierMap = new HashMap<Long, DSRow>();
for (Object[] e : existing) {
qualifierMap.put((Long) e[0], (DSRow) e[1]);
}
dsrs.putAll(qualifierMap);
// Make missing rows
Set<Long> missing = new HashSet<Long>();
for (Object x : qualifiers) {
missing.add((Long) x);
}
for (DSRow dsr : existing) {
missing.remove(dsr.getRowQualifiers().get(0).getValue());
for (Long dsrq1id : qualifierMap.keySet()) {
missing.remove(dsrq1id);
}
for (Long miss : missing) {
// LOG.info("Adding missing row for " + miss);
dsrs.add(makeRow(ds, miss));
dsrs.put(miss, makeRow(ds, miss));
}
}
......@@ -333,10 +338,7 @@ public class DSServiceImpl implements DSService {
LOG.info("\n\nUpdating " + dsd.getDescriptor().getCode() + " for tileIndexes: " + tileIndexes);
}
Map<Long, DSRow> rowMap = new HashMap<Long, DSRow>();
for (DSRow dsr : findOrMakeRows(dataset, tileIndexes)) {
rowMap.put((Long) dsr.getRowQualifiers().get(0).getValue(), dsr);
}
Map<Object, DSRow> rowMap = findOrMakeRows(dataset, tileIndexes);
// Load all existing values for rows
Map<Long, DSValue<?>> values = dsRowRepo.rowValueMap(rowMap.values(), dsd);
......@@ -382,7 +384,16 @@ public class DSServiceImpl implements DSService {
}
}
}
saveRows(rowMap.values());
// Only save new rows
Collection<DSRow> newRows = new ArrayList<DSRow>();
for (DSRow dsr : rowMap.values()) {
if (dsr.getId() == null)
newRows.add(dsr);
}
saveRows(newRows);
// Save values
dsValueRepo.save(toSave);
LOG.info("Done saving.");
......@@ -429,13 +440,13 @@ public class DSServiceImpl implements DSService {
}
return wc;
}
@Override
public void download(DS ds, OutputStream outputStream) throws IOException {
public void download(DS ds, List<DSDescriptor> dsds, OutputStream outputStream) throws IOException {
XSSFWorkbook template = new XSSFWorkbook();
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SXSSFWorkbook wb = new SXSSFWorkbook(template, 1000);
SXSSFWorkbook wb = new SXSSFWorkbook(template, 100);
SXSSFSheet sheet = (SXSSFSheet) wb.createSheet("worldclim.org");
......@@ -446,10 +457,10 @@ public class DSServiceImpl implements DSService {
r.createCell(cellIndex++).setCellValue(dsq.getDescriptor().getCode());
}
// Prepare descriptor mapping
Long[] columnDescriptors = new Long[ds.getDescriptors().size()];
Long[] columnDescriptors = new Long[dsds.size()];
{
int columnIndex = 0;
for (DSDescriptor dsd : ds.getDescriptors()) {
for (DSDescriptor dsd : dsds) {
r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode());
columnDescriptors[columnIndex++] = dsd.getId();
}
......@@ -488,12 +499,11 @@ public class DSServiceImpl implements DSService {
}
}
}
if (rowIndex > 10000) {
LOG.warn("Breaking");
break;
}
//
// if (rowIndex > 10000) {
// LOG.warn("Breaking");
// break;
// }
}
sheet.flushRows();
......@@ -503,6 +513,11 @@ public class DSServiceImpl implements DSService {
LOG.info("Done");
}
@Override
public void download(DS ds, OutputStream outputStream) throws IOException {
download(ds, ds.getDescriptors(), outputStream);
}
private void addCell(Row r, int cellIndex, Object v) {
if (v == null)
return;
......
......@@ -4,9 +4,12 @@ import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.impl.client.HttpClientBuilder;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl;
import org.genesys2.server.service.DSService;
......@@ -100,4 +103,24 @@ public class WorldClimUpdaterTest {
fos.close();
// file.delete();
}
@Test
public void testDownloadDesc() throws IOException {
DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
List<DSDescriptor> dsds = new ArrayList<DSDescriptor>();
for (DSDescriptor dsd : ds.getDescriptors()) {
if (dsd.getDescriptor().getCode().startsWith("tmin")) {
dsds.add(dsd);
}
}
System.out.println("Downloading tmin data: " + dsds.size());
File file = new File("worldclim " + System.currentTimeMillis() + ".xls");
BufferedOutputStream fos=new BufferedOutputStream(new FileOutputStream(file));
dsService.download(ds, dsds, fos);
fos.flush();
fos.close();
// file.delete();
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment