Commit d5cc8be7 authored by Matija Obreza's avatar Matija Obreza
Browse files

DS2 code organization

parent 3da7cb03
...@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset; ...@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import javax.persistence.CascadeType;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.OneToMany; import javax.persistence.OneToMany;
...@@ -23,11 +24,11 @@ public class DS extends BasicModel { ...@@ -23,11 +24,11 @@ public class DS extends BasicModel {
private UUID uuid; private UUID uuid;
@OrderBy("index") @OrderBy("index")
@OneToMany(mappedBy = "dataset") @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL })
private List<DSQualifier> qualifiers; private List<DSQualifier> qualifiers;
@OrderBy("index") @OrderBy("index")
@OneToMany(mappedBy = "dataset") @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL })
private List<DSColumn> columns; private List<DSColumn> columns;
/** /**
......
package org.genesys2.server.model.dataset; package org.genesys2.server.model.dataset;
import java.nio.ByteBuffer;
import java.util.List; import java.util.List;
import javax.persistence.CascadeType; import javax.persistence.CascadeType;
...@@ -12,9 +13,12 @@ import javax.persistence.Id; ...@@ -12,9 +13,12 @@ import javax.persistence.Id;
import javax.persistence.JoinColumn; import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne; import javax.persistence.ManyToOne;
import javax.persistence.OneToMany; import javax.persistence.OneToMany;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.UniqueConstraint; import javax.persistence.UniqueConstraint;
import org.apache.commons.codec.digest.DigestUtils;
import org.genesys.blocks.model.EntityId; import org.genesys.blocks.model.EntityId;
/** /**
...@@ -31,14 +35,14 @@ import org.genesys.blocks.model.EntityId; ...@@ -31,14 +35,14 @@ import org.genesys.blocks.model.EntityId;
@Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) }) @Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) })
public class DSRow implements EntityId { public class DSRow implements EntityId {
@ManyToOne(fetch=FetchType.LAZY, optional = false) @ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(name="ds") @JoinColumn(name = "ds")
private DS dataset; private DS dataset;
@OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.REMOVE }) @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL })
private List<DSRowQualifier<?>> rowQualifiers; private List<DSRowQualifier<?>> rowQualifiers;
@OneToMany(fetch = FetchType.LAZY, mappedBy = "row") @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL })
private List<DSValue<?>> values; private List<DSValue<?>> values;
@Id @Id
...@@ -51,6 +55,19 @@ public class DSRow implements EntityId { ...@@ -51,6 +55,19 @@ public class DSRow implements EntityId {
@Column(columnDefinition = "binary(20)", updatable = false) @Column(columnDefinition = "binary(20)", updatable = false)
private byte[] sha1; private byte[] sha1;
@PrePersist
@PreUpdate
private void prePersist() {
ByteBuffer keyBuffer = ByteBuffer.allocate(500);
for (DSRowQualifier<?> dsq : this.getRowQualifiers()) {
dsq.putKey(keyBuffer);
}
byte[] array = keyBuffer.array();
this.md5 = DigestUtils.md5(array);
this.sha1 = DigestUtils.sha1(array);
}
@Override @Override
public Long getId() { public Long getId() {
return this.id; return this.id;
...@@ -63,11 +80,11 @@ public class DSRow implements EntityId { ...@@ -63,11 +80,11 @@ public class DSRow implements EntityId {
public DS getDataset() { public DS getDataset() {
return dataset; return dataset;
} }
public void setDataset(DS dataset) { public void setDataset(DS dataset) {
this.dataset = dataset; this.dataset = dataset;
} }
public List<DSRowQualifier<?>> getRowQualifiers() { public List<DSRowQualifier<?>> getRowQualifiers() {
return rowQualifiers; return rowQualifiers;
} }
......
package org.genesys2.server.model.dataset; package org.genesys2.server.model.dataset;
import java.nio.ByteBuffer;
import javax.persistence.DiscriminatorColumn; import javax.persistence.DiscriminatorColumn;
import javax.persistence.DiscriminatorType; import javax.persistence.DiscriminatorType;
import javax.persistence.Entity; import javax.persistence.Entity;
...@@ -80,4 +82,5 @@ public abstract class DSRowQualifier<T> { ...@@ -80,4 +82,5 @@ public abstract class DSRowQualifier<T> {
return null; return null;
} }
public abstract void putKey(ByteBuffer keyBuffer);
} }
package org.genesys2.server.model.dataset; package org.genesys2.server.model.dataset;
import java.nio.ByteBuffer;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.DiscriminatorValue; import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity; import javax.persistence.Entity;
...@@ -20,5 +22,9 @@ public class DSRowQualifierLong extends DSRowQualifier<Long> { ...@@ -20,5 +22,9 @@ public class DSRowQualifierLong extends DSRowQualifier<Long> {
public void setValue(Long value) { public void setValue(Long value) {
this.value = value; this.value = value;
} }
@Override
public void putKey(ByteBuffer keyBuffer) {
keyBuffer.putLong(this.value);
}
} }
...@@ -21,12 +21,15 @@ import javax.persistence.Entity; ...@@ -21,12 +21,15 @@ import javax.persistence.Entity;
import javax.persistence.FetchType; import javax.persistence.FetchType;
import javax.persistence.JoinColumn; import javax.persistence.JoinColumn;
import javax.persistence.OneToOne; import javax.persistence.OneToOne;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Version; import javax.persistence.Version;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.genesys.blocks.auditlog.annotations.Audited; import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.BasicModel; import org.genesys.blocks.model.BasicModel;
import org.genesys.worldclim.WorldClimUtil;
import org.genesys2.server.model.impl.GeoReferencedEntity; import org.genesys2.server.model.impl.GeoReferencedEntity;
@Entity @Entity
...@@ -57,6 +60,15 @@ public class AccessionGeo extends BasicModel implements GeoReferencedEntity, Acc ...@@ -57,6 +60,15 @@ public class AccessionGeo extends BasicModel implements GeoReferencedEntity, Acc
private Long tileIndex; private Long tileIndex;
/**
* Recalculate {@link #tileIndex} on insert and update
*/
@PrePersist
@PreUpdate
private void prePersist() {
tileIndex = WorldClimUtil.getTileIndex(5, this.longitude, this.latitude);
}
public long getVersion() { public long getVersion() {
return version; return version;
} }
......
...@@ -43,4 +43,7 @@ public interface AccessionGeoRepository extends JpaRepository<AccessionGeo, Long ...@@ -43,4 +43,7 @@ public interface AccessionGeoRepository extends JpaRepository<AccessionGeo, Long
@Query("select distinct(ag.tileIndex) from AccessionGeo ag where ag.tileIndex is not null") @Query("select distinct(ag.tileIndex) from AccessionGeo ag where ag.tileIndex is not null")
Collection<Long> getTileIndexes(); Collection<Long> getTileIndexes();
@Query("select distinct(ag.accession.id) from AccessionGeo ag where ag.longitude between -180 and 180 and ag.latitude between -90 and 90 and ag.tileIndex is null")
Set<Long> withMissingTileIndex();
} }
...@@ -16,7 +16,7 @@ import org.genesys2.server.model.json.WorldclimJson; ...@@ -16,7 +16,7 @@ import org.genesys2.server.model.json.WorldclimJson;
public interface DSService { public interface DSService {
void saveDataset(DS ds); DS saveDataset(DS ds);
DSQualifier addQualifier(DS ds, DSDescriptor d1); DSQualifier addQualifier(DS ds, DSDescriptor d1);
DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor); DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor);
......
...@@ -24,7 +24,7 @@ public interface DescriptorService { ...@@ -24,7 +24,7 @@ public interface DescriptorService {
List<DSDescriptor> list(); List<DSDescriptor> list();
void saveDescriptor(DSDescriptor dSDescriptor); DSDescriptor saveDescriptor(DSDescriptor dSDescriptor);
DSDescriptor getDescriptor(String variableName); DSDescriptor getDescriptor(String variableName);
} }
...@@ -2,9 +2,7 @@ package org.genesys2.server.service.impl; ...@@ -2,9 +2,7 @@ package org.genesys2.server.service.impl;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer; import java.nio.MappedByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Date; import java.util.Date;
...@@ -18,7 +16,6 @@ import java.util.UUID; ...@@ -18,7 +16,6 @@ import java.util.UUID;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFSheet;
...@@ -26,11 +23,11 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; ...@@ -26,11 +23,11 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.genesys2.server.model.dataset.DS; import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSColumn;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSRow;
import org.genesys2.server.model.dataset.DSRowQualifier; import org.genesys2.server.model.dataset.DSRowQualifier;
import org.genesys2.server.model.dataset.DSValue; import org.genesys2.server.model.dataset.DSValue;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.json.WorldclimJson; import org.genesys2.server.model.json.WorldclimJson;
import org.genesys2.server.persistence.domain.AccessionGeoRepository; import org.genesys2.server.persistence.domain.AccessionGeoRepository;
...@@ -40,7 +37,6 @@ import org.genesys2.server.persistence.domain.dataset.DSRepository; ...@@ -40,7 +37,6 @@ import org.genesys2.server.persistence.domain.dataset.DSRepository;
import org.genesys2.server.persistence.domain.dataset.DSRowQualifierRepository; import org.genesys2.server.persistence.domain.dataset.DSRowQualifierRepository;
import org.genesys2.server.persistence.domain.dataset.DSRowRepository; import org.genesys2.server.persistence.domain.dataset.DSRowRepository;
import org.genesys2.server.persistence.domain.dataset.DSValueRepository; import org.genesys2.server.persistence.domain.dataset.DSValueRepository;
import org.genesys2.server.persistence.domain.dataset.DSDescriptorRepository;
import org.genesys2.server.service.DSService; import org.genesys2.server.service.DSService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -53,12 +49,10 @@ import org.springframework.transaction.annotation.Transactional; ...@@ -53,12 +49,10 @@ import org.springframework.transaction.annotation.Transactional;
public class DSServiceImpl implements DSService { public class DSServiceImpl implements DSService {
public static final Logger LOG = LoggerFactory.getLogger(DSServiceImpl.class); public static final Logger LOG = LoggerFactory.getLogger(DSServiceImpl.class);
private static final Charset CHARSET_UTF8 = Charset.forName("UTF8");
@Autowired @Autowired
private DSRepository dsRepo; private DSRepository dsRepo;
@Autowired @Autowired
private DSColumnRepository dsDescRepo; private DSColumnRepository dsColRepo;
@Autowired @Autowired
private DSQualifierRepository dsQualRepo; private DSQualifierRepository dsQualRepo;
@Autowired @Autowired
...@@ -67,16 +61,14 @@ public class DSServiceImpl implements DSService { ...@@ -67,16 +61,14 @@ public class DSServiceImpl implements DSService {
private DSRowQualifierRepository dsRowQualiRepo; private DSRowQualifierRepository dsRowQualiRepo;
@Autowired @Autowired
private DSValueRepository dsValueRepo; private DSValueRepository dsValueRepo;
@Autowired
private DSDescriptorRepository descriptorRepo;
@Autowired @Autowired
private AccessionGeoRepository accessionGeoRepository; private AccessionGeoRepository accessionGeoRepository;
@Transactional @Transactional
@Override @Override
public void saveDataset(DS ds) { public DS saveDataset(DS ds) {
dsRepo.save(ds); return dsRepo.save(ds);
} }
@Transactional @Transactional
...@@ -98,7 +90,7 @@ public class DSServiceImpl implements DSService { ...@@ -98,7 +90,7 @@ public class DSServiceImpl implements DSService {
dsd.setDescriptor(d1); dsd.setDescriptor(d1);
dsd.setIndex(ds.getColumns().size() + 1); dsd.setIndex(ds.getColumns().size() + 1);
ds.getColumns().add(dsd); ds.getColumns().add(dsd);
return dsDescRepo.save(dsd); return dsColRepo.save(dsd);
} }
@Override @Override
...@@ -222,12 +214,12 @@ public class DSServiceImpl implements DSService { ...@@ -222,12 +214,12 @@ public class DSServiceImpl implements DSService {
} }
private void saveRows(Collection<DSRow> dsrs) { private void saveRows(Collection<DSRow> dsrs) {
// ArrayList<DSRowQualifier<?>> quali = new ArrayList<DSRowQualifier<?>>();
// for (DSRow dsr : dsrs) {
// quali.addAll(dsr.getRowQualifiers());
// }
// dsRowQualiRepo.save(quali);
dsRowRepo.save(dsrs); dsRowRepo.save(dsrs);
ArrayList<DSRowQualifier<?>> quali = new ArrayList<DSRowQualifier<?>>();
for (DSRow dsr : dsrs) {
quali.addAll(dsr.getRowQualifiers());
}
dsRowQualiRepo.save(quali);
} }
private DSRow findOrMakeRow(DS ds, Object[] qualifiers) { private DSRow findOrMakeRow(DS ds, Object[] qualifiers) {
...@@ -288,32 +280,17 @@ public class DSServiceImpl implements DSService { ...@@ -288,32 +280,17 @@ public class DSServiceImpl implements DSService {
private DSRow makeRow(DS ds, Object... qualifiers) { private DSRow makeRow(DS ds, Object... qualifiers) {
DSRow dsr = new DSRow(); DSRow dsr = new DSRow();
dsr.setDataset(ds);
ByteBuffer keyBuffer = ByteBuffer.allocate(500);
List<DSRowQualifier<?>> rowQualifiers = new ArrayList<DSRowQualifier<?>>(); List<DSRowQualifier<?>> rowQualifiers = new ArrayList<DSRowQualifier<?>>();
int i = 0; int i = 0;
for (DSQualifier dsq : ds.getQualifiers()) { for (DSQualifier dsq : ds.getQualifiers()) {
DSRowQualifier<?> dsrq = DSRowQualifier.make(qualifiers[i]); DSRowQualifier<?> dsrq = DSRowQualifier.make(qualifiers[i]);
if (qualifiers[i] instanceof Long) {
keyBuffer.putLong(((Long) qualifiers[i]).longValue());
} else if (qualifiers[i] instanceof String) {
keyBuffer.put(((String) qualifiers[i]).getBytes(CHARSET_UTF8));
} else if (qualifiers[i] instanceof Double) {
keyBuffer.putDouble(((Double) qualifiers[i]).doubleValue());
}
dsrq.setDatasetQualifier(dsq); dsrq.setDatasetQualifier(dsq);
dsrq.setRow(dsr); dsrq.setRow(dsr);
rowQualifiers.add(dsrq); rowQualifiers.add(dsrq);
i++; i++;
} }
// LOG.info("MD4 length: " + DigestUtils.md5(keyBuffer.array()).length);
// LOG.info("SHA1 length: " +
// DigestUtils.sha1(keyBuffer.array()).length);
dsr.setMd5(DigestUtils.md5(keyBuffer.array()));
dsr.setSha1(DigestUtils.sha1(keyBuffer.array()));
dsr.setDataset(ds);
dsr.setRowQualifiers(rowQualifiers); dsr.setRowQualifiers(rowQualifiers);
return dsr; return dsr;
} }
...@@ -335,15 +312,13 @@ public class DSServiceImpl implements DSService { ...@@ -335,15 +312,13 @@ public class DSServiceImpl implements DSService {
*/ */
@Override @Override
@Transactional @Transactional
public void worldclimUpdate(DS dataset, DSColumn dsd, Set<Long> tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) { public void worldclimUpdate(DS dataset, DSColumn dsc, Set<Long> tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) {
if (LOG.isInfoEnabled()) { LOG.debug("Updating {} for tileIndexes: {}", dsc.getDescriptor().getCode(), tileIndexes.size());
LOG.info("Updating {} for tileIndexes: {}", dsd.getDescriptor().getCode(), tileIndexes.size());
}
Map<Object, DSRow> rowMap = findOrMakeRows(dataset, tileIndexes); Map<Object, DSRow> rowMap = findOrMakeRows(dataset, tileIndexes);
// Load all existing values for rows // Load all existing values for rows
Map<Long, DSValue<?>> values = dsRowRepo.rowValueMap(rowMap.values(), dsd); Map<Long, DSValue<?>> values = dsRowRepo.rowValueMap(rowMap.values(), dsc);
List<DSValue<?>> toSave = new ArrayList<DSValue<?>>(); List<DSValue<?>> toSave = new ArrayList<DSValue<?>>();
for (final Long tileIndex : tileIndexes) { for (final Long tileIndex : tileIndexes) {
...@@ -353,9 +328,7 @@ public class DSServiceImpl implements DSService { ...@@ -353,9 +328,7 @@ public class DSServiceImpl implements DSService {
try { try {
short val = buffer.getShort((int) (tileIndex * 2)); short val = buffer.getShort((int) (tileIndex * 2));
if (val != nullValue) { if (val != nullValue) {
if (LOG.isDebugEnabled()) { LOG.trace("tile={} val={}", tileIndex, val);
LOG.debug("tile={} val={}", tileIndex, val);
}
Object value = null; Object value = null;
if (factor < 1.0) { if (factor < 1.0) {
...@@ -372,9 +345,9 @@ public class DSServiceImpl implements DSService { ...@@ -372,9 +345,9 @@ public class DSServiceImpl implements DSService {
dsrv = values.get(dsr.getId()); dsrv = values.get(dsr.getId());
if (dsrv == null) { if (dsrv == null) {
LOG.debug("Making new value for row {} and {}", dsr, dsd); LOG.debug("Making new value for row {} and {}", dsr, dsc);
dsrv = DSValue.make(value); dsrv = DSValue.make(value);
dsrv.setDatasetColumn(dsd); dsrv.setDatasetColumn(dsc);
dsrv.setRow(dsr); dsrv.setRow(dsr);
} }
dsrv.setValue2(value); dsrv.setValue2(value);
...@@ -398,7 +371,7 @@ public class DSServiceImpl implements DSService { ...@@ -398,7 +371,7 @@ public class DSServiceImpl implements DSService {
// Save values // Save values
dsValueRepo.save(toSave); dsValueRepo.save(toSave);
LOG.info("Done saving."); LOG.debug("Done saving.");
} }
@Override @Override
...@@ -553,6 +526,6 @@ public class DSServiceImpl implements DSService { ...@@ -553,6 +526,6 @@ public class DSServiceImpl implements DSService {
LOG.info("Clearing DSValues for {}", dsd.getDescriptor().getCode()); LOG.info("Clearing DSValues for {}", dsd.getDescriptor().getCode());
int count = dsValueRepo.deleteFor(dsd); int count = dsValueRepo.deleteFor(dsd);
LOG.info("Deleted {} DSValue cells", count); LOG.info("Deleted {} DSValue cells", count);
dsDescRepo.delete(dsd); dsColRepo.delete(dsd);
} }
} }
...@@ -33,21 +33,21 @@ public class DescriptorServiceImpl implements DescriptorService { ...@@ -33,21 +33,21 @@ public class DescriptorServiceImpl implements DescriptorService {
public static final Logger LOG = LoggerFactory.getLogger(DescriptorServiceImpl.class); public static final Logger LOG = LoggerFactory.getLogger(DescriptorServiceImpl.class);
@Autowired @Autowired
private DSDescriptorRepository dSDescriptorRepository; private DSDescriptorRepository descriptorRepository;
@Override @Override
public List<DSDescriptor> list() { public List<DSDescriptor> list() {
return dSDescriptorRepository.findAll(); return descriptorRepository.findAll();
} }
@Transactional @Transactional
@Override @Override
public void saveDescriptor(DSDescriptor descriptor) { public DSDescriptor saveDescriptor(DSDescriptor descriptor) {
dSDescriptorRepository.save(descriptor); return descriptorRepository.save(descriptor);
} }
@Override @Override
public DSDescriptor getDescriptor(String variableName) { public DSDescriptor getDescriptor(String variableName) {
return dSDescriptorRepository.findByCode(variableName); return descriptorRepository.findByCode(variableName);
} }
} }
...@@ -25,18 +25,12 @@ import java.io.IOException; ...@@ -25,18 +25,12 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;