Commit 31d94144 authored by Matija Obreza's avatar Matija Obreza

WorldClim work with Datasets

parent 2a5449fb
......@@ -498,6 +498,11 @@
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>
<groupId>org.genesys</groupId>
<artifactId>worldclim-reader</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
......
......@@ -76,4 +76,46 @@ public abstract class AuditedModel extends BusinessModel {
this.lastModifiedDate = lastModifiedDate;
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((createdBy == null) ? 0 : createdBy.hashCode());
result = prime * result + ((createdDate == null) ? 0 : createdDate.hashCode());
result = prime * result + ((lastModifiedBy == null) ? 0 : lastModifiedBy.hashCode());
result = prime * result + ((lastModifiedDate == null) ? 0 : lastModifiedDate.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!super.equals(obj))
return false;
if (getClass() != obj.getClass())
return false;
AuditedModel other = (AuditedModel) obj;
if (createdBy == null) {
if (other.createdBy != null)
return false;
} else if (!createdBy.equals(other.createdBy))
return false;
if (createdDate == null) {
if (other.createdDate != null)
return false;
} else if (!createdDate.equals(other.createdDate))
return false;
if (lastModifiedBy == null) {
if (other.lastModifiedBy != null)
return false;
} else if (!lastModifiedBy.equals(other.lastModifiedBy))
return false;
if (lastModifiedDate == null) {
if (other.lastModifiedDate != null)
return false;
} else if (!lastModifiedDate.equals(other.lastModifiedDate))
return false;
return true;
}
}
......@@ -44,4 +44,28 @@ public abstract class BusinessModel implements EntityId, HibernateModel {
this.id = id;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((id == null) ? 0 : id.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
BusinessModel other = (BusinessModel) obj;
if (id == null) {
if (other.id != null)
return false;
} else if (!id.equals(other.id))
return false;
return true;
}
}
/**
* Copyright 2015 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.model;
import java.util.UUID;
import javax.persistence.Column;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import javax.persistence.PrePersist;
@MappedSuperclass
public abstract class UuidModel {
@Id
@Column(name = "uuid", unique = true, nullable = false)
protected UUID uuid;
public UUID getUuid() {
return uuid;
}
protected void setUuid(UUID uuid) {
this.uuid = uuid;
}
@PrePersist
protected void pre() {
if (uuid == null)
uuid = UUID.randomUUID();
}
}
......@@ -33,4 +33,27 @@ public abstract class VersionedAuditedModel extends AuditedModel {
public void setVersion(long version) {
this.version = version;
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + (int) (version ^ (version >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!super.equals(obj))
return false;
if (getClass() != obj.getClass())
return false;
VersionedAuditedModel other = (VersionedAuditedModel) obj;
if (version != other.version)
return false;
return true;
}
}
......@@ -92,7 +92,7 @@ public class Accession extends VersionedAuditedModel {
@Column(name = "available")
private Boolean availability;
@Column(name = "historic")
@Column(name = "historic", nullable = false)
private boolean historic = false;
@Column(name = "mlsStat")
......@@ -280,11 +280,11 @@ public class Accession extends VersionedAuditedModel {
protected void setStorage(String storage) {
this.storage = storage;
}
public void setHistoric(boolean historic) {
this.historic = historic;
}
public boolean getHistoric() {
return this.historic;
}
......
......@@ -19,6 +19,7 @@ package org.genesys2.server.model.impl;
import java.text.MessageFormat;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import javax.persistence.Column;
import javax.persistence.Entity;
......@@ -26,6 +27,7 @@ import javax.persistence.Lob;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.OrderBy;
import javax.persistence.PrePersist;
import javax.persistence.Table;
import org.genesys2.server.model.BusinessModel;
......@@ -63,6 +65,19 @@ public class Dataset extends BusinessModel {
@OneToMany(cascade = {}, mappedBy = "dataset")
private List<DatasetDescriptor> datasetDescriptors;
@Column(unique = true, columnDefinition = "BINARY(16)")
private UUID uuid;
/**
* Generate UUID if missing
*/
@PrePersist
protected void prepersist() {
if (this.uuid == null) {
this.uuid = UUID.randomUUID();
}
}
public License getLicense() {
return license;
}
......@@ -141,4 +156,12 @@ public class Dataset extends BusinessModel {
public String toString() {
return MessageFormat.format("Dataset id={0,number,#} date={1,date,dd/MM/yyyy} name={2}", id, uploadDate, name);
}
public void setUuid(UUID uuid) {
this.uuid = uuid;
}
public UUID getUuid() {
return uuid;
}
}
......@@ -17,6 +17,8 @@
package org.genesys2.server.model.impl;
import javax.persistence.Column;
import javax.persistence.DiscriminatorColumn;
import javax.persistence.DiscriminatorType;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
......@@ -26,7 +28,6 @@ import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
import org.genesys2.server.model.EntityId;
......@@ -37,14 +38,14 @@ import org.genesys2.server.model.genesys.Accession;
*
* @author mobreza
*/
@Entity(name = "datasetval")
@Table(indexes = { @Index(name = "IX_datasetrow_dataset", unique = false, columnList = "datasetId, descriptorId, accessionId"),
@Entity
@Table(name = "datasetval", indexes = { @Index(name = "IX_datasetrow_dataset", unique = false, columnList = "datasetId, descriptorId, accessionId"),
@Index(name = "IX_datasetrow_accn", unique = false, columnList = "accessionId, descriptorId, datasetId") })
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
@DiscriminatorColumn(discriminatorType = DiscriminatorType.INTEGER, name = "typ")
public abstract class DatasetRow<T> implements EntityId {
@Id
@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "datasetrow_seq")
@SequenceGenerator(name = "datasetrow_seq", sequenceName = "datasetrow_seq", allocationSize = 10)
@GeneratedValue(strategy = GenerationType.TABLE)
@Column(name = "id", unique = true, nullable = false, length = 20)
protected Long id;
......
......@@ -17,17 +17,21 @@
package org.genesys2.server.model.impl;
import javax.persistence.Column;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import javax.persistence.Table;
/**
* A Descriptor represents a dimension of a sparse vector.
*
* @author mobreza
*/
@Entity(name = "datasetdbl")
@Entity
//@Table(name = "datasetdbl")
@DiscriminatorValue("2")
public class TraitDblValue extends DatasetRow<Double> {
@Column(nullable = false, name="vald")
@Column(name="vald")
private Double val;
@Override
......
......@@ -17,19 +17,19 @@
package org.genesys2.server.model.impl;
import javax.persistence.Column;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.Table;
/**
* A Descriptor represents a dimension of a sparse vector.
*
* @author mobreza
*/
@Entity(name = "datasetint")
@Table(indexes={ @Index(name="IX_datasetint_val", unique=false, columnList="descriptorId, vali") })
@Entity
//@Table(name = "datasetint", indexes={ @Index(name="IX_datasetint_val", unique=false, columnList="descriptorId, vali") })
@DiscriminatorValue("1")
public class TraitIntValue extends DatasetRow<Integer> {
@Column(nullable = false, name="vali")
@Column(name="vali")
private Integer val;
@Override
......
......@@ -17,17 +17,21 @@
package org.genesys2.server.model.impl;
import javax.persistence.Column;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import javax.persistence.Table;
/**
* A Descriptor represents a dimension of a sparse vector.
*
* @author mobreza
*/
@Entity(name = "datasetstr")
@Entity
//@Table(name = "datasetstr")
@DiscriminatorValue("3")
public class TraitStrValue extends DatasetRow<String> {
@Column(nullable = false, name="vals")
@Column(name="vals")
private String val;
@Override
......
......@@ -16,8 +16,12 @@
package org.genesys2.server.persistence.domain;
import java.util.UUID;
import org.genesys2.server.model.impl.Dataset;
import org.springframework.data.jpa.repository.JpaRepository;
public interface DatasetRepository extends JpaRepository<Dataset, Long> {
Dataset findByUuid(UUID uuid);
}
......@@ -16,13 +16,21 @@
package org.genesys2.server.persistence.domain;
import java.util.List;
import java.util.Set;
import org.genesys2.server.model.impl.Dataset;
import org.genesys2.server.model.impl.DatasetRow;
import org.genesys2.server.model.impl.Descriptor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
public interface DatasetRowRepository extends JpaRepository<DatasetRow<?>, Long> {
Page<DatasetRow<?>> findByDataset(Dataset ds, Pageable page);
@Query("select dr from DatasetRow dr where dr.dataset=?1 and dr.descriptor=?2 and dr.accession.id in ( ?3 )")
List<DatasetRow<?>> list(Dataset dataset, Descriptor descriptor, Set<Long> ids);
}
......@@ -41,9 +41,8 @@ import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
@Repository
@Transactional(readOnly = true)
public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository {
public static final Log LOG = LogFactory.getLog(GenesysLowlevelRepositoryImpl.class);
public class GenesysLowlevelRepositoryCustomImpl implements GenesysLowlevelRepository {
public static final Log LOG = LogFactory.getLog(GenesysLowlevelRepositoryCustomImpl.class);
@Autowired
private JdbcTemplate jdbcTemplate;
......@@ -217,7 +216,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
......@@ -231,7 +230,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
......@@ -258,7 +257,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
directQuery.sort(sort);
......@@ -286,7 +285,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
......@@ -314,7 +313,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
......@@ -356,7 +355,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
}
});
......
......@@ -16,6 +16,30 @@
package org.genesys2.server.service;
import java.nio.MappedByteBuffer;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.genesys2.server.model.impl.Dataset;
import org.genesys2.server.model.impl.DatasetRow;
import org.genesys2.server.model.impl.Descriptor;
public interface Dataset2Service {
Dataset getDataSet(UUID uuid);
Dataset saveDataset(Dataset dataset);
Descriptor getDescriptor(Dataset dataset, String variableName);
Descriptor saveDescriptor(Descriptor descriptor);
List<DatasetRow<?>> getForAccessionIds(Dataset dataset, Descriptor descriptor, Set<Long> ids);
void saveData(List<DatasetRow<?>> toSave);
void worldclimUpdate(Dataset dataset, Descriptor descriptor, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor);
}
......@@ -16,12 +16,184 @@
package org.genesys2.server.service.impl;
import java.nio.MappedByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys.worldclim.WorldClimUtil;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.impl.Dataset;
import org.genesys2.server.model.impl.DatasetRow;
import org.genesys2.server.model.impl.Descriptor;
import org.genesys2.server.model.impl.TraitDblValue;
import org.genesys2.server.model.impl.TraitIntValue;
import org.genesys2.server.persistence.domain.AccessionGeoRepository;
import org.genesys2.server.persistence.domain.AccessionRepository;
import org.genesys2.server.persistence.domain.DatasetRepository;
import org.genesys2.server.persistence.domain.DatasetRowRepository;
import org.genesys2.server.persistence.domain.DescriptorRepository;
import org.genesys2.server.service.Dataset2Service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@Transactional
@Transactional(readOnly = true)
public class Dataset2ServiceImpl implements Dataset2Service {
private final Log LOG = LogFactory.getLog(getClass());
@Autowired
private DatasetRepository datasetRepository;
@Autowired
private DescriptorRepository descriptorRepository;
@Autowired
private DatasetRowRepository datasetRowRepository;
@Autowired
private AccessionRepository accessionRepository;
@Autowired
private AccessionGeoRepository accessionGeoRepository;
@Override
@Transactional
public Dataset getDataSet(UUID uuid) {
return datasetRepository.findByUuid(uuid);
}
@Override
@Transactional
public Dataset saveDataset(Dataset dataset) {
return datasetRepository.save(dataset);
}
@Override
public Descriptor getDescriptor(Dataset dataset, String variableName) {
return descriptorRepository.findByCode(variableName);
}
@Override
@Transactional
public Descriptor saveDescriptor(Descriptor descriptor) {
return descriptorRepository.save(descriptor);
}
@Override
public List<DatasetRow<?>> getForAccessionIds(Dataset dataset, Descriptor descriptor, Set<Long> ids) {
return datasetRowRepository.list(dataset, descriptor, ids);
}
@Override
@Transactional
public void saveData(List<DatasetRow<?>> toSave) {
datasetRowRepository.save(toSave);
}
/**
* Updates worldclim data for accessions
*
* @param ids
* @param buffer
* @param nullValue
*/
@Override
@Transactional
public void worldclimUpdate(Dataset dataset, Descriptor descriptor, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor) {
if (LOG.isDebugEnabled()) {
LOG.debug("Updating " + ids + " accessionIds with " + descriptor.getCode());
LOG.debug("IDs: " + ArrayUtils.toString(ids.toArray(), "!"));
}
List<AccessionGeo> acGeo = accessionGeoRepository.findForAccessions(ids);
List<DatasetRow<?>> existing = getForAccessionIds(dataset, descriptor, ids);
LOG.info("Found " + existing.size() + " existing datasetRows");
for (DatasetRow<?> dr : existing) {
ids.remove(dr.getAccession().getId());
}
List<DatasetRow<?>> all = new ArrayList<DatasetRow<?>>(existing);
List<DatasetRow<?>> toSave = new ArrayList<DatasetRow<?>>();
// New ones
if (LOG.isDebugEnabled()) {
LOG.debug("Generating " + ids.size() + " new datasetRows");
}
for (Long id : ids) {
DatasetRow<?> tv = null;
// TODO FIXME Figure this out!
if (factor < 1.0) {
tv = new TraitDblValue();
} else if (factor == 1.0) {
tv = new TraitIntValue();
}
tv.setAccession(accessionRepository.findOne(id));
tv.setDataset(dataset);
tv.setDescriptor(descriptor);
all.add(tv);
}