WorldClim work with Datasets

pom.xml

@@ -498,6 +498,11 @@
 		 	<artifactId>httpclient</artifactId>
 		 	<version>4.3.6</version>
 		 </dependency>
+		 <dependency>
+		 	<groupId>org.genesys</groupId>
+		 	<artifactId>worldclim-reader</artifactId>
+		 	<version>0.0.1-SNAPSHOT</version>
+		 </dependency>
 	</dependencies>
 
 	<build>

src/main/java/org/genesys2/server/model/AuditedModel.java

@@ -76,4 +76,46 @@ public abstract class AuditedModel extends BusinessModel {
 		this.lastModifiedDate = lastModifiedDate;
 	}
 
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((createdBy == null) ? 0 : createdBy.hashCode());
+		result = prime * result + ((createdDate == null) ? 0 : createdDate.hashCode());
+		result = prime * result + ((lastModifiedBy == null) ? 0 : lastModifiedBy.hashCode());
+		result = prime * result + ((lastModifiedDate == null) ? 0 : lastModifiedDate.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		AuditedModel other = (AuditedModel) obj;
+		if (createdBy == null) {
+			if (other.createdBy != null)
+				return false;
+		} else if (!createdBy.equals(other.createdBy))
+			return false;
+		if (createdDate == null) {
+			if (other.createdDate != null)
+				return false;
+		} else if (!createdDate.equals(other.createdDate))
+			return false;
+		if (lastModifiedBy == null) {
+			if (other.lastModifiedBy != null)
+				return false;
+		} else if (!lastModifiedBy.equals(other.lastModifiedBy))
+			return false;
+		if (lastModifiedDate == null) {
+			if (other.lastModifiedDate != null)
+				return false;
+		} else if (!lastModifiedDate.equals(other.lastModifiedDate))
+			return false;
+		return true;
+	}
 }

src/main/java/org/genesys2/server/model/BusinessModel.java

@@ -44,4 +44,28 @@ public abstract class BusinessModel implements EntityId, HibernateModel {
 		this.id = id;
 	}
 
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((id == null) ? 0 : id.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		BusinessModel other = (BusinessModel) obj;
+		if (id == null) {
+			if (other.id != null)
+				return false;
+		} else if (!id.equals(other.id))
+			return false;
+		return true;
+	}
 }

src/main/java/org/genesys2/server/model/UuidModel.java

+/**
+ * Copyright 2015 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.model;
+
+import java.util.UUID;
+
+import javax.persistence.Column;
+import javax.persistence.Id;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.PrePersist;
+
+@MappedSuperclass
+public abstract class UuidModel {
+	@Id
+	@Column(name = "uuid", unique = true, nullable = false)
+	protected UUID uuid;
+
+	public UUID getUuid() {
+		return uuid;
+	}
+
+	protected void setUuid(UUID uuid) {
+		this.uuid = uuid;
+	}
+
+	@PrePersist
+	protected void pre() {
+		if (uuid == null)
+			uuid = UUID.randomUUID();
+	}
+}

src/main/java/org/genesys2/server/model/VersionedAuditedModel.java

@@ -33,4 +33,27 @@ public abstract class VersionedAuditedModel extends AuditedModel {
 	public void setVersion(long version) {
 		this.version = version;
 	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + (int) (version ^ (version >>> 32));
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		VersionedAuditedModel other = (VersionedAuditedModel) obj;
+		if (version != other.version)
+			return false;
+		return true;
+	}
+
 }

src/main/java/org/genesys2/server/model/genesys/Accession.java

@@ -92,7 +92,7 @@ public class Accession extends VersionedAuditedModel {
 	@Column(name = "available")
 	private Boolean availability;
 
-	@Column(name = "historic")
+	@Column(name = "historic", nullable = false)
 	private boolean historic = false;
 
 	@Column(name = "mlsStat")
@@ -280,11 +280,11 @@ public class Accession extends VersionedAuditedModel {
 	protected void setStorage(String storage) {
 		this.storage = storage;
 	}
-	
+
 	public void setHistoric(boolean historic) {
 		this.historic = historic;
 	}
-	
+
 	public boolean getHistoric() {
 		return this.historic;
 	}

src/main/java/org/genesys2/server/model/impl/Dataset.java

@@ -19,6 +19,7 @@ package org.genesys2.server.model.impl;
 import java.text.MessageFormat;
 import java.util.Date;
 import java.util.List;
+import java.util.UUID;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -26,6 +27,7 @@ import javax.persistence.Lob;
 import javax.persistence.ManyToOne;
 import javax.persistence.OneToMany;
 import javax.persistence.OrderBy;
+import javax.persistence.PrePersist;
 import javax.persistence.Table;
 
 import org.genesys2.server.model.BusinessModel;
@@ -63,6 +65,19 @@ public class Dataset extends BusinessModel {
 	@OneToMany(cascade = {}, mappedBy = "dataset")
 	private List<DatasetDescriptor> datasetDescriptors;
 
+	@Column(unique = true, columnDefinition = "BINARY(16)")
+	private UUID uuid;
+
+	/**
+	 * Generate UUID if missing
+	 */
+	@PrePersist
+	protected void prepersist() {
+		if (this.uuid == null) {
+			this.uuid = UUID.randomUUID();
+		}
+	}
+
 	public License getLicense() {
 		return license;
 	}
@@ -141,4 +156,12 @@ public class Dataset extends BusinessModel {
 	public String toString() {
 		return MessageFormat.format("Dataset id={0,number,#} date={1,date,dd/MM/yyyy} name={2}", id, uploadDate, name);
 	}
+
+	public void setUuid(UUID uuid) {
+		this.uuid = uuid;
+	}
+
+	public UUID getUuid() {
+		return uuid;
+	}
 }

src/main/java/org/genesys2/server/model/impl/DatasetRow.java

@@ -17,6 +17,8 @@
 package org.genesys2.server.model.impl;
 
 import javax.persistence.Column;
+import javax.persistence.DiscriminatorColumn;
+import javax.persistence.DiscriminatorType;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -26,7 +28,6 @@ import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
-import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
 
 import org.genesys2.server.model.EntityId;
@@ -37,14 +38,14 @@ import org.genesys2.server.model.genesys.Accession;
  *
  * @author mobreza
  */
-@Entity(name = "datasetval")
-@Table(indexes = { @Index(name = "IX_datasetrow_dataset", unique = false, columnList = "datasetId, descriptorId, accessionId"),
+@Entity
+@Table(name = "datasetval", indexes = { @Index(name = "IX_datasetrow_dataset", unique = false, columnList = "datasetId, descriptorId, accessionId"),
 		@Index(name = "IX_datasetrow_accn", unique = false, columnList = "accessionId, descriptorId, datasetId") })
-@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
+@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
+@DiscriminatorColumn(discriminatorType = DiscriminatorType.INTEGER, name = "typ")
 public abstract class DatasetRow<T> implements EntityId {
 	@Id
-	@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "datasetrow_seq")
-	@SequenceGenerator(name = "datasetrow_seq", sequenceName = "datasetrow_seq", allocationSize = 10)
+	@GeneratedValue(strategy = GenerationType.TABLE)
 	@Column(name = "id", unique = true, nullable = false, length = 20)
 	protected Long id;
 

src/main/java/org/genesys2/server/model/impl/TraitDblValue.java

@@ -17,17 +17,21 @@
 package org.genesys2.server.model.impl;
 
 import javax.persistence.Column;
+import javax.persistence.DiscriminatorValue;
 import javax.persistence.Entity;
+import javax.persistence.Table;
 
 /**
  * A Descriptor represents a dimension of a sparse vector.
  *
  * @author mobreza
  */
-@Entity(name = "datasetdbl")
+@Entity
+//@Table(name = "datasetdbl")
+@DiscriminatorValue("2")
 public class TraitDblValue extends DatasetRow<Double> {
 
-	@Column(nullable = false, name="vald")
+	@Column(name="vald")
 	private Double val;
 
 	@Override

src/main/java/org/genesys2/server/model/impl/TraitIntValue.java

@@ -17,19 +17,19 @@
 package org.genesys2.server.model.impl;
 
 import javax.persistence.Column;
+import javax.persistence.DiscriminatorValue;
 import javax.persistence.Entity;
-import javax.persistence.Index;
-import javax.persistence.Table;
 
 /**
  * A Descriptor represents a dimension of a sparse vector.
  *
  * @author mobreza
  */
-@Entity(name = "datasetint")
-@Table(indexes={ @Index(name="IX_datasetint_val", unique=false, columnList="descriptorId, vali") }) 
+@Entity
+//@Table(name = "datasetint", indexes={ @Index(name="IX_datasetint_val", unique=false, columnList="descriptorId, vali") }) 
+@DiscriminatorValue("1")
 public class TraitIntValue extends DatasetRow<Integer> {
-	@Column(nullable = false, name="vali")
+	@Column(name="vali")
 	private Integer val;
 
 	@Override

src/main/java/org/genesys2/server/model/impl/TraitStrValue.java

@@ -17,17 +17,21 @@
 package org.genesys2.server.model.impl;
 
 import javax.persistence.Column;
+import javax.persistence.DiscriminatorValue;
 import javax.persistence.Entity;
+import javax.persistence.Table;
 
 /**
  * A Descriptor represents a dimension of a sparse vector.
  *
  * @author mobreza
  */
-@Entity(name = "datasetstr")
+@Entity
+//@Table(name = "datasetstr")
+@DiscriminatorValue("3")
 public class TraitStrValue extends DatasetRow<String> {
 
-	@Column(nullable = false, name="vals")
+	@Column(name="vals")
 	private String val;
 
 	@Override

src/main/java/org/genesys2/server/persistence/domain/DatasetRepository.java

@@ -16,8 +16,12 @@
 
 package org.genesys2.server.persistence.domain;
 
+import java.util.UUID;
+
 import org.genesys2.server.model.impl.Dataset;
 import org.springframework.data.jpa.repository.JpaRepository;
 
 public interface DatasetRepository extends JpaRepository<Dataset, Long> {
+
+	Dataset findByUuid(UUID uuid);
 }

src/main/java/org/genesys2/server/persistence/domain/DatasetRowRepository.java

@@ -16,13 +16,21 @@
 
 package org.genesys2.server.persistence.domain;
 
+import java.util.List;
+import java.util.Set;
+
 import org.genesys2.server.model.impl.Dataset;
 import org.genesys2.server.model.impl.DatasetRow;
+import org.genesys2.server.model.impl.Descriptor;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
 
 public interface DatasetRowRepository extends JpaRepository<DatasetRow<?>, Long> {
 
 	Page<DatasetRow<?>> findByDataset(Dataset ds, Pageable page);
+
+	@Query("select dr from DatasetRow dr where dr.dataset=?1 and dr.descriptor=?2 and dr.accession.id in ( ?3 )")
+	List<DatasetRow<?>> list(Dataset dataset, Descriptor descriptor, Set<Long> ids);
 }

src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepositoryImpl.java → src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepositoryCustomImpl.java

@@ -41,9 +41,8 @@ import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
 @Repository
-@Transactional(readOnly = true)
-public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository {
-	public static final Log LOG = LogFactory.getLog(GenesysLowlevelRepositoryImpl.class);
+public class GenesysLowlevelRepositoryCustomImpl implements GenesysLowlevelRepository {
+	public static final Log LOG = LogFactory.getLog(GenesysLowlevelRepositoryCustomImpl.class);
 
 	@Autowired
 	private JdbcTemplate jdbcTemplate;
@@ -217,7 +216,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 
@@ -231,7 +230,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 
@@ -258,7 +257,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 		directQuery.sort(sort);
@@ -286,7 +285,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 
@@ -314,7 +313,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 
@@ -356,7 +355,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 		directQuery.jsonFilter(filter, new MethodResolver() {
 			@Override
 			public Method getMethod(final long methodId) {
-				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+				return GenesysLowlevelRepositoryCustomImpl.this.methodRepository.findOne(methodId);
 			}
 		});
 

src/main/java/org/genesys2/server/service/Dataset2Service.java

@@ -16,6 +16,30 @@
 
 package org.genesys2.server.service;
 
+import java.nio.MappedByteBuffer;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.genesys2.server.model.impl.Dataset;
+import org.genesys2.server.model.impl.DatasetRow;
+import org.genesys2.server.model.impl.Descriptor;
+
 public interface Dataset2Service {
 
+	Dataset getDataSet(UUID uuid);
+
+	Dataset saveDataset(Dataset dataset);
+
+	Descriptor getDescriptor(Dataset dataset, String variableName);
+
+	Descriptor saveDescriptor(Descriptor descriptor);
+
+	List<DatasetRow<?>> getForAccessionIds(Dataset dataset, Descriptor descriptor, Set<Long> ids);
+
+	void saveData(List<DatasetRow<?>> toSave);
+
+	void worldclimUpdate(Dataset dataset, Descriptor descriptor, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor);
+
+
 }

src/main/java/org/genesys2/server/service/impl/Dataset2ServiceImpl.java

@@ -16,12 +16,184 @@
 
 package org.genesys2.server.service.impl;
 
+import java.nio.MappedByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.Predicate;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.genesys.worldclim.WorldClimUtil;
+import org.genesys2.server.model.genesys.AccessionGeo;
+import org.genesys2.server.model.impl.Dataset;
+import org.genesys2.server.model.impl.DatasetRow;
+import org.genesys2.server.model.impl.Descriptor;
+import org.genesys2.server.model.impl.TraitDblValue;
+import org.genesys2.server.model.impl.TraitIntValue;
+import org.genesys2.server.persistence.domain.AccessionGeoRepository;
+import org.genesys2.server.persistence.domain.AccessionRepository;
+import org.genesys2.server.persistence.domain.DatasetRepository;
+import org.genesys2.server.persistence.domain.DatasetRowRepository;
+import org.genesys2.server.persistence.domain.DescriptorRepository;
 import org.genesys2.server.service.Dataset2Service;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 @Service
-@Transactional
+@Transactional(readOnly = true)
 public class Dataset2ServiceImpl implements Dataset2Service {
+	private final Log LOG = LogFactory.getLog(getClass());
+
+	@Autowired
+	private DatasetRepository datasetRepository;
+
+	@Autowired
+	private DescriptorRepository descriptorRepository;
+
+	@Autowired
+	private DatasetRowRepository datasetRowRepository;
+
+	@Autowired
+	private AccessionRepository accessionRepository;
+
+	@Autowired
+	private AccessionGeoRepository accessionGeoRepository;
+
+	@Override
+	@Transactional
+	public Dataset getDataSet(UUID uuid) {
+		return datasetRepository.findByUuid(uuid);
+	}
+
+	@Override
+	@Transactional
+	public Dataset saveDataset(Dataset dataset) {
+		return datasetRepository.save(dataset);
+	}
+
+	@Override
+	public Descriptor getDescriptor(Dataset dataset, String variableName) {
+		return descriptorRepository.findByCode(variableName);
+	}
+
+	@Override
+	@Transactional
+	public Descriptor saveDescriptor(Descriptor descriptor) {
+		return descriptorRepository.save(descriptor);
+	}
+
+	@Override
+	public List<DatasetRow<?>> getForAccessionIds(Dataset dataset, Descriptor descriptor, Set<Long> ids) {
+		return datasetRowRepository.list(dataset, descriptor, ids);
+	}
+
+	@Override
+	@Transactional
+	public void saveData(List<DatasetRow<?>> toSave) {
+		datasetRowRepository.save(toSave);
+	}
+
+	/**
+	 * Updates worldclim data for accessions
+	 * 
+	 * @param ids
+	 * @param buffer
+	 * @param nullValue
+	 */
+	@Override
+	@Transactional
+	public void worldclimUpdate(Dataset dataset, Descriptor descriptor, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor) {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Updating " + ids + " accessionIds with " + descriptor.getCode());
+			LOG.debug("IDs: " + ArrayUtils.toString(ids.toArray(), "!"));
+		}
+		List<AccessionGeo> acGeo = accessionGeoRepository.findForAccessions(ids);
+		List<DatasetRow<?>> existing = getForAccessionIds(dataset, descriptor, ids);
+		LOG.info("Found " + existing.size() + " existing datasetRows");
+
+		for (DatasetRow<?> dr : existing) {
+			ids.remove(dr.getAccession().getId());
+		}
+
+		List<DatasetRow<?>> all = new ArrayList<DatasetRow<?>>(existing);
+		List<DatasetRow<?>> toSave = new ArrayList<DatasetRow<?>>();
+
+		// New ones
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Generating " + ids.size() + " new datasetRows");
+		}
+		for (Long id : ids) {
+			DatasetRow<?> tv = null;
+			// TODO FIXME Figure this out!
+			if (factor < 1.0) {
+				tv = new TraitDblValue();
+			} else if (factor == 1.0) {
+				tv = new TraitIntValue();
+			}
+			tv.setAccession(accessionRepository.findOne(id));
+			tv.setDataset(dataset);
+			tv.setDescriptor(descriptor);
+			all.add(tv);
+		}
+