diff --git a/src/main/java/org/genesys2/server/model/dataset/DS.java b/src/main/java/org/genesys2/server/model/dataset/DS.java index bd8be10c3427a2ae2a6e4c596e6ad5040e9fc1dc..d0b3a8b8ebca04ee6b8a92e967fa82df379495b2 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DS.java +++ b/src/main/java/org/genesys2/server/model/dataset/DS.java @@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset; import java.util.List; import java.util.UUID; +import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.OneToMany; @@ -23,12 +24,12 @@ public class DS extends BasicModel { private UUID uuid; @OrderBy("index") - @OneToMany(mappedBy = "dataset") + @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL }) private List qualifiers; @OrderBy("index") - @OneToMany(mappedBy = "dataset") - private List descriptors; + @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL }) + private List columns; /** * Generate UUID if missing @@ -56,11 +57,11 @@ public class DS extends BasicModel { this.qualifiers = qualifiers; } - public List getDescriptors() { - return descriptors; + public List getColumns() { + return columns; } - public void setDescriptors(List descriptors) { - this.descriptors = descriptors; + public void setColumns(List columns) { + this.columns = columns; } } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSColumn.java b/src/main/java/org/genesys2/server/model/dataset/DSColumn.java new file mode 100644 index 0000000000000000000000000000000000000000..b117ac25f94f1d88a05fbd1ed90314902e621a56 --- /dev/null +++ b/src/main/java/org/genesys2/server/model/dataset/DSColumn.java @@ -0,0 +1,62 @@ +package org.genesys2.server.model.dataset; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import org.genesys.blocks.model.BasicModel; + +/** + * Dataset column. + */ +@Entity +@Table(name = "ds2column") +public class DSColumn extends BasicModel { + + /** + * + */ + private static final long serialVersionUID = -7693331325536594367L; + + @Column(name="idx") + private float index = 1.0f; + + @ManyToOne(optional = false) + @JoinColumn(name="ds") + private DS dataset; + + @ManyToOne(optional = false) + @JoinColumn(name="d") + private DSDescriptor descriptor; + + public float getIndex() { + return index; + } + + public void setIndex(float index) { + this.index = index; + } + + public DS getDataset() { + return dataset; + } + + public void setDataset(DS dataset) { + this.dataset = dataset; + } + + public DSDescriptor getDescriptor() { + return descriptor; + } + + public void setDescriptor(DSDescriptor descriptor) { + this.descriptor = descriptor; + } + + @Override + public String toString() { + return "DSD " + this.getId(); + } +} diff --git a/src/main/java/org/genesys2/server/model/dataset/DSDescriptor.java b/src/main/java/org/genesys2/server/model/dataset/DSDescriptor.java index 9f39fc801efd232b6f90f3feca4b70dddb8c787c..fec64b5991899283ca7ef6950adf2055829b5b27 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSDescriptor.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSDescriptor.java @@ -1,63 +1,114 @@ +/** + * Copyright 2014 Global Crop Diversity Trust + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + package org.genesys2.server.model.dataset; +import java.text.MessageFormat; +import java.util.UUID; + import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; +import javax.persistence.Lob; +import javax.persistence.PrePersist; import javax.persistence.Table; import org.genesys.blocks.model.BasicModel; -import org.genesys2.server.model.impl.Descriptor; +import org.hibernate.annotations.Type; /** - * Dataset descriptor + * A DSDescriptor represents a dimension of a sparse vector. + * + * @author mobreza */ @Entity @Table(name = "ds2descriptor") public class DSDescriptor extends BasicModel { + private static final long serialVersionUID = 3832200593904442940L; + + @Column(unique = true, nullable=false, updatable = false) + @Type(type = "uuid-binary") + private UUID uuid; + + @Lob + @Type(type = "org.hibernate.type.TextType") + private String title; + + @Column(nullable = false, length = 200) + private String code; + + @Lob + @Type(type = "org.hibernate.type.TextType") + private String description; + + @Column(length = 20) + private String uom; /** - * + * Generate UUID if missing */ - private static final long serialVersionUID = -7693331325536594367L; + @PrePersist + protected void prepersist() { + if (this.uuid == null) { + this.uuid = UUID.randomUUID(); + } + } - @Column(name="idx") - private float index = 1.0f; + public UUID getUuid() { + return uuid; + } + + public void setUuid(UUID uuid) { + this.uuid = uuid; + } - @ManyToOne(optional = false) - @JoinColumn(name="ds") - private DS dataset; + + public String getTitle() { + return title; + } - @ManyToOne(optional = false) - @JoinColumn(name="d") - private Descriptor descriptor; + public void setTitle(String title) { + this.title = title; + } - public float getIndex() { - return index; + public String getCode() { + return code; } - public void setIndex(float index) { - this.index = index; + public void setCode(String code) { + this.code = code; } - public DS getDataset() { - return dataset; + public String getDescription() { + return description; } - public void setDataset(DS dataset) { - this.dataset = dataset; + public void setDescription(final String description) { + this.description = description; } - public Descriptor getDescriptor() { - return descriptor; + public String getUom() { + return uom; } - public void setDescriptor(Descriptor descriptor) { - this.descriptor = descriptor; + public void setUom(String uom) { + this.uom = uom; } - + @Override public String toString() { - return "DSD " + this.getId(); + return MessageFormat.format("DSDescriptor id={0,number,#} name={1}", getId(), code); } } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSQualifier.java b/src/main/java/org/genesys2/server/model/dataset/DSQualifier.java index 39761e5d38dc9036710501fecc11c3dce11bc5e9..cb1d93fd7a00865442b5a017c2e43fef8813f360 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSQualifier.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSQualifier.java @@ -7,10 +7,9 @@ import javax.persistence.ManyToOne; import javax.persistence.Table; import org.genesys.blocks.model.BasicModel; -import org.genesys2.server.model.impl.Descriptor; /** - * Dataset qualifier specification + * The dataset qualifier is a key lookup component for dataset rows. */ @Entity @Table(name = "ds2qualifier") @@ -30,7 +29,7 @@ public class DSQualifier extends BasicModel { @ManyToOne(optional = false) @JoinColumn(name="d") - private Descriptor descriptor; + private DSDescriptor descriptor; public float getIndex() { return index; @@ -48,11 +47,11 @@ public class DSQualifier extends BasicModel { this.dataset = dataset; } - public Descriptor getDescriptor() { + public DSDescriptor getDescriptor() { return descriptor; } - public void setDescriptor(Descriptor descriptor) { + public void setDescriptor(DSDescriptor descriptor) { this.descriptor = descriptor; } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRow.java b/src/main/java/org/genesys2/server/model/dataset/DSRow.java index 90405d5c1eaccf6a90485f74c15a38968a29d115..2fbcd61f4b0265152cb437d8a060ffc0dc9683c3 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRow.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRow.java @@ -1,5 +1,6 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; import java.util.List; import javax.persistence.CascadeType; @@ -12,23 +13,36 @@ import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.OneToMany; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.apache.commons.codec.digest.DigestUtils; import org.genesys.blocks.model.EntityId; +/** + * Represents a row in a DS2 dataset. The row has a unique {@link #id} and + * groups together different {@link DSValue} values (column values). + * + * The row is qualified by a list of row qualifiers ({@link DSRowQualifier}) + * that represent the "keys" of the row. Their SHA-1 and MD5 sums are calculated + * and stored in {@link #sha1} and {@link #md5} respectively. + * + * @author Matija Obreza + */ @Entity @Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) }) public class DSRow implements EntityId { - @ManyToOne(fetch=FetchType.LAZY, optional = false) - @JoinColumn(name="ds") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "ds") private DS dataset; - - @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.REMOVE }) + + @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL }) private List> rowQualifiers; - @OneToMany(fetch = FetchType.LAZY, mappedBy = "row") + @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL }) private List> values; @Id @@ -41,6 +55,19 @@ public class DSRow implements EntityId { @Column(columnDefinition = "binary(20)", updatable = false) private byte[] sha1; + @PrePersist + @PreUpdate + private void prePersist() { + ByteBuffer keyBuffer = ByteBuffer.allocate(500); + for (DSRowQualifier dsq : this.getRowQualifiers()) { + dsq.putKey(keyBuffer); + } + + byte[] array = keyBuffer.array(); + this.md5 = DigestUtils.md5(array); + this.sha1 = DigestUtils.sha1(array); + } + @Override public Long getId() { return this.id; @@ -53,11 +80,11 @@ public class DSRow implements EntityId { public DS getDataset() { return dataset; } - + public void setDataset(DS dataset) { this.dataset = dataset; } - + public List> getRowQualifiers() { return rowQualifiers; } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java index 404ac7bd1729211bc533adc152205a031f3e71c5..6a0fe161b69f77deddcb60fe1480c2a71c36da97 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java @@ -1,5 +1,7 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; + import javax.persistence.DiscriminatorColumn; import javax.persistence.DiscriminatorType; import javax.persistence.Entity; @@ -14,6 +16,16 @@ import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; +/** + * Abstract class {@link DSRowQualifier} represents the "columns" of a + * {@link DSRow} that serve as "keys" or qualifying columns to a row. A row may + * have more than one qualifier and are not necessarily unique, but provide a + * fast mechanism to look up values in a row ({@link DSValue}). + * + * @author Matija Obreza + * + * @param type of qualifier value + */ @Entity @Table(name = "ds2rowqualifier", indexes = { @Index(columnList = "dsq, vall") }) @Inheritance(strategy = InheritanceType.SINGLE_TABLE) @@ -70,4 +82,5 @@ public abstract class DSRowQualifier { return null; } + public abstract void putKey(ByteBuffer keyBuffer); } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java index 2d8757bc477b994200049a941d884adfa0e04837..85e32fecd61b471bfa64a108fe082dad143b10aa 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java @@ -1,5 +1,7 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; + import javax.persistence.Column; import javax.persistence.DiscriminatorValue; import javax.persistence.Entity; @@ -20,5 +22,9 @@ public class DSRowQualifierLong extends DSRowQualifier { public void setValue(Long value) { this.value = value; } - + + @Override + public void putKey(ByteBuffer keyBuffer) { + keyBuffer.putLong(this.value); + } } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSValue.java b/src/main/java/org/genesys2/server/model/dataset/DSValue.java index 3014f9f542ceced2e7e63717104e0291f9afb2f6..3179528140bfab9aae9aeecf1b090f5e1f56093f 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSValue.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSValue.java @@ -15,7 +15,7 @@ import javax.persistence.ManyToOne; import javax.persistence.Table; @Entity -@Table(name = "ds2value", indexes = { @Index(columnList = "r,dsd"), @Index(columnList = "dsd,vall"), @Index(columnList = "dsd,vald") }) +@Table(name = "ds2value", indexes = { @Index(columnList = "r,dsc"), @Index(columnList = "dsc,vall"), @Index(columnList = "dsc,vald") }) @Inheritance(strategy = InheritanceType.SINGLE_TABLE) @DiscriminatorColumn(discriminatorType = DiscriminatorType.INTEGER, name = "typ") public abstract class DSValue { @@ -28,8 +28,8 @@ public abstract class DSValue { private DSRow row; @ManyToOne(fetch=FetchType.LAZY, optional = false) - @JoinColumn(name = "dsd") - private DSDescriptor datasetDescriptor; + @JoinColumn(name = "dsc") + private DSColumn datasetColumn; public abstract T getValue(); @@ -51,12 +51,12 @@ public abstract class DSValue { this.row = row; } - public DSDescriptor getDatasetDescriptor() { - return datasetDescriptor; + public DSColumn getDatasetColumn() { + return datasetColumn; } - public void setDatasetDescriptor(DSDescriptor datasetDescriptor) { - this.datasetDescriptor = datasetDescriptor; + public void setDatasetColumn(DSColumn datasetColumn) { + this.datasetColumn = datasetColumn; } public static DSValue make(Object v) { diff --git a/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java b/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java index c39bfb29da26141dfeacaca76b0d516896187d50..f264214117f28fe32d5423d57a0851822bd0fbec 100644 --- a/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java +++ b/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java @@ -21,12 +21,15 @@ import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; import javax.persistence.OneToOne; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; import javax.persistence.Table; import javax.persistence.Version; import org.apache.commons.lang.StringUtils; import org.genesys.blocks.auditlog.annotations.Audited; import org.genesys.blocks.model.BasicModel; +import org.genesys.worldclim.WorldClimUtil; import org.genesys2.server.model.impl.GeoReferencedEntity; @Entity @@ -57,6 +60,15 @@ public class AccessionGeo extends BasicModel implements GeoReferencedEntity, Acc private Long tileIndex; + /** + * Recalculate {@link #tileIndex} on insert and update + */ + @PrePersist + @PreUpdate + private void prePersist() { + tileIndex = WorldClimUtil.getTileIndex(5, this.longitude, this.latitude); + } + public long getVersion() { return version; } diff --git a/src/main/java/org/genesys2/server/model/impl/Descriptor.java b/src/main/java/org/genesys2/server/model/impl/Descriptor.java deleted file mode 100644 index f99be0de1715e69c6cd104c8baeab9021dd937c3..0000000000000000000000000000000000000000 --- a/src/main/java/org/genesys2/server/model/impl/Descriptor.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2014 Global Crop Diversity Trust - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -package org.genesys2.server.model.impl; - -import java.text.MessageFormat; -import java.util.UUID; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Lob; -import javax.persistence.PrePersist; -import javax.persistence.Table; - -import org.genesys.blocks.model.BasicModel; -import org.hibernate.annotations.Type; - -/** - * A Descriptor represents a dimension of a sparse vector. - * - * @author mobreza - */ -@Entity -@Table(name = "descriptor") -public class Descriptor extends BasicModel { - private static final long serialVersionUID = 3832200593904442940L; - - @Column(unique = true, nullable=false, updatable = false) - @Type(type = "uuid-binary") - private UUID uuid; - - @Lob - @Type(type = "org.hibernate.type.TextType") - private String title; - - @Column(nullable = false, length = 200) - private String code; - - @Lob - @Type(type = "org.hibernate.type.TextType") - private String description; - - @Column(length = 20) - private String uom; - - /** - * Generate UUID if missing - */ - @PrePersist - protected void prepersist() { - if (this.uuid == null) { - this.uuid = UUID.randomUUID(); - } - } - - public UUID getUuid() { - return uuid; - } - - public void setUuid(UUID uuid) { - this.uuid = uuid; - } - - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public String getDescription() { - return description; - } - - public void setDescription(final String description) { - this.description = description; - } - - public String getUom() { - return uom; - } - - public void setUom(String uom) { - this.uom = uom; - } - - @Override - public String toString() { - return MessageFormat.format("Descriptor id={0,number,#} name={1}", getId(), code); - } -} diff --git a/src/main/java/org/genesys2/server/model/json/WorldclimJson.java b/src/main/java/org/genesys2/server/model/json/WorldclimJson.java index e5b8a9389be706dbf6184a246993a83fff14148d..228f2323762ae78de22c1bf3ab02e48cb0cf9181 100644 --- a/src/main/java/org/genesys2/server/model/json/WorldclimJson.java +++ b/src/main/java/org/genesys2/server/model/json/WorldclimJson.java @@ -3,7 +3,7 @@ package org.genesys2.server.model.json; import java.util.HashMap; import java.util.Map; -import org.genesys2.server.model.impl.Descriptor; +import org.genesys2.server.model.dataset.DSDescriptor; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -18,7 +18,7 @@ public class WorldclimJson { private Double[] tempMax = new Double[12]; @JsonIgnore - private Map other = new HashMap(); + private Map other = new HashMap(); public Long[] getPrecipitation() { return precipitation; @@ -36,11 +36,11 @@ public class WorldclimJson { return tempMin; } - public void addOther(Descriptor descriptor, Object val) { - other.put(descriptor, val); + public void addOther(DSDescriptor dSDescriptor, Object val) { + other.put(dSDescriptor, val); } - public Map getOther() { + public Map getOther() { return this.other; } } diff --git a/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java b/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java index 780cc4f009742d7f491a28885ceb40687e6ac9e5..f8bfc3cfd063fbc70015a080a1a1d57800085357 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java @@ -43,4 +43,7 @@ public interface AccessionGeoRepository extends JpaRepository getTileIndexes(); + @Query("select distinct(ag.accession.id) from AccessionGeo ag where ag.longitude between -180 and 180 and ag.latitude between -90 and 90 and ag.tileIndex is null") + Set withMissingTileIndex(); + } diff --git a/src/main/java/org/genesys2/server/persistence/domain/DescriptorRepository.java b/src/main/java/org/genesys2/server/persistence/domain/DescriptorRepository.java deleted file mode 100644 index c7ee46025143b63289346183a71f8e4b44600853..0000000000000000000000000000000000000000 --- a/src/main/java/org/genesys2/server/persistence/domain/DescriptorRepository.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2014 Global Crop Diversity Trust - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -package org.genesys2.server.persistence.domain; - -import org.genesys2.server.model.impl.Descriptor; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface DescriptorRepository extends JpaRepository { - Descriptor findByCode(String code); -} diff --git a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSColumnRepository.java b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSColumnRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..10b8ae31e8c5077a104563b17a511f875ed7f1f0 --- /dev/null +++ b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSColumnRepository.java @@ -0,0 +1,8 @@ +package org.genesys2.server.persistence.domain.dataset; + +import org.genesys2.server.model.dataset.DSColumn; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface DSColumnRepository extends JpaRepository { + +} diff --git a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSDescriptorRepository.java b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSDescriptorRepository.java index ed310aef1bee5c87b8ba219c7c76baabaaccfa7c..0ee45ecb28920a9387a4d70d2df4982f5d9f6358 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSDescriptorRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSDescriptorRepository.java @@ -1,8 +1,24 @@ +/** + * Copyright 2014 Global Crop Diversity Trust + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + package org.genesys2.server.persistence.domain.dataset; import org.genesys2.server.model.dataset.DSDescriptor; import org.springframework.data.jpa.repository.JpaRepository; public interface DSDescriptorRepository extends JpaRepository { - + DSDescriptor findByCode(String code); } diff --git a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java index a390e14ccaadc05fe284d9fdf5eb2c8007578598..195284c6f757227b1e53f1de3cba5d73901ddd25 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java @@ -5,7 +5,7 @@ import java.util.List; import java.util.Map; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSValue; @@ -13,12 +13,12 @@ public interface DSRowCustomRepository { DSRow findRow(DS ds, Object[] qualifiers); - Map> rowValueMap(Collection dsrs, DSDescriptor dsd); + Map> rowValueMap(Collection dsrs, DSColumn dsd); List getQualifiers(DS ds); /** - * Array of dsv.values as per DSDescriptor order + * Array of dsv.values as per DSColumn order * * @param rowId * @param columnDescriptors @@ -27,7 +27,7 @@ public interface DSRowCustomRepository { Object[] getRowValues(long rowId, Long[] columnDescriptors); /** - * List of arrays of dsv.values as per rowId order and DSDescriptor order + * List of arrays of dsv.values as per rowId order and DSColumn order * * @param rowId * @param columnDescriptors diff --git a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java index 3ca5dbd098745d3f49df0a5a00ae606f7667a2de..285b08b6fc40f50ec3152bcfe4c6a442e8c6cf32 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java +++ b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java @@ -13,7 +13,7 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSValue; @@ -74,7 +74,7 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { } @Override - public Map> rowValueMap(Collection dsrs, DSDescriptor dsd) { + public Map> rowValueMap(Collection dsrs, DSColumn dsc) { List validRows = new ArrayList(dsrs.size()); for (DSRow dsr : dsrs) { if (dsr.getId() != null) { @@ -86,9 +86,9 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { if (!validRows.isEmpty()) { // LOG.info("getting row value map"); - Query q = entityManager.createQuery("select dsv.row.id, dsv from DSValue dsv where dsv.row in (?1) and dsv.datasetDescriptor=?2"); + Query q = entityManager.createQuery("select dsv.row.id, dsv from DSValue dsv where dsv.row in (?1) and dsv.datasetColumn=?2"); q.setParameter(1, validRows); - q.setParameter(2, dsd); + q.setParameter(2, dsc); @SuppressWarnings("unchecked") List res = q.getResultList(); @@ -102,7 +102,7 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { } /** - * Return row.id, dsdq1.value, dsdq2.value... + * Return row.id, dscq1.value, dscq2.value... */ @SuppressWarnings("unchecked") @Override @@ -151,19 +151,19 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { @Override public Object[] getRowValues(long rowId, Long[] columnDescriptors) { - Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv from DSValue dsrv where dsrv.row.id = ?1"); + Query q = entityManager.createQuery("select dsrv.datasetColumn.id, dsrv from DSValue dsrv where dsrv.row.id = ?1"); q.setParameter(1, rowId); - // List of [dsd.id, dsrv] + // List of [dsc.id, dsrv] @SuppressWarnings("unchecked") List values = q.getResultList(); Object[] result = new Object[columnDescriptors.length]; for (Object[] v : values) { - Long dsdId = (Long) v[0]; + Long dscId = (Long) v[0]; for (int i = 0; i < columnDescriptors.length; i++) { - if (columnDescriptors[i].equals(dsdId)) { + if (columnDescriptors[i].equals(dscId)) { result[i] = ((DSValue) v[1]).getValue(); } } @@ -174,9 +174,9 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { @Override public List getRowValues(List rowIds, Long[] datasetDescriptors) { - Query q = entityManager.createQuery("select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids) and dsrv.datasetDescriptor.id in (:dsds)"); + Query q = entityManager.createQuery("select dsrv.datasetColumn.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids) and dsrv.datasetColumn.id in (:dscs)"); q.setParameter("ids", rowIds); - q.setParameter("dsds", Arrays.asList(datasetDescriptors)); + q.setParameter("dscs", Arrays.asList(datasetDescriptors)); // LOG.info("Query ready"); // Initialize results list with blank arrays @@ -186,17 +186,17 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository { } List values = q.getResultList(); - // List of [dsd.id, dsrv] + // List of [dsc.id, dsrv] for (Object[] v : values) { // LOG.info("Got next"); - Long dsdId = (Long) v[0]; + Long dscId = (Long) v[0]; Long rowId = (Long) v[1]; DSValue dsv = (DSValue) v[2]; entityManager.detach(dsv); Object[] result = results.get(rowIds.indexOf(rowId)); for (int i = 0; i < datasetDescriptors.length; i++) { - if (datasetDescriptors[i].equals(dsdId)) { + if (datasetDescriptors[i].equals(dscId)) { result[i] = dsv.getValue(); } } diff --git a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSValueRepository.java b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSValueRepository.java index 361ad3003cef8c5efbf7707ce1d7bde79e717f81..5a3686bcae470ea12d3c4c867e8c67382f9efec8 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/dataset/DSValueRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/dataset/DSValueRepository.java @@ -1,6 +1,6 @@ package org.genesys2.server.persistence.domain.dataset; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSValue; import org.springframework.data.jpa.repository.JpaRepository; @@ -9,10 +9,10 @@ import org.springframework.data.jpa.repository.Query; public interface DSValueRepository extends JpaRepository, Long> { - DSValue findByRowAndDatasetDescriptor(DSRow dsr, DSDescriptor dsd); + DSValue findByRowAndDatasetColumn(DSRow dsr, DSColumn dsd); @Modifying - @Query("delete from DSValue dsv where dsv.datasetDescriptor = ?1") - int deleteFor(DSDescriptor dsd); + @Query("delete from DSValue dsv where dsv.datasetColumn = ?1") + int deleteFor(DSColumn dsd); } diff --git a/src/main/java/org/genesys2/server/service/DSService.java b/src/main/java/org/genesys2/server/service/DSService.java index 0a98c7310edae06ba06e87fc39266cdc494d7e5e..2d6dbe1cb4e7ebd75708a8fdd7a459cba4b4e36c 100644 --- a/src/main/java/org/genesys2/server/service/DSService.java +++ b/src/main/java/org/genesys2/server/service/DSService.java @@ -8,27 +8,27 @@ import java.util.Set; import java.util.UUID; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSQualifier; +import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.genesys.AccessionGeo; -import org.genesys2.server.model.impl.Descriptor; import org.genesys2.server.model.json.WorldclimJson; public interface DSService { - void saveDataset(DS ds); + DS saveDataset(DS ds); - DSQualifier addQualifier(DS ds, Descriptor d1); - DSDescriptor addDescriptor(DS ds, Descriptor descriptor); + DSQualifier addQualifier(DS ds, DSDescriptor d1); + DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor); DS loadDatasetByUuid(UUID uuid); void updateRow(DS ds, Object[] row); void updateRows(DS ds, List rows); - void updateRow(DS ds, Object[] qualifiers, DSDescriptor dsd, Object value); + void updateRow(DS ds, Object[] qualifiers, DSColumn dsd, Object value); - void worldclimUpdate(DS dataset, DSDescriptor dsd, Set ids, MappedByteBuffer buffer, short nullValue, double factor); + void worldclimUpdate(DS dataset, DSColumn dsd, Set ids, MappedByteBuffer buffer, short nullValue, double factor); void updateAccessionTileIndex(List toSave); @@ -36,11 +36,11 @@ public interface DSService { void download(DS ds, OutputStream outputStream) throws IOException; - void download(DS ds, List dsds, OutputStream outputStream) throws IOException; + void download(DS ds, List dsds, OutputStream outputStream) throws IOException; void deleteDataset(DS ds); - void deleteDescriptor(DSDescriptor dsd); + void deleteDescriptor(DSColumn dsd); void deleteRows(DS ds); } diff --git a/src/main/java/org/genesys2/server/service/DescriptorService.java b/src/main/java/org/genesys2/server/service/DescriptorService.java index 07978c68e81d17703aaecddc1686e19be982c74e..34bff213b4995dc484a7e2a6e6e683ef316ed09f 100644 --- a/src/main/java/org/genesys2/server/service/DescriptorService.java +++ b/src/main/java/org/genesys2/server/service/DescriptorService.java @@ -18,13 +18,13 @@ package org.genesys2.server.service; import java.util.List; -import org.genesys2.server.model.impl.Descriptor; +import org.genesys2.server.model.dataset.DSDescriptor; public interface DescriptorService { - List list(); + List list(); - void saveDescriptor(Descriptor descriptor); + DSDescriptor saveDescriptor(DSDescriptor dSDescriptor); - Descriptor getDescriptor(String variableName); + DSDescriptor getDescriptor(String variableName); } diff --git a/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java index 247f7cc7b2700be625d6fff6738c5a0d775de009..241f5828cb1248420e7b8416b1521df292fd33a8 100644 --- a/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java @@ -2,9 +2,7 @@ package org.genesys2.server.service.impl; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -18,24 +16,22 @@ import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.genesys2.server.model.dataset.DS; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSRowQualifier; import org.genesys2.server.model.dataset.DSValue; import org.genesys2.server.model.genesys.AccessionGeo; -import org.genesys2.server.model.impl.Descriptor; import org.genesys2.server.model.json.WorldclimJson; import org.genesys2.server.persistence.domain.AccessionGeoRepository; -import org.genesys2.server.persistence.domain.DescriptorRepository; -import org.genesys2.server.persistence.domain.dataset.DSDescriptorRepository; +import org.genesys2.server.persistence.domain.dataset.DSColumnRepository; import org.genesys2.server.persistence.domain.dataset.DSQualifierRepository; import org.genesys2.server.persistence.domain.dataset.DSRepository; import org.genesys2.server.persistence.domain.dataset.DSRowQualifierRepository; @@ -53,12 +49,10 @@ import org.springframework.transaction.annotation.Transactional; public class DSServiceImpl implements DSService { public static final Logger LOG = LoggerFactory.getLogger(DSServiceImpl.class); - private static final Charset CHARSET_UTF8 = Charset.forName("UTF8"); - @Autowired private DSRepository dsRepo; @Autowired - private DSDescriptorRepository dsDescRepo; + private DSColumnRepository dsColRepo; @Autowired private DSQualifierRepository dsQualRepo; @Autowired @@ -67,21 +61,19 @@ public class DSServiceImpl implements DSService { private DSRowQualifierRepository dsRowQualiRepo; @Autowired private DSValueRepository dsValueRepo; - @Autowired - private DescriptorRepository descriptorRepo; @Autowired private AccessionGeoRepository accessionGeoRepository; @Transactional @Override - public void saveDataset(DS ds) { - dsRepo.save(ds); + public DS saveDataset(DS ds) { + return dsRepo.save(ds); } @Transactional @Override - public DSQualifier addQualifier(DS ds, Descriptor d1) { + public DSQualifier addQualifier(DS ds, DSDescriptor d1) { DSQualifier dsq = new DSQualifier(); dsq.setDataset(ds); dsq.setDescriptor(d1); @@ -92,13 +84,13 @@ public class DSServiceImpl implements DSService { @Transactional @Override - public DSDescriptor addDescriptor(DS ds, Descriptor d1) { - DSDescriptor dsd = new DSDescriptor(); + public DSColumn addDescriptor(DS ds, DSDescriptor d1) { + DSColumn dsd = new DSColumn(); dsd.setDataset(ds); dsd.setDescriptor(d1); - dsd.setIndex(ds.getDescriptors().size() + 1); - ds.getDescriptors().add(dsd); - return dsDescRepo.save(dsd); + dsd.setIndex(ds.getColumns().size() + 1); + ds.getColumns().add(dsd); + return dsColRepo.save(dsd); } @Override @@ -106,7 +98,7 @@ public class DSServiceImpl implements DSService { DS ds = dsRepo.findByUuid(uuid); if (ds != null) { ds.getQualifiers().size(); - ds.getDescriptors().size(); + ds.getColumns().size(); } return ds; } @@ -119,7 +111,7 @@ public class DSServiceImpl implements DSService { public void updateRow(DS ds, Object[] row) { List dsqs = ds.getQualifiers(); LOG.info("DS.qualifiers: {}", dsqs); - List dsds = ds.getDescriptors(); + List dsds = ds.getColumns(); LOG.info("DS.descriptors: {}", dsds); Object[] qualifiers = new Object[dsqs.size()]; @@ -165,11 +157,11 @@ public class DSServiceImpl implements DSService { } DSValue dsrv = null; - DSDescriptor dsd = dsds.get(i); + DSColumn dsd = dsds.get(i); LOG.info("Looking for {}", dsd); for (DSValue dsrv1 : dsrvs) { - if (dsd.getId().equals(dsrv1.getDatasetDescriptor().getId())) { - LOG.info("Found matching dsd {}", dsrv1.getDatasetDescriptor()); + if (dsd.getId().equals(dsrv1.getDatasetColumn().getId())) { + LOG.info("Found matching dsd {}", dsrv1.getDatasetColumn()); dsrv = dsrv1; break; } @@ -178,7 +170,7 @@ public class DSServiceImpl implements DSService { if (dsrv == null) { LOG.info("Making new value for row {} and {}", dsr, dsd); dsrv = DSValue.make(v); - dsrv.setDatasetDescriptor(dsd); + dsrv.setDatasetColumn(dsd); dsrv.setRow(dsr); dsr.getValues().add(dsrv); } @@ -194,7 +186,7 @@ public class DSServiceImpl implements DSService { */ @Transactional @Override - public void updateRow(DS ds, Object[] qualifiers, DSDescriptor dsd, Object value) { + public void updateRow(DS ds, Object[] qualifiers, DSColumn dsd, Object value) { if (value == null) { return; } @@ -202,11 +194,11 @@ public class DSServiceImpl implements DSService { DSRow dsr = findOrMakeRow(ds, qualifiers); saveRow(dsr); - DSValue dsrv = dsValueRepo.findByRowAndDatasetDescriptor(dsr, dsd); + DSValue dsrv = dsValueRepo.findByRowAndDatasetColumn(dsr, dsd); if (dsrv == null) { LOG.debug("Making new value for row {} and {}", dsr, dsd); dsrv = DSValue.make(value); - dsrv.setDatasetDescriptor(dsd); + dsrv.setDatasetColumn(dsd); dsrv.setRow(dsr); } dsrv.setValue2(value); @@ -222,12 +214,12 @@ public class DSServiceImpl implements DSService { } private void saveRows(Collection dsrs) { +// ArrayList> quali = new ArrayList>(); +// for (DSRow dsr : dsrs) { +// quali.addAll(dsr.getRowQualifiers()); +// } +// dsRowQualiRepo.save(quali); dsRowRepo.save(dsrs); - ArrayList> quali = new ArrayList>(); - for (DSRow dsr : dsrs) { - quali.addAll(dsr.getRowQualifiers()); - } - dsRowQualiRepo.save(quali); } private DSRow findOrMakeRow(DS ds, Object[] qualifiers) { @@ -288,32 +280,17 @@ public class DSServiceImpl implements DSService { private DSRow makeRow(DS ds, Object... qualifiers) { DSRow dsr = new DSRow(); - - ByteBuffer keyBuffer = ByteBuffer.allocate(500); + dsr.setDataset(ds); List> rowQualifiers = new ArrayList>(); int i = 0; for (DSQualifier dsq : ds.getQualifiers()) { DSRowQualifier dsrq = DSRowQualifier.make(qualifiers[i]); - if (qualifiers[i] instanceof Long) { - keyBuffer.putLong(((Long) qualifiers[i]).longValue()); - } else if (qualifiers[i] instanceof String) { - keyBuffer.put(((String) qualifiers[i]).getBytes(CHARSET_UTF8)); - } else if (qualifiers[i] instanceof Double) { - keyBuffer.putDouble(((Double) qualifiers[i]).doubleValue()); - } dsrq.setDatasetQualifier(dsq); dsrq.setRow(dsr); rowQualifiers.add(dsrq); i++; } - // LOG.info("MD4 length: " + DigestUtils.md5(keyBuffer.array()).length); - // LOG.info("SHA1 length: " + - // DigestUtils.sha1(keyBuffer.array()).length); - dsr.setMd5(DigestUtils.md5(keyBuffer.array())); - dsr.setSha1(DigestUtils.sha1(keyBuffer.array())); - dsr.setDataset(ds); - dsr.setRowQualifiers(rowQualifiers); return dsr; } @@ -335,15 +312,13 @@ public class DSServiceImpl implements DSService { */ @Override @Transactional - public void worldclimUpdate(DS dataset, DSDescriptor dsd, Set tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) { - if (LOG.isInfoEnabled()) { - LOG.info("Updating {} for tileIndexes: {}", dsd.getDescriptor().getCode(), tileIndexes.size()); - } + public void worldclimUpdate(DS dataset, DSColumn dsc, Set tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) { + LOG.debug("Updating {} for tileIndexes: {}", dsc.getDescriptor().getCode(), tileIndexes.size()); Map rowMap = findOrMakeRows(dataset, tileIndexes); // Load all existing values for rows - Map> values = dsRowRepo.rowValueMap(rowMap.values(), dsd); + Map> values = dsRowRepo.rowValueMap(rowMap.values(), dsc); List> toSave = new ArrayList>(); for (final Long tileIndex : tileIndexes) { @@ -353,9 +328,7 @@ public class DSServiceImpl implements DSService { try { short val = buffer.getShort((int) (tileIndex * 2)); if (val != nullValue) { - if (LOG.isDebugEnabled()) { - LOG.debug("tile={} val={}", tileIndex, val); - } + LOG.trace("tile={} val={}", tileIndex, val); Object value = null; if (factor < 1.0) { @@ -372,9 +345,9 @@ public class DSServiceImpl implements DSService { dsrv = values.get(dsr.getId()); if (dsrv == null) { - LOG.debug("Making new value for row {} and {}", dsr, dsd); + LOG.debug("Making new value for row {} and {}", dsr, dsc); dsrv = DSValue.make(value); - dsrv.setDatasetDescriptor(dsd); + dsrv.setDatasetColumn(dsc); dsrv.setRow(dsr); } dsrv.setValue2(value); @@ -398,7 +371,7 @@ public class DSServiceImpl implements DSService { // Save values dsValueRepo.save(toSave); - LOG.info("Done saving."); + LOG.debug("Done saving."); } @Override @@ -415,11 +388,11 @@ public class DSServiceImpl implements DSService { WorldclimJson wc = new WorldclimJson(); for (DSValue v : dsr.getValues()) { - String variable = v.getDatasetDescriptor().getDescriptor().getCode(); + String variable = v.getDatasetColumn().getDescriptor().getCode(); Matcher matcher = Pattern.compile("(prec|tmin|tmean|tmax)(\\d{1,2})").matcher(variable); if (!matcher.matches()) { // Add to "Other" - wc.addOther(v.getDatasetDescriptor().getDescriptor(), v.getValue()); + wc.addOther(v.getDatasetColumn().getDescriptor(), v.getValue()); continue; } String varName = matcher.group(1); @@ -439,7 +412,7 @@ public class DSServiceImpl implements DSService { } @Override - public void download(DS ds, List dsds, OutputStream outputStream) throws IOException { + public void download(DS ds, List dsds, OutputStream outputStream) throws IOException { XSSFWorkbook template = new XSSFWorkbook(); // keep 1000 rows in memory, exceeding rows will be flushed to disk @@ -457,7 +430,7 @@ public class DSServiceImpl implements DSService { Long[] columnDescriptors = new Long[dsds.size()]; { int columnIndex = 0; - for (DSDescriptor dsd : dsds) { + for (DSColumn dsd : dsds) { r.createCell(cellIndex++).setCellValue(dsd.getDescriptor().getCode()); columnDescriptors[columnIndex++] = dsd.getId(); } @@ -512,7 +485,7 @@ public class DSServiceImpl implements DSService { @Override public void download(DS ds, OutputStream outputStream) throws IOException { - download(ds, ds.getDescriptors(), outputStream); + download(ds, ds.getColumns(), outputStream); } private void addCell(Row r, int cellIndex, Object v) { @@ -549,10 +522,10 @@ public class DSServiceImpl implements DSService { @Override @Transactional - public void deleteDescriptor(DSDescriptor dsd) { + public void deleteDescriptor(DSColumn dsd) { LOG.info("Clearing DSValues for {}", dsd.getDescriptor().getCode()); int count = dsValueRepo.deleteFor(dsd); LOG.info("Deleted {} DSValue cells", count); - dsDescRepo.delete(dsd); + dsColRepo.delete(dsd); } } diff --git a/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java index 77628c4e435a330cb2b2e38dc28771bc5364605a..6f2ec4024a01e784e3494f41bdb9e0f9e13b0659 100644 --- a/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java @@ -18,8 +18,8 @@ package org.genesys2.server.service.impl; import java.util.List; -import org.genesys2.server.model.impl.Descriptor; -import org.genesys2.server.persistence.domain.DescriptorRepository; +import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.persistence.domain.dataset.DSDescriptorRepository; import org.genesys2.server.service.DescriptorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,21 +33,21 @@ public class DescriptorServiceImpl implements DescriptorService { public static final Logger LOG = LoggerFactory.getLogger(DescriptorServiceImpl.class); @Autowired - private DescriptorRepository descriptorRepository; + private DSDescriptorRepository descriptorRepository; @Override - public List list() { + public List list() { return descriptorRepository.findAll(); } @Transactional @Override - public void saveDescriptor(Descriptor descriptor) { - descriptorRepository.save(descriptor); + public DSDescriptor saveDescriptor(DSDescriptor descriptor) { + return descriptorRepository.save(descriptor); } @Override - public Descriptor getDescriptor(String variableName) { + public DSDescriptor getDescriptor(String variableName) { return descriptorRepository.findByCode(variableName); } } diff --git a/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java b/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java index 4e7ac20762d3965db9fde1bc66a64174364f8fee..934f49149ab7dda335f2f970a835cd1b04a9ce9b 100644 --- a/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java +++ b/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java @@ -25,18 +25,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.MappedByteBuffer; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; - import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -50,24 +44,26 @@ import org.genesys.worldclim.grid.generic.GenericGridZipFile; import org.genesys.worldclim.grid.generic.Header; import org.genesys2.server.aspect.AsAdmin; import org.genesys2.server.model.dataset.DS; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.genesys.AccessionGeo; -import org.genesys2.server.model.impl.Descriptor; import org.genesys2.server.persistence.domain.AccessionGeoRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepository; import org.genesys2.server.service.DSService; import org.genesys2.server.service.DescriptorService; -import org.genesys2.server.service.impl.FilterHandler.AppliedFilters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; -import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; + /** * Download data from WorldClim.org and update local cache. * {@link WorldClimUpdaterImpl#WORLDCLIM_ORG}. @@ -93,6 +89,9 @@ public class WorldClimUpdater implements InitializingBean { @Value("${worldclim.dir}") private String worldclimDir; + @Autowired + private TaskExecutor taskExecutor; + private File worldClimDir; @Autowired @@ -102,7 +101,7 @@ public class WorldClimUpdater implements InitializingBean { private DescriptorService descriptorService; @Autowired - @Qualifier("genesysLowlevelRepositoryCustomImpl") + @Qualifier("genesysLowlevelRepositoryCustomImpl") private GenesysLowlevelRepository genesysLowlevel; @Autowired @@ -116,16 +115,48 @@ public class WorldClimUpdater implements InitializingBean { @AsAdmin public void downloadAll() { - try { - downloadAndExtract(worldClimDir, "alt_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "prec_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmin_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmax_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmean_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "bio_2-5m_bil.zip"); - } catch (IOException e) { - LOG.error("Failed to download and import", e); - } + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "alt_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "prec_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmin_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmax_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmean_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "bio_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); } /** @@ -141,26 +172,16 @@ public class WorldClimUpdater implements InitializingBean { downloadAndExtract(worldClimDir, getFileName(variableName)); } - DS dataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET); - if (dataset == null) { - // Ensure dataset - dataset = createWorldClimDataset(); - Descriptor foo = descriptorService.getDescriptor("tileIndex"); - if (foo == null) { - foo = createDescriptor(dataset, "tileIndex"); - } - dsService.addQualifier(dataset, foo); - } else { - } + DS dataset = ensureWorldClimDataset(); - Descriptor descriptor = descriptorService.getDescriptor(variableName); - DSDescriptor worldclimDescriptor = null; + DSDescriptor descriptor = descriptorService.getDescriptor(variableName); + DSColumn worldclimDescriptor = null; if (descriptor == null) { - descriptor = createDescriptor(dataset, variableName); + descriptor = createDescriptor(variableName); worldclimDescriptor = dsService.addDescriptor(dataset, descriptor); LOG.info("Created worldclimDescriptor {}", variableName); } else { - for (DSDescriptor dsd : dataset.getDescriptors()) { + for (DSColumn dsd : dataset.getColumns()) { if (dsd.getDescriptor().getCode().equals(variableName)) { LOG.debug("Found worldclimDescriptor {}", variableName); worldclimDescriptor = dsd; @@ -171,7 +192,7 @@ public class WorldClimUpdater implements InitializingBean { worldclimDescriptor = dsService.addDescriptor(dataset, descriptor); } - LOG.info("Using worldClim {}", worldclimDescriptor); + LOG.debug("Using worldClim {}", worldclimDescriptor); GenericGridFile ggf = new GenericGridFile(worldClimDir, variableName); Header header = ggf.readHeader(); @@ -189,60 +210,70 @@ public class WorldClimUpdater implements InitializingBean { } Set tileIndexSet = getExistingTileIndexes(); - if (tileIndexSet.isEmpty()) + if (missingTileIndexes()) tileIndexSet = generateTileIndexes(); - + List tileIndexes = new ArrayList(tileIndexSet); int batchSize = 1000; for (int fromIndex = 0; fromIndex < tileIndexes.size(); fromIndex += batchSize) { HashSet ids = new HashSet(tileIndexes.subList(fromIndex, Math.min(fromIndex + batchSize, tileIndexes.size()))); - LOG.info("Processing tileIndexes: {} of {}", fromIndex, tileIndexes.size()); + LOG.info("Processing variable {} for tileIndexes: {}-{} of {}", variableName, fromIndex, fromIndex + ids.size(), tileIndexes.size()); dsService.worldclimUpdate(dataset, worldclimDescriptor, ids, buffer, nullValue, factor); } - + LOG.info("Done processing {}", variableName); } + private boolean missingTileIndexes() { + Set missingIds = accessionGeoRepository.withMissingTileIndex(); + if (missingIds.size() > 0) { + LOG.warn("{} accession records don't have tileIndex assigned", missingIds.size()); + } + return missingIds.size() > 0; + } + private Set generateTileIndexes() throws JsonParseException, JsonMappingException, IOException { // read accessions with lat/lng and update in batch - final List accessionIds = new ArrayList(20000); - ObjectMapper mapper = new ObjectMapper(); - AppliedFilters filters = mapper.readValue("{\"-geo.longitude\":[null]}", AppliedFilters.class); - - genesysLowlevel.listAccessionIds(filters, null, new RowCallbackHandler() { - @Override - public void processRow(ResultSet rs) throws SQLException { - accessionIds.add(rs.getLong(1)); - } - }); - - LOG.info("Retrieved accession ids for filter {}: {}", filters.toString(), accessionIds.size()); + final List accessionIds = new ArrayList<>(accessionGeoRepository.withMissingTileIndex()); + LOG.info("Retrieved {} accession ids without tileIndex", accessionIds.size()); final Set tileIndexSet = new HashSet(20000); - int batchSize = 50; + int batchSize = 1000; for (int fromIndex = 0; fromIndex < accessionIds.size(); fromIndex += batchSize) { - HashSet ids = new HashSet(accessionIds.subList(fromIndex, Math.min(fromIndex + batchSize, accessionIds.size()))); - LOG.info("Processing position: {} of {}", fromIndex, accessionIds.size()); - List acGeo = accessionGeoRepository.findForAccessions(ids); - List toSave = new ArrayList(); - for (AccessionGeo geo : acGeo) { - Double lat = geo.getLatitude(); - Double lon = geo.getLongitude(); - - if (lat != null && lon != null) { - Long tileIndex = WorldClimUtil.getTileIndex(5, lon, lat); - if (tileIndex != null) { - tileIndexSet.add(tileIndex); - if (geo.getTileIndex() == null) { - geo.setTileIndex(tileIndex); + final HashSet ids = new HashSet(accessionIds.subList(fromIndex, Math.min(fromIndex + batchSize, accessionIds.size()))); + + final int startIndex = fromIndex; + taskExecutor.execute(() -> { + try { + List toSave = new ArrayList(ids.size()); + LOG.debug("Calculating tileIndex at position {} of {}", startIndex, accessionIds.size()); + List acGeo = accessionGeoRepository.findForAccessions(ids); + for (AccessionGeo geo : acGeo) { + Double lon = geo.getLongitude(); + Double lat = geo.getLatitude(); + + Long tileIndex = WorldClimUtil.getTileIndex(5, lon, lat); + if (tileIndex != null) { + if (! tileIndex.equals(geo.getTileIndex())) { + geo.setTileIndex(tileIndex); + toSave.add(geo); + } + tileIndexSet.add(tileIndex); + } else if (geo.getTileIndex() != null) { + geo.setTileIndex(null); toSave.add(geo); } } + if (toSave.size() > 0) { + LOG.info("Updating {} accesssions' tileIndex at position {} of {}", toSave.size(), startIndex, accessionIds.size()); + dsService.updateAccessionTileIndex(toSave); + } + } catch (Throwable e) { + LOG.error(e.getMessage(), e); } - } - dsService.updateAccessionTileIndex(toSave); + }); } return tileIndexSet; } @@ -254,30 +285,37 @@ public class WorldClimUpdater implements InitializingBean { return tileIndexSet; } - private Descriptor createDescriptor(DS dataset, String variableName) { - Descriptor descriptor = new Descriptor(); + private DSDescriptor createDescriptor(String variableName) { + DSDescriptor descriptor = new DSDescriptor(); descriptor.setCode(variableName); descriptor.setTitle("WorldClim " + variableName); - descriptorService.saveDescriptor(descriptor); + descriptor = descriptorService.saveDescriptor(descriptor); LOG.info("Created descriptor {}", descriptor); return descriptor; } private DS createWorldClimDataset() { DS dataset = new DS(); - // dataset.setDescription("Genesys accessions linked with worldclim.org data. Maintained automatically by Genesys"); + // dataset.setDescription("Genesys accessions linked with worldclim.org data. + // Maintained automatically by Genesys"); // dataset.setName("WorldClim.org Data"); // dataset.setSource("worldclim.org + genesys-pgr.org"); // dataset.setUploadDate(new Date()); dataset.setUuid(WORLDCLIM_DATASET); dataset.setQualifiers(new ArrayList()); - dataset.setDescriptors(new ArrayList()); - dsService.saveDataset(dataset); + dataset.setColumns(new ArrayList()); + dataset = dsService.saveDataset(dataset); LOG.info("Created dataset {}", WORLDCLIM_DATASET); + + DSDescriptor tileIndex = descriptorService.getDescriptor("tileIndex"); + if (tileIndex == null) { + tileIndex = createDescriptor("tileIndex"); + } + + dsService.addQualifier(dataset, tileIndex); return dataset; } - // TODO Move to worldclim! private String getFileName(String variableName) throws FileNotFoundException { if ("alt".equalsIgnoreCase(variableName)) { return "alt_2-5m_bil.zip"; @@ -364,4 +402,69 @@ public class WorldClimUpdater implements InitializingBean { } } + /** + * Update all WorldClim variables + */ + public void update() { + ensureWorldClimDataset(); + + try { + update("alt"); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable alt", e); + } + + for (int i = 1; i <= 12; i++) { + final int variableIndex = i; + taskExecutor.execute(() -> { + try { + update("tmin" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmin" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("tmax" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmax" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("tmean" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmean" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("prec" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable prec" + variableIndex, e); + } + }); + } + + for (int i = 1; i <= 19; i++) { + final int variableIndex = i; + taskExecutor.execute(() -> { + try { + update("bio" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable bio" + variableIndex, e); + } + }); + } + } + + protected synchronized DS ensureWorldClimDataset() { + DS dataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET); + if (dataset == null) { + // Ensure dataset + dataset = createWorldClimDataset(); + } + return dataset; + } + } diff --git a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java index f963b224eb93514ad7eb010f85847203fbcd6ded..dbac92b2ef9d148301f9e22d1574d9d704b8def1 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java +++ b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java @@ -349,23 +349,6 @@ public class AdminController { return "redirect:/admin/"; } - @RequestMapping("/worldClim") - public String worldClim() throws IOException { - worldClimUpdater.update("alt"); - - for (int i = 1; i <= 12; i++) { - worldClimUpdater.update("tmin" + i); - worldClimUpdater.update("tmax" + i); - worldClimUpdater.update("tmean" + i); - worldClimUpdater.update("prec" + i); - } - - for (int i = 1; i <= 19; i++) { - worldClimUpdater.update("bio" + i); - } - return "redirect:/admin/"; - } - @RequestMapping(value = "/assign-uuid", method = RequestMethod.POST) public String assignUuid() { while (genesysService.assignMissingUuid(100) > 0) { diff --git a/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java b/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java index 23af56d109d5a042000f4abc68d823d2e0af2926..3416d4aa5464b1e10a1c43730c5df3ad323123d9 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java +++ b/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java @@ -19,7 +19,7 @@ package org.genesys2.server.servlet.controller.admin; import java.io.IOException; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.service.DSService; import org.genesys2.server.service.worker.WorldClimUpdater; import org.slf4j.Logger; @@ -50,18 +50,7 @@ public class DS2Controller { @RequestMapping(value = "/worldclim/update", method = RequestMethod.POST) public String worldClim() throws IOException { - worldClimUpdater.update("alt"); - - for (int i = 1; i <= 12; i++) { - worldClimUpdater.update("tmin" + i); - worldClimUpdater.update("tmax" + i); - worldClimUpdater.update("tmean" + i); - worldClimUpdater.update("prec" + i); - } - - for (int i = 1; i <= 19; i++) { - worldClimUpdater.update("bio" + i); - } + worldClimUpdater.update(); return "redirect:/admin/ds2/"; } @@ -70,7 +59,7 @@ public class DS2Controller { DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET); if (ds != null) { LOG.info("Deleting dataset {}", ds.getUuid()); - for (DSDescriptor dsd : ds.getDescriptors()) { + for (DSColumn dsd : ds.getColumns()) { dsService.deleteDescriptor(dsd); } dsService.deleteRows(ds); diff --git a/src/main/resources/liquibase/liquibase-changeLog.yml b/src/main/resources/liquibase/liquibase-changeLog.yml index f08e5a670e104f81055665bb3dc4dbfb69362131..1c17c1edce541f70f45c89d3d94da20c77df5bed 100644 --- a/src/main/resources/liquibase/liquibase-changeLog.yml +++ b/src/main/resources/liquibase/liquibase-changeLog.yml @@ -1147,3 +1147,23 @@ databaseChangeLog: sql: >- update acl_sid inner join user on user.id=acl_sid.id set acl_sid.sid = user.email; + +# DS2 +- changeSet: + id: 1525172348000-1 + author: Matija Obreza + comment: Renaming DS2 tables + changes: + - renameTable: + newTableName: ds2column + oldTableName: ds2descriptor + - renameTable: + newTableName: ds2descriptor + oldTableName: descriptor + - renameColumn: + columnDataType: BIGINT + newColumnName: dsc + oldColumnName: dsd + remarks: Dataset column + tableName: ds2value + diff --git a/src/test/java/org/genesys2/server/service/impl/DSTest.java b/src/test/java/org/genesys2/server/service/impl/DSTest.java index 68381b1e0f0a70ea410e84a6bda4a233c7db0f6f..8afe9ec1396a096f3cf8dac2d318e12a22d48878 100644 --- a/src/test/java/org/genesys2/server/service/impl/DSTest.java +++ b/src/test/java/org/genesys2/server/service/impl/DSTest.java @@ -8,9 +8,9 @@ import java.util.ArrayList; import java.util.List; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSQualifier; -import org.genesys2.server.model.impl.Descriptor; +import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.service.DSService; import org.genesys2.server.service.DescriptorService; import org.genesys2.server.test.JpaDataConfig; @@ -67,17 +67,17 @@ public class DSTest { dsService.saveDataset(ds); assertThat("Dataset must have a UUID", ds.getUuid(), notNullValue()); - Descriptor d1 = generateDescriptor("D1"); + DSDescriptor d1 = generateDescriptor("D1"); descriptorService.saveDescriptor(d1); - Descriptor d2 = generateDescriptor("D2"); + DSDescriptor d2 = generateDescriptor("D2"); descriptorService.saveDescriptor(d2); DSQualifier dsq1 = dsService.addQualifier(ds, d1); DSQualifier dsq2 = dsService.addQualifier(ds, d2); - Descriptor d3 = generateDescriptor("D3"); + DSDescriptor d3 = generateDescriptor("D3"); descriptorService.saveDescriptor(d3); - Descriptor d4 = generateDescriptor("D4"); + DSDescriptor d4 = generateDescriptor("D4"); descriptorService.saveDescriptor(d4); dsService.addDescriptor(ds, d3); @@ -89,7 +89,7 @@ public class DSTest { System.out.println(dsq1.getId() + " " + dsq1.getDescriptor()); assertThat("DSQualifier must be loaded by id", dsq1, notNullValue()); assertThat("DSQualifier must have a Dataset", dsq1.getDataset(), notNullValue()); - assertThat("DSQualifier must have a Descriptor", dsq1.getDescriptor(), notNullValue()); + assertThat("DSQualifier must have a DSDescriptor", dsq1.getDescriptor(), notNullValue()); DS loaded = dsService.loadDatasetByUuid(ds.getUuid()); assertThat("Dataset must be loaded by UUID", loaded, notNullValue()); @@ -146,15 +146,15 @@ public class DSTest { } - private Descriptor generateDescriptor(String code) { - Descriptor d = new Descriptor(); + private DSDescriptor generateDescriptor(String code) { + DSDescriptor d = new DSDescriptor(); d.setCode(code); return d; } private DS generateDataset() { DS ds = new DS(); - ds.setDescriptors(new ArrayList()); + ds.setColumns(new ArrayList()); ds.setQualifiers(new ArrayList()); return ds; } diff --git a/src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java b/src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java index 29e1de90fe272a755f9977b9475593c7ae9733ba..fbc53599f3ede9f6e75ab9b8f8de9e2b438cae42 100644 --- a/src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java +++ b/src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java @@ -11,7 +11,7 @@ import com.hazelcast.core.HazelcastInstance; import com.hazelcast.spring.cache.HazelcastCacheManager; import org.genesys2.server.model.dataset.DS; -import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.persistence.domain.GenesysLowlevelRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl; import org.genesys2.server.service.DSService; @@ -93,7 +93,7 @@ public class WorldClimUpdaterTest { public void deleteDataset() { DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET); if (ds != null) { - for (DSDescriptor dsd : ds.getDescriptors()) { + for (DSColumn dsd : ds.getColumns()) { dsService.deleteDescriptor(dsd); } dsService.deleteRows(ds); @@ -115,8 +115,8 @@ public class WorldClimUpdaterTest { @Test public void testDownloadDesc() throws IOException { DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET); - List dsds = new ArrayList(); - for (DSDescriptor dsd : ds.getDescriptors()) { + List dsds = new ArrayList(); + for (DSColumn dsd : ds.getColumns()) { if (dsd.getDescriptor().getCode().startsWith("tmin")) { dsds.add(dsd); } diff --git a/src/test/java/org/genesys2/tests/unit/AbstractServicesTest.java b/src/test/java/org/genesys2/tests/unit/AbstractServicesTest.java index d51b4db974f99a97898b95362dc86025c8830898..ec5a86f01ec88cd1f85209ab4656b9ccf99a598f 100644 --- a/src/test/java/org/genesys2/tests/unit/AbstractServicesTest.java +++ b/src/test/java/org/genesys2/tests/unit/AbstractServicesTest.java @@ -34,7 +34,6 @@ import org.genesys2.server.persistence.domain.ArticleRepository; import org.genesys2.server.persistence.domain.CropRepository; import org.genesys2.server.persistence.domain.CropRuleRepository; import org.genesys2.server.persistence.domain.CropTaxonomyRepository; -import org.genesys2.server.persistence.domain.DescriptorRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl; import org.genesys2.server.persistence.domain.Taxonomy2Repository; @@ -43,6 +42,7 @@ import org.genesys2.server.persistence.domain.TraitValueRepository; import org.genesys2.server.persistence.domain.TraitValueRepositoryImpl; import org.genesys2.server.persistence.domain.UserRepository; import org.genesys2.server.persistence.domain.VerificationTokenRepository; +import org.genesys2.server.persistence.domain.dataset.DSDescriptorRepository; import org.genesys2.server.service.ContentService; import org.genesys2.server.service.CropService; import org.genesys2.server.service.DescriptorService; @@ -311,7 +311,7 @@ public abstract class AbstractServicesTest extends BaseSpringTest { public DescriptorService descriptorService; @Autowired - public DescriptorRepository descriptorRepository; + public DSDescriptorRepository dSDescriptorRepository; @Autowired public JavaMailSender mailSender; diff --git a/src/test/java/org/genesys2/tests/unit/DescriptorServiceTest.java b/src/test/java/org/genesys2/tests/unit/DS2DescriptorServiceTest.java similarity index 79% rename from src/test/java/org/genesys2/tests/unit/DescriptorServiceTest.java rename to src/test/java/org/genesys2/tests/unit/DS2DescriptorServiceTest.java index 58f7b94644133590bfffa92707591bd54841aef2..5e531629ebbfce943071095ec99033c30e86be89 100644 --- a/src/test/java/org/genesys2/tests/unit/DescriptorServiceTest.java +++ b/src/test/java/org/genesys2/tests/unit/DS2DescriptorServiceTest.java @@ -17,22 +17,22 @@ package org.genesys2.tests.unit; import static org.junit.Assert.*; -import org.genesys2.server.model.impl.Descriptor; +import org.genesys2.server.model.dataset.DSDescriptor; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DescriptorServiceTest extends AbstractServicesTest { +public class DS2DescriptorServiceTest extends AbstractServicesTest { - private static final Logger LOG = LoggerFactory.getLogger(DescriptorServiceTest.class); + private static final Logger LOG = LoggerFactory.getLogger(DS2DescriptorServiceTest.class); - private Descriptor descriptor; + private DSDescriptor descriptor; @Before public void setUp() { - descriptor = new Descriptor(); + descriptor = new DSDescriptor(); descriptor.setCode("code of descriptor"); descriptorService.saveDescriptor(descriptor); @@ -40,7 +40,7 @@ public class DescriptorServiceTest extends AbstractServicesTest { @After public void teardown() { - descriptorRepository.deleteAll(); + dSDescriptorRepository.deleteAll(); } @Test @@ -58,7 +58,7 @@ public class DescriptorServiceTest extends AbstractServicesTest { assertTrue(descriptorService.list().size() == 1); - Descriptor newDescriptor = new Descriptor(); + DSDescriptor newDescriptor = new DSDescriptor(); newDescriptor.setCode("code of newDescriptor"); descriptorService.saveDescriptor(newDescriptor); @@ -72,7 +72,7 @@ public class DescriptorServiceTest extends AbstractServicesTest { public void getDescriptor() { LOG.info("Start test-method getDescriptor"); - Descriptor descriptorForTest = descriptorService.getDescriptor(descriptor.getCode()); + DSDescriptor descriptorForTest = descriptorService.getDescriptor(descriptor.getCode()); assertTrue(descriptorForTest != null); assertTrue(descriptorForTest.getCode().equals("code of descriptor"));