diff --git a/src/main/java/org/genesys2/server/model/dataset/DS.java b/src/main/java/org/genesys2/server/model/dataset/DS.java index 65f31d265f46649d85a644672e78c4ead08df26c..d0b3a8b8ebca04ee6b8a92e967fa82df379495b2 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DS.java +++ b/src/main/java/org/genesys2/server/model/dataset/DS.java @@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset; import java.util.List; import java.util.UUID; +import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.OneToMany; @@ -23,11 +24,11 @@ public class DS extends BasicModel { private UUID uuid; @OrderBy("index") - @OneToMany(mappedBy = "dataset") + @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL }) private List qualifiers; @OrderBy("index") - @OneToMany(mappedBy = "dataset") + @OneToMany(mappedBy = "dataset", cascade = { CascadeType.ALL }) private List columns; /** diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRow.java b/src/main/java/org/genesys2/server/model/dataset/DSRow.java index 2520f9ceacd25d45610845ebc45c76902586cdd4..2fbcd61f4b0265152cb437d8a060ffc0dc9683c3 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRow.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRow.java @@ -1,5 +1,6 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; import java.util.List; import javax.persistence.CascadeType; @@ -12,9 +13,12 @@ import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.OneToMany; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; import javax.persistence.Table; import javax.persistence.UniqueConstraint; +import org.apache.commons.codec.digest.DigestUtils; import org.genesys.blocks.model.EntityId; /** @@ -31,14 +35,14 @@ import org.genesys.blocks.model.EntityId; @Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) }) public class DSRow implements EntityId { - @ManyToOne(fetch=FetchType.LAZY, optional = false) - @JoinColumn(name="ds") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "ds") private DS dataset; - - @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.REMOVE }) + + @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL }) private List> rowQualifiers; - @OneToMany(fetch = FetchType.LAZY, mappedBy = "row") + @OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.ALL }) private List> values; @Id @@ -51,6 +55,19 @@ public class DSRow implements EntityId { @Column(columnDefinition = "binary(20)", updatable = false) private byte[] sha1; + @PrePersist + @PreUpdate + private void prePersist() { + ByteBuffer keyBuffer = ByteBuffer.allocate(500); + for (DSRowQualifier dsq : this.getRowQualifiers()) { + dsq.putKey(keyBuffer); + } + + byte[] array = keyBuffer.array(); + this.md5 = DigestUtils.md5(array); + this.sha1 = DigestUtils.sha1(array); + } + @Override public Long getId() { return this.id; @@ -63,11 +80,11 @@ public class DSRow implements EntityId { public DS getDataset() { return dataset; } - + public void setDataset(DS dataset) { this.dataset = dataset; } - + public List> getRowQualifiers() { return rowQualifiers; } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java index c143b96a05ec04e8adc802bbf5d38644c4746dea..6a0fe161b69f77deddcb60fe1480c2a71c36da97 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifier.java @@ -1,5 +1,7 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; + import javax.persistence.DiscriminatorColumn; import javax.persistence.DiscriminatorType; import javax.persistence.Entity; @@ -80,4 +82,5 @@ public abstract class DSRowQualifier { return null; } + public abstract void putKey(ByteBuffer keyBuffer); } diff --git a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java index 2d8757bc477b994200049a941d884adfa0e04837..85e32fecd61b471bfa64a108fe082dad143b10aa 100644 --- a/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java +++ b/src/main/java/org/genesys2/server/model/dataset/DSRowQualifierLong.java @@ -1,5 +1,7 @@ package org.genesys2.server.model.dataset; +import java.nio.ByteBuffer; + import javax.persistence.Column; import javax.persistence.DiscriminatorValue; import javax.persistence.Entity; @@ -20,5 +22,9 @@ public class DSRowQualifierLong extends DSRowQualifier { public void setValue(Long value) { this.value = value; } - + + @Override + public void putKey(ByteBuffer keyBuffer) { + keyBuffer.putLong(this.value); + } } diff --git a/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java b/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java index c39bfb29da26141dfeacaca76b0d516896187d50..f264214117f28fe32d5423d57a0851822bd0fbec 100644 --- a/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java +++ b/src/main/java/org/genesys2/server/model/genesys/AccessionGeo.java @@ -21,12 +21,15 @@ import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; import javax.persistence.OneToOne; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; import javax.persistence.Table; import javax.persistence.Version; import org.apache.commons.lang.StringUtils; import org.genesys.blocks.auditlog.annotations.Audited; import org.genesys.blocks.model.BasicModel; +import org.genesys.worldclim.WorldClimUtil; import org.genesys2.server.model.impl.GeoReferencedEntity; @Entity @@ -57,6 +60,15 @@ public class AccessionGeo extends BasicModel implements GeoReferencedEntity, Acc private Long tileIndex; + /** + * Recalculate {@link #tileIndex} on insert and update + */ + @PrePersist + @PreUpdate + private void prePersist() { + tileIndex = WorldClimUtil.getTileIndex(5, this.longitude, this.latitude); + } + public long getVersion() { return version; } diff --git a/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java b/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java index 780cc4f009742d7f491a28885ceb40687e6ac9e5..f8bfc3cfd063fbc70015a080a1a1d57800085357 100644 --- a/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java +++ b/src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java @@ -43,4 +43,7 @@ public interface AccessionGeoRepository extends JpaRepository getTileIndexes(); + @Query("select distinct(ag.accession.id) from AccessionGeo ag where ag.longitude between -180 and 180 and ag.latitude between -90 and 90 and ag.tileIndex is null") + Set withMissingTileIndex(); + } diff --git a/src/main/java/org/genesys2/server/service/DSService.java b/src/main/java/org/genesys2/server/service/DSService.java index 2490d0ef5bf2319091152ee7807b401417ae6a42..2d6dbe1cb4e7ebd75708a8fdd7a459cba4b4e36c 100644 --- a/src/main/java/org/genesys2/server/service/DSService.java +++ b/src/main/java/org/genesys2/server/service/DSService.java @@ -16,7 +16,7 @@ import org.genesys2.server.model.json.WorldclimJson; public interface DSService { - void saveDataset(DS ds); + DS saveDataset(DS ds); DSQualifier addQualifier(DS ds, DSDescriptor d1); DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor); diff --git a/src/main/java/org/genesys2/server/service/DescriptorService.java b/src/main/java/org/genesys2/server/service/DescriptorService.java index ae874cd26d0beeed425304f5b30a995229817d24..34bff213b4995dc484a7e2a6e6e683ef316ed09f 100644 --- a/src/main/java/org/genesys2/server/service/DescriptorService.java +++ b/src/main/java/org/genesys2/server/service/DescriptorService.java @@ -24,7 +24,7 @@ public interface DescriptorService { List list(); - void saveDescriptor(DSDescriptor dSDescriptor); + DSDescriptor saveDescriptor(DSDescriptor dSDescriptor); DSDescriptor getDescriptor(String variableName); } diff --git a/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java index 30598a3e768e2d1bd82cb542bede11cb003d81b4..241f5828cb1248420e7b8416b1521df292fd33a8 100644 --- a/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java @@ -2,9 +2,7 @@ package org.genesys2.server.service.impl; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -18,7 +16,6 @@ import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.xssf.streaming.SXSSFSheet; @@ -26,11 +23,11 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.genesys2.server.model.dataset.DS; import org.genesys2.server.model.dataset.DSColumn; +import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.dataset.DSRow; import org.genesys2.server.model.dataset.DSRowQualifier; import org.genesys2.server.model.dataset.DSValue; -import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.json.WorldclimJson; import org.genesys2.server.persistence.domain.AccessionGeoRepository; @@ -40,7 +37,6 @@ import org.genesys2.server.persistence.domain.dataset.DSRepository; import org.genesys2.server.persistence.domain.dataset.DSRowQualifierRepository; import org.genesys2.server.persistence.domain.dataset.DSRowRepository; import org.genesys2.server.persistence.domain.dataset.DSValueRepository; -import org.genesys2.server.persistence.domain.dataset.DSDescriptorRepository; import org.genesys2.server.service.DSService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,12 +49,10 @@ import org.springframework.transaction.annotation.Transactional; public class DSServiceImpl implements DSService { public static final Logger LOG = LoggerFactory.getLogger(DSServiceImpl.class); - private static final Charset CHARSET_UTF8 = Charset.forName("UTF8"); - @Autowired private DSRepository dsRepo; @Autowired - private DSColumnRepository dsDescRepo; + private DSColumnRepository dsColRepo; @Autowired private DSQualifierRepository dsQualRepo; @Autowired @@ -67,16 +61,14 @@ public class DSServiceImpl implements DSService { private DSRowQualifierRepository dsRowQualiRepo; @Autowired private DSValueRepository dsValueRepo; - @Autowired - private DSDescriptorRepository descriptorRepo; @Autowired private AccessionGeoRepository accessionGeoRepository; @Transactional @Override - public void saveDataset(DS ds) { - dsRepo.save(ds); + public DS saveDataset(DS ds) { + return dsRepo.save(ds); } @Transactional @@ -98,7 +90,7 @@ public class DSServiceImpl implements DSService { dsd.setDescriptor(d1); dsd.setIndex(ds.getColumns().size() + 1); ds.getColumns().add(dsd); - return dsDescRepo.save(dsd); + return dsColRepo.save(dsd); } @Override @@ -222,12 +214,12 @@ public class DSServiceImpl implements DSService { } private void saveRows(Collection dsrs) { +// ArrayList> quali = new ArrayList>(); +// for (DSRow dsr : dsrs) { +// quali.addAll(dsr.getRowQualifiers()); +// } +// dsRowQualiRepo.save(quali); dsRowRepo.save(dsrs); - ArrayList> quali = new ArrayList>(); - for (DSRow dsr : dsrs) { - quali.addAll(dsr.getRowQualifiers()); - } - dsRowQualiRepo.save(quali); } private DSRow findOrMakeRow(DS ds, Object[] qualifiers) { @@ -288,32 +280,17 @@ public class DSServiceImpl implements DSService { private DSRow makeRow(DS ds, Object... qualifiers) { DSRow dsr = new DSRow(); - - ByteBuffer keyBuffer = ByteBuffer.allocate(500); + dsr.setDataset(ds); List> rowQualifiers = new ArrayList>(); int i = 0; for (DSQualifier dsq : ds.getQualifiers()) { DSRowQualifier dsrq = DSRowQualifier.make(qualifiers[i]); - if (qualifiers[i] instanceof Long) { - keyBuffer.putLong(((Long) qualifiers[i]).longValue()); - } else if (qualifiers[i] instanceof String) { - keyBuffer.put(((String) qualifiers[i]).getBytes(CHARSET_UTF8)); - } else if (qualifiers[i] instanceof Double) { - keyBuffer.putDouble(((Double) qualifiers[i]).doubleValue()); - } dsrq.setDatasetQualifier(dsq); dsrq.setRow(dsr); rowQualifiers.add(dsrq); i++; } - // LOG.info("MD4 length: " + DigestUtils.md5(keyBuffer.array()).length); - // LOG.info("SHA1 length: " + - // DigestUtils.sha1(keyBuffer.array()).length); - dsr.setMd5(DigestUtils.md5(keyBuffer.array())); - dsr.setSha1(DigestUtils.sha1(keyBuffer.array())); - dsr.setDataset(ds); - dsr.setRowQualifiers(rowQualifiers); return dsr; } @@ -335,15 +312,13 @@ public class DSServiceImpl implements DSService { */ @Override @Transactional - public void worldclimUpdate(DS dataset, DSColumn dsd, Set tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) { - if (LOG.isInfoEnabled()) { - LOG.info("Updating {} for tileIndexes: {}", dsd.getDescriptor().getCode(), tileIndexes.size()); - } + public void worldclimUpdate(DS dataset, DSColumn dsc, Set tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) { + LOG.debug("Updating {} for tileIndexes: {}", dsc.getDescriptor().getCode(), tileIndexes.size()); Map rowMap = findOrMakeRows(dataset, tileIndexes); // Load all existing values for rows - Map> values = dsRowRepo.rowValueMap(rowMap.values(), dsd); + Map> values = dsRowRepo.rowValueMap(rowMap.values(), dsc); List> toSave = new ArrayList>(); for (final Long tileIndex : tileIndexes) { @@ -353,9 +328,7 @@ public class DSServiceImpl implements DSService { try { short val = buffer.getShort((int) (tileIndex * 2)); if (val != nullValue) { - if (LOG.isDebugEnabled()) { - LOG.debug("tile={} val={}", tileIndex, val); - } + LOG.trace("tile={} val={}", tileIndex, val); Object value = null; if (factor < 1.0) { @@ -372,9 +345,9 @@ public class DSServiceImpl implements DSService { dsrv = values.get(dsr.getId()); if (dsrv == null) { - LOG.debug("Making new value for row {} and {}", dsr, dsd); + LOG.debug("Making new value for row {} and {}", dsr, dsc); dsrv = DSValue.make(value); - dsrv.setDatasetColumn(dsd); + dsrv.setDatasetColumn(dsc); dsrv.setRow(dsr); } dsrv.setValue2(value); @@ -398,7 +371,7 @@ public class DSServiceImpl implements DSService { // Save values dsValueRepo.save(toSave); - LOG.info("Done saving."); + LOG.debug("Done saving."); } @Override @@ -553,6 +526,6 @@ public class DSServiceImpl implements DSService { LOG.info("Clearing DSValues for {}", dsd.getDescriptor().getCode()); int count = dsValueRepo.deleteFor(dsd); LOG.info("Deleted {} DSValue cells", count); - dsDescRepo.delete(dsd); + dsColRepo.delete(dsd); } } diff --git a/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java b/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java index a3b4118921137fa1bbb6caa08b0eb4d7474aa1dc..6f2ec4024a01e784e3494f41bdb9e0f9e13b0659 100644 --- a/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java +++ b/src/main/java/org/genesys2/server/service/impl/DescriptorServiceImpl.java @@ -33,21 +33,21 @@ public class DescriptorServiceImpl implements DescriptorService { public static final Logger LOG = LoggerFactory.getLogger(DescriptorServiceImpl.class); @Autowired - private DSDescriptorRepository dSDescriptorRepository; + private DSDescriptorRepository descriptorRepository; @Override public List list() { - return dSDescriptorRepository.findAll(); + return descriptorRepository.findAll(); } @Transactional @Override - public void saveDescriptor(DSDescriptor descriptor) { - dSDescriptorRepository.save(descriptor); + public DSDescriptor saveDescriptor(DSDescriptor descriptor) { + return descriptorRepository.save(descriptor); } @Override public DSDescriptor getDescriptor(String variableName) { - return dSDescriptorRepository.findByCode(variableName); + return descriptorRepository.findByCode(variableName); } } diff --git a/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java b/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java index f992f2a0e7fc3b2a4c3da3f13855c9aaa71fa7b3..934f49149ab7dda335f2f970a835cd1b04a9ce9b 100644 --- a/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java +++ b/src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java @@ -25,18 +25,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.MappedByteBuffer; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; - import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -51,23 +45,25 @@ import org.genesys.worldclim.grid.generic.Header; import org.genesys2.server.aspect.AsAdmin; import org.genesys2.server.model.dataset.DS; import org.genesys2.server.model.dataset.DSColumn; -import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.dataset.DSDescriptor; +import org.genesys2.server.model.dataset.DSQualifier; import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.persistence.domain.AccessionGeoRepository; import org.genesys2.server.persistence.domain.GenesysLowlevelRepository; import org.genesys2.server.service.DSService; import org.genesys2.server.service.DescriptorService; -import org.genesys2.server.service.impl.FilterHandler.AppliedFilters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; -import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; + /** * Download data from WorldClim.org and update local cache. * {@link WorldClimUpdaterImpl#WORLDCLIM_ORG}. @@ -93,6 +89,9 @@ public class WorldClimUpdater implements InitializingBean { @Value("${worldclim.dir}") private String worldclimDir; + @Autowired + private TaskExecutor taskExecutor; + private File worldClimDir; @Autowired @@ -102,7 +101,7 @@ public class WorldClimUpdater implements InitializingBean { private DescriptorService descriptorService; @Autowired - @Qualifier("genesysLowlevelRepositoryCustomImpl") + @Qualifier("genesysLowlevelRepositoryCustomImpl") private GenesysLowlevelRepository genesysLowlevel; @Autowired @@ -116,16 +115,48 @@ public class WorldClimUpdater implements InitializingBean { @AsAdmin public void downloadAll() { - try { - downloadAndExtract(worldClimDir, "alt_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "prec_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmin_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmax_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "tmean_2-5m_bil.zip"); - downloadAndExtract(worldClimDir, "bio_2-5m_bil.zip"); - } catch (IOException e) { - LOG.error("Failed to download and import", e); - } + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "alt_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "prec_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmin_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmax_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "tmean_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); + taskExecutor.execute(() -> { + try { + downloadAndExtract(worldClimDir, "bio_2-5m_bil.zip"); + } catch (IOException e) { + LOG.error("Failed to download and import", e); + } + }); } /** @@ -141,23 +172,13 @@ public class WorldClimUpdater implements InitializingBean { downloadAndExtract(worldClimDir, getFileName(variableName)); } - DS dataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET); - if (dataset == null) { - // Ensure dataset - dataset = createWorldClimDataset(); - DSDescriptor foo = descriptorService.getDescriptor("tileIndex"); - if (foo == null) { - foo = createDescriptor(dataset, "tileIndex"); - } - dsService.addQualifier(dataset, foo); - } else { - } + DS dataset = ensureWorldClimDataset(); - DSDescriptor dSDescriptor = descriptorService.getDescriptor(variableName); + DSDescriptor descriptor = descriptorService.getDescriptor(variableName); DSColumn worldclimDescriptor = null; - if (dSDescriptor == null) { - dSDescriptor = createDescriptor(dataset, variableName); - worldclimDescriptor = dsService.addDescriptor(dataset, dSDescriptor); + if (descriptor == null) { + descriptor = createDescriptor(variableName); + worldclimDescriptor = dsService.addDescriptor(dataset, descriptor); LOG.info("Created worldclimDescriptor {}", variableName); } else { for (DSColumn dsd : dataset.getColumns()) { @@ -168,10 +189,10 @@ public class WorldClimUpdater implements InitializingBean { } } if (worldclimDescriptor == null) - worldclimDescriptor = dsService.addDescriptor(dataset, dSDescriptor); + worldclimDescriptor = dsService.addDescriptor(dataset, descriptor); } - LOG.info("Using worldClim {}", worldclimDescriptor); + LOG.debug("Using worldClim {}", worldclimDescriptor); GenericGridFile ggf = new GenericGridFile(worldClimDir, variableName); Header header = ggf.readHeader(); @@ -189,60 +210,70 @@ public class WorldClimUpdater implements InitializingBean { } Set tileIndexSet = getExistingTileIndexes(); - if (tileIndexSet.isEmpty()) + if (missingTileIndexes()) tileIndexSet = generateTileIndexes(); - + List tileIndexes = new ArrayList(tileIndexSet); int batchSize = 1000; for (int fromIndex = 0; fromIndex < tileIndexes.size(); fromIndex += batchSize) { HashSet ids = new HashSet(tileIndexes.subList(fromIndex, Math.min(fromIndex + batchSize, tileIndexes.size()))); - LOG.info("Processing tileIndexes: {} of {}", fromIndex, tileIndexes.size()); + LOG.info("Processing variable {} for tileIndexes: {}-{} of {}", variableName, fromIndex, fromIndex + ids.size(), tileIndexes.size()); dsService.worldclimUpdate(dataset, worldclimDescriptor, ids, buffer, nullValue, factor); } - + LOG.info("Done processing {}", variableName); } + private boolean missingTileIndexes() { + Set missingIds = accessionGeoRepository.withMissingTileIndex(); + if (missingIds.size() > 0) { + LOG.warn("{} accession records don't have tileIndex assigned", missingIds.size()); + } + return missingIds.size() > 0; + } + private Set generateTileIndexes() throws JsonParseException, JsonMappingException, IOException { // read accessions with lat/lng and update in batch - final List accessionIds = new ArrayList(20000); - ObjectMapper mapper = new ObjectMapper(); - AppliedFilters filters = mapper.readValue("{\"-geo.longitude\":[null]}", AppliedFilters.class); - - genesysLowlevel.listAccessionIds(filters, null, new RowCallbackHandler() { - @Override - public void processRow(ResultSet rs) throws SQLException { - accessionIds.add(rs.getLong(1)); - } - }); - - LOG.info("Retrieved accession ids for filter {}: {}", filters.toString(), accessionIds.size()); + final List accessionIds = new ArrayList<>(accessionGeoRepository.withMissingTileIndex()); + LOG.info("Retrieved {} accession ids without tileIndex", accessionIds.size()); final Set tileIndexSet = new HashSet(20000); - int batchSize = 50; + int batchSize = 1000; for (int fromIndex = 0; fromIndex < accessionIds.size(); fromIndex += batchSize) { - HashSet ids = new HashSet(accessionIds.subList(fromIndex, Math.min(fromIndex + batchSize, accessionIds.size()))); - LOG.info("Processing position: {} of {}", fromIndex, accessionIds.size()); - List acGeo = accessionGeoRepository.findForAccessions(ids); - List toSave = new ArrayList(); - for (AccessionGeo geo : acGeo) { - Double lat = geo.getLatitude(); - Double lon = geo.getLongitude(); - - if (lat != null && lon != null) { - Long tileIndex = WorldClimUtil.getTileIndex(5, lon, lat); - if (tileIndex != null) { - tileIndexSet.add(tileIndex); - if (geo.getTileIndex() == null) { - geo.setTileIndex(tileIndex); + final HashSet ids = new HashSet(accessionIds.subList(fromIndex, Math.min(fromIndex + batchSize, accessionIds.size()))); + + final int startIndex = fromIndex; + taskExecutor.execute(() -> { + try { + List toSave = new ArrayList(ids.size()); + LOG.debug("Calculating tileIndex at position {} of {}", startIndex, accessionIds.size()); + List acGeo = accessionGeoRepository.findForAccessions(ids); + for (AccessionGeo geo : acGeo) { + Double lon = geo.getLongitude(); + Double lat = geo.getLatitude(); + + Long tileIndex = WorldClimUtil.getTileIndex(5, lon, lat); + if (tileIndex != null) { + if (! tileIndex.equals(geo.getTileIndex())) { + geo.setTileIndex(tileIndex); + toSave.add(geo); + } + tileIndexSet.add(tileIndex); + } else if (geo.getTileIndex() != null) { + geo.setTileIndex(null); toSave.add(geo); } } + if (toSave.size() > 0) { + LOG.info("Updating {} accesssions' tileIndex at position {} of {}", toSave.size(), startIndex, accessionIds.size()); + dsService.updateAccessionTileIndex(toSave); + } + } catch (Throwable e) { + LOG.error(e.getMessage(), e); } - } - dsService.updateAccessionTileIndex(toSave); + }); } return tileIndexSet; } @@ -254,30 +285,37 @@ public class WorldClimUpdater implements InitializingBean { return tileIndexSet; } - private DSDescriptor createDescriptor(DS dataset, String variableName) { - DSDescriptor dSDescriptor = new DSDescriptor(); - dSDescriptor.setCode(variableName); - dSDescriptor.setTitle("WorldClim " + variableName); - descriptorService.saveDescriptor(dSDescriptor); - LOG.info("Created descriptor {}", dSDescriptor); - return dSDescriptor; + private DSDescriptor createDescriptor(String variableName) { + DSDescriptor descriptor = new DSDescriptor(); + descriptor.setCode(variableName); + descriptor.setTitle("WorldClim " + variableName); + descriptor = descriptorService.saveDescriptor(descriptor); + LOG.info("Created descriptor {}", descriptor); + return descriptor; } private DS createWorldClimDataset() { DS dataset = new DS(); - // dataset.setDescription("Genesys accessions linked with worldclim.org data. Maintained automatically by Genesys"); + // dataset.setDescription("Genesys accessions linked with worldclim.org data. + // Maintained automatically by Genesys"); // dataset.setName("WorldClim.org Data"); // dataset.setSource("worldclim.org + genesys-pgr.org"); // dataset.setUploadDate(new Date()); dataset.setUuid(WORLDCLIM_DATASET); dataset.setQualifiers(new ArrayList()); dataset.setColumns(new ArrayList()); - dsService.saveDataset(dataset); + dataset = dsService.saveDataset(dataset); LOG.info("Created dataset {}", WORLDCLIM_DATASET); + + DSDescriptor tileIndex = descriptorService.getDescriptor("tileIndex"); + if (tileIndex == null) { + tileIndex = createDescriptor("tileIndex"); + } + + dsService.addQualifier(dataset, tileIndex); return dataset; } - // TODO Move to worldclim! private String getFileName(String variableName) throws FileNotFoundException { if ("alt".equalsIgnoreCase(variableName)) { return "alt_2-5m_bil.zip"; @@ -364,4 +402,69 @@ public class WorldClimUpdater implements InitializingBean { } } + /** + * Update all WorldClim variables + */ + public void update() { + ensureWorldClimDataset(); + + try { + update("alt"); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable alt", e); + } + + for (int i = 1; i <= 12; i++) { + final int variableIndex = i; + taskExecutor.execute(() -> { + try { + update("tmin" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmin" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("tmax" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmax" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("tmean" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable tmean" + variableIndex, e); + } + }); + taskExecutor.execute(() -> { + try { + update("prec" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable prec" + variableIndex, e); + } + }); + } + + for (int i = 1; i <= 19; i++) { + final int variableIndex = i; + taskExecutor.execute(() -> { + try { + update("bio" + variableIndex); + } catch (IOException e) { + LOG.error("Error updating worldClim data for variable bio" + variableIndex, e); + } + }); + } + } + + protected synchronized DS ensureWorldClimDataset() { + DS dataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET); + if (dataset == null) { + // Ensure dataset + dataset = createWorldClimDataset(); + } + return dataset; + } + } diff --git a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java index f963b224eb93514ad7eb010f85847203fbcd6ded..dbac92b2ef9d148301f9e22d1574d9d704b8def1 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java +++ b/src/main/java/org/genesys2/server/servlet/controller/admin/AdminController.java @@ -349,23 +349,6 @@ public class AdminController { return "redirect:/admin/"; } - @RequestMapping("/worldClim") - public String worldClim() throws IOException { - worldClimUpdater.update("alt"); - - for (int i = 1; i <= 12; i++) { - worldClimUpdater.update("tmin" + i); - worldClimUpdater.update("tmax" + i); - worldClimUpdater.update("tmean" + i); - worldClimUpdater.update("prec" + i); - } - - for (int i = 1; i <= 19; i++) { - worldClimUpdater.update("bio" + i); - } - return "redirect:/admin/"; - } - @RequestMapping(value = "/assign-uuid", method = RequestMethod.POST) public String assignUuid() { while (genesysService.assignMissingUuid(100) > 0) { diff --git a/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java b/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java index 5c37466de1f2568ca9c7934e505d11040c76d845..3416d4aa5464b1e10a1c43730c5df3ad323123d9 100644 --- a/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java +++ b/src/main/java/org/genesys2/server/servlet/controller/admin/DS2Controller.java @@ -50,18 +50,7 @@ public class DS2Controller { @RequestMapping(value = "/worldclim/update", method = RequestMethod.POST) public String worldClim() throws IOException { - worldClimUpdater.update("alt"); - - for (int i = 1; i <= 12; i++) { - worldClimUpdater.update("tmin" + i); - worldClimUpdater.update("tmax" + i); - worldClimUpdater.update("tmean" + i); - worldClimUpdater.update("prec" + i); - } - - for (int i = 1; i <= 19; i++) { - worldClimUpdater.update("bio" + i); - } + worldClimUpdater.update(); return "redirect:/admin/ds2/"; }