Commit f180ebab authored by Matija Obreza's avatar Matija Obreza

Introducing TileClimate for worldclim.org data

- liquibase
parent 19595ea7
...@@ -201,7 +201,7 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> { ...@@ -201,7 +201,7 @@ public class ElasticQueryBuilder implements Visitor<Void, Void> {
private void handleEquals(Path<?> path, Expression<?> value) { private void handleEquals(Path<?> path, Expression<?> value) {
PathMetadata pmd = path.getMetadata(); PathMetadata pmd = path.getMetadata();
if (pmd.getPathType() == PathType.COLLECTION_ANY) { if (pmd.getPathType() == PathType.COLLECTION_ANY) {
LOG.error("Path ANY for {}={}", pmd.getParent(), value); LOG.debug("Path ANY for {}={}", pmd.getParent(), value);
mustClauses.add(termsQuery(customizedPath(pmd.getParent().toString()), toValues(value))); mustClauses.add(termsQuery(customizedPath(pmd.getParent().toString()), toValues(value)));
} else { } else {
mustClauses.add(termsQuery(customizedPath(getParentPath(pmd.getParent()) + "." + pmd.getName()), toValues(value))); mustClauses.add(termsQuery(customizedPath(getParentPath(pmd.getParent()) + "." + pmd.getName()), toValues(value)));
......
This diff is collapsed.
...@@ -12,7 +12,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; ...@@ -12,7 +12,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
*/ */
public class WorldclimJson { public class WorldclimJson {
// Monthlies // Monthlies
private Long[] precipitation = new Long[12]; private Double[] precipitation = new Double[12];
private Double[] tempMin = new Double[12]; private Double[] tempMin = new Double[12];
private Double[] tempMean = new Double[12]; private Double[] tempMean = new Double[12];
private Double[] tempMax = new Double[12]; private Double[] tempMax = new Double[12];
...@@ -20,7 +20,7 @@ public class WorldclimJson { ...@@ -20,7 +20,7 @@ public class WorldclimJson {
@JsonIgnore @JsonIgnore
private Map<DSDescriptor, Object> other = new HashMap<DSDescriptor, Object>(); private Map<DSDescriptor, Object> other = new HashMap<DSDescriptor, Object>();
public Long[] getPrecipitation() { public Double[] getPrecipitation() {
return precipitation; return precipitation;
} }
...@@ -43,4 +43,20 @@ public class WorldclimJson { ...@@ -43,4 +43,20 @@ public class WorldclimJson {
public Map<DSDescriptor, Object> getOther() { public Map<DSDescriptor, Object> getOther() {
return this.other; return this.other;
} }
public void setPrecipitation(Double[] doubles) {
this.precipitation = doubles;
}
public void setTempMin(Double[] doubles) {
this.tempMin = doubles;
}
public void setTempMean(Double[] doubles) {
this.tempMean = doubles;
}
public void setTempMax(Double[] doubles) {
this.tempMax = doubles;
}
} }
...@@ -30,8 +30,6 @@ import org.genesys.blocks.auditlog.service.AuditTrailService; ...@@ -30,8 +30,6 @@ import org.genesys.blocks.auditlog.service.AuditTrailService;
import org.genesys.catalog.service.DatasetService; import org.genesys.catalog.service.DatasetService;
import org.genesys.filerepository.InvalidRepositoryPathException; import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.model.ImageGallery; import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.service.ImageGalleryService;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.elastic.AccessionDetails; import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession; import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionCollect; import org.genesys2.server.model.genesys.AccessionCollect;
...@@ -41,7 +39,7 @@ import org.genesys2.server.model.genesys.Taxonomy2; ...@@ -41,7 +39,7 @@ import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Crop; import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute; import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.AccessionService; import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.DSService; import org.genesys2.server.service.ClimateDataService;
import org.genesys2.server.service.DownloadService; import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.FilterConstants; import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
...@@ -85,14 +83,11 @@ public class AccessionController extends BaseController { ...@@ -85,14 +83,11 @@ public class AccessionController extends BaseController {
@Autowired @Autowired
private TaxonomyService taxonomyService; private TaxonomyService taxonomyService;
@Autowired
private DSService dsService;
@Autowired @Autowired
private InstituteFilesService instituteFilesService; private InstituteFilesService instituteFilesService;
@Autowired @Autowired
private ImageGalleryService imageGalleryService; private ClimateDataService climateDataService;
@Autowired @Autowired
private DownloadService downloadService; private DownloadService downloadService;
...@@ -189,10 +184,7 @@ public class AccessionController extends BaseController { ...@@ -189,10 +184,7 @@ public class AccessionController extends BaseController {
// Worldclim data // Worldclim data
if (accessionGeo != null && accessionGeo.getTileIndex() != null) { if (accessionGeo != null && accessionGeo.getTileIndex() != null) {
DS worldClimDataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET_UUID); model.addAttribute("worldclimJson", climateDataService.jsonForTile(accessionGeo.getTileIndex()));
if (worldClimDataset != null) {
model.addAttribute("worldclimJson", dsService.jsonForTile(worldClimDataset, accessionGeo.getTileIndex()));
}
} }
model.addAttribute("pdci", accessionId.getPdci()); model.addAttribute("pdci", accessionId.getPdci());
......
...@@ -18,13 +18,11 @@ package org.genesys2.server.mvc; ...@@ -18,13 +18,11 @@ package org.genesys2.server.mvc;
import java.util.UUID; import java.util.UUID;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionHistoric; import org.genesys2.server.model.genesys.AccessionHistoric;
import org.genesys2.server.service.ClimateDataService;
import org.genesys2.server.service.CropService; import org.genesys2.server.service.CropService;
import org.genesys2.server.service.DSService;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TraitService; import org.genesys2.server.service.TraitService;
import org.genesys2.spring.ResourceNotFoundException; import org.genesys2.spring.ResourceNotFoundException;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -44,14 +42,11 @@ public class ArchiveController extends BaseController { ...@@ -44,14 +42,11 @@ public class ArchiveController extends BaseController {
@Autowired @Autowired
private TraitService traitService; private TraitService traitService;
@Autowired
private TaxonomyService taxonomyService;
@Autowired @Autowired
private CropService cropService; private CropService cropService;
@Autowired @Autowired
private DSService dsService; private ClimateDataService climateDataService;
@RequestMapping(value = "/{uuid:.{36}}", method = RequestMethod.GET) @RequestMapping(value = "/{uuid:.{36}}", method = RequestMethod.GET)
public String view(ModelMap model, @PathVariable(value = "uuid") UUID uuid) { public String view(ModelMap model, @PathVariable(value = "uuid") UUID uuid) {
...@@ -91,10 +86,7 @@ public class ArchiveController extends BaseController { ...@@ -91,10 +86,7 @@ public class ArchiveController extends BaseController {
// Worldclim data // Worldclim data
if (accessionGeo != null && accessionGeo.getTileIndex() != null) { if (accessionGeo != null && accessionGeo.getTileIndex() != null) {
DS worldClimDataset = dsService.loadDatasetByUuid(AccessionController.WORLDCLIM_DATASET_UUID); model.addAttribute("worldclimJson", climateDataService.jsonForTile(accessionGeo.getTileIndex()));
if (worldClimDataset != null) {
model.addAttribute("worldclimJson", dsService.jsonForTile(worldClimDataset, accessionGeo.getTileIndex()));
}
} }
} }
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.persistence;
import org.genesys2.server.model.impl.TileClimate;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.querydsl.QueryDslPredicateExecutor;
/**
* The JPA TileClimateRepository.
*/
public interface TileClimateRepository extends JpaRepository<TileClimate, Long>, QueryDslPredicateExecutor<TileClimate> {
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service;
import java.nio.MappedByteBuffer;
import java.util.HashSet;
import java.util.List;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.impl.TileClimate;
import org.genesys2.server.model.json.WorldclimJson;
/**
* The ClimateDataService interface.
*/
public interface ClimateDataService {
void worldclimUpdate(String variableName, HashSet<Long> ids, MappedByteBuffer buffer, short nullValue, double factor);
TileClimate climateForTile(Long tileIndex);
void updateAccessionTileIndex(List<AccessionGeo> toSave);
WorldclimJson jsonForTile(Long tileIndex);
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service; package org.genesys2.server.service;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.MappedByteBuffer;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import org.genesys2.server.model.dataset.DS; import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSColumn; import org.genesys2.server.model.dataset.DSColumn;
import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.dataset.DSDescriptor; import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.json.WorldclimJson;
/**
* The DSService interface.
*/
public interface DSService { public interface DSService {
/**
* Save dataset.
*
* @param ds the ds
* @return the ds
*/
DS saveDataset(DS ds); DS saveDataset(DS ds);
/**
* Adds the qualifier.
*
* @param ds the ds
* @param d1 the d 1
* @return the DS qualifier
*/
DSQualifier addQualifier(DS ds, DSDescriptor d1); DSQualifier addQualifier(DS ds, DSDescriptor d1);
/**
* Adds the descriptor.
*
* @param ds the ds
* @param dSDescriptor the d S descriptor
* @return the DS column
*/
DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor); DSColumn addDescriptor(DS ds, DSDescriptor dSDescriptor);
/**
* Load dataset by uuid.
*
* @param uuid the uuid
* @return the ds
*/
DS loadDatasetByUuid(UUID uuid); DS loadDatasetByUuid(UUID uuid);
/**
* Update row.
*
* @param ds the ds
* @param row the row
*/
void updateRow(DS ds, Object[] row); void updateRow(DS ds, Object[] row);
/**
* Update rows.
*
* @param ds the ds
* @param rows the rows
*/
void updateRows(DS ds, List<Object[]> rows); void updateRows(DS ds, List<Object[]> rows);
/**
* Update row.
*
* @param ds the ds
* @param qualifiers the qualifiers
* @param dsd the dsd
* @param value the value
*/
void updateRow(DS ds, Object[] qualifiers, DSColumn dsd, Object value); void updateRow(DS ds, Object[] qualifiers, DSColumn dsd, Object value);
void worldclimUpdate(DS dataset, DSColumn dsd, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor); /**
* Download.
void updateAccessionTileIndex(List<AccessionGeo> toSave); *
* @param ds the ds
WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex); * @param outputStream the output stream
* @throws IOException Signals that an I/O exception has occurred.
*/
void download(DS ds, OutputStream outputStream) throws IOException; void download(DS ds, OutputStream outputStream) throws IOException;
/**
* Download.
*
* @param ds the ds
* @param dsds the dsds
* @param outputStream the output stream
* @throws IOException Signals that an I/O exception has occurred.
*/
void download(DS ds, List<DSColumn> dsds, OutputStream outputStream) throws IOException; void download(DS ds, List<DSColumn> dsds, OutputStream outputStream) throws IOException;
/**
* Delete dataset.
*
* @param ds the ds
*/
void deleteDataset(DS ds); void deleteDataset(DS ds);
/**
* Delete descriptor.
*
* @param dsd the dsd
*/
void deleteDescriptor(DSColumn dsd); void deleteDescriptor(DSColumn dsd);
/**
* Delete rows.
*
* @param ds the ds
*/
void deleteRows(DS ds); void deleteRows(DS ds);
} }
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.impl;
import java.lang.reflect.InvocationTargetException;
import java.nio.MappedByteBuffer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import org.apache.commons.beanutils.BeanUtils;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.impl.TileClimate;
import org.genesys2.server.model.json.WorldclimJson;
import org.genesys2.server.persistence.AccessionGeoRepository;
import org.genesys2.server.persistence.TileClimateRepository;
import org.genesys2.server.service.ClimateDataService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* The ClimateDataService implementation.
*/
@Service
@Transactional(readOnly = true)
public class ClimateDataServiceImpl implements ClimateDataService {
public static final Logger LOG = LoggerFactory.getLogger(ClimateDataServiceImpl.class);
/** The tile climate repository. */
@Autowired
private TileClimateRepository tileClimateRepository;
/** The accession geo repository. */
@Autowired
private AccessionGeoRepository accessionGeoRepository;
/* (non-Javadoc)
* @see org.genesys2.server.service.DSService#updateAccessionTileIndex(java.util.List)
*/
@Override
@Transactional
public void updateAccessionTileIndex(List<AccessionGeo> toSave) {
accessionGeoRepository.save(toSave);
}
/* (non-Javadoc)
* @see org.genesys2.server.service.ClimateDataService#worldclimUpdate(java.lang.String, java.util.HashSet, java.nio.MappedByteBuffer, short, double)
*/
@Override
@Transactional
public void worldclimUpdate(String variableName, HashSet<Long> tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) {
LOG.debug("Updating {} for tileIndexes: {}", variableName, tileIndexes.size());
Map<Long, TileClimate> rowMap = findOrMakeRows(tileIndexes);
for (final Long tileIndex : tileIndexes) {
TileClimate tileClimate = rowMap.get(tileIndex);
if (tileIndex * 2 > buffer.capacity() - 2) {
LOG.info("OUT OF FILE tile={}", tileIndex);
rowMap.remove(tileIndex);
} else {
try {
short val = buffer.getShort((int) (tileIndex * 2));
if (val != nullValue) {
LOG.trace("tile={} val={}", tileIndex, val);
Object value = null;
if (factor < 1.0) {
// FIXME Find a better way!
value = new Double(factor * val);
} else if (factor == 1.0) {
value = new Long((long) (factor * val));
}
try {
BeanUtils.setProperty(tileClimate, variableName, value);
} catch (IllegalAccessException | InvocationTargetException e) {
LOG.error("Cannot set TileClimate#{} to {}. {}", variableName, value, e.getMessage());
}
}
} catch (IndexOutOfBoundsException e) {
LOG.error("OUT OF BOUND tile={} capacity={} limit={}", tileIndex, buffer.capacity(), buffer.limit());
throw e;
}
}
}
tileClimateRepository.save(rowMap.values());
LOG.debug("Done saving.");
}
/**
* Find or make rows.
*
* @param tileIndexes the tile indexes
* @return the map
*/
private Map<Long, TileClimate> findOrMakeRows(HashSet<Long> tileIndexes) {
List<TileClimate> existingTiles = tileClimateRepository.findAll(tileIndexes);
HashMap<Long, TileClimate> map = new HashMap<>(tileIndexes.size());
existingTiles.forEach((tile) -> {
map.put(tile.getTileIndex(), tile);
});
tileIndexes.forEach(tileIndex -> {
if (!map.containsKey(tileIndex)) {
map.put(tileIndex, new TileClimate(tileIndex));
}
});
return map;
}
/* (non-Javadoc)
* @see org.genesys2.server.service.ClimateDataService#climateForTile(java.lang.Long)
*/
@Override
public TileClimate climateForTile(Long tileIndex) {
return tileClimateRepository.findOne(tileIndex);
}
/* (non-Javadoc)
* @see org.genesys2.server.service.ClimateDataService#jsonForTile(java.lang.Long)
*/
@Override
public WorldclimJson jsonForTile(Long tileIndex) {
TileClimate tileClimate = tileClimateRepository.findOne(tileIndex);
if (tileClimate == null)
return null;
WorldclimJson wc = new WorldclimJson();
wc.setPrecipitation(new Double[] { tileClimate.getPrec1(), tileClimate.getPrec2(), tileClimate.getPrec3(), tileClimate.getPrec4(), tileClimate.getPrec5(), tileClimate.getPrec6(), tileClimate.getPrec7(), tileClimate.getPrec8(), tileClimate.getPrec9(), tileClimate.getPrec10(), tileClimate.getPrec11(), tileClimate.getPrec12() } );
wc.setTempMin(new Double[] { tileClimate.getTmin1(), tileClimate.getTmin2(), tileClimate.getTmin3(), tileClimate.getTmin4(), tileClimate.getTmin5(), tileClimate.getTmin6(), tileClimate.getTmin7(), tileClimate.getTmin8(), tileClimate.getTmin9(), tileClimate.getTmin10(), tileClimate.getTmin11(), tileClimate.getTmin12() } );
wc.setTempMean(new Double[] { tileClimate.getTmean1(), tileClimate.getTmean2(), tileClimate.getTmean3(), tileClimate.getTmean4(), tileClimate.getTmean5(), tileClimate.getTmean6(), tileClimate.getTmean7(), tileClimate.getTmean8(), tileClimate.getTmean9(), tileClimate.getTmean10(), tileClimate.getTmean11(), tileClimate.getTmean12() } );
wc.setTempMax(new Double[] { tileClimate.getTmax1(), tileClimate.getTmax2(), tileClimate.getTmax3(), tileClimate.getTmax4(), tileClimate.getTmax5(), tileClimate.getTmax6(), tileClimate.getTmax7(), tileClimate.getTmax8(), tileClimate.getTmax9(), tileClimate.getTmax10(), tileClimate.getTmax11(), tileClimate.getTmax12() } );
return wc;
}
}
...@@ -43,15 +43,10 @@ import org.genesys.worldclim.grid.generic.GenericGridFile; ...@@ -43,15 +43,10 @@ import org.genesys.worldclim.grid.generic.GenericGridFile;
import org.genesys.worldclim.grid.generic.GenericGridZipFile; import org.genesys.worldclim.grid.generic.GenericGridZipFile;
import org.genesys.worldclim.grid.generic.Header; import org.genesys.worldclim.grid.generic.Header;
import org.genesys2.server.component.aspect.AsAdmin; import org.genesys2.server.component.aspect.AsAdmin;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSColumn;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.persistence.AccessionGeoRepository; import org.genesys2.server.persistence.AccessionGeoRepository;
import org.genesys2.server.persistence.GenesysLowlevelRepository; import org.genesys2.server.persistence.GenesysLowlevelRepository;
import org.genesys2.server.service.DSService;