Commit d605110d authored by Matija Obreza's avatar Matija Obreza

Update all worldclim.org data from admin

parent 9db3f803
......@@ -335,7 +335,7 @@ public class DSServiceImpl implements DSService {
@Transactional
public void worldclimUpdate(DS dataset, DSDescriptor dsd, Set<Long> tileIndexes, MappedByteBuffer buffer, short nullValue, double factor) {
if (LOG.isInfoEnabled()) {
LOG.info("\n\nUpdating " + dsd.getDescriptor().getCode() + " for tileIndexes: " + tileIndexes);
LOG.info("Updating " + dsd.getDescriptor().getCode() + " for tileIndexes: " + tileIndexes);
}
Map<Object, DSRow> rowMap = findOrMakeRows(dataset, tileIndexes);
......
......@@ -159,11 +159,11 @@ public class WorldClimUpdater implements InitializingBean {
if (descriptor == null) {
descriptor = createDescriptor(dataset, variableName);
worldclimDescriptor = dsService.addDescriptor(dataset, descriptor);
System.err.println("Created worldclimDescriptor " + variableName);
LOG.info("Created worldclimDescriptor " + variableName);
} else {
for (DSDescriptor dsd : dataset.getDescriptors()) {
if (dsd.getDescriptor().getCode().equals(variableName)) {
System.err.println("Found worldclimDescriptor " + variableName);
LOG.debug("Found worldclimDescriptor " + variableName);
worldclimDescriptor = dsd;
break;
}
......@@ -190,17 +190,9 @@ public class WorldClimUpdater implements InitializingBean {
}
Set<Long> tileIndexSet = getExistingTileIndexes();
// Set<Long> tileIndexSet = new HashSet<Long>();
// SecureRandom sr = new SecureRandom();
// for (int i = 0; i < 100; i++) {
// tileIndexSet.add((long) 24592511 + i);
// tileIndexSet.add((long) 4658056 + i);
// }
// for (int i = 0; i < 10; i++) {
// // add some out-of-bound values
// tileIndexSet.add((long) 8640 * 3600 + 1 + sr.nextInt(20000));
// }
if (tileIndexSet.isEmpty())
tileIndexSet = generateTileIndexes();
List<Long> tileIndexes = new ArrayList<Long>(tileIndexSet);
int batchSize = 1000;
for (int fromIndex = 0; fromIndex < tileIndexes.size(); fromIndex += batchSize) {
......@@ -208,9 +200,11 @@ public class WorldClimUpdater implements InitializingBean {
LOG.info("Processing tileIndexes: " + fromIndex + " of " + tileIndexes.size());
dsService.worldclimUpdate(dataset, worldclimDescriptor, ids, buffer, nullValue, factor);
}
LOG.info("Done processing " + variableName);
}
private Set<Long> getTile2Indexes() throws JsonParseException, JsonMappingException, IOException {
private Set<Long> generateTileIndexes() throws JsonParseException, JsonMappingException, IOException {
// read accessions with lat/lng and update in batch
final List<Long> accessionIds = new ArrayList<Long>(20000);
ObjectMapper mapper = new ObjectMapper();
......@@ -266,7 +260,7 @@ public class WorldClimUpdater implements InitializingBean {
descriptor.setCode(variableName);
descriptor.setTitle("WorldClim " + variableName);
descriptorService.saveDescriptor(descriptor);
System.err.println("Created descriptor " + descriptor);
LOG.info("Created descriptor " + descriptor);
return descriptor;
}
......
......@@ -16,14 +16,25 @@
package org.genesys2.server.servlet.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.service.*;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.impl.ContentSanitizer;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.server.service.worker.ElasticUpdater;
......@@ -41,13 +52,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
@Controller
@RequestMapping("/admin")
......@@ -334,7 +339,19 @@ public class AdminController {
}
@RequestMapping("/worldClim")
public void worldClim() throws IOException {
public String worldClim() throws IOException {
worldClimUpdater.update("alt");
for (int i = 1; i <= 12; i++) {
worldClimUpdater.update("tmin" + i);
worldClimUpdater.update("tmax" + i);
worldClimUpdater.update("tmean" + i);
worldClimUpdater.update("prec" + i);
}
for (int i = 1; i <= 19; i++) {
worldClimUpdater.update("bio" + i);
}
return "redirect:/admin/";
}
}
......@@ -310,6 +310,7 @@
</c:if>
</c:if>
<security:authorize access="hasRole('ADMINISTRATOR')">
<!-- WorldClim.org -->
<c:if test="${worldclimJson ne null}">
<h4><spring:message code="worldclim.monthly.title" /></h4>
......@@ -387,6 +388,7 @@
</c:forEach>
--%>
</c:if>
</security:authorize>
<c:if test="${svalbardData ne null}">
<h4><spring:message code="accession.svalbard-data" /></h4>
......@@ -488,6 +490,7 @@
});
</script>
</c:if>
<security:authorize access="hasRole('ADMINISTRATOR')">
<c:if test="${worldclimJson ne null}">
<script type="text/javascript">
var monthNames = ${jspHelper.toJson(jspHelper.monthShortNames(pageContext.response.locale))};
......@@ -540,6 +543,8 @@
});
</script>
</c:if>
</security:authorize>
<script type="text/javascript">
<%@include file="/WEB-INF/jsp/wiews/ga.jsp"%>
_pageDim = { institute: '${accession.instituteCode}', genus: '${accession.taxonomy.genus}' };
......
......@@ -18,7 +18,7 @@
--%>
<c:if test="${path ne null}">
<form id="downlodform" method="post" action="${path}">
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
<input type="hidden" name="${_csrf.parameterName}" value="<c:out value="${_csrf.token}" />"/>
<c:forEach items="${query.keySet()}" var="name">
<input type="hidden" name="${name}" value='<c:out value="${query[name]}" />' />
</c:forEach>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment