Commit 81233e9f authored by Matija Obreza's avatar Matija Obreza

Deleting DS2, DS2 admin controller and view

parent d605110d
......@@ -2,12 +2,15 @@ package org.genesys2.server.model.dataset;
import java.util.List;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
......@@ -17,20 +20,25 @@ import org.genesys2.server.model.EntityId;
@Entity
@Table(name = "ds2row", uniqueConstraints = { @UniqueConstraint(columnNames = { "md5", "sha1" }) })
public class DSRow implements EntityId {
@OneToMany(fetch=FetchType.LAZY, mappedBy = "row")
@ManyToOne(fetch=FetchType.LAZY, optional = false)
@JoinColumn(name="ds")
private DS dataset;
@OneToMany(fetch = FetchType.LAZY, mappedBy = "row", cascade = { CascadeType.REMOVE })
private List<DSRowQualifier<?>> rowQualifiers;
@OneToMany(fetch=FetchType.LAZY, mappedBy = "row")
@OneToMany(fetch = FetchType.LAZY, mappedBy = "row")
private List<DSValue<?>> values;
@Id
@GeneratedValue(strategy = GenerationType.TABLE)
private Long id;
@Column(columnDefinition="binary(16)", updatable=false)
@Column(columnDefinition = "binary(16)", updatable = false)
private byte[] md5;
@Column(columnDefinition="binary(20)", updatable=false)
@Column(columnDefinition = "binary(20)", updatable = false)
private byte[] sha1;
@Override
......@@ -42,6 +50,14 @@ public class DSRow implements EntityId {
this.id = id;
}
public DS getDataset() {
return dataset;
}
public void setDataset(DS dataset) {
this.dataset = dataset;
}
public List<DSRowQualifier<?>> getRowQualifiers() {
return rowQualifiers;
}
......
package org.genesys2.server.persistence.domain.dataset;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSQualifier;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface DSQualifierRepository extends JpaRepository<DSQualifier, Long> {
@Modifying
@Query("delete from DSQualifier dsq where dsq.dataset = ?1")
int deleteFor(DS ds);
}
package org.genesys2.server.persistence.domain.dataset;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSRowQualifier;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface DSRowQualifierRepository extends JpaRepository<DSRowQualifier<?>, Long> {
@Modifying
@Query("delete from DSRowQualifier dsrq where dsrq.datasetQualifier in (select dsq from DSQualifier dsq where dsq.dataset = ?1)")
int deleteFor(DS ds);
}
......@@ -7,6 +7,7 @@ import java.util.Set;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSRow;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustomRepository {
......@@ -23,4 +24,8 @@ public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustom
@Query("select distinct dsrq.row from DSRowQualifier dsrq where dsrq.datasetQualifier.dataset = ?1")
List<DSRow> findByDataset(DS ds);
@Modifying
@Query("delete from DSRow dsr where dsr.dataset = ?1")
int deleteFor(DS ds);
}
......@@ -4,9 +4,15 @@ import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSRow;
import org.genesys2.server.model.dataset.DSValue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface DSValueRepository extends JpaRepository<DSValue<?>, Long> {
DSValue<?> findByRowAndDatasetDescriptor(DSRow dsr, DSDescriptor dsd);
@Modifying
@Query("delete from DSValue dsv where dsv.datasetDescriptor = ?1")
int deleteFor(DSDescriptor dsd);
}
......@@ -30,11 +30,17 @@ public interface DSService {
void worldclimUpdate(DS dataset, DSDescriptor dsd, Set<Long> ids, MappedByteBuffer buffer, short nullValue, double factor);
void tempStore(List<AccessionGeo> toSave);
void updateAccessionTileIndex(List<AccessionGeo> toSave);
WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex);
void download(DS ds, OutputStream outputStream) throws IOException;
void download(DS ds, List<DSDescriptor> dsds, OutputStream outputStream) throws IOException;
void deleteDataset(DS ds);
void deleteDescriptor(DSDescriptor dsd);
void deleteRows(DS ds);
}
......@@ -130,6 +130,7 @@ public class DSServiceImpl implements DSService {
DSRow dsr = dsRowRepo.findRow(ds, qualifiers);
if (dsr == null) {
dsr = new DSRow();
dsr.setDataset(ds);
dsRowRepo.save(dsr);
LOG.info("Made new row " + dsr.getId());
......@@ -311,7 +312,8 @@ public class DSServiceImpl implements DSService {
// DigestUtils.sha1(keyBuffer.array()).length);
dsr.setMd5(DigestUtils.md5(keyBuffer.array()));
dsr.setSha1(DigestUtils.sha1(keyBuffer.array()));
dsr.setDataset(ds);
dsr.setRowQualifiers(rowQualifiers);
return dsr;
}
......@@ -384,7 +386,7 @@ public class DSServiceImpl implements DSService {
}
}
}
// Only save new rows
Collection<DSRow> newRows = new ArrayList<DSRow>();
for (DSRow dsr : rowMap.values()) {
......@@ -392,26 +394,21 @@ public class DSServiceImpl implements DSService {
newRows.add(dsr);
}
saveRows(newRows);
// Save values
dsValueRepo.save(toSave);
LOG.info("Done saving.");
}
private List<DSRow> getForQualifier(DS dataset, Set<Long> ids) {
return dsRowRepo.findByQualifier(dataset, ids);
}
@Override
@Transactional
public void tempStore(List<AccessionGeo> toSave) {
public void updateAccessionTileIndex(List<AccessionGeo> toSave) {
accessionGeoRepository.save(toSave);
}
@Override
public WorldclimJson jsonForTile(DS worldClimDataset, Long tileIndex) {
DSRow dsr = dsRowRepo.findByLongQualifier(worldClimDataset, tileIndex);
if (dsr == null || dsr.getValues().isEmpty())
return null;
......@@ -440,7 +437,7 @@ public class DSServiceImpl implements DSService {
}
return wc;
}
@Override
public void download(DS ds, List<DSDescriptor> dsds, OutputStream outputStream) throws IOException {
XSSFWorkbook template = new XSSFWorkbook();
......@@ -499,11 +496,11 @@ public class DSServiceImpl implements DSService {
}
}
}
//
// if (rowIndex > 10000) {
// LOG.warn("Breaking");
// break;
// }
// if (rowIndex > 10000) {
// LOG.warn("Breaking");
// break;
// }
}
sheet.flushRows();
......@@ -530,4 +527,32 @@ public class DSServiceImpl implements DSService {
c.setCellValue((Date) v);
}
}
@Override
@Transactional
public void deleteDataset(DS ds) {
int count = dsQualRepo.deleteFor(ds);
LOG.info("Deleted " + count + " DSQualifiers");
dsRepo.delete(ds.getId());
LOG.info("Deleted dataset " + ds.getUuid());
}
@Override
@Transactional
public void deleteRows(DS ds) {
LOG.info("Clearing DSRowQuali for " + ds);
int count = dsRowQualiRepo.deleteFor(ds);
LOG.info("Deleted " + count + " DSRowQualifiers");
count = dsRowRepo.deleteFor(ds);
LOG.info("Deleted " + count + " DSRows");
}
@Override
@Transactional
public void deleteDescriptor(DSDescriptor dsd) {
LOG.info("Clearing DSValues for " + dsd.getDescriptor().getCode());
int count = dsValueRepo.deleteFor(dsd);
LOG.info("Deleted " + count + " DSValue cells");
dsDescRepo.delete(dsd);
}
}
......@@ -243,7 +243,7 @@ public class WorldClimUpdater implements InitializingBean {
}
}
}
dsService.tempStore(toSave);
dsService.updateAccessionTileIndex(toSave);
}
return tileIndexSet;
}
......
......@@ -88,10 +88,12 @@ public class AccessionController extends BaseController {
model.addAttribute("crops", cropService.getCrops(accession.getTaxonomy()));
if (accessionGeo.getTileIndex() != null) {
// Worldclim data
// Worldclim data
if (accessionGeo != null && accessionGeo.getTileIndex() != null) {
DS worldClimDataset = dsService.loadDatasetByUuid(WORLDCLIM_DATASET_UUID);
model.addAttribute("worldclimJson", dsService.jsonForTile(worldClimDataset, accessionGeo.getTileIndex()));
if (worldClimDataset != null) {
model.addAttribute("worldclimJson", dsService.jsonForTile(worldClimDataset, accessionGeo.getTileIndex()));
}
}
return "/accession/details";
......
......@@ -14,7 +14,7 @@
* limitations under the License.
**/
package org.genesys2.server.servlet.controller;
package org.genesys2.server.servlet.controller.admin;
import java.io.IOException;
import java.sql.ResultSet;
......
......@@ -7,7 +7,6 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.service.MappingService;
import org.genesys2.server.servlet.controller.AdminController;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
......
/**
* Copyright 2015 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.servlet.controller.admin;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.service.DSService;
import org.genesys2.server.service.worker.WorldClimUpdater;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@Controller
@RequestMapping("/admin/ds2")
@PreAuthorize("hasRole('ADMINISTRATOR')")
public class DS2Controller {
public static final Log LOG = LogFactory.getLog(DS2Controller.class);
@Autowired
private WorldClimUpdater worldClimUpdater;
@Autowired
private DSService dsService;
@RequestMapping("/")
public String root(Model model) {
return "/admin/ds2/index";
}
@RequestMapping(value = "/worldclim/update", method = RequestMethod.POST)
public String worldClim() throws IOException {
worldClimUpdater.update("alt");
for (int i = 1; i <= 12; i++) {
worldClimUpdater.update("tmin" + i);
worldClimUpdater.update("tmax" + i);
worldClimUpdater.update("tmean" + i);
worldClimUpdater.update("prec" + i);
}
for (int i = 1; i <= 19; i++) {
worldClimUpdater.update("bio" + i);
}
return "redirect:/admin/ds2/";
}
@RequestMapping(value = "/worldclim/delete", method = RequestMethod.POST)
public String worldClimDelete() throws IOException {
DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
if (ds != null) {
LOG.info("Deleting dataset " + ds.getUuid());
for (DSDescriptor dsd : ds.getDescriptors()) {
dsService.deleteDescriptor(dsd);
}
dsService.deleteRows(ds);
dsService.deleteDataset(ds);
LOG.info("Done");
}
return "redirect:/admin/ds2/";
}
}
......@@ -11,7 +11,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.genesys2.server.servlet.controller.AdminController;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
......
......@@ -586,8 +586,10 @@ logger.name=Logger name
logger.log-level=Log level
logger.appenders=Log appenders
menu.admin.index=Admin
menu.admin.loggers=Loggers
menu.admin.caches=Caches
menu.admin.ds2=DS2 Datasets
worldclim.monthly.title=Climate at collecting site
worldclim.monthly.precipitation.title=Monthly precipitation
......
......@@ -310,7 +310,7 @@
</c:if>
</c:if>
<security:authorize access="hasRole('ADMINISTRATOR')">
<security:authorize access="isAuthenticated()">
<!-- WorldClim.org -->
<c:if test="${worldclimJson ne null}">
<h4><spring:message code="worldclim.monthly.title" /></h4>
......@@ -490,7 +490,7 @@
});
</script>
</c:if>
<security:authorize access="hasRole('ADMINISTRATOR')">
<security:authorize access="isAuthenticated()">
<c:if test="${worldclimJson ne null}">
<script type="text/javascript">
var monthNames = ${jspHelper.toJson(jspHelper.monthShortNames(pageContext.response.locale))};
......
......@@ -12,6 +12,14 @@
<spring:message code="admin.cache.page.title" />
</h1>
<div class="main-col-header clearfix">
<div class="nav-header pull-left">
<a class="" href="<c:url value="/admin/" />"><spring:message code="menu.admin.index" /></a>
<a class="" href="<c:url value="/admin/logger/" />"><spring:message code="menu.admin.loggers" /></a>
<a class="" href="<c:url value="/admin/ds2/" />"><spring:message code="menu.admin.ds2" /></a>
</div>
</div>
<form method="post" action="<c:url value="/admin/cache/clearCaches" />">
<input type="submit" class="btn btn-default" value="Clear all caches" />
<!-- CSRF protection -->
......
<!DOCTYPE html>
<%@include file="/WEB-INF/jsp/init.jsp"%>
<%@ taglib prefix="sec" uri="http://www.springframework.org/security/tags"%>
<html>
<head>
<title><spring:message code="admin.page.title" /></title>
</head>
<body>
<h1>
<spring:message code="admin.page.title" />
</h1>
<div class="main-col-header clearfix">
<div class="nav-header pull-left">
<a class="" href="<c:url value="/admin/" />"><spring:message code="menu.admin.index" /></a>
<a class="" href="<c:url value="/admin/cache/" />"><spring:message code="menu.admin.caches" /></a>
<a class="" href="<c:url value="/admin/logger/" />"><spring:message code="menu.admin.loggers" /></a>
</div>
</div>
<h3>worldclim.org</h3>
<form method="post" action="<c:url value="/admin/ds2/worldclim/update" />">
<input type="submit" class="btn btn-default" value="Update worldclim.org" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
</form>
<form method="post" action="<c:url value="/admin/ds2/worldclim/delete" />">
<input type="submit" class="btn btn-default" value="Delete worldclim.org" />
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}"/>
</form>
</body>
</html>
\ No newline at end of file
......@@ -16,6 +16,7 @@
<div class="nav-header pull-left">
<a class="" href="<c:url value="/admin/cache/" />"><spring:message code="menu.admin.caches" /></a>
<a class="" href="<c:url value="/admin/logger/" />"><spring:message code="menu.admin.loggers" /></a>
<a class="" href="<c:url value="/admin/ds2/" />"><spring:message code="menu.admin.ds2" /></a>
</div>
</div>
......
......@@ -10,6 +10,7 @@ import java.util.List;
import org.apache.http.impl.client.HttpClientBuilder;
import org.genesys2.server.model.dataset.DS;
import org.genesys2.server.model.dataset.DSDescriptor;
import org.genesys2.server.model.dataset.DSQualifier;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl;
import org.genesys2.server.service.DSService;
......@@ -58,7 +59,7 @@ public class WorldClimUpdaterTest {
public DSService dsService() {
return new DSServiceImpl();
}
@Bean
public DescriptorService descriptorService() {
return new DescriptorServiceImpl();
......@@ -80,7 +81,7 @@ public class WorldClimUpdaterTest {
@Test
public void testUpdateAlt() throws IOException {
worldClimUpdater.update("alt");
for (int i = 1; i <= 12; i++) {
worldClimUpdater.update("tmin" + i);
worldClimUpdater.update("tmax" + i);
......@@ -93,17 +94,26 @@ public class WorldClimUpdaterTest {
}
}
@Test
public void deleteDataset() {
DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
for (DSDescriptor dsd : ds.getDescriptors()) {
dsService.deleteDescriptor(dsd);
}
dsService.deleteRows(ds);
dsService.deleteDataset(ds);
}
@Test
public void testDownload() throws IOException {
DS ds = dsService.loadDatasetByUuid(WorldClimUpdater.WORLDCLIM_DATASET);
File file = new File("worldclim " + System.currentTimeMillis() + ".xls");
BufferedOutputStream fos=new BufferedOutputStream(new FileOutputStream(file));
BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(file));
dsService.download(ds, fos);
fos.flush();
fos.close();
// file.delete();
// file.delete();
}
@Test
public void testDownloadDesc() throws IOException {
......@@ -115,12 +125,12 @@ public class WorldClimUpdaterTest {
}
}
System.out.println("Downloading tmin data: " + dsds.size());
File file = new File("worldclim " + System.currentTimeMillis() + ".xls");
BufferedOutputStream fos=new BufferedOutputStream(new FileOutputStream(file));
BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(file));
dsService.download(ds, dsds, fos);
fos.flush();
fos.close();
// file.delete();
// file.delete();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment