...
 
Commits (4)
......@@ -439,6 +439,11 @@
<version>${hazelcast.version}</version>
</dependency>
<dependency>
<groupId>com.healthmarketscience.jackcess</groupId>
<artifactId>jackcess</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
......
......@@ -16,16 +16,23 @@
package org.genesys2.server.api.v1;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import java.util.List;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.Api;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.persistence.dataset.DatasetRepository;
import org.genesys.catalog.service.DatasetService;
import org.genesys.catalog.service.ShortFilterService;
import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryFolder;
import org.genesys.filerepository.persistence.ImageGalleryPersistence;
......@@ -37,6 +44,8 @@ import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.FaoInstituteRepository;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.worker.ExportPassportData;
import org.genesys2.server.service.worker.ScheduledGLISUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -46,11 +55,15 @@ import org.springframework.data.domain.PageRequest;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletResponse;
/**
* @author Maxym Borodenko
*/
......@@ -90,6 +103,15 @@ public class AdminController {
@Autowired
private ScheduledGLISUpdater scheduledGLISUpdater;
@Autowired
private ExportPassportData exportPassportData;
/** The short filter service. */
@Autowired
protected ShortFilterService shortFilterService;
ObjectMapper mapper = new ObjectMapper();
@PostMapping(value = "/ensure-inst-folders")
public Boolean ensureInstituteFolders() throws Exception {
LOG.info("Ensure institute folders");
......@@ -157,8 +179,36 @@ public class AdminController {
scheduledGLISUpdater.notifyGLIS(from);
return true;
}
@RequestMapping(value = "/download-passport-data", method = RequestMethod.POST)
public void downloadPassportData(@RequestParam(name = "filter") String filters, HttpServletResponse response) throws IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
// get AccessionFilter from filterCode
ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(null, filter, AccessionFilter.class);
File temp = File.createTempFile("passport-data", ".mdb");
try {
LOG.warn("Generate passport data for accessions matching filter: {}", filterInfo.filter);
exportPassportData.exportToAccess(filterInfo.filter, temp);
response.setContentType("application/x-msaccess");
response.setContentLength((int) temp.length());
response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1s.mdb\"", filterInfo.filterCode));
InputStream inputStream = new BufferedInputStream(new FileInputStream(temp));
FileCopyUtils.copy(inputStream, response.getOutputStream());
response.flushBuffer();
} catch (Throwable e) {
LOG.warn("Download was aborted: {}", e.getMessage());
} finally {
if (! temp.delete()) {
LOG.warn("Can't delete temporary file: {}", temp.getAbsolutePath());
}
}
}
@PostMapping(value = "/acl", params = { "institutes" })
@Transactional
public void aclFixInstitutesAcl() throws Exception {
......
......@@ -16,7 +16,11 @@
package org.genesys2.server.mvc.admin;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
......@@ -29,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.servlet.http.HttpServletResponse;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch;
......@@ -36,6 +41,7 @@ import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.persistence.dataset.DatasetRepository;
import org.genesys.catalog.service.DatasetService;
import org.genesys.catalog.service.ShortFilterService;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryFolder;
......@@ -71,6 +77,7 @@ import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
import org.genesys2.server.service.worker.AccessionCounter;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.genesys2.server.service.worker.ExportPassportData;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
......@@ -87,6 +94,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.ui.Model;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
......@@ -188,6 +196,13 @@ public class AdminController {
@Autowired
private ScheduledGLISUpdater scheduledGLISUpdater;
@Autowired
private ExportPassportData exportPassportData;
/** The short filter service. */
@Autowired
protected ShortFilterService shortFilterService;
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -611,6 +626,35 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(value = "/download-passport-data", method = RequestMethod.POST)
public void downloadPassportData(@RequestParam(name = "filter") String filters, HttpServletResponse response) throws IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
// get AccessionFilter from filterCode
ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(null, filter, AccessionFilter.class);
File temp = File.createTempFile("passport-data", ".mdb");
try {
LOG.warn("Generate passport data for accessions matching filter: {}", filterInfo.filter);
exportPassportData.exportToAccess(filterInfo.filter, temp);
response.setContentType("application/x-msaccess");
response.setContentLength((int) temp.length());
response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1s.mdb\"", filterInfo.filterCode));
InputStream inputStream = new BufferedInputStream(new FileInputStream(temp));
FileCopyUtils.copy(inputStream, response.getOutputStream());
response.flushBuffer();
} catch (Throwable e) {
LOG.warn("Download was aborted: {}", e.getMessage());
} finally {
if (! temp.delete()) {
LOG.warn("Can't delete temporary file: {}", temp.getAbsolutePath());
}
}
}
@Autowired
private AccessionIdRepository accessionIdRepository;
......
/*
* Copyright 2019 Global Crop Diversity Trust
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.worker;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.sql.Types;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.healthmarketscience.jackcess.ColumnBuilder;
import com.healthmarketscience.jackcess.Database;
import com.healthmarketscience.jackcess.DatabaseBuilder;
import com.healthmarketscience.jackcess.Table;
import com.healthmarketscience.jackcess.TableBuilder;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionAlias;
import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.service.filter.AccessionFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* @author Maxym Borodenko
*/
@Component
public class ExportPassportData {
public static final Logger LOG = LoggerFactory.getLogger(ExportPassportData.class);
@Autowired
private AccessionProcessor accessionProcessor;
public void exportToAccess(final AccessionFilter filter, File targetFile) throws IOException, SQLException {
StopWatch stopWatch = new StopWatch();
try (Database db = DatabaseBuilder.create(Database.FileFormat.V2010, targetFile)) {
stopWatch.start();
writeAccessionsCore(filter, db);
stopWatch.split();
LOG.info("Writing core took {}", stopWatch.getSplitTime());
writeAccessionsGeo(filter, db);
stopWatch.split();
LOG.info("Writing geo took {}", stopWatch.getSplitTime());
writeAccessionsColl(filter, db);
stopWatch.split();
LOG.info("Writing coll took {}", stopWatch.getSplitTime());
writeAccessionsNames(filter, db);
stopWatch.split();
LOG.info("Writing names took {}", stopWatch.getSplitTime());
// TODO Add foreign keys
// TODO Add MCPD view
}
LOG.info("Dumping to Access completed: {}", targetFile.getAbsolutePath());
}
private void writeAccessionsCore(final AccessionFilter filter, Database db) throws IOException, SQLException {
LOG.info("Exporting data to core table ...");
Set<ColumnBuilder> columns = Stream.of(
new ColumnBuilder("genesysId").setSQLType(Types.BIGINT),
new ColumnBuilder("uuid").setSQLType(Types.VARCHAR),
new ColumnBuilder("instCode").setSQLType(Types.VARCHAR),
new ColumnBuilder("acceNumb").setSQLType(Types.VARCHAR),
new ColumnBuilder("doi").setSQLType(Types.VARCHAR),
new ColumnBuilder("genus").setSQLType(Types.VARCHAR),
new ColumnBuilder("species").setSQLType(Types.VARCHAR),
new ColumnBuilder("spAuthor").setSQLType(Types.VARCHAR),
new ColumnBuilder("subtaxa").setSQLType(Types.VARCHAR),
new ColumnBuilder("subtAuthor").setSQLType(Types.VARCHAR),
new ColumnBuilder("fullTaxa").setSQLType(Types.VARCHAR),
new ColumnBuilder("origCty").setSQLType(Types.VARCHAR),
new ColumnBuilder("acqSrc").setSQLType(Types.VARCHAR),
new ColumnBuilder("acqDate").setSQLType(Types.VARCHAR),
new ColumnBuilder("mlsStat").setSQLType(Types.VARCHAR),
new ColumnBuilder("available").setSQLType(Types.VARCHAR),
new ColumnBuilder("historic").setSQLType(Types.VARCHAR),
new ColumnBuilder("storage").setSQLType(Types.VARCHAR),
new ColumnBuilder("sampStat").setSQLType(Types.INTEGER),
new ColumnBuilder("duplSite").setSQLType(Types.VARCHAR),
new ColumnBuilder("createdBy").setSQLType(Types.BIGINT),
new ColumnBuilder("createdDate").setSQLType(Types.TIMESTAMP),
new ColumnBuilder("lastModifiedBy").setSQLType(Types.BIGINT),
new ColumnBuilder("lastModifiedDate").setSQLType(Types.TIMESTAMP)
).collect(Collectors.toCollection(LinkedHashSet::new));
Table coreTable = new TableBuilder("core").addColumns(columns).toTable(db);
Writer writer = new Writer();
// Write accession information
try {
accessionProcessor.process(filter, (accession) -> {
writer.processAccessionCore(coreTable, accession);
db.flush();
return accession;
});
} catch (Exception e) {
LOG.error("Error while exporting to table core: {}", e.getMessage());
} finally {
db.flush();
}
}
private void writeAccessionsGeo(final AccessionFilter filter, Database db) throws IOException, SQLException {
LOG.info("Exporting data to geo table ...");
Set<ColumnBuilder> columns = Stream.of(
new ColumnBuilder("genesysId").setSQLType(Types.BIGINT),
new ColumnBuilder("latitude").setSQLType(Types.DOUBLE),
new ColumnBuilder("longitude").setSQLType(Types.DOUBLE),
new ColumnBuilder("elevation").setSQLType(Types.DOUBLE),
new ColumnBuilder("datum").setSQLType(Types.VARCHAR),
new ColumnBuilder("uncertainty").setSQLType(Types.DOUBLE),
new ColumnBuilder("method").setSQLType(Types.VARCHAR),
new ColumnBuilder("version").setSQLType(Types.BIGINT)
).collect(Collectors.toCollection(LinkedHashSet::new));
Table geoTable = new TableBuilder("geo").addColumns(columns).toTable(db);
Writer writer = new Writer();
try {
accessionProcessor.process(filter, (accession) -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
writer.processAccessionGeo(geoTable, geo);
}
db.flush();
return accession;
});
} catch (Exception e) {
LOG.error("Error while exporting to table geo: {}", e.getMessage());
} finally {
db.flush();
}
}
private void writeAccessionsColl(final AccessionFilter filter, Database db) throws IOException, SQLException {
LOG.info("Exporting to coll table ...");
Set<ColumnBuilder> columns = Stream.of(
new ColumnBuilder("genesysId").setSQLType(Types.BIGINT),
new ColumnBuilder("collMissId").setSQLType(Types.VARCHAR),
new ColumnBuilder("collNumb").setSQLType(Types.VARCHAR),
new ColumnBuilder("collDate").setSQLType(Types.VARCHAR),
new ColumnBuilder("collSrc").setSQLType(Types.INTEGER),
new ColumnBuilder("collSite").setSQLType(Types.VARCHAR),
new ColumnBuilder("version").setSQLType(Types.BIGINT)
).collect(Collectors.toCollection(LinkedHashSet::new));
Table callTable = new TableBuilder("coll").addColumns(columns).toTable(db);
Writer writer = new Writer();
// Write accession collections information
try {
accessionProcessor.process(filter, (accession) -> {
AccessionCollect accessionCollect = accession.getAccessionId().getColl();
if (accessionCollect != null) {
writer.processAccessionCollect(callTable, accessionCollect);
}
db.flush();
return accession;
});
} catch (Exception e) {
LOG.error("Error while exporting to table coll: {}", e.getMessage());
} finally {
db.flush();
}
}
private void writeAccessionsNames(final AccessionFilter filter, Database db) throws IOException, SQLException {
LOG.info("Exporting data to names table ...");
Set<ColumnBuilder> columns = Stream.of(
new ColumnBuilder("genesysId").setSQLType(Types.BIGINT),
new ColumnBuilder("instCode").setSQLType(Types.VARCHAR),
new ColumnBuilder("name").setSQLType(Types.VARCHAR),
new ColumnBuilder("aliasType").setSQLType(Types.VARCHAR),
new ColumnBuilder("lang").setSQLType(Types.VARCHAR),
new ColumnBuilder("version").setSQLType(Types.BIGINT)
).collect(Collectors.toCollection(LinkedHashSet::new));
Table namesTable = new TableBuilder("names").addColumns(columns).toTable(db);
Writer writer = new Writer();
// Write accession aliases information
try {
accessionProcessor.process(filter, (accession) -> {
List<AccessionAlias> accessionAliases = accession.getAccessionId().getAliases();
if (accessionAliases != null && accessionAliases.size() > 0) {
writer.processAccessionAliases(namesTable, accessionAliases);
}
db.flush();
return accession;
});
} catch (Exception e) {
LOG.error("Error while exporting to table names: {}", e.getMessage());
} finally {
db.flush();
}
}
private class Writer {
private void processAccessionGeo(final Table table, final AccessionGeo geo) {
try {
final Object[] row = new Object[8];
// genesysId
row[0] = geo.getAccession().getId();
// latitude
row[1] = geo.getLatitude();
// longitude
row[2] = geo.getLongitude();
// elevation
row[3] = geo.getElevation();
// datum
row[4] = geo.getDatum();
// uncertainty
row[5] = geo.getUncertainty();
// method
row[6] = geo.getMethod();
// version
row[7] = geo.getVersion();
table.addRow(row);
} catch (IOException e) {
LOG.warn("Error: {}", e.getMessage());
}
}
private void processAccessionAliases(final Table table, final List<AccessionAlias> aliases) {
try {
for (AccessionAlias alias : aliases) {
final Object[] row = new Object[6];
// genesysId
row[0] = alias.getAccession().getId();
// instCode
row[1] = alias.getUsedBy();
// name
row[2] = alias.getName();
// aliasType
row[3] = alias.getAliasType().name();
// lang
row[4] = alias.getLang();
// version
row[5] = alias.getVersion();
table.addRow(row);
}
} catch (IOException e) {
LOG.warn("Error: {}", e.getMessage());
}
}
private void processAccessionCollect(final Table table, final AccessionCollect collect) {
try {
final Object[] row = new Object[7];
// genesysId
row[0] = collect.getAccession().getId();
// collMissId
row[1] = collect.getCollMissId();
// collNumb
row[2] = collect.getCollNumb();
// collDate
row[3] = collect.getCollDate();
// collSrc
row[4] = collect.getCollSrc();
// collSite
row[5] = collect.getCollSite();
// version
row[6] = collect.getVersion();
table.addRow(row);
} catch (IOException e) {
LOG.warn("Error: {}", e.getMessage());
}
}
private void processAccessionCore(final Table table, final Accession accession) {
try {
final Object[] row = new Object[24];
// genesysId
row[0] = accession.getId();
// uuid
row[1] = accession.getUuid().toString();
// instCode
row[2] = accession.getInstCode();
// acceNumb
row[3] = accession.getAccessionNumber();
// doi
row[4] = accession.getDoi();
// genus
row[5] = accession.getGenus();
if (accession.getTaxonomy() != null) {
// species
row[6] = accession.getTaxonomy().getSpecies();
// spAuthor
row[7] = accession.getTaxonomy().getSpAuthor();
// subtaxa
row[8] = accession.getTaxonomy().getSubtaxa();
// subtAuthor
row[9] = accession.getTaxonomy().getSubtAuthor();
// fullTaxa
row[10] = accession.getTaxonomy().getTaxonName();
}
// origCty
row[11] = accession.getOrigCty();
// acqSrc
row[12] = accession.getAcquisitionSource();
// acqDate
row[13] = accession.getAcquisitionDate();
// mlsStat
row[14] = accession.getMlsStatus();
// available
row[15] = accession.getAvailable();
// historic
row[16] = accession.getHistoric();
// storage
row[17] = accession.getStorageStr();
// sampStat
row[18] = accession.getSampStat();
// duplSite
row[19] = accession.getDuplSiteStr();
// createdBy
row[20] = accession.getCreatedBy();
// createdDate
row[21] = accession.getCreatedDate();
// lastModifiedBy
row[22] = accession.getLastModifiedBy();
// lastModifiedDate
row[23] = accession.getLastModifiedDate();
table.addRow(row);
} catch (IOException e) {
LOG.warn("Error: {}", e.getMessage());
}
}
}
}
......@@ -99,6 +99,12 @@
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<form method="post" action="<c:url value="/admin/download-passport-data" />">
<input type="text" name="filter" placeholder="{}" value="{}" />
<input type="submit" class="btn btn-default" value="Export to MS Access"/>
<!-- CSRF protection -->
<input type="hidden" name="${_csrf.parameterName}" value="${_csrf.token}" />
</form>
<h3>Dataset</h3>
<form method="post" action="<c:url value="/admin/relinkDatasetAccessions" />">
......