Skip to content
GitLab
Commit c054dac5 authored by Matija Obreza's avatar Matija Obreza
Browse files

MCPD XLSX

parent c117b8ce
Hide whitespace changes
Inline Side-by-side
pom.xml
View file @ c054dac5
......@@ -522,7 +522,12 @@
<artifactId>transifex-client</artifactId>
<version>0.2-SNAPSHOT</version>
</dependency>
</dependencies>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.10.1</version>
</dependency>
</dependencies>
<build>
<plugins>
......
src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepository.java
View file @ c054dac5
......@@ -20,6 +20,7 @@ import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.springframework.data.domain.Sort;
import org.springframework.jdbc.core.RowCallbackHandler;
public interface GenesysLowlevelRepository {
......@@ -53,4 +54,6 @@ public interface GenesysLowlevelRepository {
int countAccessions(AppliedFilters filter);
void listAccessionIds(AppliedFilters filter, Sort sort, RowCallbackHandler rowCallbackHandler);
}
src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepositoryImpl.java
View file @ c054dac5
......@@ -32,6 +32,7 @@ import org.genesys2.server.service.impl.DirectMysqlQuery;
import org.genesys2.server.service.impl.DirectMysqlQuery.MethodResolver;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.ArgumentPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
......@@ -255,6 +256,34 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
}
}, rowCallbackHandler);
}
@Override
public void listAccessionIds(final AppliedFilters filter, final Sort sort, final RowCallbackHandler rowCallbackHandler) {
final DirectMysqlQuery directQuery = new DirectMysqlQuery("accession", "a");
directQuery.jsonFilter(filter, new MethodResolver() {
@Override
public Method getMethod(final long methodId) {
return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
}
});
directQuery.sort(sort);
this.jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(final Connection con) throws SQLException {
final PreparedStatement stmt = con.prepareStatement(directQuery
.getQuery("a.id"));
final ArgumentPreparedStatementSetter apss = new ArgumentPreparedStatementSetter(directQuery.getParameters());
apss.setValues(stmt);
// Set mysql JConnector to stream results
// stmt.setFetchSize(Integer.MIN_VALUE);
return stmt;
}
}, rowCallbackHandler);
}
@Override
public void listAccessionsGeo(final AppliedFilters filter, final RowCallbackHandler rowCallbackHandler) {
......
src/main/java/org/genesys2/server/service/DownloadService.java 0 → 100644
View file @ c054dac5
/**
* Copyright 2014 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service;
import java.io.IOException;
import java.io.OutputStream;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
/**
* Defines available downloads
*
* @author matijaobreza
*
*/
public interface DownloadService {
void writeXlsxMCPD(AppliedFilters filters, OutputStream outputStream) throws IOException;
}
src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java
View file @ c054dac5
......@@ -37,6 +37,7 @@ import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Order;
public class DirectMysqlQuery {
......@@ -85,7 +86,9 @@ public class DirectMysqlQuery {
if (StringUtils.isNotBlank(alias)) {
sb.append(alias).append(StringUtils.SPACE);
}
sb.append("on ").append(onExpr).append(StringUtils.SPACE);
if (StringUtils.isNotBlank(onExpr)) {
sb.append("on ").append(onExpr).append(StringUtils.SPACE);
}
return this;
}
......@@ -397,6 +400,30 @@ public class DirectMysqlQuery {
return params.toArray();
}
public DirectMysqlQuery sort(Sort sort) {
if (sortBuffer.length() != 0) {
throw new RuntimeException("sortBuffer is not blank, invalid use of #pageable(Pageable)");
}
sortBuffer.append("\n order by ");
for (final Order o : sort) {
if (LOG.isDebugEnabled()) {
LOG.debug("Order: " + o);
}
// ClassMetadata md =
// sessionFactory.getClassMetadata(Accession.class);
// md.
// EntityType<Accession> x =
// entityManager.getMetamodel().entity(Accession.class);
// System.err.println(x.getAttribute(o.getProperty()).getName());
// sb.append(x.getAttribute(o.getProperty()).getName());
sortBuffer.append("a.").append(o.getProperty());
sortBuffer.append(" ").append(o.getDirection());
}
return this;
}
public DirectMysqlQuery pageable(Pageable pageable) {
if (sortBuffer.length() != 0) {
throw new RuntimeException("sortBuffer is not blank, invalid use of #pageable(Pageable)");
......@@ -407,21 +434,7 @@ public class DirectMysqlQuery {
}
if (pageable.getSort() != null) {
sortBuffer.append("\n order by ");
for (final Order o : pageable.getSort()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Order: " + o);
}
// ClassMetadata md =
// sessionFactory.getClassMetadata(Accession.class);
// md.
// EntityType<Accession> x =
// entityManager.getMetamodel().entity(Accession.class);
// System.err.println(x.getAttribute(o.getProperty()).getName());
// sb.append(x.getAttribute(o.getProperty()).getName());
sortBuffer.append("a.").append(o.getProperty());
sortBuffer.append(" ").append(o.getDirection());
}
sort(pageable.getSort());
}
sortBuffer.append(" limit ");
if (LOG.isDebugEnabled()) {
......
src/main/java/org/genesys2/server/service/impl/DownloadServiceImpl.java 0 → 100644
View file @ c054dac5
/**
* Copyright 2014 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.impl;
import java.io.IOException;
import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import javax.persistence.EntityManager;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionAlias;
import org.genesys2.server.model.genesys.AccessionBreeding;
import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.server.model.genesys.AccessionExchange;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionRemark;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.MethodRepository;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
public class DownloadServiceImpl implements DownloadService {
private static final int XLSX_ROWS_LIMIT = 65536;
public static final Log LOG = LogFactory.getLog(DownloadServiceImpl.class);
@Autowired
private EntityManager entityManager;
@Autowired
private GenesysService genesysService;
@Autowired
private GenesysLowlevelRepository genesysLowlevelRepository;
@Autowired
private MethodRepository methodRepository;
@Value("${base.url}")
private String baseUrl;
@Override
@Transactional(readOnly = true)
public void writeXlsxMCPD(final AppliedFilters filters, OutputStream outputStream) throws IOException {
XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/MCPD.xlsx"));
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SXSSFWorkbook wb = new SXSSFWorkbook(template, -1);
CellStyle dateStyle = wb.createCellStyle();
dateStyle.setDataFormat(wb.createDataFormat().getFormat("dd/mm/yyyy"));
dateStyle.setAlignment(CellStyle.ALIGN_LEFT);
Sheet legal = wb.getSheet("Legal information");
System.err.println("Rows: " + legal.getLastRowNum());
Row r;
Cell c;
r = legal.createRow(0);
r.createCell(0).setCellValue("Server URL");
r.createCell(1).setCellValue(baseUrl);
r = legal.createRow(1);
r.createCell(0).setCellValue("Filters");
r.createCell(1).setCellValue(filters.toString());
r = legal.createRow(2);
r.createCell(0).setCellValue("Data source");
c = r.createCell(1);
c.setCellValue(baseUrl + "/explore?filter=" + filters.toString());
r = legal.createRow(3);
r.createCell(0).setCellValue("Date");
c = r.createCell(1);
c.setCellStyle(dateStyle);
c.setCellValue(new Date());
r = legal.createRow(4);
r.createCell(0).setCellValue("Server URL");
r.createCell(1).setCellValue(baseUrl + "/content/terms");
((SXSSFSheet) legal).flushRows();
final Sheet sheet = wb.getSheet("MCPD");
// Write accession information
genesysLowlevelRepository.listAccessionIds(filters, new Sort("acceNumb"), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
if (i > XLSX_ROWS_LIMIT) {
LOG.info("Row limit exceeded");
return;
}
long accessionId = rs.getLong(1);
Accession accession = entityManager.find(Accession.class, accessionId);
Row row = sheet.createRow(i + 1);
AccessionGeo geo = genesysService.listAccessionGeo(accession);
AccessionCollect collect = genesysService.listAccessionCollect(accession);
AccessionBreeding bred = genesysService.listAccessionBreeding(accession);
List<AccessionAlias> names = genesysService.listAccessionAliases(accession);
AccessionExchange exch = genesysService.listAccessionExchange(accession);
List<AccessionRemark> remarks = genesysService.listAccessionRemarks(accession);
writeMCPDRow(sheet, row, accession, geo, collect, bred, names, exch, remarks);
detach(accession);
detach(geo);
detach(collect);
detach(bred);
detach(names);
detach(exch);
detach(remarks);
i++;
if (i % 100 == 0 && LOG.isDebugEnabled()) {
LOG.debug("Writing MCPD row " + i);
}
if (i % 1000 == 0) {
try {
((SXSSFSheet) sheet).flushRows();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
}
private void detach(Object obj) {
if (obj != null) {
if (obj instanceof Collection) {
for (Object o : (Collection<?>) obj)
detach(o);
} else {
entityManager.detach(obj);
}
}
}
});
((SXSSFSheet) sheet).flushRows();
LOG.info("Writing to output stream");
wb.write(outputStream);
wb.dispose();
LOG.info("Done");
}
private void writeMCPDRow(Sheet sheet, Row row, Accession accession, AccessionGeo geo, AccessionCollect collect, AccessionBreeding bred,
List<AccessionAlias> names, AccessionExchange exch, List<AccessionRemark> remarks) {
// Process and write result
row.createCell(0).setCellValue(accession.getInstitute().getCode());
row.createCell(1).setCellValue(accession.getAccessionName());
if (collect != null) {
createCell(row, 2, collect.getCollNumb());
createCell(row, 3, collect.getCollCode());
createCell(row, 4, collect.getCollName());
createCell(row, 5, collect.getCollInstAddress());
createCell(row, 6, collect.getCollMissId());
createCell(row, 16, collect.getCollSite());
createCell(row, 23, collect.getCollDate());
}
if (bred != null) {
createCell(row, 24, bred.getBreederCode());
createCell(row, 27, bred.getPedigree());
}
Taxonomy2 taxonomy = accession.getTaxonomy();
if (taxonomy != null) {
createCell(row, 7, taxonomy.getGenus());
createCell(row, 8, taxonomy.getSpecies());
createCell(row, 9, taxonomy.getSpAuthor());
createCell(row, 10, taxonomy.getSubtaxa());
createCell(row, 11, taxonomy.getSubtAuthor());
}
createCell(row, 14, accession.getAcquisitionDate());
Country origin = accession.getCountryOfOrigin();
if (origin != null) {
createCell(row, 15, origin.getCode3());
}
if (geo != null) {
createCell(row, 17, geo.getLatitude());
createCell(row, 18, geo.getLongitude());
createCell(row, 19, geo.getUncertainty());
createCell(row, 20, geo.getDatum());
createCell(row, 21, geo.getMethod());
createCell(row, 22, geo.getElevation());
}
createCell(row, 26, accession.getSampleStatus());
if (StringUtils.isNotBlank(accession.getAcquisitionSource())) {
Cell c = row.createCell(28);
try {
c.setCellValue(Integer.parseInt(accession.getAcquisitionSource()));
} catch (NumberFormatException e) {
c.setCellValue(accession.getAcquisitionSource());
}
}
createCell(row, 33, accession.getDuplSite());
createCell(row, 35, toMcpdArray(accession.getStoRage()));
createCell(row, 36, accession.getMlsStatus());
if (names != null && names.size() > 0) {
String acceName = null;
String otherNumb = null;
String donorNumb = null;
for (AccessionAlias a : names) {
switch (a.getAliasType()) {
case ACCENAME:
acceName = addName(acceName, a.getName());
break;
case DONORNUMB:
donorNumb = a.getName();
break;
case OTHERNUMB:
otherNumb = addName(otherNumb, a.getName(), a.getUsedBy());
break;
default:
break;
}
}
createCell(row, 13, acceName);
createCell(row, 31, donorNumb);
createCell(row, 32, otherNumb);
}
if (exch != null) {
createCell(row, 29, exch.getDonorInstitute());
createCell(row, 30, exch.getDonorName());
// row.createCell(32).setCellValue(exch.getAccNumbDonor());
}
if (remarks != null && remarks.size() > 0) {
String r = "";
for (AccessionRemark remark : remarks) {
if (r.length() > 0)
r += ";";
if (StringUtils.isNotBlank(remark.getFieldName())) {
r += remark.getFieldName() + ":" + remark.getRemark();
} else {
r += remark.getRemark();
}
}
createCell(row, 37, r);
}
}
private Cell createCell(Row row, int column, Number integer) {
if (integer == null)
return null;
Cell c = row.createCell(column);
c.setCellValue(integer.doubleValue());
return c;
}
private Cell createCell(Row row, int column, Boolean bool) {
if (bool == null)
return null;
Cell c = row.createCell(column);
c.setCellValue(bool);
return c;
}
private Cell createCell(Row row, int column, String value) {
if (StringUtils.isBlank(value))
return null;
Cell c = row.createCell(column);
c.setCellValue(value);
return c;
}
private String addName(String otherNames, String name, String usedBy) {
if (StringUtils.isBlank(name)) {
return otherNames;
} else if (StringUtils.isBlank(usedBy)) {
return addName(otherNames, name);
} else {
return addName(otherNames, usedBy + ":" + name);
}
}
private String addName(String otherNames, String name) {
if (StringUtils.isBlank(name))
return otherNames;
if (otherNames == null)
return name;
return otherNames + ";" + name;
}
private String toMcpdArray(List<Integer> stoRage) {
if (stoRage == null || stoRage.size() == 0)
return null;
String s = "";
for (Integer i : stoRage) {
if (s.length() > 0)
s += ";";
s += i;
}
return s;
}
}
src/main/java/org/genesys2/server/servlet/controller/ExplorerController.java
View file @ c054dac5
......@@ -38,6 +38,7 @@ import org.genesys2.server.model.genesys.Parameter;
import org.genesys2.server.model.genesys.ParameterCategory;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
......@@ -57,6 +58,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.PathVariable;
......@@ -71,9 +73,11 @@ import com.jhlabs.image.MapColorsFilter;
@Controller
public class ExplorerController extends BaseController {
private static final int DOWNLOAD_LIMIT = 100000;
@Autowired
private GenesysFilterService filterService;
@Autowired
private ElasticService elasticService;
......@@ -98,6 +102,9 @@ public class ExplorerController extends BaseController {
@Autowired
private FilterHandler filterHandler;
@Autowired
private DownloadService downloadService;
private final ObjectMapper mapper = new ObjectMapper();
/**
......@@ -161,11 +168,9 @@ public class ExplorerController extends BaseController {
}
model.addAttribute("crop", crop);
// JSP works with JsonObject
final Map<?, ?> filters = mapper.readValue(appliedFilters.toString(), Map.class);
model.addAttribute("filters", filters);
selectedFilters = appliedFilters.getFilterNames();
final List<GenesysFilter> currentFilters = filterHandler.selectFilters(selectedFilters);
......@@ -187,7 +192,6 @@ public class ExplorerController extends BaseController {
return "/accession/explore";