MCPD XLSX

pom.xml

@@ -522,7 +522,12 @@
             <artifactId>transifex-client</artifactId>
             <version>0.2-SNAPSHOT</version>
         </dependency>
-    </dependencies>
+		 <dependency>
+		 	<groupId>org.apache.poi</groupId>
+		 	<artifactId>poi-ooxml</artifactId>
+		 	<version>3.10.1</version>
+		 </dependency>
+	</dependencies>
 
 	<build>
 		<plugins>

src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepository.java

@@ -20,6 +20,7 @@ import org.genesys2.server.model.genesys.Method;
 import org.genesys2.server.model.impl.Country;
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
+import org.springframework.data.domain.Sort;
 import org.springframework.jdbc.core.RowCallbackHandler;
 
 public interface GenesysLowlevelRepository {
@@ -53,4 +54,6 @@ public interface GenesysLowlevelRepository {
 
 	int countAccessions(AppliedFilters filter);
 
+	void listAccessionIds(AppliedFilters filter, Sort sort, RowCallbackHandler rowCallbackHandler);
+
 }

src/main/java/org/genesys2/server/persistence/domain/GenesysLowlevelRepositoryImpl.java

@@ -32,6 +32,7 @@ import org.genesys2.server.service.impl.DirectMysqlQuery;
 import org.genesys2.server.service.impl.DirectMysqlQuery.MethodResolver;
 import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Sort;
 import org.springframework.jdbc.BadSqlGrammarException;
 import org.springframework.jdbc.core.ArgumentPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -255,6 +256,34 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
 			}
 		}, rowCallbackHandler);
 	}
+	
+
+	@Override
+	public void listAccessionIds(final AppliedFilters filter, final Sort sort, final RowCallbackHandler rowCallbackHandler) {
+		final DirectMysqlQuery directQuery = new DirectMysqlQuery("accession", "a");
+		directQuery.jsonFilter(filter, new MethodResolver() {
+			@Override
+			public Method getMethod(final long methodId) {
+				return GenesysLowlevelRepositoryImpl.this.methodRepository.findOne(methodId);
+			}
+		});
+		directQuery.sort(sort);
+
+		this.jdbcTemplate.query(new PreparedStatementCreator() {
+			@Override
+			public PreparedStatement createPreparedStatement(final Connection con) throws SQLException {
+				final PreparedStatement stmt = con.prepareStatement(directQuery
+						.getQuery("a.id"));
+
+				final ArgumentPreparedStatementSetter apss = new ArgumentPreparedStatementSetter(directQuery.getParameters());
+				apss.setValues(stmt);
+
+				// Set mysql JConnector to stream results
+//				stmt.setFetchSize(Integer.MIN_VALUE);
+				return stmt;
+			}
+		}, rowCallbackHandler);
+	}
 
 	@Override
 	public void listAccessionsGeo(final AppliedFilters filter, final RowCallbackHandler rowCallbackHandler) {

src/main/java/org/genesys2/server/service/DownloadService.java

+/**
+ * Copyright 2014 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
+
+/**
+ * Defines available downloads
+ * 
+ * @author matijaobreza
+ * 
+ */
+public interface DownloadService {
+
+	void writeXlsxMCPD(AppliedFilters filters, OutputStream outputStream) throws IOException;
+
+}

src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java

@@ -37,6 +37,7 @@ import org.genesys2.server.service.impl.FilterHandler.MinValueFilter;
 import org.genesys2.server.service.impl.FilterHandler.StartsWithFilter;
 import org.genesys2.server.service.impl.FilterHandler.ValueRangeFilter;
 import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Sort;
 import org.springframework.data.domain.Sort.Order;
 
 public class DirectMysqlQuery {
@@ -85,7 +86,9 @@ public class DirectMysqlQuery {
 		if (StringUtils.isNotBlank(alias)) {
 			sb.append(alias).append(StringUtils.SPACE);
 		}
-		sb.append("on ").append(onExpr).append(StringUtils.SPACE);
+		if (StringUtils.isNotBlank(onExpr)) {
+			sb.append("on ").append(onExpr).append(StringUtils.SPACE);
+		}
 		return this;
 	}
 
@@ -397,6 +400,30 @@ public class DirectMysqlQuery {
 		return params.toArray();
 	}
 
+	public DirectMysqlQuery sort(Sort sort) {
+		if (sortBuffer.length() != 0) {
+			throw new RuntimeException("sortBuffer is not blank, invalid use of #pageable(Pageable)");
+		}
+
+		sortBuffer.append("\n order by ");
+		for (final Order o : sort) {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("Order: " + o);
+			}
+			// ClassMetadata md =
+			// sessionFactory.getClassMetadata(Accession.class);
+			// md.
+			// EntityType<Accession> x =
+			// entityManager.getMetamodel().entity(Accession.class);
+			// System.err.println(x.getAttribute(o.getProperty()).getName());
+			// sb.append(x.getAttribute(o.getProperty()).getName());
+			sortBuffer.append("a.").append(o.getProperty());
+			sortBuffer.append(" ").append(o.getDirection());
+		}
+
+		return this;
+	}
+
 	public DirectMysqlQuery pageable(Pageable pageable) {
 		if (sortBuffer.length() != 0) {
 			throw new RuntimeException("sortBuffer is not blank, invalid use of #pageable(Pageable)");
@@ -407,21 +434,7 @@ public class DirectMysqlQuery {
 		}
 
 		if (pageable.getSort() != null) {
-			sortBuffer.append("\n order by ");
-			for (final Order o : pageable.getSort()) {
-				if (LOG.isDebugEnabled()) {
-					LOG.debug("Order: " + o);
-				}
-				// ClassMetadata md =
-				// sessionFactory.getClassMetadata(Accession.class);
-				// md.
-				// EntityType<Accession> x =
-				// entityManager.getMetamodel().entity(Accession.class);
-				// System.err.println(x.getAttribute(o.getProperty()).getName());
-				// sb.append(x.getAttribute(o.getProperty()).getName());
-				sortBuffer.append("a.").append(o.getProperty());
-				sortBuffer.append(" ").append(o.getDirection());
-			}
+			sort(pageable.getSort());
 		}
 		sortBuffer.append(" limit ");
 		if (LOG.isDebugEnabled()) {

src/main/java/org/genesys2/server/service/impl/DownloadServiceImpl.java

+/**
+ * Copyright 2014 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service.impl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+
+import javax.persistence.EntityManager;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.xssf.streaming.SXSSFSheet;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.genesys.AccessionAlias;
+import org.genesys2.server.model.genesys.AccessionBreeding;
+import org.genesys2.server.model.genesys.AccessionCollect;
+import org.genesys2.server.model.genesys.AccessionExchange;
+import org.genesys2.server.model.genesys.AccessionGeo;
+import org.genesys2.server.model.genesys.AccessionRemark;
+import org.genesys2.server.model.genesys.Taxonomy2;
+import org.genesys2.server.model.impl.Country;
+import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
+import org.genesys2.server.persistence.domain.MethodRepository;
+import org.genesys2.server.service.DownloadService;
+import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.domain.Sort;
+import org.springframework.jdbc.core.RowCallbackHandler;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+@Service
+public class DownloadServiceImpl implements DownloadService {
+	private static final int XLSX_ROWS_LIMIT = 65536;
+
+	public static final Log LOG = LogFactory.getLog(DownloadServiceImpl.class);
+
+	@Autowired
+	private EntityManager entityManager;
+
+	@Autowired
+	private GenesysService genesysService;
+
+	@Autowired
+	private GenesysLowlevelRepository genesysLowlevelRepository;
+
+	@Autowired
+	private MethodRepository methodRepository;
+
+	@Value("${base.url}")
+	private String baseUrl;
+
+	@Override
+	@Transactional(readOnly = true)
+	public void writeXlsxMCPD(final AppliedFilters filters, OutputStream outputStream) throws IOException {
+		XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/MCPD.xlsx"));
+
+		// keep 1000 rows in memory, exceeding rows will be flushed to disk
+		SXSSFWorkbook wb = new SXSSFWorkbook(template, -1);
+
+		CellStyle dateStyle = wb.createCellStyle();
+		dateStyle.setDataFormat(wb.createDataFormat().getFormat("dd/mm/yyyy"));
+		dateStyle.setAlignment(CellStyle.ALIGN_LEFT);
+
+		Sheet legal = wb.getSheet("Legal information");
+		System.err.println("Rows: " + legal.getLastRowNum());
+		Row r;
+		Cell c;
+		r = legal.createRow(0);
+		r.createCell(0).setCellValue("Server URL");
+		r.createCell(1).setCellValue(baseUrl);
+		r = legal.createRow(1);
+		r.createCell(0).setCellValue("Filters");
+		r.createCell(1).setCellValue(filters.toString());
+		r = legal.createRow(2);
+		r.createCell(0).setCellValue("Data source");
+		c = r.createCell(1);
+		c.setCellValue(baseUrl + "/explore?filter=" + filters.toString());
+
+		r = legal.createRow(3);
+		r.createCell(0).setCellValue("Date");
+		c = r.createCell(1);
+		c.setCellStyle(dateStyle);
+		c.setCellValue(new Date());
+		r = legal.createRow(4);
+		r.createCell(0).setCellValue("Server URL");
+		r.createCell(1).setCellValue(baseUrl + "/content/terms");
+		((SXSSFSheet) legal).flushRows();
+
+		final Sheet sheet = wb.getSheet("MCPD");
+
+		// Write accession information
+		genesysLowlevelRepository.listAccessionIds(filters, new Sort("acceNumb"), new RowCallbackHandler() {
+			int i = 0;
+
+			@Override
+			public void processRow(ResultSet rs) throws SQLException {
+				if (i > XLSX_ROWS_LIMIT) {
+					LOG.info("Row limit exceeded");
+					return;
+				}
+
+				long accessionId = rs.getLong(1);
+				Accession accession = entityManager.find(Accession.class, accessionId);
+				Row row = sheet.createRow(i + 1);
+
+				AccessionGeo geo = genesysService.listAccessionGeo(accession);
+				AccessionCollect collect = genesysService.listAccessionCollect(accession);
+				AccessionBreeding bred = genesysService.listAccessionBreeding(accession);
+				List<AccessionAlias> names = genesysService.listAccessionAliases(accession);
+				AccessionExchange exch = genesysService.listAccessionExchange(accession);
+				List<AccessionRemark> remarks = genesysService.listAccessionRemarks(accession);
+
+				writeMCPDRow(sheet, row, accession, geo, collect, bred, names, exch, remarks);
+
+				detach(accession);
+				detach(geo);
+				detach(collect);
+				detach(bred);
+				detach(names);
+				detach(exch);
+				detach(remarks);
+
+				i++;
+
+				if (i % 100 == 0 && LOG.isDebugEnabled()) {
+					LOG.debug("Writing MCPD row " + i);
+				}
+
+				if (i % 1000 == 0) {
+					try {
+						((SXSSFSheet) sheet).flushRows();
+					} catch (IOException e) {
+						LOG.error(e.getMessage(), e);
+					}
+				}
+
+			}
+
+			private void detach(Object obj) {
+				if (obj != null) {
+					if (obj instanceof Collection) {
+						for (Object o : (Collection<?>) obj)
+							detach(o);
+					} else {
+						entityManager.detach(obj);
+					}
+				}
+			}
+		});
+
+		((SXSSFSheet) sheet).flushRows();
+
+		LOG.info("Writing to output stream");
+		wb.write(outputStream);
+		wb.dispose();
+		LOG.info("Done");
+	}
+
+	private void writeMCPDRow(Sheet sheet, Row row, Accession accession, AccessionGeo geo, AccessionCollect collect, AccessionBreeding bred,
+			List<AccessionAlias> names, AccessionExchange exch, List<AccessionRemark> remarks) {
+
+		// Process and write result
+		row.createCell(0).setCellValue(accession.getInstitute().getCode());
+		row.createCell(1).setCellValue(accession.getAccessionName());
+
+		if (collect != null) {
+			createCell(row, 2, collect.getCollNumb());
+			createCell(row, 3, collect.getCollCode());
+			createCell(row, 4, collect.getCollName());
+			createCell(row, 5, collect.getCollInstAddress());
+			createCell(row, 6, collect.getCollMissId());
+			createCell(row, 16, collect.getCollSite());
+			createCell(row, 23, collect.getCollDate());
+		}
+
+		if (bred != null) {
+			createCell(row, 24, bred.getBreederCode());
+			createCell(row, 27, bred.getPedigree());
+		}
+
+		Taxonomy2 taxonomy = accession.getTaxonomy();
+		if (taxonomy != null) {
+			createCell(row, 7, taxonomy.getGenus());
+			createCell(row, 8, taxonomy.getSpecies());
+			createCell(row, 9, taxonomy.getSpAuthor());
+			createCell(row, 10, taxonomy.getSubtaxa());
+			createCell(row, 11, taxonomy.getSubtAuthor());
+		}
+
+		createCell(row, 14, accession.getAcquisitionDate());
+
+		Country origin = accession.getCountryOfOrigin();
+		if (origin != null) {
+			createCell(row, 15, origin.getCode3());
+		}
+
+		if (geo != null) {
+			createCell(row, 17, geo.getLatitude());
+			createCell(row, 18, geo.getLongitude());
+			createCell(row, 19, geo.getUncertainty());
+			createCell(row, 20, geo.getDatum());
+			createCell(row, 21, geo.getMethod());
+			createCell(row, 22, geo.getElevation());
+
+		}
+
+		createCell(row, 26, accession.getSampleStatus());
+
+		if (StringUtils.isNotBlank(accession.getAcquisitionSource())) {
+			Cell c = row.createCell(28);
+			try {
+				c.setCellValue(Integer.parseInt(accession.getAcquisitionSource()));
+			} catch (NumberFormatException e) {
+				c.setCellValue(accession.getAcquisitionSource());
+			}
+		}
+
+		createCell(row, 33, accession.getDuplSite());
+		createCell(row, 35, toMcpdArray(accession.getStoRage()));
+		createCell(row, 36, accession.getMlsStatus());
+
+		if (names != null && names.size() > 0) {
+			String acceName = null;
+			String otherNumb = null;
+			String donorNumb = null;
+			for (AccessionAlias a : names) {
+				switch (a.getAliasType()) {
+				case ACCENAME:
+					acceName = addName(acceName, a.getName());
+					break;
+				case DONORNUMB:
+					donorNumb = a.getName();
+					break;
+				case OTHERNUMB:
+					otherNumb = addName(otherNumb, a.getName(), a.getUsedBy());
+					break;
+				default:
+					break;
+				}
+			}
+
+			createCell(row, 13, acceName);
+			createCell(row, 31, donorNumb);
+			createCell(row, 32, otherNumb);
+		}
+
+		if (exch != null) {
+			createCell(row, 29, exch.getDonorInstitute());
+			createCell(row, 30, exch.getDonorName());
+			// row.createCell(32).setCellValue(exch.getAccNumbDonor());
+		}
+
+		if (remarks != null && remarks.size() > 0) {
+			String r = "";
+			for (AccessionRemark remark : remarks) {
+				if (r.length() > 0)
+					r += ";";
+				if (StringUtils.isNotBlank(remark.getFieldName())) {
+					r += remark.getFieldName() + ":" + remark.getRemark();
+				} else {
+					r += remark.getRemark();
+				}
+			}
+
+			createCell(row, 37, r);
+		}
+	}
+
+	private Cell createCell(Row row, int column, Number integer) {
+		if (integer == null)
+			return null;
+		Cell c = row.createCell(column);
+		c.setCellValue(integer.doubleValue());
+		return c;
+	}
+
+	private Cell createCell(Row row, int column, Boolean bool) {
+		if (bool == null)
+			return null;
+		Cell c = row.createCell(column);
+		c.setCellValue(bool);
+		return c;
+	}
+
+	private Cell createCell(Row row, int column, String value) {
+		if (StringUtils.isBlank(value))
+			return null;
+		Cell c = row.createCell(column);
+		c.setCellValue(value);
+		return c;
+	}
+
+	private String addName(String otherNames, String name, String usedBy) {
+		if (StringUtils.isBlank(name)) {
+			return otherNames;
+		} else if (StringUtils.isBlank(usedBy)) {
+			return addName(otherNames, name);
+		} else {
+			return addName(otherNames, usedBy + ":" + name);
+		}
+	}
+
+	private String addName(String otherNames, String name) {
+		if (StringUtils.isBlank(name))
+			return otherNames;
+		if (otherNames == null)
+			return name;
+		return otherNames + ";" + name;
+	}
+
+	private String toMcpdArray(List<Integer> stoRage) {
+		if (stoRage == null || stoRage.size() == 0)
+			return null;
+		String s = "";
+		for (Integer i : stoRage) {
+			if (s.length() > 0)
+				s += ";";
+			s += i;
+		}
+		return s;
+	}
+}

src/main/java/org/genesys2/server/servlet/controller/ExplorerController.java

@@ -38,6 +38,7 @@ import org.genesys2.server.model.genesys.Parameter;
 import org.genesys2.server.model.genesys.ParameterCategory;
 import org.genesys2.server.model.impl.Crop;
 import org.genesys2.server.service.CropService;
+import org.genesys2.server.service.DownloadService;
 import org.genesys2.server.service.ElasticService;
 import org.genesys2.server.service.FilterConstants;
 import org.genesys2.server.service.GenesysFilterService;
@@ -57,6 +58,7 @@ import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.http.MediaType;
+import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.ModelMap;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -71,9 +73,11 @@ import com.jhlabs.image.MapColorsFilter;
 @Controller
 public class ExplorerController extends BaseController {
 
+	private static final int DOWNLOAD_LIMIT = 100000;
+
 	@Autowired
 	private GenesysFilterService filterService;
-	
+
 	@Autowired
 	private ElasticService elasticService;
 
@@ -98,6 +102,9 @@ public class ExplorerController extends BaseController {
 	@Autowired
 	private FilterHandler filterHandler;
 
+	@Autowired
+	private DownloadService downloadService;
+
 	private final ObjectMapper mapper = new ObjectMapper();
 
 	/**
@@ -161,11 +168,9 @@ public class ExplorerController extends BaseController {
 		}
 		model.addAttribute("crop", crop);
 
-		
 		// JSP works with JsonObject
 		final Map<?, ?> filters = mapper.readValue(appliedFilters.toString(), Map.class);
 		model.addAttribute("filters", filters);
-		
 
 		selectedFilters = appliedFilters.getFilterNames();