Commit 3fdae5c3 authored by Matija Obreza's avatar Matija Obreza

Beta: apache POI 5.0.0-SNAPSHOT with Emitting SXSSF

parent 7fd8ea37
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>5.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache POI</name>
<url>https://poi.apache.org/</url>
<description>Apache POI - Java API To Access Microsoft Format Files</description>
<repositories>
<repository>
<id>apache-releases-repo</id>
<name>apache releases repo</name>
<url>https://repository.apache.org/content/repositories/releases</url>
</repository>
</repositories>
<mailingLists>
<mailingList>
<name>POI Users List</name>
<subscribe>user-subscribe@poi.apache.org</subscribe>
<unsubscribe>user-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-user/</archive>
</mailingList>
<mailingList>
<name>POI Developer List</name>
<subscribe>dev-subscribe@poi.apache.org</subscribe>
<unsubscribe>dev-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-dev/</archive>
</mailingList>
</mailingLists>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<organization>
<name>Apache Software Foundation</name>
<url>http://www.apache.org/</url>
</organization>
<dependencies>
<dependency>
<groupId>org.apache.xmlbeans</groupId>
<artifactId>xmlbeans</artifactId>
<version>3.1.0</version>
</dependency>
</dependencies>
</project>
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache POI</name>
<url>https://poi.apache.org/</url>
<description>Apache POI - Java API To Access Microsoft Format Files</description>
<mailingLists>
<mailingList>
<name>POI Users List</name>
<subscribe>user-subscribe@poi.apache.org</subscribe>
<unsubscribe>user-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-user/</archive>
</mailingList>
<mailingList>
<name>POI Developer List</name>
<subscribe>dev-subscribe@poi.apache.org</subscribe>
<unsubscribe>dev-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-dev/</archive>
</mailingList>
</mailingLists>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<organization>
<name>Apache Software Foundation</name>
<url>http://www.apache.org/</url>
</organization>
<dependencies>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>5.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>5.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.20</version>
</dependency>
<dependency>
<groupId>com.github.virtuald</groupId>
<artifactId>curvesapi</artifactId>
<version>1.06</version>
</dependency>
</dependencies>
</project>
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>5.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache POI</name>
<url>https://poi.apache.org/</url>
<description>Apache POI - Java API To Access Microsoft Format Files</description>
<mailingLists>
<mailingList>
<name>POI Users List</name>
<subscribe>user-subscribe@poi.apache.org</subscribe>
<unsubscribe>user-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-user/</archive>
</mailingList>
<mailingList>
<name>POI Developer List</name>
<subscribe>dev-subscribe@poi.apache.org</subscribe>
<unsubscribe>dev-unsubscribe@poi.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/poi-dev/</archive>
</mailingList>
</mailingLists>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<organization>
<name>Apache Software Foundation</name>
<url>http://www.apache.org/</url>
</organization>
<dependencies>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.14</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>SparseBitSet</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<scope>test</scope>
<version>1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
<version>4.13</version>
</dependency>
</dependencies>
</project>
......@@ -508,7 +508,7 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.17</version>
<version>5.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.ocpsoft.prettytime</groupId>
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
/**
* IRowGenerator for Super-streaming XSSF sheets
*/
public interface IRowGenerator {
/**
* Generate rows
*
* @param sheet the sheet
* @throws Exception the exception
*/
void generateRows(SXSSFSheet sheet) throws Exception;
}
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.xssf.streaming;
import org.apache.poi.ss.formula.EvaluationCell;
import org.apache.poi.ss.formula.IStabilityClassifier;
import org.apache.poi.ss.formula.WorkbookEvaluator;
import org.apache.poi.ss.formula.udf.UDFFinder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.xssf.usermodel.BaseXSSFFormulaEvaluator;
/**
* Streaming-specific Formula Evaluator, which is able to
* lookup cells within the current Window.
*/
public final class SXSSFFormulaEvaluator extends BaseXSSFFormulaEvaluator {
private static final POILogger logger = POILogFactory.getLogger(SXSSFFormulaEvaluator.class);
private SXSSFWorkbook wb;
public SXSSFFormulaEvaluator(SXSSFWorkbook workbook) {
this(workbook, null, null);
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
this(workbook, new WorkbookEvaluator(SXSSFEvaluationWorkbook.create(workbook), stabilityClassifier, udfFinder));
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, WorkbookEvaluator bookEvaluator) {
super(bookEvaluator);
this.wb = workbook;
}
/**
* @param stabilityClassifier used to optimise caching performance. Pass <code>null</code>
* for the (conservative) assumption that any cell may have its definition changed after
* evaluation begins.
* @param udfFinder pass <code>null</code> for default (AnalysisToolPak only)
*/
public static SXSSFFormulaEvaluator create(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
return new SXSSFFormulaEvaluator(workbook, stabilityClassifier, udfFinder);
}
/**
* Turns a SXSSFCell into a SXSSFEvaluationCell
*/
@Override
protected EvaluationCell toEvaluationCell(Cell cell) {
if (!(cell instanceof SXSSFCell)){
throw new IllegalArgumentException("Unexpected type of cell: " + cell.getClass() + "." +
" Only SXSSFCells can be evaluated.");
}
return new SXSSFEvaluationCell((SXSSFCell)cell);
}
@Override
public SXSSFCell evaluateInCell(Cell cell) {
return (SXSSFCell) super.evaluateInCell(cell);
}
/**
* For active worksheets only, will loop over rows and
* cells, evaluating formula cells there.
* If formula cells are outside the window for that sheet,
* it can either skip them silently, or give an exception
*/
public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWindow) {
SXSSFFormulaEvaluator eval = new SXSSFFormulaEvaluator(wb);
// Check they're all available
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet && ((SXSSFSheet)sheet).areAllRowsFlushed()) {
throw new SheetsFlushedException();
}
}
// Process the sheets as best we can
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet) {
// Check if any rows have already been flushed out
int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum();
if (lastFlushedRowNum > -1) {
if (! skipOutOfWindow) throw new RowFlushedException(0);
logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping");
}
}
// Evaluate what we have
for (Row r : sheet) {
for (Cell c : r) {
if (c.getCellTypeEnum() == CellType.FORMULA) {
eval.evaluateFormulaCellEnum(c);
}
}
}
}
}
/**
* Loops over rows and cells, evaluating formula cells there.
* If any sheets are inactive, or any cells outside of the window,
* will give an Exception.
* For SXSSF, you generally don't want to use this method, instead
* evaluate your formulas as you go before they leave the window.
*/
public void evaluateAll() {
// Have the evaluation done, with exceptions
evaluateAllFormulaCells(wb, false);
}
public static class SheetsFlushedException extends IllegalStateException {
protected SheetsFlushedException() {
super("One or more sheets have been flushed, cannot evaluate all cells");
}
}
public static class RowFlushedException extends IllegalStateException {
protected RowFlushedException(int rowNum) {
super("Row " + rowNum + " has been flushed, cannot evaluate all cells");
}
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
public class StreamingSheetWriter extends SheetDataWriter {
private static final POILogger logger = POILogFactory.getLogger(StreamingSheetWriter.class);
public StreamingSheetWriter() throws IOException {
throw new RuntimeException("StreamingSheetWriter requires OutputStream");
}
public StreamingSheetWriter(OutputStream out) throws IOException {
super(createWriter(out));
logger.log(POILogger.DEBUG, "Preparing SSXSSF sheet writer");
}
@Override
public File createTempFile() throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
@Override
public Writer createWriter(File fd) throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
/**
* Create a writer for the sheet data.
*
* @param out the output stream to write to
*/
protected static Writer createWriter(OutputStream out) throws IOException {
return new BufferedWriter(
new OutputStreamWriter(out, "UTF-8"));
}
@Override
public void close() throws IOException {
_out.flush();
}
@Override
public InputStream getWorksheetXMLInputStream() throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
@Override
protected void finalize() throws Throwable {
// super.finalize();
}
@Override
boolean dispose() throws IOException {
try {
_out.close();
} finally {
}
return true;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.poi.xssf.usermodel.XSSFSheet;
public class SuperSXSSFSheet extends SXSSFSheet {
private IRowGenerator rowGenerator;
public SuperSXSSFSheet(SuperSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException {
super(workbook, xSheet, workbook.getRandomAccessWindowSize());
}
@Override
public InputStream getWorksheetXMLInputStream() throws IOException {
throw new RuntimeException("Not supported by SuperSXSSFSheet");
}
public void setRowGenerator(IRowGenerator rowGenerator) {
this.rowGenerator = rowGenerator;
}
public void writeRows(OutputStream out) throws IOException {
// delayed creation of SheetDataWriter
_writer = ((SuperSXSSFWorkbook)_workbook).createSheetDataWriter(out);
try {
if (this.rowGenerator != null) {
this.rowGenerator.generateRows(this);
}
} catch (Exception e) {
throw new IOException("Error generating Excel rows", e);
} finally {
// flush buffered rows
flushRows(0);
// flush writer buffer
_writer.close();
out.flush();
}
}
}
......@@ -28,23 +28,23 @@ import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.POIXMLProperties;
import org.apache.poi.openxml4j.util.Nullable;
import org.apache.poi.ooxml.POIXMLProperties;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.streaming.EmittingSXSSFSheet;
import org.apache.poi.xssf.streaming.EmittingSXSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SuperSXSSFSheet;
import org.apache.poi.xssf.streaming.SuperSXSSFWorkbook;
// NOTE Excel 2016 has a limit of 66,530 hyperlinks
// import org.apache.poi.xssf.usermodel.XSSFHyperlink;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
......@@ -214,7 +214,7 @@ public class DownloadServiceImpl implements DownloadService {
POIXMLProperties props = template.getProperties();
POIXMLProperties.CoreProperties coreProp = props.getCoreProperties();
coreProp.setCreated(new Nullable<Date>(new Date()));
coreProp.setCreated(Optional.of(new Date()));
POIXMLProperties.CustomProperties custProp = props.getCustomProperties();
if (StringUtils.isNotBlank(shortFilter)) {
......@@ -223,7 +223,7 @@ public class DownloadServiceImpl implements DownloadService {
custProp.addProperty("Genesys URL", frontendUrl);
// keep 50 rows in memory, exceeding rows will be flushed to disk
SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(template, 50);
EmittingSXSSFWorkbook wb = new EmittingSXSSFWorkbook(template, 50);
WorkbookStyles wbStyles = WorkbookStyles.create(wb);
CellStyle dateStyle = wbStyles.dateStyle;
......@@ -258,7 +258,7 @@ public class DownloadServiceImpl implements DownloadService {
r.createCell(1).setCellValue(frontendUrl + "/content/terms");
// Register a streaming sheet
final SuperSXSSFSheet sheet = wb.getStreamingSheet("MCPD");
final EmittingSXSSFSheet sheet = wb.getStreamingSheet("MCPD");
sheet.setRowGenerator((sheet2) -> {
// Write accession information
......@@ -450,13 +450,13 @@ public class DownloadServiceImpl implements DownloadService {
public void writeXlsxDescriptorList(final DescriptorList descriptorList, final OutputStream outputStream) throws IOException {
final XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/CATALOG-DESCRIPTORLIST.xlsx"));
final SuperSXSSFWorkbook workbook = new SuperSXSSFWorkbook(template, 50);
final EmittingSXSSFWorkbook workbook = new EmittingSXSSFWorkbook(template, 50);
WorkbookStyles wbStyles = WorkbookStyles.create(workbook);
final Sheet metadataSheet = workbook.getXSSFSheet("metadata");
descriptorListWriteMetadata(descriptorList, metadataSheet, wbStyles);
final SuperSXSSFSheet descriptorsSheet = workbook.getStreamingSheet("descriptors");
final EmittingSXSSFSheet descriptorsSheet = workbook.getStreamingSheet("descriptors");
descriptorsSheet.setRowGenerator((sheet) -> {
writeDescriptorsSheet(descriptorList.getDescriptors(), sheet);
});
......@@ -521,28 +521,28 @@ public class DownloadServiceImpl implements DownloadService {
public void writeXlsxDataset(final Dataset dataset, final OutputStream outputStream) throws IOException {
final XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/CATALOG-DATASET.xlsx"));