Commit 41f2b41e authored by Matija Obreza's avatar Matija Obreza

Merge branch '376-streaming-generated-excel' into 'master'

Resolve "Streaming generated Excel"

Closes #376

See merge request genesys-pgr/genesys-server!298
parents 29a3b548 20f0ed13
......@@ -412,7 +412,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
<version>4.1</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
......@@ -480,7 +480,7 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.10.1</version>
<version>3.17</version>
</dependency>
<dependency>
<groupId>org.ocpsoft.prettytime</groupId>
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
/**
* IRowGenerator for Super-streaming XSSF sheets
*/
public interface IRowGenerator {
/**
* Generate rows
*
* @param sheet the sheet
* @throws Exception the exception
*/
void generateRows(SXSSFSheet sheet) throws Exception;
}
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.xssf.streaming;
import org.apache.poi.ss.formula.EvaluationCell;
import org.apache.poi.ss.formula.IStabilityClassifier;
import org.apache.poi.ss.formula.WorkbookEvaluator;
import org.apache.poi.ss.formula.udf.UDFFinder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.xssf.usermodel.BaseXSSFFormulaEvaluator;
/**
* Streaming-specific Formula Evaluator, which is able to
* lookup cells within the current Window.
*/
public final class SXSSFFormulaEvaluator extends BaseXSSFFormulaEvaluator {
private static final POILogger logger = POILogFactory.getLogger(SXSSFFormulaEvaluator.class);
private SXSSFWorkbook wb;
public SXSSFFormulaEvaluator(SXSSFWorkbook workbook) {
this(workbook, null, null);
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
this(workbook, new WorkbookEvaluator(SXSSFEvaluationWorkbook.create(workbook), stabilityClassifier, udfFinder));
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, WorkbookEvaluator bookEvaluator) {
super(bookEvaluator);
this.wb = workbook;
}
/**
* @param stabilityClassifier used to optimise caching performance. Pass <code>null</code>
* for the (conservative) assumption that any cell may have its definition changed after
* evaluation begins.
* @param udfFinder pass <code>null</code> for default (AnalysisToolPak only)
*/
public static SXSSFFormulaEvaluator create(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
return new SXSSFFormulaEvaluator(workbook, stabilityClassifier, udfFinder);
}
/**
* Turns a SXSSFCell into a SXSSFEvaluationCell
*/
@Override
protected EvaluationCell toEvaluationCell(Cell cell) {
if (!(cell instanceof SXSSFCell)){
throw new IllegalArgumentException("Unexpected type of cell: " + cell.getClass() + "." +
" Only SXSSFCells can be evaluated.");
}
return new SXSSFEvaluationCell((SXSSFCell)cell);
}
@Override
public SXSSFCell evaluateInCell(Cell cell) {
return (SXSSFCell) super.evaluateInCell(cell);
}
/**
* For active worksheets only, will loop over rows and
* cells, evaluating formula cells there.
* If formula cells are outside the window for that sheet,
* it can either skip them silently, or give an exception
*/
public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWindow) {
SXSSFFormulaEvaluator eval = new SXSSFFormulaEvaluator(wb);
// Check they're all available
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet && ((SXSSFSheet)sheet).areAllRowsFlushed()) {
throw new SheetsFlushedException();
}
}
// Process the sheets as best we can
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet) {
// Check if any rows have already been flushed out
int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum();
if (lastFlushedRowNum > -1) {
if (! skipOutOfWindow) throw new RowFlushedException(0);
logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping");
}
}
// Evaluate what we have
for (Row r : sheet) {
for (Cell c : r) {
if (c.getCellTypeEnum() == CellType.FORMULA) {
eval.evaluateFormulaCellEnum(c);
}
}
}
}
}
/**
* Loops over rows and cells, evaluating formula cells there.
* If any sheets are inactive, or any cells outside of the window,
* will give an Exception.
* For SXSSF, you generally don't want to use this method, instead
* evaluate your formulas as you go before they leave the window.
*/
public void evaluateAll() {
// Have the evaluation done, with exceptions
evaluateAllFormulaCells(wb, false);
}
public static class SheetsFlushedException extends IllegalStateException {
protected SheetsFlushedException() {
super("One or more sheets have been flushed, cannot evaluate all cells");
}
}
public static class RowFlushedException extends IllegalStateException {
protected RowFlushedException(int rowNum) {
super("Row " + rowNum + " has been flushed, cannot evaluate all cells");
}
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
public class StreamingSheetWriter extends SheetDataWriter {
private static final POILogger logger = POILogFactory.getLogger(StreamingSheetWriter.class);
public StreamingSheetWriter() throws IOException {
throw new RuntimeException("StreamingSheetWriter requires OutputStream");
}
public StreamingSheetWriter(OutputStream out) throws IOException {
super(createWriter(out));
logger.log(POILogger.DEBUG, "Preparing SSXSSF sheet writer");
}
@Override
public File createTempFile() throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
@Override
public Writer createWriter(File fd) throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
/**
* Create a writer for the sheet data.
*
* @param out the output stream to write to
*/
protected static Writer createWriter(OutputStream out) throws IOException {
return new BufferedWriter(
new OutputStreamWriter(out, "UTF-8"));
}
@Override
public void close() throws IOException {
_out.flush();
}
@Override
public InputStream getWorksheetXMLInputStream() throws IOException {
throw new RuntimeException("Not supported with StreamingSheetWriter");
}
@Override
protected void finalize() throws Throwable {
// super.finalize();
}
@Override
boolean dispose() throws IOException {
try {
_out.close();
} finally {
}
return true;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.poi.xssf.usermodel.XSSFSheet;
public class SuperSXSSFSheet extends SXSSFSheet {
private IRowGenerator rowGenerator;
public SuperSXSSFSheet(SuperSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException {
super(workbook, xSheet, workbook.getRandomAccessWindowSize());
}
@Override
public InputStream getWorksheetXMLInputStream() throws IOException {
throw new RuntimeException("Not supported by SuperSXSSFSheet");
}
public void setRowGenerator(IRowGenerator rowGenerator) {
this.rowGenerator = rowGenerator;
}
public void writeRows(OutputStream out) throws IOException {
// delayed creation of SheetDataWriter
_writer = ((SuperSXSSFWorkbook)_workbook).createSheetDataWriter(out);
try {
if (this.rowGenerator != null) {
this.rowGenerator.generateRows(this);
}
} catch (Exception e) {
throw new IOException("Error generating Excel rows", e);
} finally {
// flush buffered rows
flushRows(0);
// flush writer buffer
_writer.close();
out.flush();
}
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.poi.openxml4j.util.ZipEntrySource;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.util.IOUtils;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.xssf.usermodel.XSSFChartSheet;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class SuperSXSSFWorkbook extends SXSSFWorkbook {
private static final POILogger logger = POILogFactory.getLogger(SuperSXSSFWorkbook.class);
public SuperSXSSFWorkbook() {
this(null);
}
public SuperSXSSFWorkbook(XSSFWorkbook workbook) {
this(workbook, SXSSFWorkbook.DEFAULT_WINDOW_SIZE);
}
public SuperSXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize) {
setRandomAccessWindowSize(rowAccessWindowSize);
_sharedStringSource = null;
if (workbook == null) {
_wb = new XSSFWorkbook();
} else {
_wb = workbook;
}
}
@Override
public void setCompressTempFiles(boolean compress) {
// NOOP
}
@Override
protected SheetDataWriter createSheetDataWriter() throws IOException {
throw new RuntimeException("Not supported by SuperSXSSFWorkbook");
}
protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IOException {
return new StreamingSheetWriter(out);
}
@Override
protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throws IOException {
try {
ZipOutputStream zos = new ZipOutputStream(out);
try {
Enumeration<? extends ZipEntry> en = zipEntrySource.getEntries();
while (en.hasMoreElements()) {
ZipEntry ze = en.nextElement();
zos.putNextEntry(new ZipEntry(ze.getName()));
InputStream is = zipEntrySource.getInputStream(ze);
XSSFSheet xSheet = getSheetFromZipEntryName(ze.getName());
// See bug 56557, we should not inject data into the special ChartSheets
if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) {
SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet);
if (sxSheet != null) {
copyStreamAndInjectWorksheet(is, zos, sxSheet);
} else {
IOUtils.copy(is, zos);
}
} else {
IOUtils.copy(is, zos);
}
is.close();
}
} finally {
zos.close();
}
} finally {
zipEntrySource.close();
}
}
private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, SuperSXSSFSheet sxSheet) throws IOException {
InputStreamReader inReader = new InputStreamReader(in, "UTF-8");
OutputStreamWriter outWriter = new OutputStreamWriter(out, "UTF-8");
boolean needsStartTag = true;
int c;
int pos = 0;
String s = "<sheetData";
int n = s.length();
// Copy from "in" to "out" up to the string "<sheetData/>" or "</sheetData>"
// (excluding).
while (((c = inReader.read()) != -1)) {
if (c == s.charAt(pos)) {
pos++;
if (pos == n) {
if ("<sheetData".equals(s)) {
c = inReader.read();
if (c == -1) {
outWriter.write(s);
break;
}
if (c == '>') {
// Found <sheetData>
outWriter.write(s);
outWriter.write(c);
s = "</sheetData>";
n = s.length();
pos = 0;
needsStartTag = false;
continue;
}
if (c == '/') {
// Found <sheetData/
c = inReader.read();
if (c == -1) {
outWriter.write(s);
break;
}
if (c == '>') {
// Found <sheetData/>
break;
}
outWriter.write(s);
outWriter.write('/');
outWriter.write(c);
pos = 0;
continue;
}
outWriter.write(s);
outWriter.write('/');
outWriter.write(c);
pos = 0;
continue;
} else {
// Found </sheetData>
break;
}
}
} else {
if (pos > 0) {
outWriter.write(s, 0, pos);
}
if (c == s.charAt(0)) {
pos = 1;
} else {
outWriter.write(c);
pos = 0;
}
}
}
outWriter.flush();
if (needsStartTag) {
outWriter.write("<sheetData>\n");
outWriter.flush();
}
// IOUtils.copy(worksheetData,out);
sxSheet.writeRows(out);
outWriter.write("</sheetData>");
outWriter.flush();
// Copy the rest of "in" to "out".
while (((c = inReader.read()) != -1)) {
outWriter.write(c);
}
outWriter.flush();
}
@Override
SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) {
final SuperSXSSFSheet sxSheet;
try {
sxSheet = new SuperSXSSFSheet(this, xSheet);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
registerSheetMapping(sxSheet, xSheet);
return sxSheet;
}
/**
* Returns an iterator of the sheets in the workbook in sheet order. Includes
* hidden and very hidden sheets.
*
* @return an iterator of the sheets.
*/
@Override
public Iterator<Sheet> sheetIterator() {
return new SheetIterator<Sheet>();
}
private final class SheetIterator<T extends Sheet> implements Iterator<T> {
final private Iterator<XSSFSheet> it;
@SuppressWarnings("unchecked")
public SheetIterator() {
it = (Iterator<XSSFSheet>) (Iterator<? extends Sheet>) _wb.iterator();
}
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
@SuppressWarnings("unchecked")
public T next() throws NoSuchElementException {
final XSSFSheet xssfSheet = it.next();
SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xssfSheet);
return (T) (sxSheet == null ? xssfSheet : sxSheet);
}
/**
* Unexpected behavior may occur if sheets are reordered after iterator has been
* created. Support for the remove method may be added in the future if someone
* can figure out a reliable implementation.
*/
@Override
public void remove() throws IllegalStateException {
throw new UnsupportedOperationException("remove method not supported on XSSFWorkbook.iterator(). " + "Use Sheet.removeSheetAt(int) instead.");
}
}
/**
* Alias for {@link #sheetIterator()} to allow foreach loops
*/
@Override
public Iterator<Sheet> iterator() {
return sheetIterator();
}
@Override
public SXSSFSheet getSheetAt(int index) {
throw new RuntimeException("Not supported by SuperSXSSFWorkbook");
}
public XSSFSheet getXSSFSheetAt(int index) {
return _wb.getSheetAt(index);
}
/**
* Gets the sheet at the given index for streaming.
*
* @param index the index
* @return the streaming sheet at
*/
public SXSSFSheet getStreamingSheetAt(