Merge branch '376-streaming-generated-excel' into 'master'

Resolve "Streaming generated Excel"

Closes #376

See merge request genesys-pgr/genesys-server!298

pom.xml

@@ -412,7 +412,7 @@
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-collections4</artifactId>
-			<version>4.0</version>
+			<version>4.1</version>
 		</dependency>
 		<dependency>
 			<groupId>javax.mail</groupId>
@@ -480,7 +480,7 @@
 		<dependency>
 			<groupId>org.apache.poi</groupId>
 			<artifactId>poi-ooxml</artifactId>
-			<version>3.10.1</version>
+			<version>3.17</version>
 		</dependency>
 		<dependency>
 			<groupId>org.ocpsoft.prettytime</groupId>

src/main/java/org/apache/poi/xssf/streaming/IRowGenerator.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.poi.xssf.streaming;
+
+/**
+ * IRowGenerator for Super-streaming XSSF sheets
+ */
+public interface IRowGenerator {
+	
+	/**
+	 * Generate rows
+	 *
+	 * @param sheet the sheet
+	 * @throws Exception the exception
+	 */
+	void generateRows(SXSSFSheet sheet) throws Exception;
+}

src/main/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java

+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.xssf.streaming;
+
+import org.apache.poi.ss.formula.EvaluationCell;
+import org.apache.poi.ss.formula.IStabilityClassifier;
+import org.apache.poi.ss.formula.WorkbookEvaluator;
+import org.apache.poi.ss.formula.udf.UDFFinder;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellType;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+import org.apache.poi.xssf.usermodel.BaseXSSFFormulaEvaluator;
+
+/**
+ * Streaming-specific Formula Evaluator, which is able to 
+ *  lookup cells within the current Window.
+ */
+public final class SXSSFFormulaEvaluator extends BaseXSSFFormulaEvaluator {
+    private static final POILogger logger = POILogFactory.getLogger(SXSSFFormulaEvaluator.class);
+    
+    private SXSSFWorkbook wb;
+    
+    public SXSSFFormulaEvaluator(SXSSFWorkbook workbook) {
+        this(workbook, null, null);
+    }
+    private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
+        this(workbook, new WorkbookEvaluator(SXSSFEvaluationWorkbook.create(workbook), stabilityClassifier, udfFinder));
+    }
+    private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, WorkbookEvaluator bookEvaluator) {
+        super(bookEvaluator);
+        this.wb = workbook;
+    }
+    
+    /**
+     * @param stabilityClassifier used to optimise caching performance. Pass <code>null</code>
+     * for the (conservative) assumption that any cell may have its definition changed after
+     * evaluation begins.
+     * @param udfFinder pass <code>null</code> for default (AnalysisToolPak only)
+     */
+    public static SXSSFFormulaEvaluator create(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
+        return new SXSSFFormulaEvaluator(workbook, stabilityClassifier, udfFinder);
+    }
+    
+    /**
+     * Turns a SXSSFCell into a SXSSFEvaluationCell
+     */
+    @Override
+    protected EvaluationCell toEvaluationCell(Cell cell) {
+        if (!(cell instanceof SXSSFCell)){
+            throw new IllegalArgumentException("Unexpected type of cell: " + cell.getClass() + "." +
+                    " Only SXSSFCells can be evaluated.");
+        }
+
+        return new SXSSFEvaluationCell((SXSSFCell)cell);
+    }
+    
+    @Override
+    public SXSSFCell evaluateInCell(Cell cell) {
+        return (SXSSFCell) super.evaluateInCell(cell);
+    }
+    
+    /**
+     * For active worksheets only, will loop over rows and
+     *  cells, evaluating formula cells there.
+     * If formula cells are outside the window for that sheet,
+     *  it can either skip them silently, or give an exception
+     */
+    public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWindow) {
+        SXSSFFormulaEvaluator eval = new SXSSFFormulaEvaluator(wb);
+        
+        // Check they're all available
+        for (Sheet sheet : wb) {
+            if (sheet instanceof SXSSFSheet && ((SXSSFSheet)sheet).areAllRowsFlushed()) {
+                throw new SheetsFlushedException();
+            }
+        }
+        
+        // Process the sheets as best we can
+        for (Sheet sheet : wb) {
+            
+        	if (sheet instanceof SXSSFSheet) {
+	            // Check if any rows have already been flushed out
+	            int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum();
+	            if (lastFlushedRowNum > -1) {
+	                if (! skipOutOfWindow) throw new RowFlushedException(0);
+	                logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping");
+	            }
+        	}
+        	
+            // Evaluate what we have
+            for (Row r : sheet) {
+                for (Cell c : r) {
+                    if (c.getCellTypeEnum() == CellType.FORMULA) {
+                        eval.evaluateFormulaCellEnum(c);
+                    }
+                }
+            }
+        }
+    }
+    
+    /**
+     * Loops over rows and cells, evaluating formula cells there.
+     * If any sheets are inactive, or any cells outside of the window,
+     *  will give an Exception.
+     * For SXSSF, you generally don't want to use this method, instead
+     *  evaluate your formulas as you go before they leave the window.
+     */
+    public void evaluateAll() {
+        // Have the evaluation done, with exceptions
+        evaluateAllFormulaCells(wb, false);
+    }
+    
+    public static class SheetsFlushedException extends IllegalStateException {
+        protected SheetsFlushedException() {
+            super("One or more sheets have been flushed, cannot evaluate all cells");
+        }
+    }
+    public static class RowFlushedException extends IllegalStateException {
+        protected RowFlushedException(int rowNum) {
+            super("Row " + rowNum + " has been flushed, cannot evaluate all cells");
+        }
+    }
+}

src/main/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.poi.xssf.streaming;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+
+public class StreamingSheetWriter extends SheetDataWriter {
+    private static final POILogger logger = POILogFactory.getLogger(StreamingSheetWriter.class);
+
+    public StreamingSheetWriter() throws IOException {
+    	throw new RuntimeException("StreamingSheetWriter requires OutputStream");
+	}
+
+    public StreamingSheetWriter(OutputStream out) throws IOException {
+    	super(createWriter(out));
+        logger.log(POILogger.DEBUG, "Preparing SSXSSF sheet writer");
+    }
+
+    @Override
+    public File createTempFile() throws IOException {
+    	throw new RuntimeException("Not supported with StreamingSheetWriter");
+    }
+    
+    @Override
+    public Writer createWriter(File fd) throws IOException {
+    	throw new RuntimeException("Not supported with StreamingSheetWriter");
+    }
+    
+    /**
+     * Create a writer for the sheet data.
+     * 
+     * @param  out the output stream to write to
+     */
+    protected static Writer createWriter(OutputStream out) throws IOException {
+        return new BufferedWriter(
+                new OutputStreamWriter(out, "UTF-8"));
+    }
+    
+    @Override
+    public void close() throws IOException {
+        _out.flush();
+    }
+    
+    @Override
+    public InputStream getWorksheetXMLInputStream() throws IOException {
+    	throw new RuntimeException("Not supported with StreamingSheetWriter");
+    }
+    
+    @Override
+    protected void finalize() throws Throwable {
+    	// super.finalize();
+    }
+    
+    
+    @Override
+    boolean dispose() throws IOException {
+    	try {
+            _out.close();
+        } finally {
+        }
+        return true;
+    }
+}

src/main/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.poi.xssf.streaming;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+
+public class SuperSXSSFSheet extends SXSSFSheet {
+	private IRowGenerator rowGenerator;
+
+	public SuperSXSSFSheet(SuperSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException {
+		super(workbook, xSheet, workbook.getRandomAccessWindowSize());
+	}
+	
+	@Override
+    public InputStream getWorksheetXMLInputStream() throws IOException {
+		throw new RuntimeException("Not supported by SuperSXSSFSheet");
+	}
+
+    public void setRowGenerator(IRowGenerator rowGenerator) {
+    	this.rowGenerator = rowGenerator;
+    }
+
+    public void writeRows(OutputStream out) throws IOException {
+        // delayed creation of SheetDataWriter
+		_writer = ((SuperSXSSFWorkbook)_workbook).createSheetDataWriter(out);
+		try {
+			if (this.rowGenerator != null) {
+				this.rowGenerator.generateRows(this);
+			}
+		} catch (Exception e) {
+			throw new IOException("Error generating Excel rows", e);
+		} finally {
+			// flush buffered rows
+			flushRows(0);
+			// flush writer buffer
+			_writer.close();
+			out.flush();
+		}
+	}
+
+}

src/main/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.poi.xssf.streaming;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.poi.openxml4j.util.ZipEntrySource;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.util.IOUtils;
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+import org.apache.poi.xssf.usermodel.XSSFChartSheet;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+public class SuperSXSSFWorkbook extends SXSSFWorkbook {
+	private static final POILogger logger = POILogFactory.getLogger(SuperSXSSFWorkbook.class);
+
+	public SuperSXSSFWorkbook() {
+		this(null);
+	}
+
+	public SuperSXSSFWorkbook(XSSFWorkbook workbook) {
+		this(workbook, SXSSFWorkbook.DEFAULT_WINDOW_SIZE);
+	}
+
+	public SuperSXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize) {
+		setRandomAccessWindowSize(rowAccessWindowSize);
+		_sharedStringSource = null;
+		if (workbook == null) {
+			_wb = new XSSFWorkbook();
+		} else {
+			_wb = workbook;
+		}
+	}
+
+	@Override
+	public void setCompressTempFiles(boolean compress) {
+		// NOOP
+	}
+
+	@Override
+	protected SheetDataWriter createSheetDataWriter() throws IOException {
+		throw new RuntimeException("Not supported by SuperSXSSFWorkbook");
+	}
+
+	protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IOException {
+		return new StreamingSheetWriter(out);
+	}
+
+	@Override
+	protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throws IOException {
+		try {
+			ZipOutputStream zos = new ZipOutputStream(out);
+			try {
+				Enumeration<? extends ZipEntry> en = zipEntrySource.getEntries();
+				while (en.hasMoreElements()) {
+					ZipEntry ze = en.nextElement();
+					zos.putNextEntry(new ZipEntry(ze.getName()));
+					InputStream is = zipEntrySource.getInputStream(ze);
+					XSSFSheet xSheet = getSheetFromZipEntryName(ze.getName());
+					// See bug 56557, we should not inject data into the special ChartSheets
+					if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) {
+						SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet);
+						if (sxSheet != null) {
+							copyStreamAndInjectWorksheet(is, zos, sxSheet);
+						} else {
+							IOUtils.copy(is, zos);
+						}
+					} else {
+						IOUtils.copy(is, zos);
+					}
+					is.close();
+				}
+			} finally {
+				zos.close();
+			}
+		} finally {
+			zipEntrySource.close();
+		}
+	}
+
+	private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, SuperSXSSFSheet sxSheet) throws IOException {
+		InputStreamReader inReader = new InputStreamReader(in, "UTF-8");
+		OutputStreamWriter outWriter = new OutputStreamWriter(out, "UTF-8");
+		boolean needsStartTag = true;
+		int c;
+		int pos = 0;
+		String s = "<sheetData";
+		int n = s.length();
+		// Copy from "in" to "out" up to the string "<sheetData/>" or "</sheetData>"
+		// (excluding).
+		while (((c = inReader.read()) != -1)) {
+			if (c == s.charAt(pos)) {
+				pos++;
+				if (pos == n) {
+					if ("<sheetData".equals(s)) {
+						c = inReader.read();
+						if (c == -1) {
+							outWriter.write(s);
+							break;
+						}
+						if (c == '>') {
+							// Found <sheetData>
+							outWriter.write(s);
+							outWriter.write(c);
+							s = "</sheetData>";
+							n = s.length();
+							pos = 0;
+							needsStartTag = false;
+							continue;
+						}
+						if (c == '/') {
+							// Found <sheetData/
+							c = inReader.read();
+							if (c == -1) {
+								outWriter.write(s);
+								break;
+							}
+							if (c == '>') {
+								// Found <sheetData/>
+								break;
+							}
+
+							outWriter.write(s);
+							outWriter.write('/');
+							outWriter.write(c);
+							pos = 0;
+							continue;
+						}
+
+						outWriter.write(s);
+						outWriter.write('/');
+						outWriter.write(c);
+						pos = 0;
+						continue;
+					} else {
+						// Found </sheetData>
+						break;
+					}
+				}
+			} else {
+				if (pos > 0) {
+					outWriter.write(s, 0, pos);
+				}
+				if (c == s.charAt(0)) {
+					pos = 1;
+				} else {
+					outWriter.write(c);
+					pos = 0;
+				}
+			}
+		}
+		outWriter.flush();
+		if (needsStartTag) {
+			outWriter.write("<sheetData>\n");
+			outWriter.flush();
+		}
+
+		// IOUtils.copy(worksheetData,out);
+		sxSheet.writeRows(out);
+
+		outWriter.write("</sheetData>");
+		outWriter.flush();
+		// Copy the rest of "in" to "out".
+		while (((c = inReader.read()) != -1)) {
+			outWriter.write(c);
+		}
+		outWriter.flush();
+	}
+	
+	@Override
+	SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) {
+		final SuperSXSSFSheet sxSheet;
+		try {
+			sxSheet = new SuperSXSSFSheet(this, xSheet);
+		} catch (IOException ioe) {
+			throw new RuntimeException(ioe);
+		}
+		registerSheetMapping(sxSheet, xSheet);
+		return sxSheet;
+	}
+
+	/**
+	 * Returns an iterator of the sheets in the workbook in sheet order. Includes
+	 * hidden and very hidden sheets.
+	 *
+	 * @return an iterator of the sheets.
+	 */
+	@Override
+	public Iterator<Sheet> sheetIterator() {
+		return new SheetIterator<Sheet>();
+	}
+
+	private final class SheetIterator<T extends Sheet> implements Iterator<T> {
+		final private Iterator<XSSFSheet> it;
+
+		@SuppressWarnings("unchecked")
+		public SheetIterator() {
+			it = (Iterator<XSSFSheet>) (Iterator<? extends Sheet>) _wb.iterator();
+		}
+
+		@Override
+		public boolean hasNext() {
+			return it.hasNext();
+		}
+
+		@Override
+		@SuppressWarnings("unchecked")
+		public T next() throws NoSuchElementException {
+			final XSSFSheet xssfSheet = it.next();
+			SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xssfSheet);
+			return (T) (sxSheet == null ? xssfSheet : sxSheet);
+		}
+
+		/**
+		 * Unexpected behavior may occur if sheets are reordered after iterator has been
+		 * created. Support for the remove method may be added in the future if someone
+		 * can figure out a reliable implementation.
+		 */
+		@Override
+		public void remove() throws IllegalStateException {
+			throw new UnsupportedOperationException("remove method not supported on XSSFWorkbook.iterator(). " + "Use Sheet.removeSheetAt(int) instead.");
+		}
+	}
+
+	/**
+	 * Alias for {@link #sheetIterator()} to allow foreach loops
+	 */
+	@Override
+	public Iterator<Sheet> iterator() {
+		return sheetIterator();
+	}
+
+	@Override
+	public SXSSFSheet getSheetAt(int index) {
+		throw new RuntimeException("Not supported by SuperSXSSFWorkbook");
+	}
+
+	public XSSFSheet getXSSFSheetAt(int index) {
+		return _wb.getSheetAt(index);
+	}
+
+	/**
+	 * Gets the sheet at the given index for streaming.
+	 *
+	 * @param index the index
+	 * @return the streaming sheet at
+	 */
+	public SXSSFSheet getStreamingSheetAt(