Merge branch '376-streaming-generated-excel' into 'master'

Resolve "Streaming generated Excel"

Closes #376

See merge request genesys-pgr/genesys-server!298

pom.xml

@@ -412,7 +412,7 @@
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-collections4</artifactId>
-			<version>4.0</version>
+			<version>4.1</version>
 		</dependency>
 		<dependency>
 			<groupId>javax.mail</groupId>
@@ -480,7 +480,7 @@
 		<dependency>
 			<groupId>org.apache.poi</groupId>
 			<artifactId>poi-ooxml</artifactId>
-			<version>3.10.1</version>
+			<version>3.17</version>
 		</dependency>
 		<dependency>
 			<groupId>org.ocpsoft.prettytime</groupId>

src/main/java/org/apache/poi/xssf/streaming/IRowGenerator.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.poi.xssf.streaming;
+
+/**
+ * IRowGenerator for Super-streaming XSSF sheets
+ */
+public interface IRowGenerator {
+	
+	/**
+	 * Generate rows
+	 *
+	 * @param sheet the sheet
+	 * @throws Exception the exception
+	 */
+	void generateRows(SXSSFSheet sheet) throws Exception;
+}

src/main/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java

+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.xssf.streaming;
+
+import org.apache.poi.ss.formula.EvaluationCell;
+import org.apache.poi.ss.formula.IStabilityClassifier;
+import org.apache.poi.ss.formula.WorkbookEvaluator;
+import org.apache.poi.ss.formula.udf.UDFFinder;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellType;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+import org.apache.poi.xssf.usermodel.BaseXSSFFormulaEvaluator;
+
+/**
+ * Streaming-specific Formula Evaluator, which is able to 
+ *  lookup cells within the current Window.
+ */
+public final class SXSSFFormulaEvaluator extends BaseXSSFFormulaEvaluator {
+    private static final POILogger logger = POILogFactory.getLogger(SXSSFFormulaEvaluator.class);
+    
+    private SXSSFWorkbook wb;
+    
+    public SXSSFFormulaEvaluator(SXSSFWorkbook workbook) {
+        this(workbook, null, null);
+    }
+    private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
+        this(workbook, new WorkbookEvaluator(SXSSFEvaluationWorkbook.create(workbook), stabilityClassifier, udfFinder));
+    }
+    private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, WorkbookEvaluator bookEvaluator) {
+        super(bookEvaluator);
+        this.wb = workbook;
+    }
+    
+    /**
+     * @param stabilityClassifier used to optimise caching performance. Pass <code>null</code>
+     * for the (conservative) assumption that any cell may have its definition changed after
+     * evaluation begins.
+     * @param udfFinder pass <code>null</code> for default (AnalysisToolPak only)
+     */
+    public static SXSSFFormulaEvaluator create(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
+        return new SXSSFFormulaEvaluator(workbook, stabilityClassifier, udfFinder);
+    }
+    
+    /**
+     * Turns a SXSSFCell into a SXSSFEvaluationCell
+     */
+    @Override
+    protected EvaluationCell toEvaluationCell(Cell cell) {
+        if (!(cell instanceof SXSSFCell)){
+            throw new IllegalArgumentException("Unexpected type of cell: " + cell.getClass() + "." +
+                    " Only SXSSFCells can be evaluated.");
+        }
+
+        return new SXSSFEvaluationCell((SXSSFCell)cell);
+    }
+    
+    @Override
+    public SXSSFCell evaluateInCell(Cell cell) {
+        return (SXSSFCell) super.evaluateInCell(cell);
+    }
+    
+    /**
+     * For active worksheets only, will loop over rows and
+     *  cells, evaluating formula cells there.
+     * If formula cells are outside the window for that sheet,
+     *  it can either skip them silently, or give an exception
+     */
+    public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWindow) {
+        SXSSFFormulaEvaluator eval = new SXSSFFormulaEvaluator(wb);
+        
+        // Check they're all available
+        for (Sheet sheet : wb) {
+            if (sheet instanceof SXSSFSheet && ((SXSSFSheet)sheet).areAllRowsFlushed()) {
+                throw new SheetsFlushedException();
+            }
+        }
+        
+        // Process the sheets as best we can
+        for (Sheet sheet : wb) {
+            
+        	if (sheet instanceof SXSSFSheet) {
+	            // Check if any rows have already been flushed out
+	            int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum();
+	            if (lastFlushedRowNum > -1) {
+	                if (! skipOutOfWindow) throw new RowFlushedException(0);
+	                logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping");
+	            }
+        	}
+        	
+            // Evaluate what we have
+            for (Row r : sheet) {
+                for (Cell c : r) {
+                    if (c.getCellTypeEnum() == CellType.FORMULA) {
+                        eval.evaluateFormulaCellEnum(c);
+                    }
+                }
+            }
+        }
+    }
+    
+    /**
+     * Loops over rows and cells, evaluating formula cells there.
+     * If any sheets are inactive, or any cells outside of the window,
+     *  will give an Exception.
+     * For SXSSF, you generally don't want to use this method, instead
+     *  evaluate your formulas as you go before they leave the window.
+     */
+    public void evaluateAll() {
+        // Have the evaluation done, with exceptions
+        evaluateAllFormulaCells(wb, false);
+    }
+    
+    public static class SheetsFlushedException extends IllegalStateException {
+        protected SheetsFlushedException() {
+            super("One or more sheets have been flushed, cannot evaluate all cells");
+        }
+    }
+    public static class RowFlushedException extends IllegalStateException {
+        protected RowFlushedException(int rowNum) {
+            super("Row " + rowNum + " has been flushed, cannot evaluate all cells");
+        }
+    }
+}

src/main/java/org/apache/poi/xssf/streaming/SXSSFPicture.java

+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.xssf.streaming;
+
+import java.awt.Dimension;
+import java.io.IOException;
+
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.ss.usermodel.Picture;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Shape;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.util.ImageUtils;
+import org.apache.poi.util.Internal;
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+import org.apache.poi.util.Units;
+import org.apache.poi.xssf.usermodel.XSSFAnchor;
+import org.apache.poi.xssf.usermodel.XSSFClientAnchor;
+import org.apache.poi.xssf.usermodel.XSSFDrawing;
+import org.apache.poi.xssf.usermodel.XSSFPicture;
+import org.apache.poi.xssf.usermodel.XSSFPictureData;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveSize2D;
+import org.openxmlformats.schemas.drawingml.x2006.main.CTShapeProperties;
+import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTPicture;
+import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCol;
+
+/**
+ * Streaming version of Picture.
+ * Most of the code is a copy of the non-streaming XSSFPicture code.
+ * This is necessary as a private method getRowHeightInPixels of that class needs to be changed, which is called by a method call chain nested several levels.
+ *
+ * The main change is to access the rows in the SXSSF sheet, not the always empty rows in the XSSF sheet when checking the row heights.
+ */
+public final class SXSSFPicture implements Picture {
+    private static final POILogger logger = POILogFactory.getLogger(SXSSFPicture.class);
+    /**
+     * Column width measured as the number of characters of the maximum digit width of the
+     * numbers 0, 1, 2, ..., 9 as rendered in the normal style's font. There are 4 pixels of margin
+     * padding (two on each side), plus 1 pixel padding for the gridlines.
+     *
+     * This value is the same for default font in Office 2007 (Calibry) and Office 2003 and earlier (Arial)
+     */
+    private static float DEFAULT_COLUMN_WIDTH = 9.140625f;
+
+    private final SXSSFWorkbook _wb;
+    private final XSSFPicture _picture;
+
+    SXSSFPicture(SXSSFWorkbook _wb, XSSFPicture _picture) {
+        this._wb = _wb;
+        this._picture = _picture;
+    }
+
+    /**
+     * Return the underlying CTPicture bean that holds all properties for this picture
+     *
+     * @return the underlying CTPicture bean
+     */
+    @Internal
+    public CTPicture getCTPicture(){
+        return _picture.getCTPicture();
+    }
+
+    /**
+     * Reset the image to the original size.
+     *
+     * <p>
+     * Please note, that this method works correctly only for workbooks
+     * with the default font size (Calibri 11pt for .xlsx).
+     * If the default font is changed the resized image can be streched vertically or horizontally.
+     * </p>
+     */
+    @Override
+    public void resize(){
+        resize(1.0);
+    }
+
+    /**
+     * Reset the image to the original size.
+     * <p>
+     * Please note, that this method works correctly only for workbooks
+     * with the default font size (Calibri 11pt for .xlsx).
+     * If the default font is changed the resized image can be streched vertically or horizontally.
+     * </p>
+     *
+     * @param scale the amount by which image dimensions are multiplied relative to the original size.
+     * <code>resize(1.0)</code> sets the original size, <code>resize(0.5)</code> resize to 50% of the original,
+     * <code>resize(2.0)</code> resizes to 200% of the original.
+     */
+    @Override
+    public void resize(double scale){
+        XSSFClientAnchor anchor = getClientAnchor();
+
+        XSSFClientAnchor pref = getPreferredSize(scale);
+
+        int row2 = anchor.getRow1() + (pref.getRow2() - pref.getRow1());
+        int col2 = anchor.getCol1() + (pref.getCol2() - pref.getCol1());
+
+        anchor.setCol2(col2);
+        anchor.setDx1(0);
+        anchor.setDx2(pref.getDx2());
+
+        anchor.setRow2(row2);
+        anchor.setDy1(0);
+        anchor.setDy2(pref.getDy2());
+    }
+
+    /**
+     * Calculate the preferred size for this picture.
+     *
+     * @return XSSFClientAnchor with the preferred size for this image
+     */
+    @Override
+    public XSSFClientAnchor getPreferredSize(){
+        return getPreferredSize(1.0);
+    }
+
+    /**
+     * Calculate the preferred size for this picture.
+     *
+     * @param scale the amount by which image dimensions are multiplied relative to the original size.
+     * @return XSSFClientAnchor with the preferred size for this image
+     */
+    public XSSFClientAnchor getPreferredSize(double scale){
+        XSSFClientAnchor anchor = getClientAnchor();
+
+        XSSFPictureData data = getPictureData();
+        Dimension size = getImageDimension(data.getPackagePart(), data.getPictureType());
+        double scaledWidth = size.getWidth() * scale;
+        double scaledHeight = size.getHeight() * scale;
+
+        float w = 0;
+        int col2 = anchor.getCol1()-1;
+
+        while (w <= scaledWidth) {
+            w += getColumnWidthInPixels(++col2);
+        }
+
+        assert (w > scaledWidth);
+        double cw = getColumnWidthInPixels(col2);
+        double deltaW = w - scaledWidth;
+        int dx2 = (int)(Units.EMU_PER_PIXEL * (cw - deltaW));
+
+        anchor.setCol2(col2);
+        anchor.setDx2(dx2);
+
+        double h = 0;
+        int row2 = anchor.getRow1()-1;
+
+        while (h <= scaledHeight) {
+            h += getRowHeightInPixels(++row2);
+        }
+
+        assert (h > scaledHeight);
+        double ch = getRowHeightInPixels(row2);
+        double deltaH = h - scaledHeight;
+        int dy2 = (int)(Units.EMU_PER_PIXEL * (ch - deltaH));
+        anchor.setRow2(row2);
+        anchor.setDy2(dy2);
+
+        CTPositiveSize2D size2d =  getCTPicture().getSpPr().getXfrm().getExt();
+        size2d.setCx((long)(scaledWidth * Units.EMU_PER_PIXEL));
+        size2d.setCy((long)(scaledHeight * Units.EMU_PER_PIXEL));
+
+        return anchor;
+    }
+
+    private float getColumnWidthInPixels(int columnIndex){
+        XSSFSheet sheet = getSheet();
+
+        CTCol col = sheet.getColumnHelper().getColumn(columnIndex, false);
+        double numChars = col == null || !col.isSetWidth() ? DEFAULT_COLUMN_WIDTH : col.getWidth();
+
+        return (float)numChars*Units.DEFAULT_CHARACTER_WIDTH;
+    }
+
+    private float getRowHeightInPixels(int rowIndex) {
+        // THE FOLLOWING THREE LINES ARE THE MAIN CHANGE compared to the non-streaming version: use the SXSSF sheet,
+		// not the XSSF sheet (which never contais rows when using SXSSF)
+        XSSFSheet xssfSheet = getSheet();
+        SXSSFSheet sxSheet = _wb.getSXSSFSheet(xssfSheet);
+        Sheet sheet = sxSheet == null ? xssfSheet : sxSheet;
+        Row row = sheet.getRow(rowIndex);
+        float height = row != null ?  row.getHeightInPoints() : sheet.getDefaultRowHeightInPoints();
+        return height * Units.PIXEL_DPI / Units.POINT_DPI;
+    }
+    /**
+     * Return the dimension of this image
+     *
+     * @param part the package part holding raw picture data
+     * @param type type of the picture: {@link Workbook#PICTURE_TYPE_JPEG},
+     * {@link Workbook#PICTURE_TYPE_PNG} or {@link Workbook#PICTURE_TYPE_DIB}
+     *
+     * @return image dimension in pixels
+     */
+    protected static Dimension getImageDimension(PackagePart part, int type){
+        try {
+            return ImageUtils.getImageDimension(part.getInputStream(), type);
+        } catch (IOException e){
+            //return a "singulariry" if ImageIO failed to read the image
+            logger.log(POILogger.WARN, e);
+            return new Dimension();
+        }
+    }
+
+    /**
+     * Return picture data for this shape
+     *
+     * @return picture data for this shape
+     */
+    @Override
+    public XSSFPictureData getPictureData() {
+        return _picture.getPictureData();
+    }
+
+    protected CTShapeProperties getShapeProperties(){
+        return getCTPicture().getSpPr();
+    }
+
+    @Override
+    public XSSFAnchor getAnchor() {
+        return _picture.getAnchor();
+    }
+
+    @Override
+    public void resize(double scaleX, double scaleY) {
+        _picture.resize(scaleX, scaleY);
+    }
+
+    @Override
+    public XSSFClientAnchor getPreferredSize(double scaleX, double scaleY) {
+        return _picture.getPreferredSize(scaleX, scaleY);
+    }
+
+    @Override
+    public Dimension getImageDimension() {
+        return _picture.getImageDimension();
+    }
+
+    @Override
+    public XSSFClientAnchor getClientAnchor() {
+        XSSFAnchor a = getAnchor();
+        return (a instanceof XSSFClientAnchor) ? (XSSFClientAnchor)a : null;
+    }
+    
+    public XSSFDrawing getDrawing() {
+        return _picture.getDrawing();
+    }
+
+    @Override
+    public XSSFSheet getSheet() {
+        return _picture.getSheet();
+    }
+
+    @Override
+    public String getShapeName() {
+        return _picture.getShapeName();
+    }
+
+    @Override
+    public Shape getParent() {
+        return _picture.getParent();
+    }
+
+    @Override
+    public boolean isNoFill() {
+        return _picture.isNoFill();
+    }
+
+    @Override
+    public void setNoFill(boolean noFill) {
+        _picture.setNoFill(noFill);
+    }
+
+    @Override
+    public void setFillColor(int red, int green, int blue) {
+        _picture.setFillColor(red, green, blue);
+    }
+    
+    @Override
+    public void setLineStyleColor( int red, int green, int blue ) {
+        _picture.setLineStyleColor(red, green, blue);
+    }
+}

src/main/java/org/apache/poi/xssf/streaming/SheetDataWriter.java

+/*
+ *  ====================================================================
+ *    Licensed to the Apache Software Foundation (ASF) under one or more
+ *    contributor license agreements.  See the NOTICE file distributed with
+ *    this work for additional information regarding copyright ownership.
+ *    The ASF licenses this file to You under the Apache License, Version 2.0
+ *    (the "License"); you may not use this file except in compliance with
+ *    the License.  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ * ====================================================================
+ */
+
+package org.apache.poi.xssf.streaming;
+
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Iterator;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.CellType;
+import org.apache.poi.ss.usermodel.FormulaError;
+import org.apache.poi.ss.util.CellReference;
+import org.apache.poi.util.POILogFactory;
+import org.apache.poi.util.POILogger;
+import org.apache.poi.util.TempFile;
+import org.apache.poi.xssf.model.SharedStringsTable;
+import org.apache.poi.xssf.usermodel.XSSFRichTextString;
+import org.openxmlformats.schemas.spreadsheetml.x2006.main.STCellType;
+
+/**
+ * Initially copied from BigGridDemo "SpreadsheetWriter".
+ * Then copied from SXSSF "SheetDataWrter".
+ * Unlike the original code which wrote the entire document,
+ * this class only writes the "sheetData" document fragment
+ * to the entire workbook being streamed.
+ */
+public class SheetDataWriter implements Closeable {
+    private static final POILogger logger = POILogFactory.getLogger(SheetDataWriter.class);
+    
+    private final File _fd;
+    protected final Writer _out;
+    private int _rownum;
+    private int _numberOfFlushedRows;
+    private int _lowestIndexOfFlushedRows; // meaningful only of _numberOfFlushedRows>0
+    private int _numberOfCellsOfLastFlushedRow; // meaningful only of _numberOfFlushedRows>0
+    private int _numberLastFlushedRow = -1; // meaningful only of _numberOfFlushedRows>0
+
+    /**
+     * Table of strings shared across this workbook.
+     * If two cells contain the same string, then the cell value is the same index into SharedStringsTable
+     */
+    private SharedStringsTable _sharedStringSource;
+
+    public SheetDataWriter() throws IOException {
+        _fd = createTempFile();
+        _out = createWriter(_fd);
+    }
+    
+    // Constructor for StreamingSheetWriter
+    public SheetDataWriter(Writer writer) throws IOException {
+        _fd = null;
+        _out = writer;
+    }
+
+    public SheetDataWriter(SharedStringsTable sharedStringsTable) throws IOException {
+        this();
+        this._sharedStringSource = sharedStringsTable;
+    }
+    /**
+     * Create a temp file to write sheet data. 
+     * By default, temp files are created in the default temporary-file directory
+     * with a prefix "poi-sxssf-sheet" and suffix ".xml".  Subclasses can override 
+     * it and specify a different temp directory or filename or suffix, e.g. <code>.gz</code>
+     * 
+     * @return temp file to write sheet data
+     */
+    public File createTempFile() throws IOException {
+        return TempFile.createTempFile("poi-sxssf-sheet", ".xml");
+    }
+
+    /**
+     * Create a writer for the sheet data.