Commit 41f2b41e authored by Matija Obreza's avatar Matija Obreza

Merge branch '376-streaming-generated-excel' into 'master'

Resolve "Streaming generated Excel"

Closes #376

See merge request genesys-pgr/genesys-server!298
parents 29a3b548 20f0ed13
......@@ -412,7 +412,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
<version>4.1</version>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
......@@ -480,7 +480,7 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.10.1</version>
<version>3.17</version>
</dependency>
<dependency>
<groupId>org.ocpsoft.prettytime</groupId>
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.poi.xssf.streaming;
/**
* IRowGenerator for Super-streaming XSSF sheets
*/
public interface IRowGenerator {
/**
* Generate rows
*
* @param sheet the sheet
* @throws Exception the exception
*/
void generateRows(SXSSFSheet sheet) throws Exception;
}
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.xssf.streaming;
import org.apache.poi.ss.formula.EvaluationCell;
import org.apache.poi.ss.formula.IStabilityClassifier;
import org.apache.poi.ss.formula.WorkbookEvaluator;
import org.apache.poi.ss.formula.udf.UDFFinder;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.xssf.usermodel.BaseXSSFFormulaEvaluator;
/**
* Streaming-specific Formula Evaluator, which is able to
* lookup cells within the current Window.
*/
public final class SXSSFFormulaEvaluator extends BaseXSSFFormulaEvaluator {
private static final POILogger logger = POILogFactory.getLogger(SXSSFFormulaEvaluator.class);
private SXSSFWorkbook wb;
public SXSSFFormulaEvaluator(SXSSFWorkbook workbook) {
this(workbook, null, null);
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
this(workbook, new WorkbookEvaluator(SXSSFEvaluationWorkbook.create(workbook), stabilityClassifier, udfFinder));
}
private SXSSFFormulaEvaluator(SXSSFWorkbook workbook, WorkbookEvaluator bookEvaluator) {
super(bookEvaluator);
this.wb = workbook;
}
/**
* @param stabilityClassifier used to optimise caching performance. Pass <code>null</code>
* for the (conservative) assumption that any cell may have its definition changed after
* evaluation begins.
* @param udfFinder pass <code>null</code> for default (AnalysisToolPak only)
*/
public static SXSSFFormulaEvaluator create(SXSSFWorkbook workbook, IStabilityClassifier stabilityClassifier, UDFFinder udfFinder) {
return new SXSSFFormulaEvaluator(workbook, stabilityClassifier, udfFinder);
}
/**
* Turns a SXSSFCell into a SXSSFEvaluationCell
*/
@Override
protected EvaluationCell toEvaluationCell(Cell cell) {
if (!(cell instanceof SXSSFCell)){
throw new IllegalArgumentException("Unexpected type of cell: " + cell.getClass() + "." +
" Only SXSSFCells can be evaluated.");
}
return new SXSSFEvaluationCell((SXSSFCell)cell);
}
@Override
public SXSSFCell evaluateInCell(Cell cell) {
return (SXSSFCell) super.evaluateInCell(cell);
}
/**
* For active worksheets only, will loop over rows and
* cells, evaluating formula cells there.
* If formula cells are outside the window for that sheet,
* it can either skip them silently, or give an exception
*/
public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWindow) {
SXSSFFormulaEvaluator eval = new SXSSFFormulaEvaluator(wb);
// Check they're all available
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet && ((SXSSFSheet)sheet).areAllRowsFlushed()) {
throw new SheetsFlushedException();
}
}
// Process the sheets as best we can
for (Sheet sheet : wb) {
if (sheet instanceof SXSSFSheet) {
// Check if any rows have already been flushed out
int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum();
if (lastFlushedRowNum > -1) {
if (! skipOutOfWindow) throw new RowFlushedException(0);
logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping");
}
}
// Evaluate what we have
for (Row r : sheet) {
for (Cell c : r) {
if (c.getCellTypeEnum() == CellType.FORMULA) {
eval.evaluateFormulaCellEnum(c);
}
}
}
}
}
/**
* Loops over rows and cells, evaluating formula cells there.
* If any sheets are inactive, or any cells outside of the window,
* will give an Exception.
* For SXSSF, you generally don't want to use this method, instead
* evaluate your formulas as you go before they leave the window.
*/
public void evaluateAll() {
// Have the evaluation done, with exceptions
evaluateAllFormulaCells(wb, false);
}
public static class SheetsFlushedException extends IllegalStateException {
protected SheetsFlushedException() {
super("One or more sheets have been flushed, cannot evaluate all cells");
}
}
public static class RowFlushedException extends IllegalStateException {
protected RowFlushedException(int rowNum) {
super("Row " + rowNum + " has been flushed, cannot evaluate all cells");
}
}
}
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.xssf.streaming;
import java.awt.Dimension;
import java.io.IOException;
import org.apache.poi.openxml4j.opc.PackagePart;
import org.apache.poi.ss.usermodel.Picture;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Shape;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.ImageUtils;
import org.apache.poi.util.Internal;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.util.Units;
import org.apache.poi.xssf.usermodel.XSSFAnchor;
import org.apache.poi.xssf.usermodel.XSSFClientAnchor;
import org.apache.poi.xssf.usermodel.XSSFDrawing;
import org.apache.poi.xssf.usermodel.XSSFPicture;
import org.apache.poi.xssf.usermodel.XSSFPictureData;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveSize2D;
import org.openxmlformats.schemas.drawingml.x2006.main.CTShapeProperties;
import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTPicture;
import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCol;
/**
* Streaming version of Picture.
* Most of the code is a copy of the non-streaming XSSFPicture code.
* This is necessary as a private method getRowHeightInPixels of that class needs to be changed, which is called by a method call chain nested several levels.
*
* The main change is to access the rows in the SXSSF sheet, not the always empty rows in the XSSF sheet when checking the row heights.
*/
public final class SXSSFPicture implements Picture {
private static final POILogger logger = POILogFactory.getLogger(SXSSFPicture.class);
/**
* Column width measured as the number of characters of the maximum digit width of the
* numbers 0, 1, 2, ..., 9 as rendered in the normal style's font. There are 4 pixels of margin
* padding (two on each side), plus 1 pixel padding for the gridlines.
*
* This value is the same for default font in Office 2007 (Calibry) and Office 2003 and earlier (Arial)
*/
private static float DEFAULT_COLUMN_WIDTH = 9.140625f;
private final SXSSFWorkbook _wb;
private final XSSFPicture _picture;
SXSSFPicture(SXSSFWorkbook _wb, XSSFPicture _picture) {
this._wb = _wb;
this._picture = _picture;
}
/**
* Return the underlying CTPicture bean that holds all properties for this picture
*
* @return the underlying CTPicture bean
*/
@Internal
public CTPicture getCTPicture(){
return _picture.getCTPicture();
}
/**
* Reset the image to the original size.
*
* <p>
* Please note, that this method works correctly only for workbooks
* with the default font size (Calibri 11pt for .xlsx).
* If the default font is changed the resized image can be streched vertically or horizontally.
* </p>
*/
@Override
public void resize(){
resize(1.0);
}
/**
* Reset the image to the original size.
* <p>
* Please note, that this method works correctly only for workbooks
* with the default font size (Calibri 11pt for .xlsx).
* If the default font is changed the resized image can be streched vertically or horizontally.
* </p>
*
* @param scale the amount by which image dimensions are multiplied relative to the original size.
* <code>resize(1.0)</code> sets the original size, <code>resize(0.5)</code> resize to 50% of the original,
* <code>resize(2.0)</code> resizes to 200% of the original.
*/
@Override
public void resize(double scale){
XSSFClientAnchor anchor = getClientAnchor();
XSSFClientAnchor pref = getPreferredSize(scale);
int row2 = anchor.getRow1() + (pref.getRow2() - pref.getRow1());
int col2 = anchor.getCol1() + (pref.getCol2() - pref.getCol1());
anchor.setCol2(col2);
anchor.setDx1(0);
anchor.setDx2(pref.getDx2());
anchor.setRow2(row2);
anchor.setDy1(0);
anchor.setDy2(pref.getDy2());
}
/**
* Calculate the preferred size for this picture.
*
* @return XSSFClientAnchor with the preferred size for this image
*/
@Override
public XSSFClientAnchor getPreferredSize(){
return getPreferredSize(1.0);
}
/**
* Calculate the preferred size for this picture.
*
* @param scale the amount by which image dimensions are multiplied relative to the original size.
* @return XSSFClientAnchor with the preferred size for this image
*/
public XSSFClientAnchor getPreferredSize(double scale){
XSSFClientAnchor anchor = getClientAnchor();
XSSFPictureData data = getPictureData();
Dimension size = getImageDimension(data.getPackagePart(), data.getPictureType());
double scaledWidth = size.getWidth() * scale;
double scaledHeight = size.getHeight() * scale;
float w = 0;
int col2 = anchor.getCol1()-1;
while (w <= scaledWidth) {
w += getColumnWidthInPixels(++col2);
}
assert (w > scaledWidth);
double cw = getColumnWidthInPixels(col2);
double deltaW = w - scaledWidth;
int dx2 = (int)(Units.EMU_PER_PIXEL * (cw - deltaW));
anchor.setCol2(col2);
anchor.setDx2(dx2);
double h = 0;
int row2 = anchor.getRow1()-1;
while (h <= scaledHeight) {
h += getRowHeightInPixels(++row2);
}
assert (h > scaledHeight);
double ch = getRowHeightInPixels(row2);
double deltaH = h - scaledHeight;
int dy2 = (int)(Units.EMU_PER_PIXEL * (ch - deltaH));
anchor.setRow2(row2);
anchor.setDy2(dy2);
CTPositiveSize2D size2d = getCTPicture().getSpPr().getXfrm().getExt();
size2d.setCx((long)(scaledWidth * Units.EMU_PER_PIXEL));
size2d.setCy((long)(scaledHeight * Units.EMU_PER_PIXEL));
return anchor;
}
private float getColumnWidthInPixels(int columnIndex){
XSSFSheet sheet = getSheet();
CTCol col = sheet.getColumnHelper().getColumn(columnIndex, false);
double numChars = col == null || !col.isSetWidth() ? DEFAULT_COLUMN_WIDTH : col.getWidth();
return (float)numChars*Units.DEFAULT_CHARACTER_WIDTH;
}
private float getRowHeightInPixels(int rowIndex) {
// THE FOLLOWING THREE LINES ARE THE MAIN CHANGE compared to the non-streaming version: use the SXSSF sheet,
// not the XSSF sheet (which never contais rows when using SXSSF)
XSSFSheet xssfSheet = getSheet();
SXSSFSheet sxSheet = _wb.getSXSSFSheet(xssfSheet);
Sheet sheet = sxSheet == null ? xssfSheet : sxSheet;
Row row = sheet.getRow(rowIndex);
float height = row != null ? row.getHeightInPoints() : sheet.getDefaultRowHeightInPoints();
return height * Units.PIXEL_DPI / Units.POINT_DPI;
}
/**
* Return the dimension of this image
*
* @param part the package part holding raw picture data
* @param type type of the picture: {@link Workbook#PICTURE_TYPE_JPEG},
* {@link Workbook#PICTURE_TYPE_PNG} or {@link Workbook#PICTURE_TYPE_DIB}
*
* @return image dimension in pixels
*/
protected static Dimension getImageDimension(PackagePart part, int type){
try {
return ImageUtils.getImageDimension(part.getInputStream(), type);
} catch (IOException e){
//return a "singulariry" if ImageIO failed to read the image
logger.log(POILogger.WARN, e);
return new Dimension();
}
}
/**
* Return picture data for this shape
*
* @return picture data for this shape
*/
@Override
public XSSFPictureData getPictureData() {
return _picture.getPictureData();
}
protected CTShapeProperties getShapeProperties(){
return getCTPicture().getSpPr();
}
@Override
public XSSFAnchor getAnchor() {
return _picture.getAnchor();
}
@Override
public void resize(double scaleX, double scaleY) {
_picture.resize(scaleX, scaleY);
}
@Override
public XSSFClientAnchor getPreferredSize(double scaleX, double scaleY) {
return _picture.getPreferredSize(scaleX, scaleY);
}
@Override
public Dimension getImageDimension() {
return _picture.getImageDimension();
}
@Override
public XSSFClientAnchor getClientAnchor() {
XSSFAnchor a = getAnchor();
return (a instanceof XSSFClientAnchor) ? (XSSFClientAnchor)a : null;
}
public XSSFDrawing getDrawing() {
return _picture.getDrawing();
}
@Override
public XSSFSheet getSheet() {
return _picture.getSheet();
}
@Override
public String getShapeName() {
return _picture.getShapeName();
}
@Override
public Shape getParent() {
return _picture.getParent();
}
@Override
public boolean isNoFill() {
return _picture.isNoFill();
}
@Override
public void setNoFill(boolean noFill) {
_picture.setNoFill(noFill);
}
@Override
public void setFillColor(int red, int green, int blue) {
_picture.setFillColor(red, green, blue);
}
@Override
public void setLineStyleColor( int red, int green, int blue ) {
_picture.setLineStyleColor(red, green, blue);
}
}
This diff is collapsed.
This diff is collapsed.
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.apache.poi.xssf.streaming;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Iterator;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.FormulaError;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.util.POILogFactory;
import org.apache.poi.util.POILogger;
import org.apache.poi.util.TempFile;
import org.apache.poi.xssf.model.SharedStringsTable;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.openxmlformats.schemas.spreadsheetml.x2006.main.STCellType;
/**
* Initially copied from BigGridDemo "SpreadsheetWriter".
* Then copied from SXSSF "SheetDataWrter".
* Unlike the original code which wrote the entire document,
* this class only writes the "sheetData" document fragment
* to the entire workbook being streamed.
*/
public class SheetDataWriter implements Closeable {
private static final POILogger logger = POILogFactory.getLogger(SheetDataWriter.class);
private final File _fd;
protected final Writer _out;
private int _rownum;
private int _numberOfFlushedRows;
private int _lowestIndexOfFlushedRows; // meaningful only of _numberOfFlushedRows>0
private int _numberOfCellsOfLastFlushedRow; // meaningful only of _numberOfFlushedRows>0
private int _numberLastFlushedRow = -1; // meaningful only of _numberOfFlushedRows>0
/**
* Table of strings shared across this workbook.
* If two cells contain the same string, then the cell value is the same index into SharedStringsTable
*/
private SharedStringsTable _sharedStringSource;
public SheetDataWriter() throws IOException {
_fd = createTempFile();
_out = createWriter(_fd);
}
// Constructor for StreamingSheetWriter
public SheetDataWriter(Writer writer) throws IOException {
_fd = null;
_out = writer;
}
public SheetDataWriter(SharedStringsTable sharedStringsTable) throws IOException {
this();
this._sharedStringSource = sharedStringsTable;
}
/**
* Create a temp file to write sheet data.
* By default, temp files are created in the default temporary-file directory
* with a prefix "poi-sxssf-sheet" and suffix ".xml". Subclasses can override
* it and specify a different temp directory or filename or suffix, e.g. <code>.gz</code>
*
* @return temp file to write sheet data
*/
public File createTempFile() throws IOException {
return TempFile.createTempFile("poi-sxssf-sheet", ".xml");
}
/**
* Create a writer for the sheet data.