Commit 955f6d33 authored by Matija Obreza's avatar Matija Obreza
Browse files

Fill template with JSON data

parent eccace08
/target/
.classpath
.project
result.xlsx
/*
* Copyright 2016 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.croptrust.excel.templates;
import org.apache.poi.ss.usermodel.Workbook;
/**
* Service to populate Excel with data from JSON objects.
*/
public interface ExcelTemplatingService {
public static final String NAMED_NAME_PREFIX = "JSON.";
void fill(Workbook workbook, String jsonObject);
}
/*
* Copyright 2016 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.croptrust.excel.templates.impl;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.Name;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.AreaReference;
import org.apache.poi.ss.util.CellReference;
import org.croptrust.excel.templates.ExcelTemplatingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
public class ExcelTemplatingServiceImpl implements ExcelTemplatingService {
final static Logger LOG = LoggerFactory.getLogger(ExcelTemplatingServiceImpl.class);
/**
* Populate Excel Workbook with data from JSON object
*/
@Override
public void fill(Workbook workbook, String jsonObject) {
if (StringUtils.isBlank(jsonObject)) {
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Filling workbook with json={}", jsonObject);
}
DocumentContext json = JsonPath.parse(jsonObject);
for (int i = 0; i < workbook.getNumberOfNames(); i++) {
Name name = workbook.getNameAt(i);
if (!name.getNameName().startsWith(ExcelTemplatingService.NAMED_NAME_PREFIX)) {
continue;
}
String jsonPath = "$." + name.getNameName().substring(ExcelTemplatingService.NAMED_NAME_PREFIX.length());
LOG.debug("Filling path={}", jsonPath);
if (LOG.isDebugEnabled()) {
LOG.debug("Name='" + name.getNameName() + "' sheet='" + name.getSheetName() + "' deleted="
+ name.isDeleted());
LOG.debug("\t" + name.getRefersToFormula() + " isFunction=" + name.isFunctionName());
}
if (name.isDeleted()) {
continue;
}
populate(workbook, name, jsonPath, json);
}
}
private void populate(Workbook workbook, Name name, String jsonPath, DocumentContext json) {
if (!AreaReference.isContiguous(name.getRefersToFormula())) {
throw new UnsupportedOperationException("We don't handle non-contiguous references");
}
AreaReference areaReference = new AreaReference(name.getRefersToFormula(), workbook.getSpreadsheetVersion());
CellReference[] crefs = areaReference.getAllReferencedCells();
LOG.info("Named range references cells={} size={}", name.getRefersToFormula(), crefs.length);
if (crefs.length == 1) {
LOG.debug("Single field needs populating");
Sheet s = workbook.getSheet(crefs[0].getSheetName());
Row r = s.getRow(crefs[0].getRow());
Cell cell = r.getCell(crefs[0].getCol());
fillCell(cell, jsonPath, json);
} else {
LOG.debug("A range of cells needs populating");
fillRange(workbook, crefs, jsonPath, json);
}
}
private void fillRange(Workbook workbook, CellReference[] crefs, String jsonPath, DocumentContext json) {
Object jsonItems = null;
try {
jsonItems = json.read(jsonPath);
} catch (PathNotFoundException e) {
LOG.debug("No data found for path={}", jsonPath);
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Refence is to multiple cells, handling as array starting at path={}", jsonPath);
LOG.debug("Json gives {} cl={}", jsonItems, jsonItems.getClass());
}
if (!(jsonItems instanceof ArrayList)) {
LOG.debug("Path={} does not refer to existing array in json, but to {}", jsonPath, jsonItems);
return;
}
// The first row will have defined properties
List<String> subs = new ArrayList<>();
int firstRowIndex = SpreadsheetUtil.minRow(crefs);
Sheet s = workbook.getSheet(crefs[0].getSheetName());
for (int c = SpreadsheetUtil.minCol(crefs); c < SpreadsheetUtil.maxCol(crefs); c++) {
// Find named range referencing this cell
Name name = SpreadsheetUtil.findNameForFromula(workbook, new CellReference(s.getSheetName(), firstRowIndex,
c, true, true).formatAsString());
if (name != null) {
String subPath = "$." + name.getNameName().substring(ExcelTemplatingService.NAMED_NAME_PREFIX.length());
LOG.info("Found name for cell name={} subPath={}", name.getNameName(), subPath);
subPath = subPath.substring(jsonPath.length());
subs.add(subPath);
} else {
subs.add(null);
}
}
for (int rowCounter = 0, r = SpreadsheetUtil.minRow(crefs); r < SpreadsheetUtil.maxRow(crefs); rowCounter++, r++) {
Row row = s.getRow(r);
for (int cellCounter = 0, c = SpreadsheetUtil.minCol(crefs); c < SpreadsheetUtil.maxCol(crefs); cellCounter++, c++) {
if (subs.size() < cellCounter || subs.get(cellCounter) == null) {
continue;
}
Cell cell = row.getCell(c);
String subPath = jsonPath + "[" + rowCounter + "]" + subs.get(cellCounter);
LOG.debug("Filling path={}", subPath);
fillCell(cell, subPath, json);
}
}
}
private void fillCell(Cell cell, String jsonPath, DocumentContext json) {
try {
Object jsonValue = json.read(jsonPath);
if (LOG.isDebugEnabled()) {
LOG.debug("Json value at path={} val={}", jsonPath, jsonValue);
}
CellStyle style = cell.getCellStyle();
LOG.trace("Cell fmt={} type={}", style.getDataFormatString(), cell.getCellType());
if (jsonValue instanceof String) {
cell.setCellValue((String) jsonValue);
} else if (jsonValue instanceof Number) {
if (looksLikeADate(style.getDataFormatString())) {
cell.setCellValue(new Date(((Number) jsonValue).longValue()));
} else {
cell.setCellValue(((Number) jsonValue).doubleValue());
}
} else {
LOG.trace("Type val={} c={}", jsonValue, jsonValue.getClass());
if (cell.getCellType() == Cell.CELL_TYPE_NUMERIC) {
}
}
} catch (PathNotFoundException e) {
LOG.debug("Value not defined path={} json={}", jsonPath, json.jsonString());
}
}
private boolean looksLikeADate(String fmt) {
if (StringUtils.isBlank(fmt)) {
return false;
}
if (fmt.contains("d") || fmt.contains("m") || fmt.contains("y")) {
return true;
}
return false;
}
}
......@@ -18,6 +18,11 @@ package org.croptrust.excel.templates.impl;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.FormulaEvaluator;
import org.apache.poi.ss.usermodel.Name;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.CellReference;
public class SpreadsheetUtil {
......@@ -31,7 +36,7 @@ public class SpreadsheetUtil {
if (cell == null) {
return null;
}
switch (cell.getCellType()) {
case Cell.CELL_TYPE_BOOLEAN:
return cell.getBooleanCellValue();
......@@ -55,4 +60,54 @@ public class SpreadsheetUtil {
}
}
public static Cell getCell(Workbook workbook, String cellRef) {
CellReference cellReference = new CellReference(cellRef);
Sheet s = workbook.getSheet(cellReference.getSheetName());
Row r = s.getRow(cellReference.getRow());
Cell c = r.getCell(cellReference.getCol());
return c;
}
public static int minRow(CellReference[] crefs) {
int min = crefs[0].getRow();
for (CellReference cr : crefs) {
min = Math.min(min, cr.getRow());
}
return min;
}
public static int maxRow(CellReference[] crefs) {
int max = crefs[0].getRow();
for (CellReference cr : crefs) {
max = Math.max(max, cr.getRow());
}
return max;
}
public static int minCol(CellReference[] crefs) {
int min = crefs[0].getCol();
for (CellReference cr : crefs) {
min = Math.min(min, cr.getCol());
}
return min;
}
public static int maxCol(CellReference[] crefs) {
int max = crefs[0].getCol();
for (CellReference cr : crefs) {
max = Math.max(max, cr.getCol());
}
return max;
}
public static Name findNameForFromula(Workbook workbook, String formula) {
for (int i = 0; i < workbook.getNumberOfNames(); i++) {
Name name = workbook.getNameAt(i);
if (name.getRefersToFormula().equalsIgnoreCase(formula)) {
return name;
}
}
return null;
}
}
/*
* Copyright 2016 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.croptrust.excel.templates;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import org.apache.poi.EncryptedDocumentException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Name;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.ss.util.AreaReference;
import org.apache.poi.ss.util.CellReference;
import org.croptrust.excel.templates.impl.ExcelTemplatingServiceImpl;
import org.croptrust.excel.templates.impl.ExcelToJSON2Impl;
import org.croptrust.excel.templates.impl.SpreadsheetUtil;
import org.junit.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
public class TemplatingTest extends JsonPathTest {
protected ExcelToJSON excelToJson = new ExcelToJSON2Impl();
protected ExcelTemplatingService excelTemplating = new ExcelTemplatingServiceImpl();
private DateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy");
@Test
public void listNamesTest() throws EncryptedDocumentException, InvalidFormatException, IOException,
URISyntaxException {
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.TR_2016_XSLX)) {
for (int i = 0; i < wb.getNumberOfNames(); i++) {
Name name = wb.getNameAt(i);
System.err.println("Name='" + name.getNameName() + "' sheet='" + name.getSheetName() + "' deleted="
+ name.isDeleted());
}
}
}
@Test
public void testJsonNames() throws EncryptedDocumentException, InvalidFormatException, IOException,
URISyntaxException {
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.TR_2016_XSLX)) {
for (int i = 0; i < wb.getNumberOfNames(); i++) {
Name name = wb.getNameAt(i);
if (!name.getNameName().startsWith("JSON.")) {
continue;
}
System.err.println("Name='" + name.getNameName() + "' sheet='" + name.getSheetName() + "' deleted="
+ name.isDeleted());
System.err.println("\t" + name.getRefersToFormula() + " isFunction=" + name.isFunctionName());
if (name.isDeleted()) {
continue;
}
AreaReference areaReference = new AreaReference(name.getRefersToFormula(), wb.getSpreadsheetVersion());
CellReference[] crefs = areaReference.getAllReferencedCells();
for (int cellRef = 0; cellRef < crefs.length; cellRef++) {
Sheet s = wb.getSheet(crefs[cellRef].getSheetName());
Row r = s.getRow(crefs[cellRef].getRow());
Cell c = r.getCell(crefs[cellRef].getCol());
assertThat("Named cell should not be be null", c, not(nullValue()));
}
}
}
}
@Test
public void fillBasicInfo() throws EncryptedDocumentException, InvalidFormatException, IOException,
URISyntaxException {
List<String> jsonStrings;
ObjectNode root = new ObjectMapper().createObjectNode();
root.put("fwd", new Date().getTime());
root.put("lwd", new Date().getTime());
root.put("purpose", "Participate in the workshop");
root.put("benefit", "No benefit");
String defaults = root.toString();
// System.err.println(defaults);
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.PARTICIPANTS_XSLX)) {
Sheet sheet = wb.getSheetAt(0);
String[] columnMapping = new String[] { "timestamp", "traveller.lastName", "traveller.firstNames",
"traveller.organization", "traveller.address", "traveller.email", "traveller.phone",
"traveller.dateOfBirth", "traveller.passportCountry", null, null, null, "itinerary[0].origin",
"itinerary[1].destination" };
// Expect converter to return a list of null objects with size matching number of rows
jsonStrings = excelToJson.readAsJson(sheet, columnMapping, defaults, true, true);
}
for (String jsonObject : jsonStrings) {
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.TR_2016_XSLX)) {
excelTemplating.fill(wb, jsonObject);
DocumentContext json = JsonPath.parse(jsonObject);
assertThat("Cell must contain the lastName", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E5")
.getStringCellValue(), is(json.read("$.traveller.lastName")));
assertThat("Cell must contain the firstNames", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E6")
.getStringCellValue(), is(json.read("$.traveller.firstNames")));
assertThat("Cell must contain the email", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E9")
.getStringCellValue(), is(json.read("$.traveller.email")));
assertThat("Cell must contain the phone", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E10")
.getStringCellValue(), is(json.read("$.traveller.phone")));
assertThat("Cell must contain the address", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E11")
.getStringCellValue(), is(json.read("$.traveller.address")));
assertThat("Cell must contain the purpose", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!B14")
.getStringCellValue(), is(json.read("$.purpose")));
assertThat("Cell must contain the benefit", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!B16")
.getStringCellValue(), is(json.read("$.benefit")));
assertThat("Cell must contain the fwd", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!D19")
.getDateCellValue().getTime(), is(((Number) json.read("$.fwd")).longValue()));
assertThat("Cell must contain the lwd", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!I19")
.getDateCellValue().getTime(), is(((Number) json.read("$.lwd")).longValue()));
}
}
}
@Test
public void fillArrayTest() throws EncryptedDocumentException, InvalidFormatException, IOException,
URISyntaxException {
List<String> jsonStrings;
ObjectNode root = new ObjectMapper().createObjectNode();
root.put("fwd", new Date().getTime());
root.put("lwd", new Date().getTime());
root.put("purpose", "Participate in the workshop");
root.put("benefit", "No benefit");
ArrayNode bcodes;
root.set("budgetCodes", bcodes = root.arrayNode());
ObjectNode bcode = bcodes.addObject();
bcode.put("unit", "CRP1");
bcode.put("account", "ACC");
bcode.put("donor", "DON");
bcode.put("project", "PRO");
bcode.put("activity", "ACT");
bcode.put("subActivity", "SUBA");
bcode.put("year", 2016);
String defaults = root.toString();
System.err.println(defaults);
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.PARTICIPANTS_XSLX)) {
Sheet sheet = wb.getSheetAt(0);
String[] columnMapping = new String[] { "timestamp", "traveller.lastName", "traveller.firstNames",
"traveller.organization", "traveller.address", "traveller.email", "traveller.phone",
"traveller.dateOfBirth", "traveller.passportCountry", null, null, null, "itinerary[0].origin",
"itinerary[1].destination" };
// Expect converter to return a list of null objects with size matching number of rows
jsonStrings = excelToJson.readAsJson(sheet, columnMapping, defaults, true, true);
}
for (String jsonObject : jsonStrings) {
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.TR_2016_XSLX)) {
excelTemplating.fill(wb, jsonObject);
DocumentContext json = JsonPath.parse(jsonObject);
assertThat("Cell must contain the unit", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!B23")
.getStringCellValue(), is(json.read("$.budgetCodes[0].unit")));
assertThat("Cell must contain the account", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!C23")
.getStringCellValue(), is(json.read("$.budgetCodes[0].account")));
assertThat("Cell must contain the donor", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!D23")
.getStringCellValue(), is(json.read("$.budgetCodes[0].donor")));
assertThat("Cell must contain the project", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!E23")
.getStringCellValue(), is(json.read("$.budgetCodes[0].project")));
assertThat("Cell must contain the activity", SpreadsheetUtil.getCell(wb, "'TA REQUEST-1'!F23")
.getStringCellValue(), is(json.read("$.budgetCodes[0].activity")));
try (FileOutputStream fileOut = new FileOutputStream("result.xlsx")) {
wb.write(fileOut);
fileOut.flush();
}
return;
}
}
}
@Test
public void fillArray2Test() throws EncryptedDocumentException, InvalidFormatException, IOException,
URISyntaxException, ParseException {
List<String> jsonStrings;
ObjectNode root = new ObjectMapper().createObjectNode();
root.put("fwd", new Date().getTime());
root.put("lwd", new Date().getTime());
root.put("purpose", "Participate in the workshop");
root.put("benefit", "No benefit");
ArrayNode itinerary;
root.set("itinerary", itinerary = root.arrayNode());
ObjectNode travel = itinerary.addObject();
travel.put("origin", "");
travel.put("destination", "TXL - Berlin, Germany");
travel.put("departure", dateFormat.parse("2-JAN-2016").getTime());
travel.put("arrival", dateFormat.parse("3-JAN-2016").getTime());
travel.put("budgetUnit", "CRP1");
travel = itinerary.addObject();
travel.put("origin", "TXL - Berlin, Germany");
travel.put("destination", "");
travel.put("departure", dateFormat.parse("2-JAN-2016").getTime());
travel.put("arrival", dateFormat.parse("3-JAN-2016").getTime());
travel.put("budgetUnit", "CRP1");
String defaults = root.toString();
System.err.println(defaults);
try (Workbook wb = WorkbookFactory.create(BasicExcelTest.PARTICIPANTS_XSLX)) {
Sheet sheet = wb.getSheetAt(0);
String[] columnMapping = new String[] { "timestamp", "traveller.lastName", "traveller.firstNames",
"traveller.organization", "traveller.address", "traveller.email", "traveller.phone",
"traveller.dateOfBirth", "traveller.passportCountry", null, null, null, "itinerary[0].origin",
"itinerary[1].destination" };
// Expect converter to return a list of null objects with size matching number of rows
jsonStrings = excelToJson.readAsJson(sheet, columnMapping, defaults, true, true);
}