Commit fb438f43 authored by Matija Obreza's avatar Matija Obreza
Browse files

To JSON using Jackson

parent 6ae541dc
......@@ -16,6 +16,7 @@
package org.croptrust.excel.templates;
import java.io.IOException;
import java.util.List;
import org.apache.poi.ss.usermodel.Sheet;
......@@ -37,7 +38,8 @@ public interface ExcelToJSON {
* @param ignoreHeaderRow Should the header row be ignored
* @param ignoreEmptyRows TODO
* @return the list
* @throws IOException
*/
List<String> readAsJson(Sheet sheet, String[] columnMapping, String objectDefaults, boolean ignoreHeaderRow, boolean ignoreEmptyRows);
List<String> readAsJson(Sheet sheet, String[] columnMapping, String objectDefaults, boolean ignoreHeaderRow, boolean ignoreEmptyRows) throws IOException;
}
......@@ -65,21 +65,23 @@ public class ExcelToJSONImpl implements ExcelToJSON {
* java.lang.String, boolean, boolean)
*/
@Override
public List<String> readAsJson(Sheet sheet, String[] columnMapping, String objectDefaults, boolean ignoreHeaderRow,
boolean ignoreEmptyRows) {
List<String> results = new ArrayList<String>(sheet.getLastRowNum());
public List<String> readAsJson(final Sheet sheet, final String[] columnMapping, String objectDefaults,
final boolean ignoreHeaderRow, final boolean ignoreEmptyRows) {
final List<String> results = new ArrayList<String>(sheet.getLastRowNum());
if (StringUtils.isBlank(objectDefaults)) {
if (LOG.isTraceEnabled())
if (LOG.isTraceEnabled()) {
LOG.trace("Blank defaults provided");
}
objectDefaults = EMPTY_JSON_OBJECT_STRING;
}
{
// Log defaults
DocumentContext defaults = JsonPath.using(jsonPathConfig).parse(objectDefaults);
if (LOG.isTraceEnabled())
final DocumentContext defaults = JsonPath.using(jsonPathConfig).parse(objectDefaults);
if (LOG.isTraceEnabled()) {
LOG.trace("Defaults: {}", defaults.jsonString());
}
// TODO Expand defaults object with all mappings defined
......@@ -88,22 +90,23 @@ public class ExcelToJSONImpl implements ExcelToJSON {
}
// Read rows
int startAt = ignoreHeaderRow ? 1 : 0;
int lastRowIndex = sheet.getLastRowNum();
final int startAt = ignoreHeaderRow ? 1 : 0;
final int lastRowIndex = sheet.getLastRowNum();
for (int rowIndex = startAt; rowIndex < lastRowIndex; rowIndex++) {
Row row = sheet.getRow(rowIndex);
final Row row = sheet.getRow(rowIndex);
if (row == null && !ignoreEmptyRows) {
results.add(null);
continue;
}
// Create object with specified defaults
DocumentContext defaults = JsonPath.using(jsonPathConfig).parse(objectDefaults);
final DocumentContext defaults = JsonPath.using(jsonPathConfig).parse(objectDefaults);
String updatedJson = rowToJson(row, columnMapping, defaults);
if (LOG.isTraceEnabled())
final String updatedJson = rowToJson(row, columnMapping, defaults);
if (LOG.isTraceEnabled()) {
LOG.trace("Row JSON: {}", updatedJson);
}
results.add(updatedJson);
}
......@@ -119,7 +122,7 @@ public class ExcelToJSONImpl implements ExcelToJSON {
* @param obj the obj
* @return the string
*/
private String rowToJson(Row row, String[] columnMapping, DocumentContext obj) {
private String rowToJson(final Row row, final String[] columnMapping, final DocumentContext obj) {
int updates = 0;
for (int i = 0; i < columnMapping.length && i < row.getLastCellNum(); i++) {
......@@ -128,25 +131,27 @@ public class ExcelToJSONImpl implements ExcelToJSON {
continue;
}
String jsonPath = "$." + columnMapping[i];
JsonPath compiledPath = JsonPath.compile(jsonPath);
final String jsonPath = "$." + columnMapping[i];
final JsonPath compiledPath = JsonPath.compile(jsonPath);
Cell cell = row.getCell(i);
Object cellValue = cellValue(cell);
final Cell cell = row.getCell(i);
final Object cellValue = cellValue(cell);
if (LOG.isTraceEnabled()) {
LOG.trace("Mapping {} to {}", cellValue, jsonPath);
}
try {
Object read = obj.read(compiledPath);
if (LOG.isTraceEnabled())
final Object read = obj.read(compiledPath);
if (LOG.isTraceEnabled()) {
LOG.trace("Updating {} old={} val={}", jsonPath, read, cellValue);
}
obj.set(compiledPath, cellValue);
updates++;
} catch (PathNotFoundException e) {
if (LOG.isTraceEnabled())
} catch (final PathNotFoundException e) {
if (LOG.isTraceEnabled()) {
LOG.trace("Setting {} val={}", jsonPath, cellValue);
}
// TODO This should be done before the loop on the defaults object
// Add missing properties!
......@@ -171,33 +176,41 @@ public class ExcelToJSONImpl implements ExcelToJSON {
* @param obj the obj
* @param compiledPath the compiled path
*/
private void ensureParents(DocumentContext obj, JsonPath compiledPath) {
String bracketNotation = compiledPath.getPath();
StringBuilder currentPath = new StringBuilder();
private void ensureParents(final DocumentContext obj, final JsonPath compiledPath) {
final String bracketNotation = compiledPath.getPath();
final StringBuilder currentPath = new StringBuilder();
currentPath.append("$");
LOG.trace("Analyzing {} for parents", bracketNotation);
Matcher matcher = Pattern.compile("\\[([^]]+)\\]").matcher(bracketNotation);
if (LOG.isTraceEnabled()) {
LOG.trace("Analyzing {} for parents", bracketNotation);
}
final Matcher matcher = Pattern.compile("\\[([^]]+)\\]").matcher(bracketNotation);
while (matcher.find()) {
String field = matcher.group(1);
LOG.trace("Inspecting field {}", field);
String parentPath = currentPath.toString();
final String field = matcher.group(1);
if (LOG.isTraceEnabled()) {
LOG.trace("Inspecting field {}", field);
}
final String parentPath = currentPath.toString();
currentPath.append("[").append(field).append("]");
try {
// System.err.println(currentPath);
Object res = obj.read(currentPath.toString());
final Object res = obj.read(currentPath.toString());
// System.err.println("-- " + res);
if (res == null) {
obj.set(currentPath.toString(), null);
// System.err.println(">+ " + obj.jsonString());
}
} catch (PathNotFoundException e) {
LOG.trace("No path found: {}", e.getMessage());
} catch (final PathNotFoundException e) {
if (LOG.isTraceEnabled()) {
LOG.trace("No path found: {}", e.getMessage());
}
if (field.startsWith("'")) {
// Is a property: ['property']
LOG.trace("Creating property at {}", currentPath);
if (LOG.isTraceEnabled()) {
LOG.trace("Creating property at {}", currentPath);
}
Object parent = obj.read(parentPath);
final Object parent = obj.read(parentPath);
// System.err.println(parent);
if (parent == null || parent instanceof ArrayNode) {
// convert to array
......@@ -211,7 +224,7 @@ public class ExcelToJSONImpl implements ExcelToJSON {
// Is an array index: [digit]
// make parent into array
Object parent = obj.read(parentPath);
final Object parent = obj.read(parentPath);
// System.err.println(parent);
if (parent == null || parent instanceof ObjectNode) {
// convert to array
......@@ -219,7 +232,9 @@ public class ExcelToJSONImpl implements ExcelToJSON {
// System.err.println("Converted "+ obj.jsonString());
}
// TODO make as many as needed!
LOG.trace("Creating new array item in {}", parentPath);
if (LOG.isTraceEnabled()) {
LOG.trace("Creating new array item in {}", parentPath);
}
obj.add(parentPath, new HashMap<>());
// System.err.println(">> " + obj.jsonString());
}
......@@ -233,7 +248,7 @@ public class ExcelToJSONImpl implements ExcelToJSON {
* @param cell the cell
* @return the object
*/
private Object cellValue(Cell cell) {
private Object cellValue(final Cell cell) {
if (cell == null) {
return null;
}
......@@ -252,7 +267,8 @@ public class ExcelToJSONImpl implements ExcelToJSON {
return null;
case Cell.CELL_TYPE_FORMULA:
LOG.debug("Cell contains a formula");
FormulaEvaluator evaluator = cell.getSheet().getWorkbook().getCreationHelper().createFormulaEvaluator();
final FormulaEvaluator evaluator = cell.getSheet().getWorkbook().getCreationHelper()
.createFormulaEvaluator();
return cellValue(evaluator.evaluateInCell(cell));
default:
LOG.warn("Unsupported cell type {}", cell.getCellType());
......
/*
* Copyright 2016 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.croptrust.excel.templates.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.FormulaEvaluator;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.croptrust.excel.templates.ExcelToJSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.DoubleNode;
import com.fasterxml.jackson.databind.node.LongNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.jayway.jsonpath.JsonPath;
/**
* The {@link ExcelToJSON} implementation using ObjectMapper
*/
public class ExcelToMapsJSONImpl implements ExcelToJSON {
/** The Constant LOG. */
private final static Logger LOG = LoggerFactory.getLogger(ExcelToMapsJSONImpl.class);
private final ObjectMapper objectMapper = new ObjectMapper();
static {
}
/*
* (non-Javadoc)
*
* @see org.croptrust.excel.templates.ExcelToJSON#readAsJson(org.apache.poi.ss.usermodel.Sheet, java.lang.String[],
* java.lang.String, boolean, boolean)
*/
@Override
public List<String> readAsJson(final Sheet sheet, final String[] columnMapping, String objectDefaults,
final boolean ignoreHeaderRow, final boolean ignoreEmptyRows) throws IOException {
final List<String> results = new ArrayList<String>(sheet.getLastRowNum());
if (StringUtils.isBlank(objectDefaults)) {
if (LOG.isTraceEnabled()) {
LOG.trace("Blank defaults provided");
}
objectDefaults = EMPTY_JSON_OBJECT_STRING;
}
{
// Log defaults
final JsonNode defaults = objectMapper.readTree(objectDefaults);
if (LOG.isTraceEnabled()) {
LOG.trace("Defaults: {}", defaults);
}
// TODO Expand defaults object with all mappings defined
// Update objectDefaults
objectDefaults = defaults.toString();
}
// Read rows
final int startAt = ignoreHeaderRow ? 1 : 0;
final int lastRowIndex = sheet.getLastRowNum();
for (int rowIndex = startAt; rowIndex < lastRowIndex; rowIndex++) {
final Row row = sheet.getRow(rowIndex);
if (row == null && !ignoreEmptyRows) {
results.add(null);
continue;
}
// Create object with specified defaults
final JsonNode defaults = objectMapper.readTree(objectDefaults);
final String updatedJson = rowToJson(row, columnMapping, defaults);
if (LOG.isTraceEnabled()) {
LOG.trace("Row JSON: {}", updatedJson);
}
results.add(updatedJson);
}
return results;
}
/**
* Row to json.
*
* @param row the row
* @param columnMapping the column mapping
* @param defaults the object with default values
* @return the string
*/
private String rowToJson(final Row row, final String[] columnMapping, final JsonNode defaults) {
int updates = 0;
for (int i = 0; i < columnMapping.length && i < row.getLastCellNum(); i++) {
if (StringUtils.isBlank(columnMapping[i])) {
// Skip blank and null mapppings
continue;
}
final String jsonPath = "$." + columnMapping[i];
final JsonPath compiledPath = JsonPath.compile(jsonPath);
final Cell cell = row.getCell(i);
final Object cellValue = cellValue(cell);
if (LOG.isTraceEnabled()) {
LOG.trace("Mapping {} to {}", cellValue, jsonPath);
}
if (updateObject(defaults, compiledPath, cellValue)) {
updates++;
if (LOG.isDebugEnabled()) {
LOG.debug("Updated object={}", defaults);
}
}
}
if (LOG.isTraceEnabled()) {
LOG.trace("Updates count={}", updates);
}
if (updates == 0) {
// When nothing was done, return null
return null;
}
return defaults.toString();
}
private boolean updateObject(final JsonNode defaults, final JsonPath compiledPath, final Object cellValue) {
final String bracketNotation = compiledPath.getPath();
final StringBuilder currentPath = new StringBuilder();
currentPath.append("$");
JsonNode root = defaults, parent = null;
if (LOG.isTraceEnabled()) {
LOG.trace("Analyzing {}", bracketNotation);
}
final Matcher matcher = Pattern.compile("\\[([^]]+)\\]").matcher(bracketNotation);
boolean updated = false;
final List<String> fields = new ArrayList<>();
while (matcher.find()) {
String fieldName = matcher.group(1);
if (fieldName.startsWith("'")) {
fieldName = fieldName.substring(1, fieldName.length() - 1);
}
fields.add(fieldName);
}
for (int i = 0; i < fields.size(); i++) {
final String fieldName = fields.get(i);
if (LOG.isTraceEnabled()) {
LOG.trace("Working on field={} root={} parent={}", fieldName, root, parent);
}
final boolean isLast = i == fields.size() - 1;
Integer arrayRef = null;
try {
arrayRef = Integer.parseInt(fieldName);
} catch (final NumberFormatException e) {
}
if (isLast) {
if (LOG.isTraceEnabled()) {
LOG.trace("Last element");
}
if (arrayRef == null) {
parent = root;
final ObjectNode obj = (ObjectNode) root;
root = obj.set(fieldName, convert(cellValue));
updated |= true;
if (LOG.isTraceEnabled()) {
LOG.trace("Leaf field {} is missing, added property val={} node={}", fieldName, cellValue, obj);
}
} else {
// ensure root is array
if (LOG.isTraceEnabled()) {
LOG.trace("**** Leaf is array element in node={} parent={}", root, parent);
}
final ArrayNode arr = (ArrayNode) root;
// root must be array
addArrayElements(arr, arrayRef, null);
// root=parent.get(arrayRef);
arr.set(arrayRef, convert(cellValue));
updated |= true;
if (LOG.isTraceEnabled()) {
LOG.trace("Leaf field {} is missing, added property val={} node={}", fieldName, cellValue, arr);
}
}
} else {
final JsonNode thisNode = root.get(fieldName);
if (LOG.isTraceEnabled()) {
LOG.trace("Looking at field={} arrayRef={} node={} parent={}", fieldName, arrayRef, thisNode,
parent);
}
if (thisNode == null) {
if (arrayRef == null) {
// make object node
final ObjectNode n = addObjectNode(fieldName, root);
updated |= true;
if (LOG.isTraceEnabled()) {
LOG.trace("Added object node field={} node={}", fieldName, root);
}
parent = root;
root = n;
} else {
// ensure root is array
final String parentName = fields.get(i - 1);
final ArrayNode arr = ensureParentArray(parentName, parent);
root = arr;
boolean nextIsArray = false;
try {
Integer.parseInt(fields.get(i + 1));
nextIsArray = true;
} catch (final NumberFormatException e) {
}
updated |= addArrayElements(arr, arrayRef, nextIsArray);
root = arr.get(arrayRef);
parent = arr;
}
} else {
// this node exists, what now?
if (arrayRef == null) {
if (LOG.isTraceEnabled()) {
LOG.trace("Node field={} exists", fieldName);
}
parent = root;
root = thisNode;
} else {
throw new UnsupportedOperationException("Whaaaaaa??!?!");
}
}
if (LOG.isTraceEnabled()) {
LOG.trace("Done with field='{}' root={} parent={} updated={}", fieldName, root, parent, updated);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Result={}", defaults);
}
}
}
return updated;
}
private boolean addArrayElements(final ArrayNode arr, final int arrayRef, final Boolean addArrays) {
if (arrayRef < arr.size()) {
if (LOG.isTraceEnabled()) {
LOG.trace("Array size={} no need to add any elements for arrayRef={} to array={}", arr.size(),
arrayRef, arr);
}
return false;
}
if (LOG.isTraceEnabled()) {
LOG.trace("Array size={} Need to add {} elements to array={} arrayRef={}", arr.size(), arr.size()
- arrayRef + 1, arr, arrayRef);
}
boolean updated = false;
for (int idx = arr.size() - arrayRef + 1; idx > 0; idx--) {
if (addArrays == null) {
arr.addNull();
} else if (addArrays) {
arr.addArray();
} else {
arr.addObject();
}
updated |= true;
if (LOG.isTraceEnabled()) {
LOG.trace("Added object node to array node idx={} node={}", idx, arr);
}
}
return updated;
}
private ArrayNode ensureParentArray(final String parentName, final JsonNode parent) {
if (LOG.isTraceEnabled()) {
LOG.trace("Ensuring parent is array field={} node={} type={}", parentName, parent, parent.getClass());
}
Integer arrayRef = null;
try {
arrayRef = Integer.parseInt(parentName);
} catch (final NumberFormatException e) {
}
ArrayNode arr;
if (parent instanceof ObjectNode) {
final ObjectNode o = (ObjectNode) parent;
JsonNode p;
if (arrayRef == null) {
p = o.get(parentName);
} else {
final ArrayNode a = (ArrayNode) parent;
p = a.get(arrayRef);
}
if (p == null || p instanceof ObjectNode) {
if (arrayRef == null) {
// if ObjectNode, we could move it to become an item in the array
o.replace(parentName, arr = o.arrayNode());
if (LOG.isTraceEnabled()) {
LOG.trace("Replaced object node with array node field={} node={}", parentName, arr);
}
} else {
// if arrayRef then make array element into an array
((ArrayNode) parent).remove(arrayRef);
((ArrayNode) parent).insert(arrayRef, arr = o.arrayNode());
if (LOG.isTraceEnabled()) {
LOG.trace("Replaced object node with array node in arrayRef={} node={}", arrayRef, arr);
}
}
} else if (p instanceof ArrayNode) {
// already an array
return (ArrayNode) p;
} else {
throw new UnsupportedOperationException("What now " + p.getClass() + " val=" + p);
}
} else if (parent instanceof ArrayNode) {
if (LOG.isTraceEnabled()) {
LOG.trace("Parent is array node={}", parent);
}