Commit 55200aba authored by Matija Obreza's avatar Matija Obreza
Browse files

Convert data sheet rows to Map<term, Object>

Convert Map<term, Object> to Genesys JSON
Preview JSON
parent f81502ec
......@@ -139,6 +139,11 @@
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.4.1.1</version>
</dependency>
</dependencies>
<build>
<plugins>
......
package org.genesys2.anno.converter;
public class GenesysJSONException extends Exception {
public GenesysJSONException(String message) {
super(message);
}
private static final long serialVersionUID = 1L;
}
package org.genesys2.anno.converter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.genesys2.anno.gui.ColumnDefs;
import org.genesys2.anno.gui.DataSourceSheet;
import org.genesys2.anno.model.Column;
import org.genesys2.anno.model.ColumnDef;
import org.genesys2.anno.predefined.GenesysJSON;
import org.genesys2.anno.predefined.GenesysJSON.JsonField;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.DoubleNode;
import com.fasterxml.jackson.databind.node.LongNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
public class RowConverter {
private static final Logger _log = Logger.getLogger(RowConverter.class);
private static final ObjectMapper mapper = new ObjectMapper();
private static final GenesysJSON genesysJson = new GenesysJSON();
public static Map<String, ?> toMap(DataSourceSheet dataSourceSheet, Object[] row, ColumnDefs columnDefs) {
HashMap<String, Object> map = new HashMap<String, Object>(columnDefs.getColumnDefs().size());
_log.debug("Converting " + ArrayUtils.toString(row));
List<Column> columns = dataSourceSheet.getColumns();
for (int i = 0; i < columns.size(); i++) {
Column column = columns.get(i);
ColumnDef columnDef = findColumnDef(column, columnDefs.getColumnDefs());
if (columnDef == null) {
continue;
}
Object rowValue = convertCellValue(row[i], column.isMultiple());
Object mapValue = map.get(columnDef.getRdfTerm());
if (mapValue == null) {
_log.debug("Addding value for " + columnDef.getRdfTerm() + " val=" + rowValue);
if (rowValue instanceof String[]) {
_log.debug("val=" + ArrayUtils.toString(rowValue));
map.put(columnDef.getRdfTerm(), new ArrayList<String>(Arrays.asList((String[]) rowValue)));
} else {
map.put(columnDef.getRdfTerm(), rowValue);
}
} else if (mapValue instanceof List<?>) {
_log.debug("Adding " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
if (rowValue instanceof String[]) {
((List<Object>) mapValue).addAll(Arrays.asList((String[]) rowValue));
} else {
((List<Object>) mapValue).add(rowValue);
}
} else {
_log.debug("Converting " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
Object currentVal = mapValue;
List<Object> newVal = new ArrayList<Object>();
newVal.add(currentVal);
if (rowValue instanceof String[]) {
newVal.addAll(Arrays.asList((String[]) rowValue));
} else {
newVal.add(rowValue);
}
map.put(columnDef.getRdfTerm(), newVal);
}
}
// try {
// StringWriter out = new StringWriter();
// mapper.writeValue(out, map);
// _log.info(out.getBuffer().toString());
// } catch (Throwable e) {
// _log.error(e.getMessage(), e);
// }
return map;
}
private static Object convertCellValue(Object value, boolean splitString) {
if (value != null) {
if (value instanceof String) {
String stringValue = ((String) value);
if (StringUtils.isBlank(stringValue)) {
return null;
}
if (splitString) {
String[] split = stringValue.split(";");
for (int i = split.length - 1; i >= 0; i--) {
split[i] = split[i].trim();
}
return split;
}
}
}
return value;
}
private static ColumnDef findColumnDef(Column column, List<ColumnDef> columnDefList) {
for (ColumnDef columnDef : columnDefList) {
if (columnDef.getRdfTerm().equals(column.getRdfTerm())) {
return columnDef;
}
}
_log.warn("ColumnDef not found for column=" + column.getPreferredName() + " term=" + column.getRdfTerm());
return null;
}
public static ArrayNode toGenesysJson(List<Map<String, ?>> list) throws GenesysJSONException, JsonGenerationException {
ArrayNode result = mapper.createArrayNode();
for (Map<String, ?> row : list) {
ObjectNode rowObject = toJson(row);
if (rowObject != null) {
result.add(rowObject);
}
}
return result;
}
private static ObjectNode toJson(Map<String, ?> row) throws GenesysJSONException, JsonGenerationException {
if (row == null)
return null;
ObjectNode rowNode = mapper.createObjectNode();
for (GenesysJSON.JsonField jsonField : genesysJson.getJsonFields()) {
Object rdfValues = row.get(jsonField.getRdfTerm());
add(rowNode, jsonField, rdfValues);
}
return rowNode;
}
private static void add(ObjectNode rowNode, JsonField jsonField, Object rdfValues) throws GenesysJSONException, JsonGenerationException {
String fieldName = jsonField.getFieldName();
ObjectNode containerNode = ensureNode(rowNode, fieldName);
int dotPost = fieldName.lastIndexOf('.');
if (dotPost > -1)
fieldName = fieldName.substring(dotPost + 1);
System.err.println(">> " + jsonField.getFieldName() + " field=" + fieldName);
JsonNode valueNode = containerNode.get(fieldName);
if (valueNode == null) {
if (jsonField.hasAllowMultiple()) {
valueNode = containerNode.putArray(fieldName);
}
} else {
if (jsonField.hasAllowMultiple() && !valueNode.isArray()) {
throw new GenesysJSONException("Expecting arrayObject for " + fieldName);
}
}
if (rdfValues == null) {
// skip
} else if (rdfValues instanceof ArrayList<?>) {
ArrayNode arrayNode = (ArrayNode) valueNode;
for (String rdfValue : (ArrayList<String>) rdfValues) {
arrayNode.add(coerce(jsonField.getType(), rdfValue));
}
} else {
System.err.println("Putting field=" + fieldName + " type=" + rdfValues.getClass());
containerNode.put(fieldName, coerce(jsonField.getType(), rdfValues));
}
}
static ObjectNode ensureNode(ObjectNode parentNode, String nodeName) throws JsonGenerationException {
if (StringUtils.isBlank(nodeName)) {
throw new JsonGenerationException("nodeName should not be blank or null");
}
int dotIndex = nodeName.indexOf('.');
if (dotIndex > -1) {
String[] path = nodeName.split("\\.");
for (int i = 0; i < path.length - 1; i++) {
JsonNode node = parentNode.get(path[i]);
if (node == null) {
node = parentNode.putObject(path[i]);
}
parentNode = (ObjectNode) node;
}
}
return parentNode;
}
static JsonNode coerce(Class<?> clazz, Object rdfValues) {
if (rdfValues == null) {
return null;
}
if (clazz == String.class) {
return new TextNode(rdfValues.toString());
}
if (clazz == double.class || clazz == float.class) {
return new DoubleNode(Double.parseDouble(rdfValues.toString()));
}
if (clazz == int.class || clazz == long.class) {
return new LongNode(Long.parseLong(rdfValues.toString()));
}
if (clazz == boolean.class) {
return BooleanNode.valueOf(toBoolean(rdfValues.toString()));
}
return null;
}
private static boolean toBoolean(String rdfValue) {
return "true".equalsIgnoreCase(rdfValue) || "yes".equalsIgnoreCase(rdfValue) || "t".equalsIgnoreCase(rdfValue) || "y".equalsIgnoreCase(rdfValue) || "1".equalsIgnoreCase(rdfValue);
}
}
package org.genesys2.anno.gui;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.genesys2.anno.parser.CsvDataSourceParser;
import org.genesys2.anno.parser.XlsxDataSourceParser;
......@@ -12,6 +15,7 @@ import org.genesys2.anno.parser.XlsxDataSourceParser;
public class AppConfig {
public static final AppConfig instance = new AppConfig();
private DataSourceLoaderImpl dataSourceLoader;
private ExecutorService executorService=Executors.newFixedThreadPool(5);
private AppConfig() {
dataSourceLoader = new DataSourceLoaderImpl();
......@@ -22,4 +26,12 @@ public class AppConfig {
public DataSourceLoaderImpl getDataSourceLoader() {
return dataSourceLoader;
}
public ExecutorService getExecutorService() {
return executorService;
}
public static void shutdown() {
instance.executorService.shutdown();
}
}
......@@ -77,6 +77,8 @@ import org.eclipse.wb.swt.SWTResourceManager;
import org.genesys2.anno.model.ColumnDef;
import swing2swt.layout.BorderLayout;
import org.eclipse.swt.events.DisposeListener;
import org.eclipse.swt.events.DisposeEvent;
public class AppWindow {
private static final Logger _log = Logger.getLogger(AppWindow.class);
......@@ -224,7 +226,7 @@ public class AppWindow {
AppWindow.archiveIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/archive.png")).createImage();
AppWindow.openPackageIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/openpackage.png")).createImage();
AppWindow.spreadsheetIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/spreadsheet.png")).createImage();
AppWindow.automapColumns= ImageDescriptor.createFromURL(this.getClass().getResource("/icon/automap.png")).createImage();
AppWindow.automapColumns = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/automap.png")).createImage();
}
public static boolean isMac() {
......@@ -239,6 +241,11 @@ public class AppWindow {
*/
protected void createContents() {
shlGenesysMagic = new Shell();
shlGenesysMagic.addDisposeListener(new DisposeListener() {
public void widgetDisposed(DisposeEvent e) {
doShutdown();
}
});
shlGenesysMagic.setMinimumSize(new Point(100, 22));
shlGenesysMagic.setSize(744, 528);
shlGenesysMagic.setText("Genesys Magic");
......@@ -465,19 +472,44 @@ public class AppWindow {
});
tltmAutomap.setText("Automap");
tltmAutomap.setImage(automapColumns);
ToolItem tltmPush = new ToolItem(toolBar, SWT.NONE);
tltmPush.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
doPush();
}
});
tltmPush.setText("Push");
m_bindingContext = initDataBindings();
}
protected void doShutdown() {
_log.info("Shutting down.");
AppConfig.shutdown();
}
protected void doPush() {
final DataSourceSheet dss = getCurrentDataSourceSheet();
if (dss == null) {
return;
}
PushDialog pushDialog = new PushDialog(shlGenesysMagic, SWT.NONE);
pushDialog.setDataSourceSheet(dss);
pushDialog.setColumnDefs(this.columnDefs);
pushDialog.open();
}
protected void doAutomap() {
// ((SheetDisplay) c).getDataSourceSheet()) {
CTabItem currentTab = tabFolder.getSelection();
if (currentTab == null) {
_log.info("No tab selected");
final DataSourceSheet dss = getCurrentDataSourceSheet();
if (dss == null) {
return;
}
final DataSourceSheet dss = ((SheetDisplay) currentTab.getControl()).getDataSourceSheet();
_log.info("Got data source sheet " + dss);
threadPool.execute(new Runnable() {
@Override
public void run() {
......@@ -486,6 +518,16 @@ public class AppWindow {
});
}
public DataSourceSheet getCurrentDataSourceSheet() {
CTabItem currentTab = tabFolder.getSelection();
if (currentTab == null) {
_log.info("No tab selected");
return null;
}
final DataSourceSheet dss = ((SheetDisplay) currentTab.getControl()).getDataSourceSheet();
return dss;
}
public void addSourceFile() {
FileDialog fd = new FileDialog(shlGenesysMagic, SWT.MULTI);
fd.setFilterNames(new String[] { "Supported source files", "Excel files", "CSV files" });
......
......@@ -32,12 +32,12 @@ public class ColumnDefs extends AbstractModelObject {
public List<ColumnDef> getColumnDefs() {
return columnDefs;
}
public void addDataSource() {
firePropertyChange("columnDefs", null, this);
}
public void removeDataSource() {
firePropertyChange("columnDefs", null, this);
}
//
// public void addDataSource() {
// firePropertyChange("columnDefs", null, this);
// }
//
// public void removeDataSource() {
// firePropertyChange("columnDefs", null, this);
// }
}
......@@ -28,4 +28,6 @@ public interface DataSourceLoader {
DataSourceFile loadDataSource(File file);
List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
List<Object[]> loadDataRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
}
......@@ -42,8 +42,7 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
@Override
public DataSourceFile loadDataSource(File file) {
if (!file.exists() || !file.canRead()) {
_log.debug("Could not find or read "
+ file.getAbsolutePath());
_log.debug("Could not find or read " + file.getAbsolutePath());
return null;
}
DataSourceFile dataSourceFile = new DataSourceFile();
......@@ -54,8 +53,7 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
return dataSourceFile;
}
private Collection<? extends DataSourceSheet> findSheets(
DataSourceFile dataSourceFile) {
private Collection<? extends DataSourceSheet> findSheets(DataSourceFile dataSourceFile) {
File sourceFile = dataSourceFile.getFile();
......@@ -75,13 +73,28 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
}
@Override
public List<Object[]> loadRows(DataSourceSheet sheet, int maxRows)
throws UnsupportedDataFormatException, FileNotFoundException, IOException {
public List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException {
// Iterate over parsers and see if something comes out
for (DataSourceParser parser : parsers) {
try {
if (parser.supports(sheet.getSourceFile())) {
return parser.loadRows(sheet, maxRows);
return parser.loadRows(sheet, maxRows, 0);
}
} catch (UnsupportedDataFormatException e) {
_log.debug(e.getMessage(), e);
}
}
throw new UnsupportedDataFormatException("No parser found", null);
}
@Override
public List<Object[]> loadDataRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException {
// Iterate over parsers and see if something comes out
for (DataSourceParser parser : parsers) {
try {
if (parser.supports(sheet.getSourceFile())) {
return parser.loadRows(sheet, maxRows, sheet.getHeaderRowIndex() + 1);
}
} catch (UnsupportedDataFormatException e) {
_log.debug(e.getMessage(), e);
......
......@@ -29,6 +29,6 @@ public interface DataSourceParser {
Collection<? extends DataSourceSheet> findSheets(File sourceFile)
throws UnsupportedDataFormatException;
List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
List<Object[]> loadRows(DataSourceSheet sheet, int maxRows, int startAt) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
}
package org.genesys2.anno.gui;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.log4j.Logger;
import org.eclipse.swt.SWT;
import org.eclipse.swt.widgets.Dialog;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.MessageBox;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Text;
import org.eclipse.swt.widgets.ToolBar;
import org.eclipse.swt.widgets.ToolItem;
import org.genesys2.anno.converter.RowConverter;
import org.genesys2.anno.model.ColumnDef;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import swing2swt.layout.BorderLayout;
public class PushDialog extends Dialog {
private static final Logger _log = Logger.getLogger(PushDialog.class);
private static final ObjectMapper mapper = new ObjectMapper();
protected DataSourceLoader dataSourceLoader = AppConfig.instance.getDataSourceLoader();
protected ExecutorService executorService = AppConfig.instance.getExecutorService();
protected Object result;
protected Shell shell;
private Text txtJson;
private DataSourceSheet dataSourceSheet;
private ColumnDefs columnDefs;
/**
* Create the dialog.
*
* @param parent
* @param style
*/
public PushDialog(Shell parent, int style) {
super(parent, SWT.DIALOG_TRIM | SWT.MAX | SWT.RESIZE | SWT.APPLICATION_MODAL);
setText("Push data to Genesys");
}
/**
* Open the dialog.
*
* @return the result
*/
public Object open() {
if (this.dataSourceSheet == null || this.columnDefs == null) {
_log.warn("Can't open PushDialog, no columnDefs or no dataSourceSheet");
return SWT.CANCEL;
}
createContents();
shell.open();
shell.layout();
// create worker
createWorker();
Display display = getParent().getDisplay();
while (!shell.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
return result;
}
/**
* Initializes a streamed reader for {@link #dataSourceSheet} and generates
* JSON
*/
private void createWorker() {
this.executorService.execute(new Runnable() {
@Override
public void run() {
try {
List<Object[]> rows = dataSourceLoader.loadDataRows(dataSourceSheet, 10);
_log.info("Loaded rows " + rows.size());
List<Map<String, ?>> list = new ArrayList<Map<String, ?>>();
int counter = 0, total = rows.size();
for (Object[] row : rows) {
counter++;
list.add(RowConverter.toMap(dataSourceSheet, row, columnDefs));
_log.info("Converted row " + counter + " of " + total);
}
final JsonNode genesysJson = RowConverter.toGenesysJson(list);
final String x = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(genesysJson);
shell.getDisplay().asyncExec(new Runnable() {
@Override
public void run() {
txtJson.setText(x);
}
});
} catch (final Throwable e) {
_log.error(e.getMessage(), e);
shell.getDisplay().syncExec(new Runnable() {
@Override
public void run() {
MessageBox mb = new MessageBox(shell);
mb.setMessage(e.getMessage());
mb.setText("Error loading rows");
mb.open();
}
});