Convert data sheet rows to Map<term, Object>

Convert Map<term, Object> to Genesys JSON
Preview JSON

anno-gui/pom.xml

@@ -139,6 +139,11 @@
 			<artifactId>log4j</artifactId>
 			<version>1.2.17</version>
 		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>2.4.1.1</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

anno-gui/src/main/java/org/genesys2/anno/converter/GenesysJSONException.java

+package org.genesys2.anno.converter;
+
+public class GenesysJSONException extends Exception {
+
+	public GenesysJSONException(String message) {
+		super(message);
+	}
+
+	private static final long serialVersionUID = 1L;
+
+}

anno-gui/src/main/java/org/genesys2/anno/converter/RowConverter.java

+package org.genesys2.anno.converter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.genesys2.anno.gui.ColumnDefs;
+import org.genesys2.anno.gui.DataSourceSheet;
+import org.genesys2.anno.model.Column;
+import org.genesys2.anno.model.ColumnDef;
+import org.genesys2.anno.predefined.GenesysJSON;
+import org.genesys2.anno.predefined.GenesysJSON.JsonField;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.BooleanNode;
+import com.fasterxml.jackson.databind.node.DoubleNode;
+import com.fasterxml.jackson.databind.node.LongNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+
+public class RowConverter {
+	private static final Logger _log = Logger.getLogger(RowConverter.class);
+
+	private static final ObjectMapper mapper = new ObjectMapper();
+	private static final GenesysJSON genesysJson = new GenesysJSON();
+
+	public static Map<String, ?> toMap(DataSourceSheet dataSourceSheet, Object[] row, ColumnDefs columnDefs) {
+		HashMap<String, Object> map = new HashMap<String, Object>(columnDefs.getColumnDefs().size());
+
+		_log.debug("Converting " + ArrayUtils.toString(row));
+
+		List<Column> columns = dataSourceSheet.getColumns();
+		for (int i = 0; i < columns.size(); i++) {
+			Column column = columns.get(i);
+			ColumnDef columnDef = findColumnDef(column, columnDefs.getColumnDefs());
+			if (columnDef == null) {
+				continue;
+			}
+
+			Object rowValue = convertCellValue(row[i], column.isMultiple());
+
+			Object mapValue = map.get(columnDef.getRdfTerm());
+			if (mapValue == null) {
+				_log.debug("Addding value for " + columnDef.getRdfTerm() + " val=" + rowValue);
+				if (rowValue instanceof String[]) {
+					_log.debug("val=" + ArrayUtils.toString(rowValue));
+					map.put(columnDef.getRdfTerm(), new ArrayList<String>(Arrays.asList((String[]) rowValue)));
+				} else {
+					map.put(columnDef.getRdfTerm(), rowValue);
+				}
+			} else if (mapValue instanceof List<?>) {
+				_log.debug("Adding " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
+				if (rowValue instanceof String[]) {
+					((List<Object>) mapValue).addAll(Arrays.asList((String[]) rowValue));
+				} else {
+					((List<Object>) mapValue).add(rowValue);
+				}
+			} else {
+				_log.debug("Converting " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
+				Object currentVal = mapValue;
+				List<Object> newVal = new ArrayList<Object>();
+				newVal.add(currentVal);
+				if (rowValue instanceof String[]) {
+					newVal.addAll(Arrays.asList((String[]) rowValue));
+				} else {
+					newVal.add(rowValue);
+				}
+				map.put(columnDef.getRdfTerm(), newVal);
+			}
+		}
+
+		// try {
+		// StringWriter out = new StringWriter();
+		// mapper.writeValue(out, map);
+		// _log.info(out.getBuffer().toString());
+		// } catch (Throwable e) {
+		// _log.error(e.getMessage(), e);
+		// }
+
+		return map;
+	}
+
+	private static Object convertCellValue(Object value, boolean splitString) {
+		if (value != null) {
+			if (value instanceof String) {
+				String stringValue = ((String) value);
+				if (StringUtils.isBlank(stringValue)) {
+					return null;
+				}
+				if (splitString) {
+					String[] split = stringValue.split(";");
+					for (int i = split.length - 1; i >= 0; i--) {
+						split[i] = split[i].trim();
+					}
+					return split;
+				}
+			}
+		}
+		return value;
+	}
+
+	private static ColumnDef findColumnDef(Column column, List<ColumnDef> columnDefList) {
+		for (ColumnDef columnDef : columnDefList) {
+			if (columnDef.getRdfTerm().equals(column.getRdfTerm())) {
+				return columnDef;
+			}
+		}
+		_log.warn("ColumnDef not found for column=" + column.getPreferredName() + " term=" + column.getRdfTerm());
+		return null;
+	}
+
+	public static ArrayNode toGenesysJson(List<Map<String, ?>> list) throws GenesysJSONException, JsonGenerationException {
+		ArrayNode result = mapper.createArrayNode();
+
+		for (Map<String, ?> row : list) {
+
+			ObjectNode rowObject = toJson(row);
+			if (rowObject != null) {
+				result.add(rowObject);
+			}
+		}
+
+		return result;
+	}
+
+	private static ObjectNode toJson(Map<String, ?> row) throws GenesysJSONException, JsonGenerationException {
+		if (row == null)
+			return null;
+
+		ObjectNode rowNode = mapper.createObjectNode();
+
+		for (GenesysJSON.JsonField jsonField : genesysJson.getJsonFields()) {
+			Object rdfValues = row.get(jsonField.getRdfTerm());
+			add(rowNode, jsonField, rdfValues);
+		}
+
+		return rowNode;
+	}
+
+	private static void add(ObjectNode rowNode, JsonField jsonField, Object rdfValues) throws GenesysJSONException, JsonGenerationException {
+		String fieldName = jsonField.getFieldName();
+		ObjectNode containerNode = ensureNode(rowNode, fieldName);
+		int dotPost = fieldName.lastIndexOf('.');
+		if (dotPost > -1)
+			fieldName = fieldName.substring(dotPost + 1);
+		System.err.println(">> " + jsonField.getFieldName() + " field=" + fieldName);
+		JsonNode valueNode = containerNode.get(fieldName);
+
+		if (valueNode == null) {
+			if (jsonField.hasAllowMultiple()) {
+				valueNode = containerNode.putArray(fieldName);
+			}
+		} else {
+			if (jsonField.hasAllowMultiple() && !valueNode.isArray()) {
+				throw new GenesysJSONException("Expecting arrayObject for " + fieldName);
+			}
+		}
+
+		if (rdfValues == null) {
+			// skip
+		} else if (rdfValues instanceof ArrayList<?>) {
+			ArrayNode arrayNode = (ArrayNode) valueNode;
+			for (String rdfValue : (ArrayList<String>) rdfValues) {
+				arrayNode.add(coerce(jsonField.getType(), rdfValue));
+			}
+		} else {
+			System.err.println("Putting field=" + fieldName + " type=" + rdfValues.getClass());
+			containerNode.put(fieldName, coerce(jsonField.getType(), rdfValues));
+		}
+	}
+
+	static ObjectNode ensureNode(ObjectNode parentNode, String nodeName) throws JsonGenerationException {
+		if (StringUtils.isBlank(nodeName)) {
+			throw new JsonGenerationException("nodeName should not be blank or null");
+		}
+		int dotIndex = nodeName.indexOf('.');
+		if (dotIndex > -1) {
+			String[] path = nodeName.split("\\.");
+			for (int i = 0; i < path.length - 1; i++) {
+				JsonNode node = parentNode.get(path[i]);
+				if (node == null) {
+					node = parentNode.putObject(path[i]);
+				}
+				parentNode = (ObjectNode) node;
+			}
+		}
+		return parentNode;
+	}
+
+	static JsonNode coerce(Class<?> clazz, Object rdfValues) {
+		if (rdfValues == null) {
+			return null;
+		}
+		if (clazz == String.class) {
+			return new TextNode(rdfValues.toString());
+		}
+		if (clazz == double.class || clazz == float.class) {
+			return new DoubleNode(Double.parseDouble(rdfValues.toString()));
+		}
+		if (clazz == int.class || clazz == long.class) {
+			return new LongNode(Long.parseLong(rdfValues.toString()));
+		}
+		if (clazz == boolean.class) {
+			return BooleanNode.valueOf(toBoolean(rdfValues.toString()));
+		}
+		return null;
+	}
+
+	private static boolean toBoolean(String rdfValue) {
+		return "true".equalsIgnoreCase(rdfValue) || "yes".equalsIgnoreCase(rdfValue) || "t".equalsIgnoreCase(rdfValue) || "y".equalsIgnoreCase(rdfValue) || "1".equalsIgnoreCase(rdfValue);
+	}
+}

anno-gui/src/main/java/org/genesys2/anno/gui/AppConfig.java

 package org.genesys2.anno.gui;
 
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
 import org.genesys2.anno.parser.CsvDataSourceParser;
 import org.genesys2.anno.parser.XlsxDataSourceParser;
 
@@ -12,6 +15,7 @@ import org.genesys2.anno.parser.XlsxDataSourceParser;
 public class AppConfig {
 	public static final AppConfig instance = new AppConfig();
 	private DataSourceLoaderImpl dataSourceLoader;
+	private ExecutorService executorService=Executors.newFixedThreadPool(5);
 
 	private AppConfig() {
 		dataSourceLoader = new DataSourceLoaderImpl();
@@ -22,4 +26,12 @@ public class AppConfig {
 	public DataSourceLoaderImpl getDataSourceLoader() {
 		return dataSourceLoader;
 	}
+
+	public ExecutorService getExecutorService() {
+		return executorService;
+	}
+
+	public static void shutdown() {
+		instance.executorService.shutdown();
+	}
 }

anno-gui/src/main/java/org/genesys2/anno/gui/AppWindow.java

@@ -77,6 +77,8 @@ import org.eclipse.wb.swt.SWTResourceManager;
 import org.genesys2.anno.model.ColumnDef;
 
 import swing2swt.layout.BorderLayout;
+import org.eclipse.swt.events.DisposeListener;
+import org.eclipse.swt.events.DisposeEvent;
 
 public class AppWindow {
 	private static final Logger _log = Logger.getLogger(AppWindow.class);
@@ -224,7 +226,7 @@ public class AppWindow {
 		AppWindow.archiveIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/archive.png")).createImage();
 		AppWindow.openPackageIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/openpackage.png")).createImage();
 		AppWindow.spreadsheetIcon = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/spreadsheet.png")).createImage();
-		AppWindow.automapColumns= ImageDescriptor.createFromURL(this.getClass().getResource("/icon/automap.png")).createImage();
+		AppWindow.automapColumns = ImageDescriptor.createFromURL(this.getClass().getResource("/icon/automap.png")).createImage();
 	}
 
 	public static boolean isMac() {
@@ -239,6 +241,11 @@ public class AppWindow {
 	 */
 	protected void createContents() {
 		shlGenesysMagic = new Shell();
+		shlGenesysMagic.addDisposeListener(new DisposeListener() {
+			public void widgetDisposed(DisposeEvent e) {
+				doShutdown();
+			}
+		});
 		shlGenesysMagic.setMinimumSize(new Point(100, 22));
 		shlGenesysMagic.setSize(744, 528);
 		shlGenesysMagic.setText("Genesys Magic");
@@ -465,19 +472,44 @@ public class AppWindow {
 		});
 		tltmAutomap.setText("Automap");
 		tltmAutomap.setImage(automapColumns);
+
+		ToolItem tltmPush = new ToolItem(toolBar, SWT.NONE);
+		tltmPush.addSelectionListener(new SelectionAdapter() {
+			@Override
+			public void widgetSelected(SelectionEvent e) {
+				doPush();
+			}
+		});
+		tltmPush.setText("Push");
 		m_bindingContext = initDataBindings();
 	}
 
+	protected void doShutdown() {
+		_log.info("Shutting down.");
+		AppConfig.shutdown();
+	}
+
+	protected void doPush() {
+		final DataSourceSheet dss = getCurrentDataSourceSheet();
+		if (dss == null) {
+			return;
+		}
+
+		PushDialog pushDialog = new PushDialog(shlGenesysMagic, SWT.NONE);
+		pushDialog.setDataSourceSheet(dss);
+		pushDialog.setColumnDefs(this.columnDefs);
+
+		pushDialog.open();
+	}
+
 	protected void doAutomap() {
-		// ((SheetDisplay) c).getDataSourceSheet()) {
-		CTabItem currentTab = tabFolder.getSelection();
-		if (currentTab == null) {
-			_log.info("No tab selected");
+		final DataSourceSheet dss = getCurrentDataSourceSheet();
+		if (dss == null) {
 			return;
 		}
-		final DataSourceSheet dss = ((SheetDisplay) currentTab.getControl()).getDataSourceSheet();
+
 		_log.info("Got data source sheet " + dss);
-		
+
 		threadPool.execute(new Runnable() {
 			@Override
 			public void run() {
@@ -486,6 +518,16 @@ public class AppWindow {
 		});
 	}
 
+	public DataSourceSheet getCurrentDataSourceSheet() {
+		CTabItem currentTab = tabFolder.getSelection();
+		if (currentTab == null) {
+			_log.info("No tab selected");
+			return null;
+		}
+		final DataSourceSheet dss = ((SheetDisplay) currentTab.getControl()).getDataSourceSheet();
+		return dss;
+	}
+
 	public void addSourceFile() {
 		FileDialog fd = new FileDialog(shlGenesysMagic, SWT.MULTI);
 		fd.setFilterNames(new String[] { "Supported source files", "Excel files", "CSV files" });

anno-gui/src/main/java/org/genesys2/anno/gui/ColumnDefs.java

@@ -32,12 +32,12 @@ public class ColumnDefs extends AbstractModelObject {
 	public List<ColumnDef> getColumnDefs() {
 		return columnDefs;
 	}
-
-	public void addDataSource() {
-		firePropertyChange("columnDefs", null, this);
-	}
-
-	public void removeDataSource() {
-		firePropertyChange("columnDefs", null, this);
-	}
+//
+//	public void addDataSource() {
+//		firePropertyChange("columnDefs", null, this);
+//	}
+//
+//	public void removeDataSource() {
+//		firePropertyChange("columnDefs", null, this);
+//	}
 }

anno-gui/src/main/java/org/genesys2/anno/gui/DataSourceLoader.java

@@ -28,4 +28,6 @@ public interface DataSourceLoader {
 	DataSourceFile loadDataSource(File file);
 
 	List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
+
+	List<Object[]> loadDataRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
 }

anno-gui/src/main/java/org/genesys2/anno/gui/DataSourceLoaderImpl.java

@@ -42,8 +42,7 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
 	@Override
 	public DataSourceFile loadDataSource(File file) {
 		if (!file.exists() || !file.canRead()) {
-			_log.debug("Could not find or read "
-					+ file.getAbsolutePath());
+			_log.debug("Could not find or read " + file.getAbsolutePath());
 			return null;
 		}
 		DataSourceFile dataSourceFile = new DataSourceFile();
@@ -54,8 +53,7 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
 		return dataSourceFile;
 	}
 
-	private Collection<? extends DataSourceSheet> findSheets(
-			DataSourceFile dataSourceFile) {
+	private Collection<? extends DataSourceSheet> findSheets(DataSourceFile dataSourceFile) {
 
 		File sourceFile = dataSourceFile.getFile();
 
@@ -75,13 +73,28 @@ public class DataSourceLoaderImpl implements DataSourceLoader {
 	}
 
 	@Override
-	public List<Object[]> loadRows(DataSourceSheet sheet, int maxRows)
-			throws UnsupportedDataFormatException, FileNotFoundException, IOException {
+	public List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException {
 		// Iterate over parsers and see if something comes out
 		for (DataSourceParser parser : parsers) {
 			try {
 				if (parser.supports(sheet.getSourceFile())) {
-					return parser.loadRows(sheet, maxRows);
+					return parser.loadRows(sheet, maxRows, 0);
+				}
+			} catch (UnsupportedDataFormatException e) {
+				_log.debug(e.getMessage(), e);
+			}
+		}
+
+		throw new UnsupportedDataFormatException("No parser found", null);
+	}
+
+	@Override
+	public List<Object[]> loadDataRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException {
+		// Iterate over parsers and see if something comes out
+		for (DataSourceParser parser : parsers) {
+			try {
+				if (parser.supports(sheet.getSourceFile())) {
+					return parser.loadRows(sheet, maxRows, sheet.getHeaderRowIndex() + 1);
 				}
 			} catch (UnsupportedDataFormatException e) {
 				_log.debug(e.getMessage(), e);

anno-gui/src/main/java/org/genesys2/anno/gui/DataSourceParser.java

@@ -29,6 +29,6 @@ public interface DataSourceParser {
 	Collection<? extends DataSourceSheet> findSheets(File sourceFile)
 			throws UnsupportedDataFormatException;
 
-	List<Object[]> loadRows(DataSourceSheet sheet, int maxRows) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
+	List<Object[]> loadRows(DataSourceSheet sheet, int maxRows, int startAt) throws UnsupportedDataFormatException, FileNotFoundException, IOException;
 
 }

anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java

+package org.genesys2.anno.gui;
+
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.log4j.Logger;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.Dialog;
+import org.eclipse.swt.widgets.Display;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.Text;
+import org.eclipse.swt.widgets.ToolBar;
+import org.eclipse.swt.widgets.ToolItem;
+import org.genesys2.anno.converter.RowConverter;
+import org.genesys2.anno.model.ColumnDef;
+
+import com.fasterxml.jackson.core.PrettyPrinter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import swing2swt.layout.BorderLayout;
+
+public class PushDialog extends Dialog {
+	private static final Logger _log = Logger.getLogger(PushDialog.class);
+	private static final ObjectMapper mapper = new ObjectMapper();
+
+	protected DataSourceLoader dataSourceLoader = AppConfig.instance.getDataSourceLoader();
+	protected ExecutorService executorService = AppConfig.instance.getExecutorService();
+
+	protected Object result;
+	protected Shell shell;
+	private Text txtJson;
+	private DataSourceSheet dataSourceSheet;
+	private ColumnDefs columnDefs;
+
+	/**
+	 * Create the dialog.
+	 * 
+	 * @param parent
+	 * @param style
+	 */
+	public PushDialog(Shell parent, int style) {
+		super(parent, SWT.DIALOG_TRIM | SWT.MAX | SWT.RESIZE | SWT.APPLICATION_MODAL);
+		setText("Push data to Genesys");
+	}
+
+	/**
+	 * Open the dialog.
+	 * 
+	 * @return the result
+	 */
+	public Object open() {
+		if (this.dataSourceSheet == null || this.columnDefs == null) {
+			_log.warn("Can't open PushDialog, no columnDefs or no dataSourceSheet");
+			return SWT.CANCEL;
+		}
+
+		createContents();
+		shell.open();
+		shell.layout();
+
+		// create worker
+		createWorker();
+
+		Display display = getParent().getDisplay();
+		while (!shell.isDisposed()) {
+			if (!display.readAndDispatch()) {
+				display.sleep();
+			}
+		}
+		return result;
+	}
+
+	/**
+	 * Initializes a streamed reader for {@link #dataSourceSheet} and generates
+	 * JSON
+	 */
+	private void createWorker() {
+		this.executorService.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				try {
+					List<Object[]> rows = dataSourceLoader.loadDataRows(dataSourceSheet, 10);
+					_log.info("Loaded rows " + rows.size());
+
+					List<Map<String, ?>> list = new ArrayList<Map<String, ?>>();
+					int counter = 0, total = rows.size();
+					for (Object[] row : rows) {
+						counter++;
+						list.add(RowConverter.toMap(dataSourceSheet, row, columnDefs));
+						_log.info("Converted row " + counter + " of " + total);
+					}
+
+					final JsonNode genesysJson = RowConverter.toGenesysJson(list);
+					final String x = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(genesysJson);
+					shell.getDisplay().asyncExec(new Runnable() {
+						@Override
+						public void run() {
+							txtJson.setText(x);
+						}
+					});
+
+				} catch (final Throwable e) {
+					_log.error(e.getMessage(), e);
+					shell.getDisplay().syncExec(new Runnable() {
+						@Override
+						public void run() {
+							MessageBox mb = new MessageBox(shell);
+							mb.setMessage(e.getMessage());
+							mb.setText("Error loading rows");
+							mb.open();