Commit 1303b9e1 authored by Matija Obreza's avatar Matija Obreza
Browse files

log4j SWT appender

Push data to Genesys PGR server
parent dd6d11d4
......@@ -154,7 +154,12 @@
<artifactId>client-api</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
</dependencies>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
</dependencies>
<build>
<plugins>
......
package org.genesys2.anno.converter;
public class GenesysJSONIncompleteException extends GenesysJSONException {
public GenesysJSONIncompleteException(String message) {
super(message);
}
}
......@@ -3,6 +3,7 @@ package org.genesys2.anno.converter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
......@@ -144,15 +145,14 @@ public class RowConverter {
return null;
}
public static List<Map<String, ?>> toMap(DataSourceSheet dataSourceSheet, List<Object[]> rows, ColumnDefs columnDefs) {
List<Map<String, ?>> list=new ArrayList<Map<String,?>>(rows.size());
List<Map<String, ?>> list = new ArrayList<Map<String, ?>>(rows.size());
for (Object[] row : rows) {
list.add(toMap(dataSourceSheet, row, columnDefs));
}
return list;
}
public static ArrayNode toGenesysJson(List<Map<String, ?>> list) throws GenesysJSONException, JsonGenerationException {
ArrayNode result = mapper.createArrayNode();
......@@ -178,6 +178,12 @@ public class RowConverter {
add(rowNode, jsonField, rdfValues);
}
for (GenesysJSON.JsonField jsonField : genesysJson.getRequiredFields()) {
if (!hasNodeValue(rowNode, jsonField.getFieldName())) {
throw new GenesysJSONIncompleteException("Missing required property '" + jsonField.getFieldName() + "' " + jsonField.getRdfTerm());
}
}
return rowNode;
}
......@@ -238,6 +244,31 @@ public class RowConverter {
return parentNode;
}
static boolean hasNodeValue(final ObjectNode parentNode, final String nodeName) throws JsonGenerationException {
if (StringUtils.isBlank(nodeName)) {
throw new JsonGenerationException("nodeName should not be blank or null");
}
String[] path = nodeName.split("\\.");
JsonNode node = parentNode;
for (int i = 0; i < path.length; i++) {
node = node.get(path[i]);
if (node == null) {
return false;
}
}
// Array nodes should have a non-null value
if (node != null && node.isArray()) {
Iterator<JsonNode> it = node.elements();
while (it.hasNext()) {
JsonNode arrVal = it.next();
if (arrVal != null && !arrVal.isNull()) {
return true;
}
}
}
return node != null && !node.isNull() && !node.isMissingNode() && !node.isArray() && node.isValueNode();
}
static JsonNode coerce(Class<?> clazz, Object rdfValues) {
if (rdfValues == null) {
return null;
......
......@@ -2,23 +2,33 @@ package org.genesys2.anno.gui;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.log4j.Logger;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.DisposeEvent;
import org.eclipse.swt.events.DisposeListener;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.widgets.Dialog;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.MessageBox;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Text;
import org.eclipse.swt.widgets.ToolBar;
import org.eclipse.swt.widgets.ToolItem;
import org.genesys2.anno.converter.GenesysJSONException;
import org.genesys2.anno.converter.GenesysJSONIncompleteException;
import org.genesys2.anno.converter.RowConverter;
import org.genesys2.anno.model.OAuthSettings;
import org.genesys2.anno.model.Settings;
......@@ -32,21 +42,25 @@ import swing2swt.layout.BorderLayout;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
public class PushDialog extends Dialog {
private static final Logger _log = Logger.getLogger(PushDialog.class);
private static final ObjectMapper mapper = new ObjectMapper();
protected static final int BATCH_SIZE = 25;
private Settings settings = AppConfig.instance.getSettings();
protected DataSourceLoader dataSourceLoader = AppConfig.instance.getDataSourceLoader();
protected ExecutorService executorService = AppConfig.instance.getExecutorService();
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(100);
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 2, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
protected Object result;
protected Shell shell;
private Text txtJson;
private DataSourceSheet dataSourceSheet;
private ColumnDefs columnDefs;
private SwtLogAppender logAppender;
/**
* Create the dialog.
......@@ -71,6 +85,10 @@ public class PushDialog extends Dialog {
}
createContents();
logAppender = new SwtLogAppender(this.txtJson);
logAppender.setConversionPattern("%d{yyyy-MM-dd HH:mm:ss} %-5p - %m%n");
shell.open();
shell.layout();
......@@ -109,12 +127,7 @@ public class PushDialog extends Dialog {
final JsonNode genesysJson = RowConverter.toGenesysJson(list);
final String x = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(genesysJson);
shell.getDisplay().asyncExec(new Runnable() {
@Override
public void run() {
txtJson.setText(x);
}
});
_log.info("Generated JSON: " + x);
} catch (final Throwable e) {
_log.error(e.getMessage(), e);
......@@ -137,6 +150,22 @@ public class PushDialog extends Dialog {
*/
private void createContents() {
shell = new Shell(getParent(), getStyle());
shell.addDisposeListener(new DisposeListener() {
public void widgetDisposed(DisposeEvent e) {
_log.warn("Terminating thread pool!");
executorService.shutdownNow();
logAppender.close();
}
});
shell.addListener(SWT.Close, new Listener() {
public void handleEvent(Event event) {
System.err.println("Killing pool");
executorService.shutdown();
executorService.shutdownNow();
event.doit = executorService.getActiveCount() == 0;
_log.warn("Jobs are still running, refusing to close.");
}
});
shell.setSize(691, 383);
shell.setText(getText());
shell.setLayout(new BorderLayout(0, 0));
......@@ -156,12 +185,13 @@ public class PushDialog extends Dialog {
});
tltmPushToGenesys.setText("Upload");
txtJson = new Text(shell, SWT.BORDER | SWT.READ_ONLY | SWT.WRAP | SWT.H_SCROLL | SWT.V_SCROLL | SWT.CANCEL | SWT.MULTI);
txtJson = new Text(shell, SWT.BORDER | SWT.READ_ONLY | SWT.H_SCROLL | SWT.V_SCROLL | SWT.CANCEL | SWT.MULTI);
txtJson.setLayoutData(BorderLayout.CENTER);
}
protected void doPush() {
OAuthSettings oauthSettings = settings.getOauthSettings();
_log.info("Starting push to " + oauthSettings.getServerUrl());
final GenesysClient genesysClient = AppConfig.createGenesysClient(oauthSettings);
......@@ -173,6 +203,7 @@ public class PushDialog extends Dialog {
rowReader.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1);
executorService.execute(new Runnable() {
final Map<String, List<ObjectNode>> instCodeMap = new HashMap<String, List<ObjectNode>>();
@Override
public void run() {
......@@ -184,41 +215,58 @@ public class PushDialog extends Dialog {
if (rows.size() == 0) {
_log.info("Exhausted data source.");
return;
break;
}
List<Map<String, ?>> mappedList = new ArrayList<Map<String, ?>>(rows.size());
System.err.println("\n\nBatch: start=" + count);
_log.info("Batch start at row " + count);
for (Object[] row : rows) {
count++;
System.err.println(count + ": " + ArrayUtils.toString(row));
mappedList.add(RowConverter.toMap(dataSourceSheet, row, columnDefs));
}
_log.debug(count + ": " + ArrayUtils.toString(row));
Map<String, ?> accnMap = RowConverter.toMap(dataSourceSheet, row, columnDefs);
try {
ObjectNode accnJson = RowConverter.toJson(accnMap);
final String instCode = accnJson.get("instCode").textValue();
List<ObjectNode> instCodeBatch = null;
synchronized (instCodeMap) {
instCodeBatch = instCodeMap.get(instCode);
if (instCodeBatch == null) {
instCodeMap.put(instCode, instCodeBatch = new ArrayList<ObjectNode>());
}
}
final ArrayNode genesysJson = RowConverter.toGenesysJson(mappedList);
final String x = mapper.writer().writeValueAsString(genesysJson);
System.err.println(x);
executorService.execute(new Runnable() {
@Override
public void run() {
try {
genesysClient.updateAccessions("USA1003", genesysJson);
} catch (OAuthAuthenticationException e) {
_log.error(e.getMessage(), e);
} catch (GenesysApiException e) {
_log.error(e.getMessage(), e);
synchronized (instCodeBatch) {
instCodeBatch.add(accnJson);
if (instCodeBatch.size() >= BATCH_SIZE) {
doAsyncPush(genesysClient, instCode, instCodeBatch);
}
}
} catch (GenesysJSONIncompleteException e) {
_log.warn("Ignoring incomplete accession " + ArrayUtils.toString(row));
}
});
}
Thread.sleep(10);
} while (true);
_log.info("Pushing queued data");
for (final String instCode : instCodeMap.keySet()) {
List<ObjectNode> accns = instCodeMap.get(instCode);
if (accns.size() > 0) {
doAsyncPush(genesysClient, instCode, accns);
}
}
_log.info("Done queuing upload jobs!");
} catch (IOException e) {
_log.error(e.getMessage(), e);
} catch (GenesysJSONException e) {
_log.error("Genesys JSON conversion failed in batch" + count);
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Execution was interrupted");
} finally {
try {
rowReader.close();
......@@ -227,11 +275,8 @@ public class PushDialog extends Dialog {
}
}
}
});
_log.info("Success!");
} catch (OAuthAuthenticationException e) {
_log.error(e.getMessage(), e);
} catch (PleaseRetryException e) {
......@@ -242,9 +287,9 @@ public class PushDialog extends Dialog {
_log.error(e.getMessage(), e);
} catch (UnsupportedDataFormatException e) {
_log.error(e.getMessage(), e);
} finally {
_log.info("Done.");
}
_log.info("Done.");
}
public void setDataSourceSheet(DataSourceSheet dss) {
......@@ -259,4 +304,29 @@ public class PushDialog extends Dialog {
this.columnDefs = columnDefs;
}
private void doAsyncPush(final GenesysClient genesysClient, final String instCode, final Collection<ObjectNode> instCodeBatch) {
if (instCodeBatch.size() == 0) {
_log.debug("Nothing to push");
return;
}
final ArrayList<ObjectNode> accns = new ArrayList<ObjectNode>(instCodeBatch);
instCodeBatch.clear();
executorService.execute(new Runnable() {
@Override
public void run() {
try {
_log.info("Pushing data for instCode=" + instCode + " size=" + accns.size());
genesysClient.updateAccessions(instCode, accns);
} catch (OAuthAuthenticationException e) {
_log.error(e.getMessage(), e);
} catch (GenesysApiException e) {
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Push was interrupted.");
}
}
});
}
}
......@@ -429,20 +429,25 @@ public class SheetDisplay extends Composite {
Map<String, ?> map1 = RowConverter.toMap(dataSourceSheet, row, AppConfig.instance.getColumnDefs());
try {
final ObjectNode jsonObject = RowConverter.toJson(map1);
getDisplay().syncExec(new Runnable() {
@Override
public void run() {
MessageBox mb = new MessageBox(getShell());
mb.setMessage(jsonObject.toString());
mb.setText("Genesys JSON Preview");
mb.open();
}
});
showMessageBox("Genesys JSON Preview", jsonObject.toString());
} catch (Exception e) {
_log.error(e.getMessage(), e);
showMessageBox("Conversion error", e.getMessage());
}
}
public void showMessageBox(final String title, final String message) {
getDisplay().syncExec(new Runnable() {
@Override
public void run() {
MessageBox mb = new MessageBox(getShell());
mb.setMessage(message);
mb.setText(title);
mb.open();
}
});
}
protected void doDataReload() {
_log.info("Force reload of CSV data sheet");
DataSourceSheet currentSheet = dsw.getDataSourceSheet();
......
package org.genesys2.anno.gui;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.log4j.LogManager;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
import org.apache.log4j.spi.LoggingEvent;
import org.eclipse.swt.events.DisposeEvent;
import org.eclipse.swt.events.DisposeListener;
import org.eclipse.swt.events.PaintEvent;
import org.eclipse.swt.events.PaintListener;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Text;
public class SwtLogAppender extends WriterAppender {
private static final String DEFAULT_LAYOUT_PATTERN = "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n";
private Text text;
private Display display;
private int maxBufferSize = 100000;
private StringBuffer logBuffer = new StringBuffer();
private AtomicBoolean resetText=new AtomicBoolean(true);
public SwtLogAppender(Text text2) {
this.text = text2;
this.text.addDisposeListener(new DisposeListener() {
@Override
public void widgetDisposed(DisposeEvent e) {
close();
}
});
text.addPaintListener(new PaintListener() {
public void paintControl(PaintEvent e) {
if (resetText.getAndSet(false)) {
text.setText(logBuffer.toString());
text.setSelection(logBuffer.length());
}
}
});
display = text.getDisplay();
LogManager.getRootLogger().addAppender(this);
PatternLayout patternLayout = new org.apache.log4j.PatternLayout();
patternLayout.setConversionPattern(DEFAULT_LAYOUT_PATTERN);
setLayout(patternLayout);
}
public void setConversionPattern(String conversionPattern) {
PatternLayout patternLayout = (PatternLayout) getLayout();
patternLayout.setConversionPattern(conversionPattern);
}
public void close() {
LogManager.getRootLogger().removeAppender(this);
this.text = null;
this.display = null;
}
public void append(LoggingEvent loggingEvent) {
// System.err.println("append!");
if (text == null || display == null) {
return;
}
String[] throwableStr = loggingEvent.getThrowableStrRep();
StringBuffer sb = new StringBuffer();
if (throwableStr != null) {
for (String thst : throwableStr) {
sb.append(thst).append('\n');
}
}
final String mes = layout.format(loggingEvent) + (throwableStr == null ? "" : sb.toString());
logBuffer.append(mes);
if (logBuffer.length() > maxBufferSize) {
// System.err.println("CLEARING POOL len=" + logBuffer.length() +
// " removing=" + (logBuffer.length() - maxBufferSize));
logBuffer.replace(0, logBuffer.length() - maxBufferSize, "");
int pos = logBuffer.indexOf("\n");
if (pos >= 0) {
logBuffer.replace(0, pos + 1, "");
}
}
resetText.set(true);
// System.err.println(mes);
display.asyncExec(new Runnable() {
@Override
public void run() {
if (text != null) {
text.redraw();
}
}
});
}
}
\ No newline at end of file
......@@ -17,15 +17,20 @@
package org.genesys2.anno.predefined;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
public class GenesysJSON {
public static class JsonField {
private String fieldName;
private String rdfTerm;
private boolean allowMultiple = false;
private Class<?> type = String.class;
private boolean required;
public JsonField(String fieldName, String rdfTerm) {
this.fieldName = fieldName;
......@@ -57,16 +62,26 @@ public class GenesysJSON {
this.type = type;
return this;
}
public JsonField setRequired(boolean b) {
this.required=b;
return this;
}
public boolean isRequired() {
return required;
}
}
private final List<JsonField> jsonFields;
private Collection<JsonField> requiredFields;
public GenesysJSON() {
List<JsonField> columns = new ArrayList<JsonField>();
JsonField instCode = new JsonField("instCode", RdfMCPD.INSTCODE);
JsonField instCode = new JsonField("instCode", RdfMCPD.INSTCODE).setRequired(true);
columns.add(instCode);
JsonField acceNumb = new JsonField("acceNumb", RdfMCPD.ACCENUMB);
JsonField acceNumb = new JsonField("acceNumb", RdfMCPD.ACCENUMB).setRequired(true);
columns.add(acceNumb);
JsonField columnDef = new JsonField("coll.collNumb", RdfMCPD.COLLNUMB);
......@@ -84,16 +99,16 @@ public class GenesysJSON {
columnDef = new JsonField("coll.collMissId", RdfMCPD.COLLMISSID);
columns.add(columnDef);
JsonField genus = new JsonField("genus", RdfMCPD.GENUS);
JsonField genus = new JsonField("genus", RdfMCPD.GENUS).setRequired(true);
columns.add(genus);
JsonField species = new JsonField("species", RdfMCPD.SPECIES);
columns.add(species);
columnDef = new JsonField("spAuthor", RdfMCPD.SPAUTHOR);
columnDef = new JsonField("spauthor", RdfMCPD.SPAUTHOR);
columns.add(columnDef);
columnDef = new JsonField("subTaxa", RdfMCPD.SUBTAXA);
columnDef = new JsonField("subtaxa", RdfMCPD.SUBTAXA);
columns.add(columnDef);
columnDef = new JsonField("subTauthor", RdfMCPD.SUBTAUTHOR);
......@@ -177,9 +192,24 @@ public class GenesysJSON {
columns.add(columnDef);
this.jsonFields = Collections.unmodifiableList(columns);
@SuppressWarnings("unchecked")
Collection<JsonField> requiredFields = CollectionUtils.select(columns, new Predicate() {
@Override
public boolean evaluate(Object object) {
JsonField jsonField=(JsonField) object;
return jsonField.isRequired();
}
});
this.requiredFields = Collections.unmodifiableCollection(requiredFields);
}
public final List<JsonField> getJsonFields() {
return jsonFields;
}
public Collection<JsonField> getRequiredFields() {
return requiredFields;
}
}
# Loggers
log4j.rootLogger=WARN, stdout
log4j.logger.org.genesys2.anno=DEBUG, stdout
log4j.additivity.org.genesys2.anno=false