Commit 6e5132c9 authored by Matija Obreza's avatar Matija Obreza
Browse files

Genesys API v1 constants

parent 78da4bbe
......@@ -35,8 +35,12 @@ public class RowConverter {
private static final ObjectMapper mapper = new ObjectMapper();
private static final GenesysJSON genesysJson = new GenesysJSON();
public static Map<String, ?> toMap(IDataSourceSheet dataSourceSheet, Object[] row, ColumnDefs columnDefs) {
HashMap<String, Object> map = new HashMap<String, Object>(columnDefs.getColumnDefs().size());
private static final Pattern REGEXP_REPLACE_PATTERN = Pattern.compile("\\$(\\d+)");
public static Map<String, ?> toMap(IDataSourceSheet dataSourceSheet,
Object[] row, ColumnDefs columnDefs) {
HashMap<String, Object> map = new HashMap<String, Object>(columnDefs
.getColumnDefs().size());
_log.debug("Converting " + ArrayUtils.toString(row));
......@@ -44,7 +48,8 @@ public class RowConverter {
for (int i = 0; i < columns.size(); i++) {
Column column = columns.get(i);
ColumnDef columnDef = findColumnDef(column, columnDefs.getColumnDefs());
ColumnDef columnDef = findColumnDef(column,
columnDefs.getColumnDefs());
if (columnDef == null) {
continue;
}
......@@ -53,31 +58,38 @@ public class RowConverter {
if (column.isIncludeNull()) {
map.put(columnDef.getRdfTerm(), null);
}
_log.debug("Row has less than " + (i + 1) + " columns: len=" + row.length + " " + ArrayUtils.toString(row));
_log.debug("Row has less than " + (i + 1) + " columns: len="
+ row.length + " " + ArrayUtils.toString(row));
continue;
}
Object rowValue = convertCellValue(row[i], column.isMultiple(), column.getSeparator(), column.getPattern(), column.getGroupPattern());
Object rowValue = convertCellValue(row[i], column.isMultiple(),
column.getSeparator(), column.getPattern(),
column.getGroupPattern());
Object mapValue = map.get(columnDef.getRdfTerm());
if (mapValue == null) {
_log.debug("Adding value for " + columnDef.getRdfTerm() + " val=" + rowValue);
_log.debug("Adding value for " + columnDef.getRdfTerm()
+ " val=" + rowValue);
if (rowValue instanceof String[]) {
_log.debug("val=" + ArrayUtils.toString(rowValue));
map.put(columnDef.getRdfTerm(), new ArrayList<String>(Arrays.asList((String[]) rowValue)));
map.put(columnDef.getRdfTerm(), new ArrayList<String>(
Arrays.asList((String[]) rowValue)));
} else {
map.put(columnDef.getRdfTerm(), rowValue);
}
} else if (mapValue instanceof List<?>) {
_log.debug("Adding " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
_log.debug("Adding " + columnDef.getRdfTerm() + " to List<?>"
+ " val=" + rowValue);
if (rowValue instanceof String[]) {
((List<Object>) mapValue).addAll(Arrays.asList((String[]) rowValue));
((List<Object>) mapValue).addAll(Arrays
.asList((String[]) rowValue));
} else if (rowValue != null) {
((List<Object>) mapValue).add(rowValue);
}
} else if (rowValue != null) {
_log.debug("Converting " + columnDef.getRdfTerm() + " to List<?>" + " val=" + rowValue);
_log.debug("Converting " + columnDef.getRdfTerm()
+ " to List<?>" + " val=" + rowValue);
Object currentVal = mapValue;
List<Object> newVal = new ArrayList<Object>();
newVal.add(currentVal);
......@@ -105,7 +117,8 @@ public class RowConverter {
return map;
}
private static Object convertCellValue(Object value, boolean splitString, String separator, String pattern, String groupPattern) {
private static Object convertCellValue(Object value, boolean splitString,
String separator, String pattern, String groupPattern) {
if (value != null) {
if (value instanceof String) {
String stringValue = ((String) value);
......@@ -116,7 +129,8 @@ public class RowConverter {
String[] split = stringValue.split(separator);
for (int i = split.length - 1; i >= 0; i--) {
if (pattern != null) {
split[i] = valueFromPattern(split[i].trim(), pattern, groupPattern);
split[i] = valueFromPattern(split[i].trim(),
pattern, groupPattern);
} else {
split[i] = split[i].trim();
}
......@@ -124,7 +138,8 @@ public class RowConverter {
return split;
}
String patVal = valueFromPattern(stringValue.trim(), pattern, groupPattern);
String patVal = valueFromPattern(stringValue.trim(), pattern,
groupPattern);
if (StringUtils.isBlank(patVal)) {
return null;
}
......@@ -136,20 +151,23 @@ public class RowConverter {
return value;
}
private static String valueFromPattern(String trim, String pattern, String groupPattern) {
private static String valueFromPattern(String trim, String pattern,
String groupPattern) {
if (StringUtils.isBlank(pattern)) {
return trim;
}
Pattern pat = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
Matcher matcher = pat.matcher(trim);
if (matcher.find()) {
String res = matcher.groupCount() == 0 ? matcher.group() : matcher.group(1);
String res = matcher.groupCount() == 0 ? matcher.group() : matcher
.group(1);
if (StringUtils.isNotBlank(groupPattern)) {
res = groupPattern;
if (res.contains("$1"))
res = res.replaceAll("\\$1", matcher.group(1));
if (res.contains("$2"))
res = res.replaceAll("\\$2", matcher.group(2));
Matcher replaceMat = REGEXP_REPLACE_PATTERN.matcher(groupPattern);
while (replaceMat.find()) {
int group = Integer.parseInt(replaceMat.group(1));
res = res.replaceAll("\\$" + group, matcher.group(group));
}
}
// System.err.println("Pattern=" + pattern + " groups=" +
// matcher.groupCount() + " res=" + res);
......@@ -159,7 +177,8 @@ public class RowConverter {
return null;
}
private static ColumnDef findColumnDef(Column column, List<ColumnDef> columnDefList) {
private static ColumnDef findColumnDef(Column column,
List<ColumnDef> columnDefList) {
for (ColumnDef columnDef : columnDefList) {
if (columnDef.getRdfTerm().equals(column.getRdfTerm())) {
return columnDef;
......@@ -170,7 +189,8 @@ public class RowConverter {
return null;
}
public static List<Map<String, ?>> toMap(IDataSourceSheet dataSourceSheet, List<Object[]> rows, ColumnDefs columnDefs) {
public static List<Map<String, ?>> toMap(IDataSourceSheet dataSourceSheet,
List<Object[]> rows, ColumnDefs columnDefs) {
List<Map<String, ?>> list = new ArrayList<Map<String, ?>>(rows.size());
for (Object[] row : rows) {
list.add(toMap(dataSourceSheet, row, columnDefs));
......@@ -178,7 +198,8 @@ public class RowConverter {
return list;
}
public static ArrayNode toGenesysJson(List<Map<String, ?>> list) throws GenesysJSONException, JsonGenerationException {
public static ArrayNode toGenesysJson(List<Map<String, ?>> list)
throws GenesysJSONException, JsonGenerationException {
ArrayNode result = mapper.createArrayNode();
for (Map<String, ?> row : list) {
......@@ -192,7 +213,8 @@ public class RowConverter {
return result;
}
public static ObjectNode toJson(Map<String, ?> row) throws GenesysJSONException, JsonGenerationException {
public static ObjectNode toJson(Map<String, ?> row)
throws GenesysJSONException, JsonGenerationException {
if (row == null)
return null;
......@@ -207,14 +229,19 @@ public class RowConverter {
for (GenesysJSON.JsonField jsonField : genesysJson.getRequiredFields()) {
if (!hasNodeValue(rowNode, jsonField.getFieldName())) {
throw new GenesysJSONIncompleteException("Missing required property '" + jsonField.getFieldName() + "' " + jsonField.getRdfTerm());
throw new GenesysJSONIncompleteException(
"Missing required property '"
+ jsonField.getFieldName() + "' "
+ jsonField.getRdfTerm());
}
}
return rowNode;
}
private static void add(ObjectNode rowNode, JsonField jsonField, Object rdfValues) throws GenesysJSONException, JsonGenerationException {
private static void add(ObjectNode rowNode, JsonField jsonField,
Object rdfValues) throws GenesysJSONException,
JsonGenerationException {
String fieldName = jsonField.getFieldName();
ObjectNode containerNode = ensureNode(rowNode, fieldName);
int dotPost = fieldName.lastIndexOf('.');
......@@ -230,7 +257,8 @@ public class RowConverter {
}
} else {
if (jsonField.hasAllowMultiple() && !valueNode.isArray()) {
throw new GenesysJSONException("Expecting arrayObject for " + fieldName);
throw new GenesysJSONException("Expecting arrayObject for "
+ fieldName);
}
}
......@@ -248,14 +276,17 @@ public class RowConverter {
} else {
// System.err.println("Putting field=" + fieldName + " type=" +
// rdfValues.getClass());
containerNode.set(fieldName, coerce(jsonField.getType(), rdfValues));
containerNode.set(fieldName,
coerce(jsonField.getType(), rdfValues));
}
}
}
static ObjectNode ensureNode(ObjectNode parentNode, String nodeName) throws JsonGenerationException {
static ObjectNode ensureNode(ObjectNode parentNode, String nodeName)
throws JsonGenerationException {
if (StringUtils.isBlank(nodeName)) {
throw new JsonGenerationException("nodeName should not be blank or null");
throw new JsonGenerationException(
"nodeName should not be blank or null");
}
int dotIndex = nodeName.indexOf('.');
if (dotIndex > -1) {
......@@ -271,9 +302,11 @@ public class RowConverter {
return parentNode;
}
static boolean hasNodeValue(final ObjectNode parentNode, final String nodeName) throws JsonGenerationException {
static boolean hasNodeValue(final ObjectNode parentNode,
final String nodeName) throws JsonGenerationException {
if (StringUtils.isBlank(nodeName)) {
throw new JsonGenerationException("nodeName should not be blank or null");
throw new JsonGenerationException(
"nodeName should not be blank or null");
}
String[] path = nodeName.split("\\.");
JsonNode node = parentNode;
......@@ -293,7 +326,8 @@ public class RowConverter {
}
}
}
return node != null && !node.isNull() && !node.isMissingNode() && !node.isArray() && node.isValueNode();
return node != null && !node.isNull() && !node.isMissingNode()
&& !node.isArray() && node.isValueNode();
}
static JsonNode coerce(Class<?> clazz, Object rdfValues) {
......@@ -316,7 +350,11 @@ public class RowConverter {
}
private static boolean toBoolean(String rdfValue) {
return "true".equalsIgnoreCase(rdfValue) || "yes".equalsIgnoreCase(rdfValue) || "t".equalsIgnoreCase(rdfValue) || "y".equalsIgnoreCase(rdfValue) || "1".equalsIgnoreCase(rdfValue);
return "true".equalsIgnoreCase(rdfValue)
|| "yes".equalsIgnoreCase(rdfValue)
|| "t".equalsIgnoreCase(rdfValue)
|| "y".equalsIgnoreCase(rdfValue)
|| "1".equalsIgnoreCase(rdfValue);
}
}
......@@ -3,8 +3,6 @@ package org.genesys2.anno.gui;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.anno.model.Column;
import org.genesys2.anno.model.ColumnDef;
import org.genesys2.anno.model.DatabaseSettings;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
......
......@@ -7,12 +7,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.eclipse.jface.resource.ImageDescriptor;
import org.eclipse.swt.SWT;
......@@ -35,6 +37,7 @@ import org.genesys2.anno.converter.RowConverter;
import org.genesys2.anno.model.OAuthSettings;
import org.genesys2.anno.model.Settings;
import org.genesys2.anno.parser.RowReader;
import org.genesys2.anno.predefined.Api1Constants;
import org.genesys2.client.oauth.GenesysApiException;
import org.genesys2.client.oauth.GenesysClient;
import org.genesys2.client.oauth.OAuthAuthenticationException;
......@@ -62,8 +65,11 @@ public class PushDialog extends Dialog {
@Autowired
protected DataSourceLoader dataSourceLoader;
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(4);
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 2, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(
4);
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 2,
30, TimeUnit.SECONDS, linkedBlockingDeque,
new ThreadPoolExecutor.CallerRunsPolicy());
protected Object result;
protected Shell shell;
......@@ -72,10 +78,13 @@ public class PushDialog extends Dialog {
private ColumnDefs columnDefs;
private SwtLogAppender logAppender;
private IDataSource dataSource;
private List<Future<?>> futures = new ArrayList<Future<?>>();
static {
deleteDataIcon = ImageDescriptor.createFromURL(PushDialog.class.getResource("/icon/delete.png")).createImage();
parseDataIcon = ImageDescriptor.createFromURL(PushDialog.class.getResource("/icon/parse.png")).createImage();
deleteDataIcon = ImageDescriptor.createFromURL(
PushDialog.class.getResource("/icon/delete.png")).createImage();
parseDataIcon = ImageDescriptor.createFromURL(
PushDialog.class.getResource("/icon/parse.png")).createImage();
}
/**
......@@ -85,7 +94,8 @@ public class PushDialog extends Dialog {
* @param style
*/
public PushDialog(Shell parent, int style) {
super(parent, SWT.DIALOG_TRIM | SWT.MAX | SWT.RESIZE | SWT.APPLICATION_MODAL);
super(parent, SWT.DIALOG_TRIM | SWT.MAX | SWT.RESIZE
| SWT.APPLICATION_MODAL);
setText("Push data to Genesys");
}
......@@ -134,10 +144,23 @@ public class PushDialog extends Dialog {
@Override
public void handleEvent(Event event) {
_log.info("Killing worker thread pool");
for (int i = futures.size() - 1; i >= 0; i--) {
Future<?> future = futures.get(i);
if (future.isDone()) {
futures.remove(future);
}
}
if (futures.size() > 0) {
_log.warn("Jobs still running: " + futures.size());
}
executorService.shutdown();
executorService.shutdownNow();
event.doit = executorService.getActiveCount() == 0;
_log.warn((executorService.getActiveCount()) + " jobs are still running, refusing to close.");
_log.warn((executorService.getActiveCount())
+ " jobs are still running, refusing to close.");
}
});
shell.setSize(691, 383);
......@@ -161,7 +184,10 @@ public class PushDialog extends Dialog {
tltmPushToGenesys.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
if (SwtUtil.showYesNoPrompt(getParent(), "Confirmation required", "Push data to Genesys server?\n" + settings.getOauthSettings().getServerUrl())) {
if (SwtUtil.showYesNoPrompt(getParent(),
"Confirmation required",
"Push data to Genesys server?\n"
+ settings.getOauthSettings().getServerUrl())) {
_log.info("Upsert confirmed");
doPush(GenesysOp.UPSERT);
} else {
......@@ -176,7 +202,10 @@ public class PushDialog extends Dialog {
tltmRemove.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent arg0) {
if (SwtUtil.showYesNoPrompt(getParent(), "Confirmation required", "Delete data from Genesys server?\n" + settings.getOauthSettings().getServerUrl())) {
if (SwtUtil.showYesNoPrompt(getParent(),
"Confirmation required",
"Delete data from Genesys server?\n"
+ settings.getOauthSettings().getServerUrl())) {
_log.info("Delete confirmed");
doPush(GenesysOp.DELETE);
} else {
......@@ -191,6 +220,7 @@ public class PushDialog extends Dialog {
tltmRadioItemDebug.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
LogManager.getRootLogger().setLevel(Level.DEBUG);
logAppender.setThreshold(Level.DEBUG);
}
});
......@@ -201,6 +231,7 @@ public class PushDialog extends Dialog {
tltmRadioItemInfo.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
LogManager.getRootLogger().setLevel(Level.INFO);
logAppender.setThreshold(Level.INFO);
}
});
......@@ -210,12 +241,14 @@ public class PushDialog extends Dialog {
tltmRadioItemWarning.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
LogManager.getRootLogger().setLevel(Level.WARN);
logAppender.setThreshold(Level.WARN);
}
});
tltmRadioItemWarning.setText("WARNING");
txtJson = new Text(shell, SWT.BORDER | SWT.READ_ONLY | SWT.H_SCROLL | SWT.V_SCROLL | SWT.CANCEL | SWT.MULTI);
txtJson = new Text(shell, SWT.BORDER | SWT.READ_ONLY | SWT.H_SCROLL
| SWT.V_SCROLL | SWT.CANCEL | SWT.MULTI);
txtJson.setLayoutData(BorderLayout.CENTER);
}
......@@ -223,10 +256,11 @@ public class PushDialog extends Dialog {
protected void doTrialRun() {
_log.info("Decoding data...");
try {
final RowReader rowReader = dataSourceLoader.createRowReader(dataSourceSheet, dataSource);
final RowReader rowReader = dataSourceLoader.createRowReader(
dataSourceSheet, dataSource);
rowReader.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1);
executorService.execute(new Runnable() {
Future<?> future = executorService.submit(new Runnable() {
@Override
public void run() {
int count = 0;
......@@ -245,12 +279,19 @@ public class PushDialog extends Dialog {
for (Object[] row : rows) {
count++;
try {
Map<String, ?> accnMap = RowConverter.toMap(dataSourceSheet, row, columnDefs);
ObjectNode accnJson = RowConverter.toJson(accnMap);
final String instCode = accnJson.get("instCode").textValue();
Map<String, ?> accnMap = RowConverter
.toMap(dataSourceSheet, row,
columnDefs);
ObjectNode accnJson = RowConverter
.toJson(accnMap);
// Check for instCode
accnJson.get(Api1Constants.Accession.INSTCODE).textValue();
} catch (Throwable e) {
_log.warn("Error in row " + count + ". " + e.getMessage() + ": " + ArrayUtils.toString(row));
_log.info("Error in row " + count + ". "
+ e.getMessage() + ": "
+ ArrayUtils.toString(row));
}
}
......@@ -260,6 +301,7 @@ public class PushDialog extends Dialog {
_log.info("Done queuing parse jobs!");
} catch (IOException e) {
_log.error("Error reading row " + rowReader.getRowCount());
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Execution was interrupted");
......@@ -267,12 +309,16 @@ public class PushDialog extends Dialog {
try {
rowReader.close();
} catch (IOException e) {
_log.error("Failed to close rowReader: " + e.getMessage(), e);
_log.error(
"Failed to close rowReader: "
+ e.getMessage(), e);
}
}
}
});
futures.add(future);
} catch (IOException e) {
_log.error(e.getMessage(), e);
} catch (UnsupportedDataFormatException e) {
......@@ -286,15 +332,17 @@ public class PushDialog extends Dialog {
OAuthSettings oauthSettings = settings.getOauthSettings();
_log.info("Starting push to " + oauthSettings.getServerUrl());
final GenesysClient genesysClient = SpringConfig.createGenesysClient(oauthSettings);
final GenesysClient genesysClient = SpringConfig
.createGenesysClient(oauthSettings);
try {
// Ping the server
genesysClient.me();
final RowReader rowReader = dataSourceLoader.createRowReader(dataSourceSheet, dataSource);
final RowReader rowReader = dataSourceLoader.createRowReader(
dataSourceSheet, dataSource);
rowReader.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1);
executorService.execute(new Runnable() {
Future<?> future = executorService.submit(new Runnable() {
final Map<String, List<ObjectNode>> instCodeMap = new HashMap<String, List<ObjectNode>>();
@Override
......@@ -313,28 +361,38 @@ public class PushDialog extends Dialog {
_log.info("Batch start at row " + count);
for (Object[] row : rows) {
count++;
_log.debug(count + ": " + ArrayUtils.toString(row));
Map<String, ?> accnMap = RowConverter.toMap(dataSourceSheet, row, columnDefs);
_log.debug(count + ": "
+ ArrayUtils.toString(row));
Map<String, ?> accnMap = RowConverter.toMap(
dataSourceSheet, row, columnDefs);
try {
ObjectNode accnJson = RowConverter.toJson(accnMap);
final String instCode = accnJson.get("instCode").textValue();
ObjectNode accnJson = RowConverter
.toJson(accnMap);
final String instCode = accnJson.get(
Api1Constants.Accession.INSTCODE).textValue();
List<ObjectNode> instCodeBatch = null;
synchronized (instCodeMap) {
instCodeBatch = instCodeMap.get(instCode);
instCodeBatch = instCodeMap
.get(instCode);
if (instCodeBatch == null) {
instCodeMap.put(instCode, instCodeBatch = new ArrayList<ObjectNode>());
instCodeMap
.put(instCode,
instCodeBatch = new ArrayList<ObjectNode>());
}
}
synchronized (instCodeBatch) {
instCodeBatch.add(accnJson);
if (instCodeBatch.size() >= BATCH_SIZE) {
doAsyncPush(operation, genesysClient, instCode, instCodeBatch);
doAsyncPush(operation,
genesysClient, instCode,
instCodeBatch);
}
}
} catch (GenesysJSONIncompleteException e) {
_log.warn("Ignoring incomplete accession " + ArrayUtils.toString(row));
_log.info("Ignoring incomplete accession "
+ ArrayUtils.toString(row));
}
}
......@@ -346,16 +404,19 @@ public class PushDialog extends Dialog {
for (final String instCode : instCodeMap.keySet()) {
List<ObjectNode> accns = instCodeMap.get(instCode);
if (accns.size() > 0) {
doAsyncPush(operation, genesysClient, instCode, accns);
doAsyncPush(operation, genesysClient, instCode,
accns);
}
}
_log.info("Done queuing upload jobs!");
} catch (IOException e) {
_log.error("Error reading row " + rowReader.getRowCount());
_log.error(e.getMessage(), e);
} catch (GenesysJSONException e) {
_log.error("Genesys JSON conversion failed in batch" + count);
_log.error("Genesys JSON conversion failed in batch"
+ count);
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Execution was interrupted");
......@@ -366,12 +427,16 @@ public class PushDialog extends Dialog {
try {
rowReader.close();
} catch (IOException e) {
_log.error("Failed to close rowReader: " + e.getMessage(), e);
_log.error(
"Failed to close rowReader: "
+ e.getMessage(), e);
}
}
}
});
futures.add(future);
} catch (OAuthAuthenticationException e) {
_log.error(e.getMessage(), e);
} catch (PleaseRetryException e) {
......@@ -401,20 +466,24 @@ public class PushDialog extends Dialog {
this.columnDefs = columnDefs;
}
private void doAsyncPush(final GenesysOp operation, final GenesysClient genesysClient, final String instCode, final Collection<ObjectNode> instCodeBatch) {
private void doAsyncPush(final GenesysOp operation,
final GenesysClient genesysClient, final String instCode,
final Collection<ObjectNode> instCodeBatch) {