Commit 43ca3f3b authored by Matija Obreza's avatar Matija Obreza
Browse files

Catch error in RowConverter when row contains less items than defined

columns (stopped the parser thread)
Increased Batch size for load and push, reduced queue size
parent 917db135
......@@ -42,6 +42,12 @@ public class RowConverter {
List<Column> columns = dataSourceSheet.getColumns();
for (int i = 0; i < columns.size(); i++) {
if (i >= row.length) {
_log.warn("Row has less than " + (i + 1) + " columns: " + ArrayUtils.toString(row));
continue;
}
Column column = columns.get(i);
ColumnDef columnDef = findColumnDef(column, columnDefs.getColumnDefs());
if (columnDef == null) {
......@@ -181,7 +187,7 @@ public class RowConverter {
ObjectNode rowNode = mapper.createObjectNode();
for (GenesysJSON.JsonField jsonField : genesysJson.getJsonFields()) {
if (! row.containsKey(jsonField.getRdfTerm()))
if (!row.containsKey(jsonField.getRdfTerm()))
continue;
Object rdfValues = row.get(jsonField.getRdfTerm());
add(rowNode, jsonField, rdfValues);
......
......@@ -45,7 +45,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
public class PushDialog extends Dialog {
private static final Logger _log = Logger.getLogger(PushDialog.class);
protected static final int BATCH_SIZE = 25;
protected static final int BATCH_SIZE = 50;
@Autowired
private Settings settings;
......@@ -53,7 +53,7 @@ public class PushDialog extends Dialog {
@Autowired
protected DataSourceLoader dataSourceLoader;
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(100);
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(4);
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 2, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
protected Object result;
......@@ -267,7 +267,7 @@ public class PushDialog extends Dialog {
try {
do {
List<Object[]> rows = rowReader.readRows(10);
List<Object[]> rows = rowReader.readRows(100);
if (rows.size() == 0) {
_log.info("Exhausted data source.");
......@@ -323,7 +323,10 @@ public class PushDialog extends Dialog {
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Execution was interrupted");
} catch (Throwable e) {
_log.error(e.getMessage(), e);
} finally {
_log.info("Reader finished.");
try {
rowReader.close();
} catch (IOException e) {
......@@ -375,6 +378,12 @@ public class PushDialog extends Dialog {
public void run() {
try {
_log.info("Pushing data for instCode=" + instCode + " size=" + accns.size());
if (_log.isDebugEnabled()) {
for (ObjectNode o : accns) {
_log.debug(o);
}
}
genesysClient.updateAccessions(instCode, accns);
} catch (OAuthAuthenticationException e) {
_log.error(e.getMessage(), e);
......
......@@ -236,7 +236,7 @@ public class SettingsDialog extends Dialog {
_log.warn(e.getMessage());
String authorizationUrl = genesysClient.getAuthorizationUrl(null);
VerifierDialog vd = new VerifierDialog(getParent(), SWT.NONE);
String verifierCode = vd.open(authorizationUrl);
String verifierCode = vd.open(authorizationUrl).trim();
_log.info("Got verifier code: " + verifierCode);
if (StringUtils.isNotBlank(verifierCode)) {
genesysClient.authenticate(verifierCode);
......
......@@ -15,7 +15,7 @@ import org.eclipse.swt.widgets.Text;
public class SwtLogAppender extends WriterAppender {
private static final String DEFAULT_LAYOUT_PATTERN = "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n";
private static final String DEFAULT_LAYOUT_PATTERN = "%d{yyyy-MM-dd HH:mm:ss} %t %-5p %c{1}:%L - %m%n";
private Text text;
private Display display;
private int maxBufferSize = 100000;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment