Skip to content

GitLab

U
Uploader
Commit 8834532a authored by Matija Obreza's avatar Matija Obreza
Browse files
Options

Added "Skip rows" and read progress indicator

parent dfe849ef
Hide whitespace changes
Inline Side-by-side
Showing with 197 additions and 21 deletions
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
View file @ 8834532a
......@@ -30,6 +30,7 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
......@@ -38,6 +39,10 @@ import org.apache.commons.lang3.time.StopWatch;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.eclipse.core.databinding.DataBindingContext;
import org.eclipse.core.databinding.beans.PojoProperties;
import org.eclipse.core.databinding.observable.value.IObservableValue;
import org.eclipse.jface.databinding.swt.WidgetProperties;
import org.eclipse.jface.resource.ImageDescriptor;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.DisposeEvent;
......@@ -45,15 +50,22 @@ import org.eclipse.swt.events.DisposeListener;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Dialog;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.ProgressBar;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Text;
import org.eclipse.swt.widgets.ToolBar;
import org.eclipse.swt.widgets.ToolItem;
import org.eclipse.wb.swt.SWTResourceManager;
import org.genesys2.anno.converter.GenesysJSONException;
import org.genesys2.anno.converter.GenesysJSONIncompleteException;
import org.genesys2.anno.converter.RowConverter;
......@@ -111,6 +123,58 @@ public class PushDialog extends Dialog {
private List<Future<?>> futures = new ArrayList<Future<?>>();
private ToolBar toolBar;
private AtomicBoolean doPushPaused = new AtomicBoolean(true); // paused by default
private AtomicInteger pushRowCount = new AtomicInteger(0); // Number of rows read during push
private AtomicInteger pushAccessionCount = new AtomicInteger(0); // Number of accessions uploaded
private PushProgress pushProgress = new PushProgress();
private Text txtSkipDataRows;
private ProgressBar progressBar;
public static class PushProgress {
private int extraSkipRows = 0;
private int sourceRowCount = 0; // Number of rows read during push
private int rowsRead = 0;
private int readProgress = 0; // % (0 - 100)
public PushProgress() {
reset();
}
public void reset() {
rowsRead = 0;
sourceRowCount = 0;
readProgress = 0;
}
public int updateReadProgress(int sourceRowCount, int rowsRead) {
this.sourceRowCount = sourceRowCount;
this.rowsRead = rowsRead;
return this.readProgress = this.sourceRowCount == 0 ? 0 : (int) (100 * ((float) rowsRead / (float) sourceRowCount));
}
public int getReadProgress() {
return readProgress;
}
public void setReadProgress(int readProgress) {
this.readProgress = readProgress;
}
public int getRowsRead() {
return rowsRead;
}
public void setRowsRead(int rowsRead) {
this.rowsRead = rowsRead;
}
public int getExtraSkipRows() {
return extraSkipRows;
}
public void setExtraSkipRows(int extraSkipRows) {
this.extraSkipRows = extraSkipRows;
}
}
static {
deleteDataIcon = ImageDescriptor.createFromURL(PushDialog.class.getResource("/icon/delete.png")).createImage();
......@@ -142,8 +206,47 @@ public class PushDialog extends Dialog {
createContents();
logAppender = new SwtLogAppender(this.txtJson);
Composite composite = new Composite(shell, SWT.NO_FOCUS);
composite.setLayoutData(BorderLayout.SOUTH);
FormLayout fl_composite = new FormLayout();
fl_composite.marginBottom = 5;
fl_composite.marginWidth = 10;
composite.setLayout(fl_composite);
Label lblUploaded = new Label(composite, SWT.NONE);
FormData fd_lblUploaded = new FormData();
fd_lblUploaded.top = new FormAttachment(0, 7);
fd_lblUploaded.left = new FormAttachment(0, 5);
lblUploaded.setLayoutData(fd_lblUploaded);
lblUploaded.setText("Uploaded:");
progressBar = new ProgressBar(composite, SWT.NONE);
FormData fd_progressBar = new FormData();
fd_progressBar.top = new FormAttachment(0, 7);
fd_progressBar.left = new FormAttachment(0, 68);
progressBar.setLayoutData(fd_progressBar);
progressBar.setForeground(SWTResourceManager.getColor(SWT.COLOR_DARK_GREEN));
progressBar.setSelection(0);
Label lblUploadStatus = new Label(composite, SWT.NONE);
lblUploadStatus.setAlignment(SWT.RIGHT);
FormData fd_lblUploadStatus = new FormData();
fd_lblUploadStatus.left = new FormAttachment(lblUploaded, 150);
fd_lblUploadStatus.top = new FormAttachment(0, 7);
lblUploadStatus.setLayoutData(fd_lblUploadStatus);
lblUploadStatus.setText("Skip rows:");
txtSkipDataRows = new Text(composite, SWT.BORDER | SWT.RIGHT);
FormData fd_txtSkipDataRows = new FormData();
fd_txtSkipDataRows.width = 100;
fd_txtSkipDataRows.left = new FormAttachment(lblUploadStatus, 5);
fd_txtSkipDataRows.top = new FormAttachment(0, 5);
txtSkipDataRows.setLayoutData(fd_txtSkipDataRows);
txtSkipDataRows.setText("0");
shell.setTabList(new Control[] { txtJson, toolBar });
logAppender.setConversionPattern("%d{yyyy-MM-dd HH:mm:ss} %t %-5p - %m%n");
initDataBindings();
shell.open();
shell.layout();
......@@ -360,16 +463,50 @@ public class PushDialog extends Dialog {
OAuthSettings oauthSettings = settings.getOauthSettings();
_log.info("Starting push to " + oauthSettings.getServerUrl());
final GenesysClient genesysClient = SpringConfig.createGenesysClient(oauthSettings);
this.pushProgress.sourceRowCount = 0;
// Manages pause/resume
doPushPaused.set(false);
pushRowCount.set(0);
pushAccessionCount.set(0);
final Thread uploadMonitor = new Thread(() -> {
// This thread runs until interrupted by end of upload
// Logs the current line
StopWatch stopWatch = new StopWatch();
stopWatch.start();
long lastTime = 0;
do {
int pushedRows = pushRowCount.get();
long currentTime = stopWatch.getTime() / 1000;
if (pushedRows > 0 && (currentTime != lastTime && currentTime % 10 == 0)) {
lastTime = currentTime;
// log every 10s
_log.warn("Read " + pushRowCount + " rows from data source");
}
int readProgress = this.pushProgress.updateReadProgress(this.pushProgress.sourceRowCount, pushedRows);
Display.getDefault().asyncExec(() -> {
progressBar.setSelection(readProgress);
});
try {
Thread.sleep(100);
} catch (InterruptedException e) {
_log.warn("Total run time: " + (stopWatch.getTime() / 1000) + "s");
break;
}
} while (true);
}, "UploadMonitor");
uploadMonitor.start();
final int extraSkipRows = this.pushProgress.extraSkipRows;
Future<?> future = executorService.submit(() -> {
try {
List<String> instCodesOrder = new ArrayList<>();
try (final RowReader rowReader1 = dataSourceLoader.createRowReader(dataSourceSheet, dataSource)) {
rowReader1.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1);
_log.warn("Skipping data rows: " + extraSkipRows);
rowReader1.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1 + extraSkipRows);
Set<String> instCodes = new HashSet<>();
// Read instCodes
......@@ -387,11 +524,14 @@ public class PushDialog extends Dialog {
Map<String, ?> accnMap = RowConverter.toMap(dataSourceSheet, row, columnDefs);
final String accessionInstCode = (String) accnMap.get(RdfMCPD.INSTCODE);
if (instCodes.add(accessionInstCode)) {
_log.info("Detected INSTCODE: " + accessionInstCode + " in line " + count);
_log.info("Detected INSTCODE: " + accessionInstCode + " in line " + (extraSkipRows + count));
instCodesOrder.add(accessionInstCode);
}
}
} while (rows != null && rows.size() > 0);
this.pushProgress.sourceRowCount = extraSkipRows + count;
_log.warn("Detected rows: " + this.pushProgress.sourceRowCount);
}
_log.warn("Have instCodes: " + instCodesOrder.stream().sorted().collect(Collectors.toList()));
......@@ -399,12 +539,39 @@ public class PushDialog extends Dialog {
// Ping the server
genesysClient.me();
_log.warn("Queueing upload for " + instCodesOrder.toString());
// Filter only for specific INSTCODES
final Set<String> instCodesFilter = new HashSet<>(); // (instCodesOrder);
final Set<String> instCodesFilter = new HashSet<>();
instCodesFilter.addAll(Arrays.asList("ALB020", "ALB021", "ALB026", "ARM005", "ARM006", "ARM008", "ARM035", "ARM059", "AUT001", "AUT005", "AUT006", "AUT007", "AUT023", "AUT024", "AUT025", "AUT046", "AUT047", "AUT048", "AUT052", "AUT054", "AUT060"));
instCodesFilter.addAll(Arrays.asList("AUT067", "AZE003", "AZE004", "AZE005", "AZE006", "AZE007", "AZE009", "AZE014", "AZE015", "BEL001", "BEL002", "BEL004", "BEL010", "BEL014", "BEL019", "BEL020", "BEL023", "BEL094", "BEL098", "BEL102"));
instCodesFilter.addAll(Arrays.asList("BEL103", "BGR001", "BGR005", "BGR029", "BIH036", "BIH039", "BLR011", "CHE001", "CHE002", "CHE006", "CHE009", "CHE016", "CHE019", "CHE063", "CHE065", "CHE066", "CHE087", "CHE088", "CHE089", "CHE090", "CHE092"));
instCodesFilter.addAll(Arrays.asList("CHE094", "CHE095", "CHE100", "CHE101", "CHE104", "CHE109", "CHE119", "CHE124", "CHE125", "CHE127", "CYP004", "CZE005", "CZE027", "CZE031", "CZE041", "CZE042", "CZE050", "CZE061", "CZE079", "CZE082", "CZE112"));
instCodesFilter.addAll(Arrays.asList("CZE122", "DEU005", "DEU011", "DEU021", "DEU022", "DEU043", "DEU069", "DEU070", "DEU072", "DEU087", "DEU098", "DEU101", "DEU159", "DEU271", "DEU363", "DEU400", "DEU401", "DEU451", "DEU454", "DEU455"));
instCodesFilter.addAll(Arrays.asList("DEU456", "DEU457", "DEU483", "DEU502", "DEU515", "DEU526", "DEU528", "DEU567", "DEU590", "DEU594", "DEU595", "DEU597", "DEU609", "DEU610", "DEU612", "DEU613", "DEU614", "DEU615", "DEU616", "DEU626", "DEU627"));
instCodesFilter.addAll(Arrays.asList("DEU628", "DEU630", "DEU631", "DEU633", "DEU634", "DEU635", "DEU637", "DEU652", "DNK020", "ESP003", "ESP004", "ESP007", "ESP009", "ESP010", "ESP014", "ESP016", "ESP025", "ESP026", "ESP027", "ESP031", "ESP032"));
instCodesFilter.addAll(Arrays.asList("ESP046", "ESP048", "ESP058", "ESP074", "ESP080", "ESP089", "ESP103", "ESP109", "ESP110", "ESP117", "ESP119", "ESP121", "ESP124", "ESP133", "ESP138", "ESP149", "ESP160", "ESP172", "ESP197", "ESP198", "ESP200"));
instCodesFilter.addAll(Arrays.asList("ESP214", "ESP216", "ESP218", "ESP220", "ESP221", "ESP222", "ESP223", "EST001", "EST006", "EST012", "EST019", "FIN027", "FRA001", "FRA010", "FRA011", "FRA015", "FRA040", "FRA041", "FRA043", "FRA065", "FRA179"));
instCodesFilter.addAll(Arrays.asList("GBR006", "GBR016", "GBR017", "GBR030", "GBR165", "GBR247", "GBR251", "GEO001", "GEO003", "GEO007", "GEO013", "GRC001", "GRC005", "GRC010", "GRC012", "HRV021", "HRV041", "HRV044", "HRV045"));
instCodesFilter.addAll(Arrays.asList("HRV048", "HRV049", "HRV050", "HRV053", "HRV054", "HUN003", "IRL001", "IRL012", "IRL029", "ISR001", "ISR002", "ISR003", "ITA034", "ITA070", "ITA226", "ITA363", "ITA365", "ITA368", "ITA378", "ITA379", "ITA380"));
instCodesFilter.addAll(Arrays.asList("ITA382", "ITA383", "ITA384", "ITA385", "ITA386", "ITA388", "ITA389", "ITA390", "ITA391", "ITA392", "ITA393", "ITA394", "ITA395", "ITA396", "ITA397", "ITA398", "ITA399", "ITA400", "ITA401", "ITA403", "ITA407"));
instCodesFilter.addAll(Arrays.asList("ITA420", "ITA422", "ITA425", "LTU001", "LTU003", "LTU008", "LTU010", "LVA006", "LVA009", "LVA014", "LVA015", "MDA005", "MDA010", "MDA011", "MKD001", "MNE001", "MNE047", "NLD037", "NLD162", "NOR010", "NOR017"));
instCodesFilter.addAll(Arrays.asList("POL001", "POL002", "POL003", "POL017", "POL022", "POL028", "POL047", "POL055", "POL101", "PRT001", "PRT018", "PRT025", "PRT051", "PRT102", "PRT201", "ROM001", "ROM002", "ROM003", "ROM007", "ROM008", "ROM009"));
instCodesFilter.addAll(Arrays.asList("ROM012", "ROM014", "ROM017", "ROM018", "ROM019", "ROM021", "ROM022", "ROM023", "ROM025", "ROM028", "ROM029", "ROM032", "ROM033", "ROM035", "ROM037", "ROM040", "SRB001", "SRB002", "SVK001", "SVN018"));
instCodesFilter.addAll(Arrays.asList("SVN019", "SWE054", "TUR001", "UKR001", "UKR002", "UKR003", "UKR004", "UKR005", "UKR006", "UKR007", "UKR008", "UKR009", "UKR011", "UKR012", "UKR013", "UKR014", "UKR015", "UKR016", "UKR017", "UKR018", "UKR019"));
instCodesFilter.addAll(Arrays.asList("UKR020", "UKR021", "UKR022", "UKR023", "UKR025", "UKR026", "UKR027", "UKR028", "UKR029", "UKR030", "UKR032", "UKR033", "UKR034", "UKR036", "UKR037", "UKR039", "UKR040", "UKR043", "UKR046", "UKR048", "UKR050"));
instCodesFilter.addAll(Arrays.asList("UKR053", "UKR054", "UKR063", "UKR065", "UKR068", "UKR070", "UKR075", "UKR077", "UKR079", "UKR081", "UKR084", "UKR085", "UKR093", "UKR095", "UKR099", "UKR100", "UKR124", "UKR130", "UKR135", "UKR142", "UKR143"));
// The big collections
instCodesFilter.addAll(Arrays.asList("DEU146"));
instCodesFilter.addAll(Arrays.asList("RUS001"));
instCodesFilter.addAll(Arrays.asList("GBR140"));
_log.warn("Queueing upload for " + instCodesFilter.toString());
Future<?> instFuture = executorService.submit(new Runnable() {
final Map<String, List<ObjectNode>> instCodeMap = new HashMap<String, List<ObjectNode>>();
......@@ -413,10 +580,9 @@ public class PushDialog extends Dialog {
public void run() {
_log.warn("Running upload for " + instCodesFilter.toString());
try (RowReader rowReader = dataSourceLoader.createRowReader(dataSourceSheet, dataSource)) {
rowReader.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1);
int rowCount = 0;
int accessionCount = 0;
_log.warn("Skipping data rows: " + extraSkipRows);
rowReader.setSkipRows(dataSourceSheet.getHeaderRowIndex() + 1 + extraSkipRows);
pushRowCount.set(extraSkipRows);
try {
do {
......@@ -448,8 +614,8 @@ public class PushDialog extends Dialog {
for (Object[] row : rows) {
rowCount++;
_log.trace(rowCount + ": " + ArrayUtils.toString(row));
pushRowCount.incrementAndGet();
_log.trace(pushRowCount.get() + ": " + ArrayUtils.toString(row));
Map<String, ?> accnMap = RowConverter.toMap(dataSourceSheet, row, columnDefs);
final String accessionInstCode = (String) accnMap.get(RdfMCPD.INSTCODE);
......@@ -462,7 +628,7 @@ public class PushDialog extends Dialog {
try {
ObjectNode accnJson = RowConverter.toJson(accnMap);
accessionCount++;
pushAccessionCount.incrementAndGet();
List<ObjectNode> instCodeBatch = null;
synchronized (instCodeMap) {
instCodeBatch = instCodeMap.get(accessionInstCode);
......@@ -497,10 +663,10 @@ public class PushDialog extends Dialog {
}
}
_log.warn("Done processing upload jobs for " + accessionCount + " accessions");
_log.warn("Done processing upload jobs for " + pushAccessionCount.get() + " accessions");
} catch (GenesysJSONException e) {
_log.error("Genesys JSON conversion failed in batch" + rowCount);
_log.error("Genesys JSON conversion failed in batch " + pushRowCount.get());
_log.error(e.getMessage(), e);
} catch (InterruptedException e) {
_log.info("Execution was interrupted");
......@@ -661,4 +827,13 @@ public class PushDialog extends Dialog {
public void setDataSource(IDataSource dataSource) {
this.dataSource = dataSource;
}
protected DataBindingContext initDataBindings() {
DataBindingContext bindingContext = new DataBindingContext();
//
IObservableValue observeTextTextObserveWidget = WidgetProperties.text(SWT.Modify).observe(txtSkipDataRows);
IObservableValue extraSkipRowsPushProgressObserveValue = PojoProperties.value("extraSkipRows").observe(pushProgress);
bindingContext.bindValue(observeTextTextObserveWidget, extraSkipRowsPushProgressObserveValue, null, null);
//
return bindingContext;
}
}
anno-gui/src/main/java/org/genesys2/anno/parser/MsAccessRowReader.java
View file @ 8834532a
......@@ -61,14 +61,15 @@ public class MsAccessRowReader implements RowReader {
/// This is called in case data starts beyond 1st row
@Override
public void setSkipRows(int skipRows) {
_log.info("Skipping rows {}", skipRows);
// The -1 is because CSV and Excel include the header row in the count
_log.info("Skipping rows {}", skipRows - 1);
cursor.reset();
// try {
// cursor.moveNextRows(skipRows);
// } catch (IOException e) {
// _log.error("Could not skip rows", e);
// }
try {
cursor.moveNextRows(skipRows - 1);
} catch (IOException e) {
_log.error("Could not skip rows", e);
}
}
@Override
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment