Improved uploader

- Automatically log push jobs still in progress
- log & report time for upsert

anno-gui-macosx/.classpath

@@ -17,7 +17,7 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
@@ -27,5 +27,10 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

anno-gui-macosx/.project

@@ -15,8 +15,19 @@
 			<arguments>
 			</arguments>
 		</buildCommand>
+		<buildCommand>
+			<name>org.springframework.ide.eclipse.core.springbuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.springframework.ide.eclipse.boot.validation.springbootbuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
 	</buildSpec>
 	<natures>
+		<nature>org.springframework.ide.eclipse.core.springnature</nature>
 		<nature>org.eclipse.jdt.core.javanature</nature>
 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
 	</natures>

anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java

@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
 import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.time.StopWatch;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
@@ -88,8 +89,8 @@ public class PushDialog extends Dialog {
 	@Autowired
 	protected DataSourceLoader dataSourceLoader;
 
-	BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(10);
-	private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 10, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
+	BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(36);
+	private ThreadPoolExecutor executorService = new ThreadPoolExecutor(4, 8, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
 
 	protected Object result;
 	protected Shell shell;
@@ -132,7 +133,7 @@ public class PushDialog extends Dialog {
 
 		logAppender = new SwtLogAppender(this.txtJson);
 		shell.setTabList(new Control[] { txtJson, toolBar });
-		logAppender.setConversionPattern("%d{yyyy-MM-dd HH:mm:ss} %-5p - %m%n");
+		logAppender.setConversionPattern("%d{yyyy-MM-dd HH:mm:ss} %t %-5p - %m%n");
 
 		shell.open();
 		shell.layout();
@@ -166,12 +167,14 @@ public class PushDialog extends Dialog {
 
 				for (int i = futures.size() - 1; i >= 0; i--) {
 					Future<?> future = futures.get(i);
-					if (future.isDone()) {
+					if (future.isDone() || future.isCancelled()) {
 						futures.remove(future);
+					} else {
+						future.cancel(true);
 					}
 				}
 
-				if (futures.size() > 0) {
+				if (futures.size() > 1) {
 					_log.warn("Jobs still running: " + futures.size());
 				}
 
@@ -349,6 +352,7 @@ public class PushDialog extends Dialog {
 				@Override
 				public void run() {
 					int count = 0;
+					
 					try {
 
 						do {
@@ -359,7 +363,7 @@ public class PushDialog extends Dialog {
 								break;
 							}
 
-							_log.info("Batch start at row " + count);
+							_log.info("Reading source data starting at row=" + count);
 							for (Object[] row : rows) {
 								count++;
 								_log.debug(count + ": " + ArrayUtils.toString(row));
@@ -388,7 +392,7 @@ public class PushDialog extends Dialog {
 
 							}
 
-							Thread.sleep(10);
+							Thread.sleep(5);
 						} while (true);
 
 						_log.info("Pushing queued data");
@@ -420,6 +424,30 @@ public class PushDialog extends Dialog {
 							_log.error("Failed to close rowReader: " + e.getMessage(), e);
 						}
 					}
+					
+					// Wait for uploads to finish
+					do {
+						try {
+							Thread.sleep(1000);
+						} catch (InterruptedException e) {
+							break;
+						}
+			
+						for (int i = futures.size() - 1; i >= 0; i--) {
+							Future<?> future = futures.get(i);
+							if (future.isDone()) {
+								futures.remove(future);
+							}
+						}
+				
+						if (futures.size() > 1) {
+							_log.warn("Waiting for uploads still running: " + futures.size());
+						}
+						
+						// We're the last job
+					} while (futures.size() > 1);
+					_log.warn("PUSH FINISHED.");
+					_log.warn("Really.");
 				}
 			});
 
@@ -459,6 +487,12 @@ public class PushDialog extends Dialog {
 			_log.debug("Nothing to push");
 			return;
 		}
+		
+		try {
+			Thread.sleep(10);
+		} catch (InterruptedException e1) {
+			return;
+		}
 
 		Future<?> future = executorService.submit(new Runnable() {
 			final ArrayList<ObjectNode> accns = new ArrayList<ObjectNode>(instCodeBatch);
@@ -467,10 +501,13 @@ public class PushDialog extends Dialog {
 			@Override
 			public void run() {
 				try {
-					_log.info("Pushing data for instCode=" + instCode + " size=" + accns.size());
-					if (_log.isDebugEnabled()) {
+					StopWatch stopWatch = new StopWatch();
+					stopWatch.start();
+					
+					_log.debug("Pushing data for instCode=" + instCode + " size=" + accns.size());
+					if (_log.isTraceEnabled()) {
 						for (ObjectNode o : accns) {
-							_log.debug(o);
+							_log.trace(o);
 						}
 					}
 
@@ -480,8 +517,8 @@ public class PushDialog extends Dialog {
 					else if (op == GenesysOp.DELETE)
 						serverResponse = genesysClient.deleteAccessionsByName(instCode, objectMapper.writeValueAsString(accns));
 
-					if (_log.isDebugEnabled())
-						_log.debug(serverResponse);
+					if (_log.isTraceEnabled())
+						_log.trace(serverResponse);
 
 					try {
 						JsonNode ri = objectMapper.readTree(serverResponse);
@@ -491,12 +528,12 @@ public class PushDialog extends Dialog {
 								if (t.has("error") && !t.get("error").isNull()) {
 									_log.error(t.get("instCode") + " " + t.get("acceNumb") + ":" + t.get("error"));
 								} else if (t.has("result") && !t.get("result").isNull()) {
-									_log.debug(t.get("instCode") + " " + t.get("acceNumb") + ":" + t.get("result"));
+									_log.trace(t.get("instCode") + " " + t.get("acceNumb") + ":" + t.get("result"));
 								}
 							}
 						});
 
-						_log.info("Push done.");
+						_log.info("Push done for instCode=" + instCode + " size=" + accns.size() + " in " + stopWatch.getTime() + "ms.");
 
 					} catch (IOException e) {
 						// TODO Auto-generated catch block