Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
U
Uploader
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
5
Issues
5
List
Boards
Labels
Service Desk
Milestones
Merge Requests
2
Merge Requests
2
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Uploader
Commits
64076cb4
Commit
64076cb4
authored
Oct 14, 2018
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improved uploader
- Automatically log push jobs still in progress - log & report time for upsert
parent
24c7b8c4
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
68 additions
and
15 deletions
+68
-15
anno-gui-macosx/.classpath
anno-gui-macosx/.classpath
+6
-1
anno-gui-macosx/.project
anno-gui-macosx/.project
+11
-0
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
+51
-14
No files found.
anno-gui-macosx/.classpath
View file @
64076cb4
...
...
@@ -17,7 +17,7 @@
<attribute
name=
"maven.pomderived"
value=
"true"
/>
</attributes>
</classpathentry>
<classpathentry
kind=
"con"
path=
"org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.
6
"
>
<classpathentry
kind=
"con"
path=
"org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.
8
"
>
<attributes>
<attribute
name=
"maven.pomderived"
value=
"true"
/>
</attributes>
...
...
@@ -27,5 +27,10 @@
<attribute
name=
"maven.pomderived"
value=
"true"
/>
</attributes>
</classpathentry>
<classpathentry
excluding=
"**"
kind=
"src"
output=
"target/test-classes"
path=
"src/test/resources"
>
<attributes>
<attribute
name=
"maven.pomderived"
value=
"true"
/>
</attributes>
</classpathentry>
<classpathentry
kind=
"output"
path=
"target/classes"
/>
</classpath>
anno-gui-macosx/.project
View file @
64076cb4
...
...
@@ -15,8 +15,19 @@
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>
org.springframework.ide.eclipse.core.springbuilder
</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>
org.springframework.ide.eclipse.boot.validation.springbootbuilder
</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>
org.springframework.ide.eclipse.core.springnature
</nature>
<nature>
org.eclipse.jdt.core.javanature
</nature>
<nature>
org.eclipse.m2e.core.maven2Nature
</nature>
</natures>
...
...
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
View file @
64076cb4
...
...
@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import
java.util.function.Consumer
;
import
org.apache.commons.lang3.ArrayUtils
;
import
org.apache.commons.lang3.time.StopWatch
;
import
org.apache.log4j.Level
;
import
org.apache.log4j.LogManager
;
import
org.apache.log4j.Logger
;
...
...
@@ -88,8 +89,8 @@ public class PushDialog extends Dialog {
@Autowired
protected
DataSourceLoader
dataSourceLoader
;
BlockingQueue
<
Runnable
>
linkedBlockingDeque
=
new
LinkedBlockingDeque
<
Runnable
>(
10
);
private
ThreadPoolExecutor
executorService
=
new
ThreadPoolExecutor
(
1
,
10
,
30
,
TimeUnit
.
SECONDS
,
linkedBlockingDeque
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
BlockingQueue
<
Runnable
>
linkedBlockingDeque
=
new
LinkedBlockingDeque
<
Runnable
>(
36
);
private
ThreadPoolExecutor
executorService
=
new
ThreadPoolExecutor
(
4
,
8
,
30
,
TimeUnit
.
SECONDS
,
linkedBlockingDeque
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
protected
Object
result
;
protected
Shell
shell
;
...
...
@@ -132,7 +133,7 @@ public class PushDialog extends Dialog {
logAppender
=
new
SwtLogAppender
(
this
.
txtJson
);
shell
.
setTabList
(
new
Control
[]
{
txtJson
,
toolBar
});
logAppender
.
setConversionPattern
(
"%d{yyyy-MM-dd HH:mm:ss} %-5p - %m%n"
);
logAppender
.
setConversionPattern
(
"%d{yyyy-MM-dd HH:mm:ss} %
t %
-5p - %m%n"
);
shell
.
open
();
shell
.
layout
();
...
...
@@ -166,12 +167,14 @@ public class PushDialog extends Dialog {
for
(
int
i
=
futures
.
size
()
-
1
;
i
>=
0
;
i
--)
{
Future
<?>
future
=
futures
.
get
(
i
);
if
(
future
.
isDone
())
{
if
(
future
.
isDone
()
||
future
.
isCancelled
()
)
{
futures
.
remove
(
future
);
}
else
{
future
.
cancel
(
true
);
}
}
if
(
futures
.
size
()
>
0
)
{
if
(
futures
.
size
()
>
1
)
{
_log
.
warn
(
"Jobs still running: "
+
futures
.
size
());
}
...
...
@@ -349,6 +352,7 @@ public class PushDialog extends Dialog {
@Override
public
void
run
()
{
int
count
=
0
;
try
{
do
{
...
...
@@ -359,7 +363,7 @@ public class PushDialog extends Dialog {
break
;
}
_log
.
info
(
"
Batch start at row
"
+
count
);
_log
.
info
(
"
Reading source data starting at row=
"
+
count
);
for
(
Object
[]
row
:
rows
)
{
count
++;
_log
.
debug
(
count
+
": "
+
ArrayUtils
.
toString
(
row
));
...
...
@@ -388,7 +392,7 @@ public class PushDialog extends Dialog {
}
Thread
.
sleep
(
10
);
Thread
.
sleep
(
5
);
}
while
(
true
);
_log
.
info
(
"Pushing queued data"
);
...
...
@@ -420,6 +424,30 @@ public class PushDialog extends Dialog {
_log
.
error
(
"Failed to close rowReader: "
+
e
.
getMessage
(),
e
);
}
}
// Wait for uploads to finish
do
{
try
{
Thread
.
sleep
(
1000
);
}
catch
(
InterruptedException
e
)
{
break
;
}
for
(
int
i
=
futures
.
size
()
-
1
;
i
>=
0
;
i
--)
{
Future
<?>
future
=
futures
.
get
(
i
);
if
(
future
.
isDone
())
{
futures
.
remove
(
future
);
}
}
if
(
futures
.
size
()
>
1
)
{
_log
.
warn
(
"Waiting for uploads still running: "
+
futures
.
size
());
}
// We're the last job
}
while
(
futures
.
size
()
>
1
);
_log
.
warn
(
"PUSH FINISHED."
);
_log
.
warn
(
"Really."
);
}
});
...
...
@@ -459,6 +487,12 @@ public class PushDialog extends Dialog {
_log
.
debug
(
"Nothing to push"
);
return
;
}
try
{
Thread
.
sleep
(
10
);
}
catch
(
InterruptedException
e1
)
{
return
;
}
Future
<?>
future
=
executorService
.
submit
(
new
Runnable
()
{
final
ArrayList
<
ObjectNode
>
accns
=
new
ArrayList
<
ObjectNode
>(
instCodeBatch
);
...
...
@@ -467,10 +501,13 @@ public class PushDialog extends Dialog {
@Override
public
void
run
()
{
try
{
_log
.
info
(
"Pushing data for instCode="
+
instCode
+
" size="
+
accns
.
size
());
if
(
_log
.
isDebugEnabled
())
{
StopWatch
stopWatch
=
new
StopWatch
();
stopWatch
.
start
();
_log
.
debug
(
"Pushing data for instCode="
+
instCode
+
" size="
+
accns
.
size
());
if
(
_log
.
isTraceEnabled
())
{
for
(
ObjectNode
o
:
accns
)
{
_log
.
debug
(
o
);
_log
.
trace
(
o
);
}
}
...
...
@@ -480,8 +517,8 @@ public class PushDialog extends Dialog {
else
if
(
op
==
GenesysOp
.
DELETE
)
serverResponse
=
genesysClient
.
deleteAccessionsByName
(
instCode
,
objectMapper
.
writeValueAsString
(
accns
));
if
(
_log
.
is
Debug
Enabled
())
_log
.
debug
(
serverResponse
);
if
(
_log
.
is
Trace
Enabled
())
_log
.
trace
(
serverResponse
);
try
{
JsonNode
ri
=
objectMapper
.
readTree
(
serverResponse
);
...
...
@@ -491,12 +528,12 @@ public class PushDialog extends Dialog {
if
(
t
.
has
(
"error"
)
&&
!
t
.
get
(
"error"
).
isNull
())
{
_log
.
error
(
t
.
get
(
"instCode"
)
+
" "
+
t
.
get
(
"acceNumb"
)
+
":"
+
t
.
get
(
"error"
));
}
else
if
(
t
.
has
(
"result"
)
&&
!
t
.
get
(
"result"
).
isNull
())
{
_log
.
debug
(
t
.
get
(
"instCode"
)
+
" "
+
t
.
get
(
"acceNumb"
)
+
":"
+
t
.
get
(
"result"
));
_log
.
trace
(
t
.
get
(
"instCode"
)
+
" "
+
t
.
get
(
"acceNumb"
)
+
":"
+
t
.
get
(
"result"
));
}
}
});
_log
.
info
(
"Push done."
);
_log
.
info
(
"Push done
for instCode="
+
instCode
+
" size="
+
accns
.
size
()
+
" in "
+
stopWatch
.
getTime
()
+
"ms
."
);
}
catch
(
IOException
e
)
{
// TODO Auto-generated catch block
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment