Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
U
Uploader
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
5
Issues
5
List
Boards
Labels
Service Desk
Milestones
Merge Requests
2
Merge Requests
2
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Uploader
Commits
729f50ed
Commit
729f50ed
authored
Jan 04, 2019
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
No INSTCODE filters
parent
da352684
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
58 additions
and
49 deletions
+58
-49
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
+58
-49
No files found.
anno-gui/src/main/java/org/genesys2/anno/gui/PushDialog.java
View file @
729f50ed
...
...
@@ -17,6 +17,7 @@ package org.genesys2.anno.gui;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collection
;
import
java.util.HashMap
;
import
java.util.HashSet
;
...
...
@@ -77,6 +78,8 @@ public class PushDialog extends Dialog {
private
static
final
Logger
_log
=
Logger
.
getLogger
(
PushDialog
.
class
);
protected
static
final
int
BATCH_SIZE
=
50
;
protected
static
final
int
MAX_UPLOAD_THREADS
=
4
;
protected
static
final
int
MAX_POOL_SIZE
=
30
;
public
static
enum
GenesysOp
{
UPSERT
,
DELETE
...
...
@@ -95,7 +98,7 @@ public class PushDialog extends Dialog {
protected
DataSourceLoader
dataSourceLoader
;
BlockingQueue
<
Runnable
>
linkedBlockingDeque
=
new
LinkedBlockingDeque
<
Runnable
>(
MAX_UPLOAD_THREADS
*
3
);
private
ThreadPoolExecutor
executorService
=
new
ThreadPoolExecutor
(
2
,
MAX_UPLOAD_THREADS
+
2
,
30
,
TimeUnit
.
SECONDS
,
linkedBlockingDeque
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
private
ThreadPoolExecutor
executorService
=
new
ThreadPoolExecutor
(
2
,
MAX_UPLOAD_THREADS
+
2
,
MAX_POOL_SIZE
,
TimeUnit
.
SECONDS
,
linkedBlockingDeque
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
protected
Object
result
;
protected
Shell
shell
;
...
...
@@ -347,12 +350,11 @@ public class PushDialog extends Dialog {
Future
<?>
future
=
executorService
.
submit
(()
->
{
try
{
final
RowReader
rowReader1
=
dataSourceLoader
.
createRowReader
(
dataSourceSheet
,
dataSource
);
rowReader1
.
setSkipRows
(
dataSourceSheet
.
getHeaderRowIndex
()
+
1
);
List
<
String
>
instCodesOrder
=
new
ArrayList
<>();
{
try
(
final
RowReader
rowReader1
=
dataSourceLoader
.
createRowReader
(
dataSourceSheet
,
dataSource
))
{
rowReader1
.
setSkipRows
(
dataSourceSheet
.
getHeaderRowIndex
()
+
1
);
Set
<
String
>
instCodes
=
new
HashSet
<>();
// Read instCodes
List
<
Object
[]>
rows
=
null
;
...
...
@@ -360,8 +362,8 @@ public class PushDialog extends Dialog {
do
{
rows
=
rowReader1
.
readRows
(
1000
);
for
(
Object
[]
row
:
rows
)
{
count
++;
count
++;
Map
<
String
,
?>
accnMap
=
RowConverter
.
toMap
(
dataSourceSheet
,
row
,
columnDefs
);
final
String
accessionInstCode
=
(
String
)
accnMap
.
get
(
RdfMCPD
.
INSTCODE
);
if
(
instCodes
.
add
(
accessionInstCode
))
{
...
...
@@ -371,38 +373,31 @@ public class PushDialog extends Dialog {
}
}
while
(
rows
!=
null
&&
rows
.
size
()
>
0
);
}
_log
.
warn
(
"Have instCodes: "
+
instCodesOrder
.
stream
().
sorted
().
collect
(
Collectors
.
toList
()));
_log
.
warn
(
"Have instCodes: "
+
instCodesOrder
.
stream
().
sorted
().
collect
(
Collectors
.
toList
()));
// Ping the server
genesysClient
.
me
();
for
(
String
instCode
:
instCodesOrder
)
{
_log
.
warn
(
"Queueing upload for "
+
instCode
);
_log
.
warn
(
"Queueing upload for "
+
instCodesOrder
.
toString
());
Future
<?>
instFuture
=
executorService
.
submit
(
new
Runnable
()
{
// Filter only for specific INSTCODES
final
Set
<
String
>
instCodesFilter
=
new
HashSet
<>();
// (instCodesOrder);
@Override
public
void
run
()
{
_log
.
warn
(
"Running upload for "
+
instCode
);
try
(
RowReader
rowReader
=
dataSourceLoader
.
createRowReader
(
dataSourceSheet
,
dataSource
))
{
rowReader
.
setSkipRows
(
dataSourceSheet
.
getHeaderRowIndex
()
+
1
);
Future
<?>
instFuture
=
executorService
.
submit
(
new
Runnable
()
{
final
Map
<
String
,
List
<
ObjectNode
>>
instCodeMap
=
new
HashMap
<
String
,
List
<
ObjectNode
>>();
uploadInsitute
(
instCode
,
rowReader
);
@Override
public
void
run
()
{
_log
.
warn
(
"Running upload for "
+
instCodesFilter
.
toString
());
try
(
RowReader
rowReader
=
dataSourceLoader
.
createRowReader
(
dataSourceSheet
,
dataSource
))
{
rowReader
.
setSkipRows
(
dataSourceSheet
.
getHeaderRowIndex
()
+
1
);
}
catch
(
IOException
|
UnsupportedDataFormatException
e
)
{
_log
.
error
(
e
.
getMessage
(),
e
);
}
}
private
void
uploadInsitute
(
String
instCode
,
RowReader
rowReader
)
{
int
rowCount
=
0
;
int
accessionCount
=
0
;
List
<
ObjectNode
>
instCodeBatch
=
new
ArrayList
<>(
BATCH_SIZE
);
_log
.
warn
(
"Uploading data for "
+
instCode
);
try
{
do
{
List
<
Object
[]>
rows
=
rowReader
.
readRows
(
100
);
...
...
@@ -413,15 +408,15 @@ public class PushDialog extends Dialog {
}
// _log.info("Reading source data for " + instCode + " starting at row=" +
// count);
for
(
Object
[]
row
:
rows
)
{
rowCount
++;
_log
.
trace
(
rowCount
+
": "
+
ArrayUtils
.
toString
(
row
));
Map
<
String
,
?>
accnMap
=
RowConverter
.
toMap
(
dataSourceSheet
,
row
,
columnDefs
);
final
String
accessionInstCode
=
(
String
)
accnMap
.
get
(
RdfMCPD
.
INSTCODE
);
if
(!
accessionInstCode
.
equals
(
instCode
))
{
// If filtering, check INSTCODE is listed
if
(
instCodesFilter
.
size
()
>
0
&&
!
instCodesFilter
.
contains
(
accessionInstCode
))
{
continue
;
}
...
...
@@ -429,27 +424,41 @@ public class PushDialog extends Dialog {
ObjectNode
accnJson
=
RowConverter
.
toJson
(
accnMap
);
accessionCount
++;
instCodeBatch
.
add
(
accnJson
);
if
(
instCodeBatch
.
size
()
>=
BATCH_SIZE
)
{
doAsyncPush
(
operation
,
genesysClient
,
accessionInstCode
,
instCodeBatch
);
instCodeBatch
.
clear
();
List
<
ObjectNode
>
instCodeBatch
=
null
;
synchronized
(
instCodeMap
)
{
instCodeBatch
=
instCodeMap
.
get
(
accessionInstCode
);
if
(
instCodeBatch
==
null
)
{
instCodeMap
.
put
(
accessionInstCode
,
instCodeBatch
=
new
ArrayList
<
ObjectNode
>());
}
}
synchronized
(
instCodeBatch
)
{
instCodeBatch
.
add
(
accnJson
);
if
(
instCodeBatch
.
size
()
>=
BATCH_SIZE
)
{
doAsyncPush
(
operation
,
genesysClient
,
accessionInstCode
,
instCodeBatch
);
}
}
}
catch
(
GenesysJSONIncompleteException
e
)
{
_log
.
info
(
"Ignoring incomplete accession "
+
ArrayUtils
.
toString
(
row
));
}
Thread
.
sleep
(
1
);
}
Thread
.
sleep
(
1
);
}
while
(
true
);
_log
.
info
(
"Pushing remaining data for "
+
instCode
);
if
(
instCodeBatch
.
size
()
>
0
)
{
doAsyncPush
(
operation
,
genesysClient
,
instCode
,
instCodeBatch
);
_log
.
info
(
"Pushing queued data"
);
for
(
final
String
instCode
:
instCodeMap
.
keySet
())
{
List
<
ObjectNode
>
accns
=
instCodeMap
.
get
(
instCode
);
if
(
accns
.
size
()
>
0
)
{
doAsyncPush
(
operation
,
genesysClient
,
instCode
,
accns
);
}
}
_log
.
warn
(
"Done processing upload jobs for "
+
accessionCount
+
" accessions
for "
+
instCode
);
_log
.
warn
(
"Done processing upload jobs for "
+
accessionCount
+
" accessions
"
);
}
catch
(
GenesysJSONException
e
)
{
_log
.
error
(
"Genesys JSON conversion failed in batch"
+
rowCount
);
...
...
@@ -461,20 +470,20 @@ public class PushDialog extends Dialog {
e
.
printStackTrace
();
}
finally
{
_log
.
info
(
"Reader finished."
);
// try {
// rowReader.close();
// } catch (IOException e) {
// _log.error("Failed to close rowReader: " + e.getMessage(), e);
// }
//
try {
//
rowReader.close();
//
} catch (IOException e) {
//
_log.error("Failed to close rowReader: " + e.getMessage(), e);
//
}
}
}
});
futures
.
add
(
instFuture
);
}
catch
(
IOException
|
UnsupportedDataFormatException
e
)
{
_log
.
error
(
e
.
getMessage
(),
e
);
}
}
});
Thread
.
sleep
(
200
);
}
futures
.
add
(
instFuture
);
// Wait for uploads to finish
do
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment