Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Genesys PGR
Genesys Backend
Commits
4e432938
Commit
4e432938
authored
Jan 02, 2015
by
Matija Obreza
Browse files
Load AccessionDetails in batches
parent
0065da86
Changes
5
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/genesys2/server/service/ElasticService.java
View file @
4e432938
...
...
@@ -32,8 +32,6 @@ public interface ElasticService {
void
remove
(
String
className
,
long
id
);
void
update
(
String
className
,
long
id
);
void
updateAll
(
String
className
,
Collection
<
Long
>
bucket
);
void
refreshIndex
(
String
className
);
...
...
src/main/java/org/genesys2/server/service/GenesysService.java
View file @
4e432938
...
...
@@ -191,8 +191,6 @@ public interface GenesysService {
List
<
AccessionBreeding
>
removeBreeding
(
List
<
AccessionBreeding
>
toRemove
);
Set
<
AccessionDetails
>
getAccessionDetails
(
Collection
<
Accession
>
accessions
);
List
<
Long
>
listAccessionsIds
(
Taxonomy2
taxonomy
);
public
static
class
AllStuff
{
...
...
@@ -208,7 +206,10 @@ public interface GenesysService {
public
List
<
AccessionAlias
>
names
=
null
;
public
AccessionExchange
exch
=
null
;
public
List
<
AccessionRemark
>
remarks
=
null
;
public
SvalbardData
svalbard
=
null
;
}
List
<
AllStuff
>
loadAllStuff
(
List
<
Long
>
accessionIds
);
List
<
AllStuff
>
loadAllStuff
(
Collection
<
Long
>
accessionIds
);
Set
<
AccessionDetails
>
getAccessionDetails
(
Collection
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @
4e432938
...
...
@@ -284,50 +284,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
@Override
public
void
update
(
String
className
,
l
ong
id
)
{
public
void
update
All
(
String
className
,
Collection
<
L
ong
>
id
s
)
{
if
(!
clazzMap
.
containsKey
(
className
))
{
return
;
}
Object
eo
=
toElasticObject
(
className
,
id
);
if
(
eo
==
null
)
if
(
ids
.
isEmpty
())
{
LOG
.
info
(
"Skipping empty updateAll."
);
return
;
IndexQuery
iq
=
new
IndexQuery
();
iq
.
setId
(
String
.
valueOf
(
id
));
iq
.
setObject
(
eo
);
if
(
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Indexing "
+
className
+
" id="
+
id
);
}
elasticsearchTemplate
.
index
(
iq
);
}
@Override
public
void
updateAll
(
String
className
,
Collection
<
Long
>
ids
)
{
if
(!
clazzMap
.
containsKey
(
className
))
{
if
(
LOG
.
isDebugEnabled
())
LOG
.
debug
(
"Updating "
+
className
+
" bulk_size="
+
ids
.
size
());
List
<
IndexQuery
>
queries
=
new
ArrayList
<
IndexQuery
>();
if
(!
Accession
.
class
.
getName
().
equals
(
className
))
{
LOG
.
warn
(
"Unsupported class "
+
className
);
return
;
}
LOG
.
info
(
"Updating "
+
className
+
" bulk_size="
+
ids
.
size
());
List
<
IndexQuery
>
queries
=
new
ArrayList
<
IndexQuery
>();
for
(
Long
id
:
ids
)
{
if
(
id
==
null
)
continue
;
// Skip null id
Set
<
AccessionDetails
>
ads
=
genesysService
.
getAccessionDetails
(
ids
);
Object
eo
=
toElasticObject
(
className
,
id
);
if
(
eo
==
null
)
continue
;
// Skip null
results, TODO perhaps delete?
for
(
AccessionDetails
ad
:
ads
)
{
if
(
ad
==
null
)
continue
;
// Skip null
id
IndexQuery
iq
=
new
IndexQuery
();
iq
.
setId
(
String
.
valueOf
(
id
));
iq
.
setObject
(
eo
);
iq
.
setId
(
String
.
valueOf
(
ad
.
getId
()
));
iq
.
setObject
(
ad
);
queries
.
add
(
iq
);
}
if
(
LOG
.
isInfoEnabled
()
&&
!
queries
.
isEmpty
())
{
LOG
.
info
(
"Indexing "
+
className
+
" count="
+
queries
.
size
());
LOG
.
info
(
"Indexing "
+
className
+
" count="
+
queries
.
size
()
+
" of provided objects count="
+
ids
.
size
()
);
elasticsearchTemplate
.
bulkIndex
(
queries
);
}
}
...
...
@@ -343,15 +332,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate
.
delete
(
clazz2
,
String
.
valueOf
(
id
));
}
private
Object
toElasticObject
(
String
className
,
long
id
)
{
if
(
Accession
.
class
.
getName
().
equals
(
className
))
{
return
genesysService
.
getAccessionDetails
(
id
);
}
LOG
.
warn
(
"Unsupported class "
+
className
);
return
null
;
}
@Override
public
void
refreshIndex
(
String
className
)
{
Class
<?>
clazz2
=
clazzMap
.
get
(
className
);
...
...
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @
4e432938
...
...
@@ -89,6 +89,7 @@ import org.genesys2.server.service.CropService;
import
org.genesys2.server.service.DatasetService
;
import
org.genesys2.server.service.GenesysService
;
import
org.genesys2.server.service.HtmlSanitizer
;
import
org.genesys2.server.service.OrganizationService
;
import
org.genesys2.server.service.TaxonomyService
;
import
org.genesys2.server.service.impl.FilterHandler.AppliedFilters
;
import
org.genesys2.spring.SecurityContextUtil
;
...
...
@@ -152,7 +153,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Autowired
private
CropTaxonomyRepository
cropTaxonomyRepository
;
@Autowired
private
Organization
Repository
organization
Repository
;
private
Organization
Service
organization
Service
;
@Autowired
private
FaoInstituteRepository
instituteRepository
;
@Autowired
...
...
@@ -311,29 +312,37 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
if
(
accession
==
null
)
return
null
;
AccessionDetails
ad
=
AccessionDetails
.
from
(
accession
);
ad
.
networks
(
organizationRepository
.
getOrganizations
(
accession
.
getInstitute
()));
ad
.
aliases
(
listAccessionAliases
(
accession
));
ad
.
exch
(
listAccessionExchange
(
accession
));
ad
.
collect
(
listAccessionCollect
(
accession
));
ad
.
breeding
(
listAccessionBreeding
(
accession
));
ad
.
geo
(
listAccessionGeo
(
accession
));
ad
.
svalbard
(
getSvalbardData
(
accession
));
ad
.
remarks
(
listAccessionRemarks
(
accession
));
return
toAccessionDetails
(
loadAllStuff
(
accession
));
}
private
AccessionDetails
toAccessionDetails
(
AllStuff
all
)
{
if
(
all
==
null
||
all
.
accession
==
null
)
return
null
;
AccessionDetails
ad
=
AccessionDetails
.
from
(
all
.
accession
);
ad
.
networks
(
organizationService
.
getOrganizations
(
all
.
accession
.
getInstitute
()));
ad
.
aliases
(
all
.
names
);
ad
.
exch
(
all
.
exch
);
ad
.
collect
(
all
.
collect
);
ad
.
breeding
(
all
.
bred
);
ad
.
geo
(
all
.
geo
);
ad
.
svalbard
(
all
.
svalbard
);
ad
.
remarks
(
all
.
remarks
);
// ad.traits(listMethods(accession),
// getAccessionTraitValues(accession));
ad
.
crops
(
cropService
.
getCrops
(
accession
.
getTaxonomy
()));
ad
.
crops
(
cropService
.
getCrops
(
all
.
accession
.
getTaxonomy
()));
return
ad
;
}
@Override
public
Set
<
AccessionDetails
>
getAccessionDetails
(
Collection
<
Accession
>
accessions
)
{
Set
<
AccessionDetails
>
set
=
new
HashSet
<
AccessionDetails
>(
accessions
.
size
());
for
(
Accession
accn
:
accessions
)
{
if
(
accn
==
null
)
public
Set
<
AccessionDetails
>
getAccessionDetails
(
Collection
<
Long
>
accessionIds
)
{
Set
<
AccessionDetails
>
set
=
new
HashSet
<
AccessionDetails
>(
accessionIds
.
size
());
List
<
AllStuff
>
alls
=
loadAllStuff
(
accessionIds
);
for
(
AllStuff
all
:
alls
)
{
if
(
all
==
null
)
continue
;
set
.
add
(
ge
tAccessionDetails
(
a
ccn
.
getId
()
));
set
.
add
(
t
o
AccessionDetails
(
a
ll
));
}
return
set
;
}
...
...
@@ -412,15 +421,23 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
public
AllStuff
loadAllStuff
(
Accession
accession
)
{
if
(
accession
==
null
)
return
null
;
AllStuff
all
=
new
AllStuff
(
1
l
);
AllStuff
all
=
new
AllStuff
(
accession
.
getId
());
all
.
bred
=
listAccessionBreeding
(
accession
);
all
.
collect
=
listAccessionCollect
(
accession
);
all
.
exch
=
listAccessionExchange
(
accession
);
all
.
geo
=
listAccessionGeo
(
accession
);
all
.
names
=
listAccessionAliases
(
accession
);
all
.
remarks
=
listAccessionRemarks
(
accession
);
all
.
svalbard
=
getSvalbardData
(
accession
);
return
all
;
}
@Override
public
List
<
AllStuff
>
loadAllStuff
(
List
<
Long
>
accessionIds
)
{
public
List
<
AllStuff
>
loadAllStuff
(
Collection
<
Long
>
accessionIds
)
{
List
<
AllStuff
>
alls
=
new
ArrayList
<
AllStuff
>(
accessionIds
.
size
());
if
(
accessionIds
==
null
||
accessionIds
.
size
()
==
0
)
{
return
alls
;
}
HashMap
<
Long
,
AllStuff
>
map
=
new
HashMap
<
Long
,
AllStuff
>();
for
(
Long
accessionId
:
accessionIds
)
{
...
...
@@ -457,12 +474,19 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
all
.
exch
=
e
;
}
for
(
SvalbardData
e
:
svalbardRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
e
.
getId
());
all
.
svalbard
=
e
;
}
for
(
AccessionAlias
aa
:
accessionAliasRepository
.
findAllFor
(
accessionIds
))
{
if
(
aa
==
null
)
{
System
.
err
.
println
(
"aa is null"
);
continue
;
}
AllStuff
all
=
map
.
get
(
aa
.
getAccession
().
getId
());
if
(
all
==
null
)
{
System
.
err
.
println
(
aa
.
getAccession
().
getId
()
+
" not in the list! "
+
aa
);
}
else
if
(
aa
==
null
)
{
System
.
err
.
println
(
"aa is null"
);
}
else
{
all
.
names
.
add
(
aa
);
}
...
...
@@ -579,9 +603,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Override
public
Page
<
Accession
>
listAccessionsByOrganization
(
Organization
organization
,
Pageable
pageable
)
{
final
List
<
FaoInstitute
>
members
=
organization
Repository
.
findInstitutesByOrganization
(
organization
);
final
List
<
FaoInstitute
>
members
=
organization
Service
.
getMembers
(
organization
);
if
(
members
==
null
||
members
.
size
()
==
0
)
{
return
new
PageImpl
<
Accession
>(
ListUtils
.
EMPTY_LIST
);
return
new
PageImpl
<
Accession
>(
Accession
.
EMPTY_LIST
);
}
return
accessionRepository
.
findByInstitute
(
members
,
pageable
);
}
...
...
@@ -649,14 +673,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Transactional
(
readOnly
=
false
)
@CacheEvict
(
value
=
"statistics"
,
allEntries
=
true
)
public
Accession
saveAccession
(
Accession
accession
)
{
if
(
LOG
.
isDebugEnabled
())
if
(
LOG
.
isDebugEnabled
())
LOG
.
debug
(
"Updating "
+
accession
);
System
.
err
.
println
(
"Saving "
+
accession
+
" ST="
+
accession
.
getStorage
());
Accession
res
=
accessionRepository
.
save
(
accession
);
updateAccessionCount
(
accession
.
getInstitute
());
return
res
;
}
}
@Override
@Transactional
(
readOnly
=
false
)
...
...
src/main/java/org/genesys2/server/service/worker/ElasticUpdaterAspect.java
View file @
4e432938
...
...
@@ -16,6 +16,7 @@
package
org.genesys2.server.service.worker
;
import
java.util.ArrayList
;
import
java.util.Collection
;
import
java.util.HashSet
;
import
java.util.List
;
...
...
@@ -151,8 +152,13 @@ public class ElasticUpdaterAspect {
LOG
.
debug
(
"Returning from "
+
joinPoint
.
toLongString
());
}
Collection
<
Long
>
accessionIds
=
new
ArrayList
<
Long
>(
accessions
.
size
());
for
(
Accession
a
:
accessions
)
{
if
(
a
!=
null
&&
a
.
getId
()
!=
null
)
accessionIds
.
add
(
a
.
getId
());
}
// Need to load AccessionDetails before deleting
Set
<
AccessionDetails
>
deletedAccessions
=
genesysService
.
getAccessionDetails
(
accessions
);
Set
<
AccessionDetails
>
deletedAccessions
=
genesysService
.
getAccessionDetails
(
accession
Id
s
);
try
{
Object
res
=
joinPoint
.
proceed
();
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment