Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Genesys Backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
19
Issues
19
List
Boards
Labels
Service Desk
Milestones
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Genesys Backend
Commits
4ee8ce1e
Commit
4ee8ce1e
authored
Oct 13, 2018
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Elasticsearch: total and missing terms
- Fix: Use single ElasticJPAListener and ElasticReindexProcessor
parent
4543c36b
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
44 additions
and
38 deletions
+44
-38
src/main/java/org/genesys2/server/component/elastic/ElasticJPAListener.java
...genesys2/server/component/elastic/ElasticJPAListener.java
+2
-2
src/main/java/org/genesys2/server/component/elastic/ElasticReindexProcessor.java
...ys2/server/component/elastic/ElasticReindexProcessor.java
+5
-1
src/main/java/org/genesys2/server/service/ElasticsearchService.java
...ava/org/genesys2/server/service/ElasticsearchService.java
+17
-16
src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java
...enesys2/server/service/impl/ElasticsearchServiceImpl.java
+14
-13
src/main/java/org/genesys2/spring/config/ElasticsearchConfig.java
.../java/org/genesys2/spring/config/ElasticsearchConfig.java
+2
-2
src/test/java/org/genesys/test/server/services/ElasticQueryBuilderTest.java
...genesys/test/server/services/ElasticQueryBuilderTest.java
+4
-4
No files found.
src/main/java/org/genesys2/server/component/elastic/ElasticJPAListener.java
View file @
4ee8ce1e
...
...
@@ -45,7 +45,7 @@ import org.springframework.stereotype.Component;
* {@link ElasticReindexProcessor}.
*/
@Aspect
@Component
@Component
(
"elasticJpaListener"
)
public
class
ElasticJPAListener
{
private
final
static
Logger
LOG
=
LoggerFactory
.
getLogger
(
ElasticJPAListener
.
class
);
...
...
@@ -149,7 +149,7 @@ public class ElasticJPAListener {
}
else
{
Class
<?>
clazz
=
toReindex
.
getClass
();
if
(
isIndexed
(
clazz
))
{
LOG
.
trace
(
"
Reindexing
{} {}"
,
clazz
.
getName
(),
toReindex
);
LOG
.
trace
(
"
Scheduling reindexing of
{} {}"
,
clazz
.
getName
(),
toReindex
);
if
(
toReindex
instanceof
BasicModel
)
{
BasicModel
entity
=
(
BasicModel
)
toReindex
;
elasticReindexQueue
.
add
(
new
ElasticReindex
(
clazz
.
getName
(),
entity
.
getId
()));
...
...
src/main/java/org/genesys2/server/component/elastic/ElasticReindexProcessor.java
View file @
4ee8ce1e
...
...
@@ -40,7 +40,7 @@ import org.springframework.stereotype.Component;
* with 2000ms delay measured from the completion time of each preceding
* invocation.
*/
@Component
@Component
(
"elasticReindexProcessor"
)
public
class
ElasticReindexProcessor
{
/** The Constant LOG. */
...
...
@@ -54,6 +54,10 @@ public class ElasticReindexProcessor {
@Autowired
private
ElasticsearchService
elasticsearch
;
public
ElasticReindexProcessor
()
{
System
.
err
.
println
(
"Making ElasticReindexProcessor"
);
}
/**
* Process queues.
*/
...
...
src/main/java/org/genesys2/server/service/ElasticsearchService.java
View file @
4ee8ce1e
...
...
@@ -94,11 +94,11 @@ public interface ElasticsearchService {
* @return the term result
* @throws SearchException the search exception
*/
TermResult
termStatisticsAuto
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
term
)
throws
SearchException
;
Map
<
String
,
TermResult
>
termStatisticsAuto
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
...
terms
)
throws
SearchException
;
TermResult
termStatisticsAuto
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
term
)
throws
SearchException
;
Map
<
String
,
TermResult
>
termStatisticsAuto
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
...
terms
)
throws
SearchException
;
TermResult
termStatistics
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
term
)
throws
SearchException
;
Map
<
String
,
TermResult
>
termStatistics
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
...
terms
)
throws
SearchException
;
TermResult
termStatistics
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
term
)
throws
SearchException
;
Map
<
String
,
TermResult
>
termStatistics
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
...
terms
)
throws
SearchException
;
void
realias
(
String
aliasName
,
String
indexName
);
...
...
@@ -110,11 +110,11 @@ public interface ElasticsearchService {
void
reindexAll
();
<
T
extends
BasicModel
>
void
reindex
(
Class
<
T
>
clazz
,
BasicModelFilter
filter
);
<
T
extends
BasicModel
>
void
reindex
(
Class
<
T
>
clazz
,
BasicModelFilter
<?,
?>
filter
);
List
<
Class
<?
extends
BasicModel
>>
getIndexedEntities
();
long
count
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
filter
);
long
count
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filter
);
/**
* Aggregate by date
...
...
@@ -123,10 +123,10 @@ public interface ElasticsearchService {
* @param indexClass the index class
* @param aggregatedDateField the name of date field to aggregate
* @param groupingByField the name of field to grouping
* @param filter the BasicModelFilter filter
* @param filter the BasicModelFilter
<?, ?>
filter
* @return the result list
*/
List
<
Object
[]>
aggregateDate
(
int
size
,
Class
<?
extends
BasicModel
>
targetClass
,
Class
<?
extends
BasicModel
>
indexClass
,
String
aggregatedDateField
,
String
groupingByField
,
BasicModelFilter
filter
);
List
<
Object
[]>
aggregateDate
(
int
size
,
Class
<?
extends
BasicModel
>
targetClass
,
Class
<?
extends
BasicModel
>
indexClass
,
String
aggregatedDateField
,
String
groupingByField
,
BasicModelFilter
<?,
?>
filter
);
public
static
class
TermResult
{
...
...
@@ -134,17 +134,18 @@ public interface ElasticsearchService {
private
Long
total
;
private
long
other
;
/**
* @deprecated This number is useless here and needs to be calculated based matches - total - other
*/
private
Long
missing
;
public
TermResult
(
String
name
,
L
ist
<
Term
>
terms
,
long
other
,
Long
missing
)
{
public
TermResult
(
String
name
,
L
ong
total
,
List
<
Term
>
terms
,
long
other
)
{
this
.
terms
=
terms
;
this
.
total
=
t
erms
.
stream
().
map
(
Term:
:
getCount
).
mapToLong
(
Long:
:
longValue
).
sum
()
;
this
.
total
=
t
otal
;
this
.
other
=
other
;
this
.
missing
=
missing
;
this
.
missing
=
total
-
terms
.
stream
().
map
(
Term:
:
getCount
).
mapToLong
(
Long:
:
longValue
).
sum
()
-
other
;
if
(
missing
<=
0
)
{
// We have some terms (storage) where the total is not the total number of all
// storage options, but count of accessions
missing
=
null
;
}
}
public
List
<
Term
>
getTerms
()
{
...
...
@@ -188,5 +189,5 @@ public interface ElasticsearchService {
}
}
<
T
extends
BasicModel
>
List
<
T
>
find
(
Class
<
T
>
clazz
,
BasicModelFilter
filter
);
<
T
extends
BasicModel
>
List
<
T
>
find
(
Class
<
T
>
clazz
,
BasicModelFilter
<?,
?>
filter
);
}
src/main/java/org/genesys2/server/service/impl/ElasticsearchServiceImpl.java
View file @
4ee8ce1e
...
...
@@ -249,7 +249,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
@Override
public
<
T
extends
BasicModel
>
void
reindex
(
Class
<
T
>
clazz
,
BasicModelFilter
filter
)
{
public
<
T
extends
BasicModel
>
void
reindex
(
Class
<
T
>
clazz
,
BasicModelFilter
<?,
?>
filter
)
{
scanDatabase
(
clazz
,
filter
);
}
...
...
@@ -275,7 +275,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return
indexName
;
}
private
<
R
>
void
scanDatabase
(
Class
<
R
>
clazz
,
BasicModelFilter
filter
)
{
private
<
R
>
void
scanDatabase
(
Class
<
R
>
clazz
,
BasicModelFilter
<?,
?>
filter
)
{
PathBuilder
<
R
>
builder
=
new
PathBuilderFactory
().
create
(
clazz
);
Querydsl
querydsl
=
new
Querydsl
(
em
,
builder
);
...
...
@@ -646,17 +646,18 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
@Override
public
TermResult
termStatistics
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
term
)
throws
SearchException
{
public
TermResult
termStatistics
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
term
)
throws
SearchException
{
return
termStatistics
(
clazz
,
filters
,
size
,
new
String
[]
{
term
}).
get
(
term
);
}
@Override
public
Map
<
String
,
TermResult
>
termStatistics
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
...
terms
)
throws
SearchException
{
public
Map
<
String
,
TermResult
>
termStatistics
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
...
terms
)
throws
SearchException
{
if
(!
indexedEntities
.
contains
(
clazz
))
{
throw
new
RuntimeException
(
"Class is not indexed "
+
clazz
);
}
Long
total
=
count
(
clazz
,
filters
);
String
indexName
=
toIndexName
(
clazz
)
+
INDEX_READ
;
SearchRequestBuilder
esQuery
=
client
.
prepareSearch
(
indexName
).
setTypes
(
COMMON_TYPE_NAME
).
setQuery
(
toEsQuery
(
clazz
,
filters
));
for
(
String
term
:
terms
)
{
...
...
@@ -681,8 +682,8 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
InternalTerms
<?,
?>
topCounts
=
(
InternalTerms
<?,
?>)
agg
;
List
<
Bucket
>
buckets
=
topCounts
.
getBuckets
();
TermResult
tr
=
new
TermResult
(
term
,
buckets
.
stream
().
map
(
bucket
->
new
Term
(
bucket
.
getKeyAsString
(),
bucket
.
getDocCount
())).
collect
(
Collectors
.
toList
()),
topCounts
.
getSumOfOtherDocCounts
()
,
topCounts
.
getDocCountError
()
);
TermResult
tr
=
new
TermResult
(
term
,
total
,
buckets
.
stream
().
map
(
bucket
->
new
Term
(
bucket
.
getKeyAsString
(),
bucket
.
getDocCount
())).
collect
(
Collectors
.
toList
()),
topCounts
.
getSumOfOtherDocCounts
());
termResults
.
put
(
term
,
tr
);
}
else
if
(
agg
instanceof
UnmappedTerms
)
{
...
...
@@ -697,7 +698,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return
termResults
;
}
private
QueryBuilder
toEsQuery
(
Class
<?>
clazz
,
BasicModelFilter
filters
)
{
private
QueryBuilder
toEsQuery
(
Class
<?>
clazz
,
BasicModelFilter
<?,
?>
filters
)
{
ElasticQueryBuilder
esQb
=
new
ElasticQueryBuilder
();
if
(
filters
!=
null
)
{
filters
.
buildQuery
().
accept
(
esQb
,
null
);
...
...
@@ -716,12 +717,12 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
* than 10%
*/
@Override
public
TermResult
termStatisticsAuto
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
term
)
throws
SearchException
{
public
TermResult
termStatisticsAuto
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
term
)
throws
SearchException
{
return
termStatisticsAuto
(
clazz
,
filters
,
size
,
new
String
[]
{
term
}).
get
(
term
);
}
@Override
public
Map
<
String
,
TermResult
>
termStatisticsAuto
(
Class
<?
>
clazz
,
BasicModelFilter
filters
,
int
size
,
String
...
terms
)
throws
SearchException
{
public
Map
<
String
,
TermResult
>
termStatisticsAuto
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filters
,
int
size
,
String
...
terms
)
throws
SearchException
{
Map
<
String
,
TermResult
>
termResult
=
termStatistics
(
clazz
,
filters
,
size
,
terms
);
...
...
@@ -731,7 +732,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
@Override
public
long
count
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
filter
)
{
public
long
count
(
Class
<?
extends
BasicModel
>
clazz
,
BasicModelFilter
<?,
?>
filter
)
{
String
indexName
=
toIndexName
(
clazz
)
+
INDEX_READ
;
SearchResponse
hits
=
client
.
prepareSearch
(
indexName
).
setTypes
(
COMMON_TYPE_NAME
).
setQuery
(
toEsQuery
(
clazz
,
filter
)).
setSize
(
0
).
get
();
return
hits
.
getHits
().
getTotalHits
();
...
...
@@ -740,7 +741,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
@Override
@Cacheable
(
value
=
"statistics"
,
unless
=
"#result == null"
,
key
=
"'stats.' + #root.methodName + '-' + #size + '-' + #targetClass.name + '-' + #indexClass.name"
)
public
List
<
Object
[]>
aggregateDate
(
final
int
size
,
final
Class
<?
extends
BasicModel
>
targetClass
,
final
Class
<?
extends
BasicModel
>
indexClass
,
final
String
aggregatedDateField
,
final
String
groupingByField
,
final
BasicModelFilter
filter
)
{
final
String
aggregatedDateField
,
final
String
groupingByField
,
final
BasicModelFilter
<?,
?>
filter
)
{
final
String
DATE_HISTOGRAM_AGG
=
"updates_over_time"
;
final
String
TERMS_AGG
=
"institute_terms"
;
final
String
TOP_HITS_AGG
=
"top_tag_hits"
;
...
...
@@ -791,7 +792,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
}
@Override
public
<
T
extends
BasicModel
>
List
<
T
>
find
(
Class
<
T
>
clazz
,
BasicModelFilter
filter
)
{
public
<
T
extends
BasicModel
>
List
<
T
>
find
(
Class
<
T
>
clazz
,
BasicModelFilter
<?,
?>
filter
)
{
String
indexName
=
toIndexName
(
clazz
)
+
INDEX_READ
;
List
<
T
>
results
=
new
ArrayList
<>();
client
.
prepareSearch
(
indexName
).
setTypes
(
COMMON_TYPE_NAME
).
setQuery
(
toEsQuery
(
clazz
,
filter
)).
get
()
...
...
src/main/java/org/genesys2/spring/config/ElasticsearchConfig.java
View file @
4ee8ce1e
...
...
@@ -179,8 +179,8 @@ public class ElasticsearchConfig {
*
* @return the elastic reindex processor
*/
@Bean
public
ElasticReindexProcessor
elasticReindex
e
r
()
{
@Bean
()
public
ElasticReindexProcessor
elasticReindex
Processo
r
()
{
return
new
ElasticReindexProcessor
();
}
...
...
src/test/java/org/genesys/test/server/services/ElasticQueryBuilderTest.java
View file @
4ee8ce1e
...
...
@@ -187,7 +187,7 @@ public class ElasticQueryBuilderTest extends AbstractElasticServiceTest {
terms
=
elasticService
.
termStatisticsAuto
(
Accession
.
class
,
af
,
10
,
"storage"
);
LOG
.
debug
(
"Storage {}"
,
objectMapper
.
writeValueAsString
(
terms
));
assertThat
(
terms
.
getTotal
(),
is
(
3
l
));
assertThat
(
terms
.
getTotal
(),
is
(
2
l
));
terms
.
getTerms
().
forEach
(
t
->
{
if
(
t
.
getTerm
().
equals
(
Integer
.
toString
(
MCPD_STORAGE_FIELD
)))
{
assertThat
(
t
.
getCount
(),
is
(
2
l
));
...
...
@@ -212,7 +212,7 @@ public class ElasticQueryBuilderTest extends AbstractElasticServiceTest {
assertThat
(
terms
.
getTerms
().
get
(
0
).
getTerm
(),
is
(
"INS001"
));
assertThat
(
terms
.
getTerms
().
get
(
0
).
getCount
(),
is
(
2
l
));
terms
=
stats
.
get
(
"storage"
);
assertThat
(
terms
.
getTotal
(),
is
(
3
l
));
assertThat
(
terms
.
getTotal
(),
is
(
2
l
));
terms
.
getTerms
().
forEach
(
t
->
{
if
(
t
.
getTerm
().
equals
(
Integer
.
toString
(
MCPD_STORAGE_FIELD
)))
{
assertThat
(
t
.
getCount
(),
is
(
1
l
));
...
...
@@ -226,7 +226,7 @@ public class ElasticQueryBuilderTest extends AbstractElasticServiceTest {
af
.
storage
=
newHashSet
(
MCPD_STORAGE_FIELD
);
stats
=
elasticService
.
termStatistics
(
Accession
.
class
,
af
,
10
,
"institute.code"
,
"storage"
);
terms
=
stats
.
get
(
"storage"
);
assertThat
(
terms
.
getTotal
(),
is
(
5
l
));
assertThat
(
terms
.
getTotal
(),
is
(
3
l
));
terms
.
getTerms
().
forEach
(
t
->
{
if
(
t
.
getTerm
().
equals
(
Integer
.
toString
(
MCPD_STORAGE_FIELD
)))
{
assertThat
(
t
.
getCount
(),
is
(
3
l
));
...
...
@@ -239,7 +239,7 @@ public class ElasticQueryBuilderTest extends AbstractElasticServiceTest {
af
.
storage
=
newHashSet
(
MCPD_STORAGE_INVITRO
,
MCPD_STORAGE_CRYO
);
terms
=
elasticService
.
termStatistics
(
Accession
.
class
,
af
,
10
,
"storage"
);
assertThat
(
terms
.
getTotal
(),
is
(
5
l
));
assertThat
(
terms
.
getTotal
(),
is
(
3
l
));
terms
.
getTerms
().
forEach
(
t
->
{
if
(
t
.
getTerm
().
equals
(
Integer
.
toString
(
MCPD_STORAGE_FIELD
)))
{
assertThat
(
t
.
getCount
(),
is
(
2
l
));
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment