Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Genesys Backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
45
Issues
45
List
Boards
Labels
Service Desk
Milestones
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Genesys Backend
Commits
98e0f20f
Commit
98e0f20f
authored
Aug 28, 2014
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Filtering with elasticsearch
parent
5dd0a047
Changes
8
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
463 additions
and
14 deletions
+463
-14
src/main/java/org/genesys2/server/service/ContentService.java
...main/java/org/genesys2/server/service/ContentService.java
+3
-0
src/main/java/org/genesys2/server/service/ElasticService.java
...main/java/org/genesys2/server/service/ElasticService.java
+2
-0
src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java
.../org/genesys2/server/service/impl/ContentServiceImpl.java
+18
-2
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
...2/server/service/impl/ElasticsearchSearchServiceImpl.java
+112
-8
src/main/java/org/genesys2/server/servlet/controller/rdf/AccessionControllerRdf.java
...server/servlet/controller/rdf/AccessionControllerRdf.java
+20
-0
src/main/webapp/WEB-INF/jsp/accession/details-turtle.jsp
src/main/webapp/WEB-INF/jsp/accession/details-turtle.jsp
+1
-1
src/test/java/org/genesys2/server/service/impl/ElasticsearchTest.java
...a/org/genesys2/server/service/impl/ElasticsearchTest.java
+301
-3
src/test/resources/spring/spring.properties
src/test/resources/spring/spring.properties
+6
-0
No files found.
src/main/java/org/genesys2/server/service/ContentService.java
View file @
98e0f20f
...
...
@@ -16,6 +16,7 @@
package
org.genesys2.server.service
;
import
java.io.Writer
;
import
java.util.List
;
import
java.util.Locale
;
import
java.util.Map
;
...
...
@@ -102,4 +103,6 @@ public interface ContentService {
Locale
getDefaultLocale
();
String
processTemplate
(
String
body
,
Map
<
String
,
Object
>
root
);
void
processTemplate
(
String
templateStr
,
Map
<
String
,
Object
>
root
,
Writer
writer
);
}
src/main/java/org/genesys2/server/service/ElasticService.java
View file @
98e0f20f
...
...
@@ -37,4 +37,6 @@ public interface ElasticService {
void
refreshIndex
(
String
className
);
Page
<
AccessionDetails
>
filter
(
String
jsonFilter
,
Pageable
pageable
)
throws
SearchException
;
}
src/main/java/org/genesys2/server/service/impl/ContentServiceImpl.java
View file @
98e0f20f
...
...
@@ -17,6 +17,7 @@
package
org.genesys2.server.service.impl
;
import
java.io.StringWriter
;
import
java.io.Writer
;
import
java.util.Calendar
;
import
java.util.List
;
import
java.util.Locale
;
...
...
@@ -169,7 +170,7 @@ public class ContentServiceImpl implements ContentService {
/**
* Creates or updates an article
*
*
* @param entity
* @param body
* @param locale
...
...
@@ -289,7 +290,22 @@ public class ContentServiceImpl implements ContentService {
* Merge data and template
*/
velocityEngine
.
evaluate
(
context
,
swOut
,
"log tag name"
,
templateStr
);
System
.
out
.
println
(
swOut
);
if
(
LOG
.
isTraceEnabled
())
{
LOG
.
trace
(
swOut
);
}
return
swOut
.
toString
();
}
@Override
public
void
processTemplate
(
String
templateStr
,
Map
<
String
,
Object
>
root
,
Writer
writer
)
{
final
VelocityContext
context
=
new
VelocityContext
();
for
(
final
String
key
:
root
.
keySet
())
{
context
.
put
(
key
,
root
.
get
(
key
));
}
/**
* Merge data and template
*/
velocityEngine
.
evaluate
(
context
,
writer
,
"log tag name"
,
templateStr
);
}
}
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @
98e0f20f
...
...
@@ -4,17 +4,23 @@ import java.io.IOException;
import
java.util.ArrayList
;
import
java.util.Collection
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Set
;
import
org.apache.commons.lang3.StringUtils
;
import
org.elasticsearch.index.query.AndFilterBuilder
;
import
org.elasticsearch.index.query.FilterBuilders
;
import
org.elasticsearch.index.query.OrFilterBuilder
;
import
org.elasticsearch.index.query.QueryBuilder
;
import
org.elasticsearch.index.query.QueryBuilders
;
import
org.genesys2.server.model.elastic.AccessionDetails
;
import
org.genesys2.server.model.genesys.Accession
;
import
org.genesys2.server.service.CropService
;
import
org.genesys2.server.service.ElasticService
;
import
org.genesys2.server.service.FilterConstants
;
import
org.genesys2.server.service.GenesysFilterService
;
import
org.genesys2.server.service.GenesysService
;
import
org.genesys2.server.service.OrganizationService
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.InitializingBean
;
...
...
@@ -45,15 +51,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired
private
GenesysFilterService
filterService
;
@Autowired
private
OrganizationService
organizationService
;
@Autowired
private
ObjectMapper
objectMapper
;
@Autowired
private
CropService
cropService
;
private
final
Map
<
String
,
Class
<?>>
clazzMap
;
public
ElasticsearchSearchServiceImpl
()
{
...
...
@@ -74,6 +74,107 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
}
@Override
public
Page
<
AccessionDetails
>
filter
(
String
jsonFilter
,
Pageable
pageable
)
throws
SearchException
{
Map
<
String
,
List
<
String
>>
filters
;
try
{
filters
=
objectMapper
.
readValue
(
jsonFilter
,
Map
.
class
);
}
catch
(
IOException
e
)
{
throw
new
SearchException
(
e
.
getMessage
(),
e
);
}
AndFilterBuilder
filterBuilder
=
FilterBuilders
.
andFilter
();
for
(
String
key
:
filters
.
keySet
())
{
List
<
String
>
filterValues
=
filters
.
get
(
key
);
if
(
filterValues
==
null
||
filterValues
.
isEmpty
())
continue
;
System
.
err
.
println
(
"key="
+
key
);
if
(
FilterConstants
.
ACCENUMB
.
equals
(
key
)
||
FilterConstants
.
COLLMISSID
.
equals
(
key
))
{
// indexed field
filterBuilder
.
add
(
FilterBuilders
.
queryFilter
(
makeQuery
(
key
,
filterValues
)));
}
else
if
(
FilterConstants
.
ALIAS
.
equals
(
key
))
{
// Nested object
filterBuilder
.
add
(
FilterBuilders
.
nestedFilter
(
"aliases"
,
makeQuery
(
"aliases.name"
,
filterValues
)));
}
else
if
(
FilterConstants
.
SGSV
.
equals
(
key
))
{
// Check for existance nested object
OrFilterBuilder
orFilter
=
FilterBuilders
.
orFilter
();
if
(
filterValues
.
contains
(
true
))
{
// true:
orFilter
.
add
(
FilterBuilders
.
notFilter
(
FilterBuilders
.
missingFilter
(
FilterConstants
.
SGSV
)));
}
if
(
filterValues
.
contains
(
false
))
{
// false:
orFilter
.
add
(
FilterBuilders
.
missingFilter
(
FilterConstants
.
SGSV
));
}
if
(
filterValues
.
contains
(
true
)
||
filterValues
.
contains
(
false
))
filterBuilder
.
add
(
orFilter
);
}
else
if
(
FilterConstants
.
GEO_ELEVATION
.
equals
(
key
)
||
FilterConstants
.
GEO_LONGITUDE
.
equals
(
key
)
||
FilterConstants
.
GEO_LATITUDE
.
equals
(
key
))
{
OrFilterBuilder
orFilter
=
FilterBuilders
.
orFilter
();
// range filters
for
(
Object
val
:
filterValues
)
{
System
.
err
.
println
(
val
.
getClass
()
+
" "
+
val
);
Map
<
String
,
Object
>
v
=
(
Map
<
String
,
Object
>)
val
;
List
<
Number
>
range
=
(
List
<
Number
>)
v
.
get
(
"range"
);
Number
max
=
(
Number
)
v
.
get
(
"max"
);
Number
min
=
(
Number
)
v
.
get
(
"min"
);
if
(
range
!=
null
)
{
System
.
err
.
println
(
"Range "
+
range
.
getClass
()
+
" "
+
range
);
orFilter
.
add
(
FilterBuilders
.
rangeFilter
(
key
).
from
(
range
.
get
(
0
)).
to
(
range
.
get
(
1
)));
}
else
if
(
max
!=
null
)
{
System
.
err
.
println
(
"Max "
+
max
);
orFilter
.
add
(
FilterBuilders
.
rangeFilter
(
key
).
to
(
max
));
}
else
if
(
min
!=
null
)
{
System
.
err
.
println
(
"Min "
+
min
);
orFilter
.
add
(
FilterBuilders
.
rangeFilter
(
key
).
from
(
min
));
}
}
filterBuilder
.
add
(
orFilter
);
}
else
{
Set
<
Object
>
nonNull
=
new
HashSet
<
Object
>(
filters
.
size
());
boolean
hasNull
=
false
;
for
(
Object
val
:
filterValues
)
{
if
(
val
!=
null
)
{
nonNull
.
add
(
val
);
}
else
{
hasNull
=
true
;
}
}
if
(
hasNull
)
{
filterBuilder
.
add
(
FilterBuilders
.
orFilter
(
FilterBuilders
.
missingFilter
(
key
),
FilterBuilders
.
termsFilter
(
key
,
nonNull
).
execution
(
"or"
)));
}
else
{
filterBuilder
.
add
(
FilterBuilders
.
termsFilter
(
key
,
filterValues
).
execution
(
"or"
));
}
}
}
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withFilter
(
filterBuilder
).
withPageable
(
pageable
).
build
();
// System.err.println("Filter query: " + searchQuery.toString());
try
{
Page
<
AccessionDetails
>
sampleEntities
=
elasticsearchTemplate
.
queryForPage
(
searchQuery
,
AccessionDetails
.
class
);
return
sampleEntities
;
}
catch
(
Throwable
e
)
{
throw
new
SearchException
(
e
.
getMessage
(),
e
);
}
}
private
QueryBuilder
makeQuery
(
String
key
,
List
<
String
>
list
)
{
// Indexed fields
StringBuilder
sb
=
new
StringBuilder
();
for
(
String
val
:
list
)
{
if
(
sb
.
length
()
>
0
)
sb
.
append
(
","
);
sb
.
append
(
"\""
+
val
+
"\""
);
}
return
QueryBuilders
.
queryString
(
key
+
":("
+
sb
.
toString
()
+
")"
);
}
@Override
public
void
update
(
String
className
,
long
id
)
{
if
(!
clazzMap
.
containsKey
(
className
))
{
...
...
@@ -255,6 +356,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate
.
createIndex
(
"genesysarchive"
);
}
Map
indexMapping
=
elasticsearchTemplate
.
getMapping
(
AccessionDetails
.
class
);
for
(
Object
key
:
indexMapping
.
keySet
())
{
System
.
err
.
println
(
"Mapping ke="
+
key
+
" val="
+
indexMapping
.
get
(
key
));
}
LOG
.
info
(
"Copying mapping"
);
elasticsearchTemplate
.
putMapping
(
"genesysarchive"
,
"mcpd"
,
indexMapping
);
}
...
...
src/main/java/org/genesys2/server/servlet/controller/rdf/AccessionControllerRdf.java
View file @
98e0f20f
...
...
@@ -16,6 +16,13 @@
package
org.genesys2.server.servlet.controller.rdf
;
import
java.io.IOException
;
import
java.io.OutputStream
;
import
javax.servlet.http.HttpServletResponse
;
import
org.codehaus.jackson.JsonNode
;
import
org.codehaus.jackson.map.ObjectMapper
;
import
org.genesys2.server.model.genesys.Accession
;
import
org.genesys2.server.service.CropService
;
import
org.genesys2.server.service.GenesysService
;
...
...
@@ -28,6 +35,7 @@ import org.springframework.ui.ModelMap;
import
org.springframework.web.bind.annotation.PathVariable
;
import
org.springframework.web.bind.annotation.RequestMapping
;
import
org.springframework.web.bind.annotation.RequestMethod
;
import
org.springframework.web.bind.annotation.RequestParam
;
@Controller
@RequestMapping
(
value
=
"/acn"
,
headers
=
"accept=text/turtle"
,
produces
=
"text/turtle"
)
...
...
@@ -67,4 +75,16 @@ public class AccessionControllerRdf {
return
"/accession/details-turtle"
;
}
@RequestMapping
(
value
=
"/search"
,
method
=
RequestMethod
.
POST
)
public
void
downloadFiltered
(
@RequestParam
(
value
=
"filter"
,
required
=
false
,
defaultValue
=
"{}"
)
String
jsonFilter
,
HttpServletResponse
response
)
throws
IOException
{
final
OutputStream
outputStream
=
response
.
getOutputStream
();
// TODO
response
.
flushBuffer
();
}
}
src/main/webapp/WEB-INF/jsp/accession/details-turtle.jsp
View file @
98e0f20f
...
...
@@ -13,7 +13,7 @@
<http:
//www.genesys-pgr.org/acn/id/${accession.id}
>
a germplasm:GermplasmAccession;
germplasm:germplasmID "
<c:out
value=
"
${
accession
.
id
}
"
/>
";
dwc:cataogNumber "
<c:out
value=
"
${
accession
.
accessionName
}
"
/>
";
dwc:cata
l
ogNumber "
<c:out
value=
"
${
accession
.
accessionName
}
"
/>
";
dwc:institutionCode "
<c:out
value=
"
${
accession
.
instituteCode
}
"
/>
";
dwc:instituteId "
<c:out
value=
"
${
accession
.
institute
.
id
}
"
/>
";
dwc:countryCode "
<c:out
value=
"
${
accession
.
institute
.
country
.
code3
}
"
/>
";
...
...
src/test/java/org/genesys2/server/service/impl/ElasticsearchTest.java
View file @
98e0f20f
This diff is collapsed.
Click to expand it.
src/test/resources/spring/spring.properties
View file @
98e0f20f
...
...
@@ -79,3 +79,9 @@ hazelcast.aws.access-key=
hazelcast.aws.secret-key
=
hazelcast.aws.region
=
eu-west-1
hazelcast.aws.security-group
=
sg-hazelcast
# TileServer Cache
cache.tileserver.max-size
=
1000
cache.tileserver.time-to-live-seconds
=
300
cache.tileserver.max-idle-seconds
=
0
cache.tileserver.eviction-policy
=
LRU
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment