Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Genesys Backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
45
Issues
45
List
Boards
Labels
Service Desk
Milestones
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Genesys Backend
Commits
5e6aa0f7
Commit
5e6aa0f7
authored
May 02, 2016
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Cleanup
parent
fad5cfa1
Changes
6
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
322 additions
and
224 deletions
+322
-224
src/main/java/org/genesys2/server/model/elastic/FullTextDocument.java
...a/org/genesys2/server/model/elastic/FullTextDocument.java
+42
-27
src/main/java/org/genesys2/server/service/FullTextSearchService.java
...va/org/genesys2/server/service/FullTextSearchService.java
+18
-3
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
...2/server/service/impl/ElasticsearchSearchServiceImpl.java
+13
-12
src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java
...nesys2/server/service/impl/FullTextSearchServiceImpl.java
+196
-168
src/main/java/org/genesys2/server/servlet/controller/admin/ElasticSearchController.java
...ver/servlet/controller/admin/ElasticSearchController.java
+33
-2
src/main/webapp/WEB-INF/jsp/admin/elastic/index.jsp
src/main/webapp/WEB-INF/jsp/admin/elastic/index.jsp
+20
-12
No files found.
src/main/java/org/genesys2/server/model/elastic/FullTextDocument.java
View file @
5e6aa0f7
/**
* Copyright 2016 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package
org.genesys2.server.model.elastic
;
import
com.eclipsesource.json.JsonObject
;
import
com.eclipsesource.json.JsonValue
;
import
com.eclipsesource.json.ParseException
;
import
org.apache.log4j.Logger
;
import
org.genesys2.server.model.impl.*
;
import
java.util.Date
;
import
java.util.Locale
;
import
org.genesys2.server.model.impl.ClassPK
;
import
org.springframework.data.annotation.Id
;
import
org.springframework.data.elasticsearch.annotations.Document
;
import
org.springframework.data.elasticsearch.annotations.Field
;
import
org.springframework.data.elasticsearch.annotations.FieldIndex
;
import
org.springframework.data.elasticsearch.annotations.FieldType
;
import
java.util.Date
;
import
java.util.Locale
;
import
com.eclipsesource.json.JsonObject
;
import
com.eclipsesource.json.JsonValue
;
import
com.eclipsesource.json.ParseException
;
/**
* <code>FullTextDocument</code> is used in Elasticsearch mapping
*/
@Document
(
indexName
=
"fulltext"
,
refreshInterval
=
"60s"
)
public
class
FullTextDocument
{
private
static
final
Logger
LOG
=
Logger
.
getLogger
(
FullTextDocument
.
class
);
// FIXME Revisit!
private
static
final
String
ENGLISH_LOCALE
=
"en"
;
@Id
...
...
@@ -47,81 +62,81 @@ public class FullTextDocument {
private
Date
lastModifiedDate
;
public
Long
getId
()
{
return
id
;
return
this
.
id
;
}
public
void
setId
(
Long
id
)
{
public
void
setId
(
final
Long
id
)
{
this
.
id
=
id
;
}
public
String
getUrlToContent
()
{
return
urlToContent
;
return
this
.
urlToContent
;
}
public
void
setUrlToContent
(
String
urlToContent
)
{
public
void
setUrlToContent
(
final
String
urlToContent
)
{
this
.
urlToContent
=
urlToContent
;
}
public
ClassPK
getClassPK
()
{
return
classPK
;
return
this
.
classPK
;
}
public
void
setClassPK
(
ClassPK
classPK
)
{
public
void
setClassPK
(
final
ClassPK
classPK
)
{
this
.
classPK
=
classPK
;
}
public
String
getBody
()
{
return
body
;
return
this
.
body
;
}
public
void
setBody
(
String
body
)
{
public
void
setBody
(
final
String
body
)
{
this
.
body
=
body
;
}
public
String
getSummary
()
{
return
summary
;
return
this
.
summary
;
}
public
void
setSummary
(
String
summary
)
{
public
void
setSummary
(
final
String
summary
)
{
this
.
summary
=
summary
;
}
public
String
getTextByLocale
(
Locale
locale
)
{
public
String
getTextByLocale
(
final
Locale
locale
)
{
try
{
JsonObject
json
=
JsonObject
.
readFrom
(
body
);
final
JsonObject
json
=
JsonObject
.
readFrom
(
this
.
body
);
JsonValue
value
=
json
.
get
(
locale
.
getLanguage
());
if
(
value
==
null
)
{
value
=
json
.
get
(
ENGLISH_LOCALE
);
}
return
value
.
toString
().
replace
(
"\""
,
""
);
}
catch
(
ParseException
ignored
)
{
}
catch
(
final
ParseException
ignored
)
{
}
return
getBody
();
}
public
String
getLanguage
()
{
return
language
;
return
this
.
language
;
}
public
void
setLanguage
(
String
language
)
{
public
void
setLanguage
(
final
String
language
)
{
this
.
language
=
language
;
}
public
Date
getCreatedDate
()
{
return
createdDate
;
return
this
.
createdDate
;
}
public
void
setCreatedDate
(
Date
createdDate
)
{
public
void
setCreatedDate
(
final
Date
createdDate
)
{
this
.
createdDate
=
createdDate
;
}
public
Date
getLastModifiedDate
()
{
return
lastModifiedDate
;
return
this
.
lastModifiedDate
;
}
public
void
setLastModifiedDate
(
Date
lastModifiedDate
)
{
public
void
setLastModifiedDate
(
final
Date
lastModifiedDate
)
{
this
.
lastModifiedDate
=
lastModifiedDate
;
}
}
src/main/java/org/genesys2/server/service/FullTextSearchService.java
View file @
5e6aa0f7
/**
* Copyright 2016 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package
org.genesys2.server.service
;
import
java.util.Collection
;
import
java.util.List
;
import
org.genesys2.server.model.elastic.FullTextDocument
;
import
org.genesys2.server.service.impl.SearchException
;
import
org.springframework.data.domain.Page
;
import
org.springframework.data.domain.Pageable
;
import
org.springframework.data.elasticsearch.ElasticsearchException
;
import
java.util.Collection
;
import
java.util.List
;
public
interface
FullTextSearchService
{
Page
<?>
search
(
String
query
,
Pageable
pageable
,
String
section
)
throws
SearchException
;
...
...
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @
5e6aa0f7
...
...
@@ -74,8 +74,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
private
static
final
Log
LOG
=
LogFactory
.
getLog
(
ElasticsearchSearchServiceImpl
.
class
);
p
rivate
static
final
String
INDEXALIAS_PASSPORT_READ
=
"passport"
;
p
ublic
static
final
String
INDEXALIAS_PASSPORT_READ
=
"passport"
;
private
static
final
String
INDEXALIAS_PASSPORT_WRITE
=
"passportWrite"
;
public
static
final
String
PASSPORT_TYPE
=
"mcpd"
;
@Autowired
private
Client
client
;
...
...
@@ -108,7 +109,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public
Page
<
AccessionDetails
>
search
(
String
query
,
Pageable
pageable
)
throws
SearchException
{
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
"mcpd"
)
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
PASSPORT_TYPE
)
.
withQuery
(
org
.
elasticsearch
.
index
.
query
.
QueryBuilders
.
queryString
(
query
).
defaultOperator
(
Operator
.
AND
)).
withPageable
(
pageable
).
build
();
try
{
...
...
@@ -121,7 +122,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public
List
<
String
>
autocompleteSearch
(
String
query
)
throws
SearchException
{
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
"mcpd"
)
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
PASSPORT_TYPE
)
.
withQuery
(
org
.
elasticsearch
.
index
.
query
.
QueryBuilders
.
queryString
(
"acceNumb:("
+
query
+
"*)"
).
defaultOperator
(
Operator
.
AND
))
.
withSort
(
SortBuilders
.
fieldSort
(
FilterConstants
.
ACCENUMB
).
order
(
SortOrder
.
ASC
)).
withPageable
(
new
PageRequest
(
0
,
10
)).
build
();
...
...
@@ -143,7 +144,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
AndFilterBuilder
filterBuilder
=
getFilterBuilder
(
appliedFilters
);
SortBuilder
sortBuilder
=
SortBuilders
.
fieldSort
(
FilterConstants
.
ACCENUMB
).
order
(
SortOrder
.
ASC
);
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
"mcpd"
)
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
PASSPORT_TYPE
)
.
withFilter
(
filterBuilder
).
withSort
(
sortBuilder
).
withPageable
(
pageable
).
build
();
try
{
...
...
@@ -167,7 +168,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
else
{
termFacetRequest
=
new
TermFacetRequestBuilder
(
"f"
).
applyQueryFilter
().
fields
(
term
).
size
(
size
).
build
();
}
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
"mcpd"
)
SearchQuery
searchQuery
=
new
NativeSearchQueryBuilder
().
withIndices
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
).
withTypes
(
PASSPORT_TYPE
)
.
withFilter
(
filterBuilder
).
withFacet
(
termFacetRequest
).
build
();
try
{
...
...
@@ -348,7 +349,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
IndexQuery
iq
=
new
IndexQuery
();
iq
.
setIndexName
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
);
iq
.
setType
(
"mcpd"
);
iq
.
setType
(
PASSPORT_TYPE
);
iq
.
setId
(
String
.
valueOf
(
ad
.
getId
()));
iq
.
setObject
(
ad
);
...
...
@@ -373,7 +374,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG
.
info
(
"Removing from index "
+
clazz2
+
" id="
+
id
);
if
(
clazz2
.
equals
(
AccessionDetails
.
class
))
{
elasticsearchTemplate
.
delete
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
,
"mcpd"
,
String
.
valueOf
(
id
));
elasticsearchTemplate
.
delete
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
,
PASSPORT_TYPE
,
String
.
valueOf
(
id
));
}
else
{
// Default
elasticsearchTemplate
.
delete
(
clazz2
,
String
.
valueOf
(
id
));
...
...
@@ -411,7 +412,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG
.
info
(
"Initializing genesysarchive"
);
elasticsearchTemplate
.
createIndex
(
"genesysarchive"
);
LOG
.
info
(
"Copying mapping to genesysarchive"
);
elasticsearchTemplate
.
putMapping
(
"genesysarchive"
,
"mcpd"
,
indexMapping
);
elasticsearchTemplate
.
putMapping
(
"genesysarchive"
,
PASSPORT_TYPE
,
indexMapping
);
}
if
(!
aliasExists
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_READ
))
{
...
...
@@ -525,7 +526,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
}
}
/**
* Delete an index
*/
...
...
@@ -540,7 +541,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG
.
info
(
"Creating index "
+
indexName
);
elasticsearchTemplate
.
createIndex
(
indexName
,
settings
);
LOG
.
info
(
"Copying mapping to genesysarchive"
);
elasticsearchTemplate
.
putMapping
(
indexName
,
"mcpd"
,
indexMapping
);
elasticsearchTemplate
.
putMapping
(
indexName
,
PASSPORT_TYPE
,
indexMapping
);
}
@Override
...
...
@@ -588,10 +589,10 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
try
{
IndexRequest
indexRequest
=
new
IndexRequest
();
indexRequest
.
index
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
).
type
(
"mcpd"
);
indexRequest
.
index
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
).
type
(
PASSPORT_TYPE
);
indexRequest
.
source
(
"seqNo"
,
o
[
2
]);
UpdateQuery
updateQuery
=
new
UpdateQueryBuilder
().
withClass
(
AccessionDetails
.
class
)
.
withIndexName
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
).
withType
(
"mcpd"
).
withId
(
o
[
0
].
toString
())
.
withIndexName
(
ElasticsearchSearchServiceImpl
.
INDEXALIAS_PASSPORT_WRITE
).
withType
(
PASSPORT_TYPE
).
withId
(
o
[
0
].
toString
())
.
withIndexRequest
(
indexRequest
).
build
();
queries
.
add
(
updateQuery
);
// LOG.debug("ES added seqNo to " + o[0].toString());
...
...
src/main/java/org/genesys2/server/service/impl/FullTextSearchServiceImpl.java
View file @
5e6aa0f7
This diff is collapsed.
Click to expand it.
src/main/java/org/genesys2/server/servlet/controller/admin/ElasticSearchController.java
View file @
5e6aa0f7
...
...
@@ -101,9 +101,19 @@ public class ElasticSearchController {
*
* @return
*/
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"regenerate"
})
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"regenerate
=accn
"
})
public
String
regenerateElastic
()
{
elasticService
.
regenerateIndexes
();
return
"redirect:/admin/elastic/"
;
}
/**
* Completely recreate Elasticsearch indexes: create, index, re-alias.
*
* @return
*/
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"regenerate=content"
})
public
String
regenerateElasticContent
()
{
fullTextSearchService
.
regenerateIndexes
();
return
"redirect:/admin/elastic/"
;
}
...
...
@@ -115,7 +125,7 @@ public class ElasticSearchController {
* @param jsonFilter
* @throws IOException
*/
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"reindex"
,
"filter"
})
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"reindex
=accn
"
,
"filter"
})
public
String
reindexElasticFiltered
(
@RequestParam
(
value
=
"filter"
,
required
=
true
)
String
jsonFilter
,
@RequestParam
(
value
=
"type"
,
required
=
true
)
String
type
)
throws
IOException
{
...
...
@@ -133,6 +143,27 @@ public class ElasticSearchController {
return
"redirect:/admin/elastic/"
;
}
/**
* This method refreshes data in the currently active index. It is very
* handy when having to refresh part of ES after direct database update.
*
* @param jsonFilter
* @throws IOException
*/
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/action"
,
params
=
{
"reindex=content"
,
"type"
})
public
String
reindexElasticContent
(
@RequestParam
(
value
=
"type"
,
required
=
true
)
String
type
)
throws
IOException
{
if
(
type
.
equals
(
"All"
))
{
fullTextSearchService
.
regenerateIndexes
();
}
else
{
fullTextSearchService
.
reindex
(
type
);
}
return
"redirect:/admin/elastic/"
;
}
/**
* Clear ES queue
*
...
...
src/main/webapp/WEB-INF/jsp/admin/elastic/index.jsp
View file @
5e6aa0f7
...
...
@@ -9,25 +9,33 @@
<body>
<%@ include
file=
"/WEB-INF/jsp/admin/menu.jsp"
%>
<h3>
Accession data
</h3>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/elastic/action"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<input
type=
"text"
name=
"filter"
placeholder=
"Genesys filter {}"
value=
"{}"
/>
<select
name=
"type"
>
<option
value=
"All"
selected
>
All entities
</option>
<c:forEach
items=
"
${
reindexTypes
}
"
var=
"type"
>
<option
value=
"${type.value}"
>
${type.key}
</option>
</c:forEach>
</select>
<button
type=
"submit"
class=
"btn btn-default"
name=
"reindex"
>
Reindex
</button>
<button
type=
"submit"
class=
"btn btn-default"
name=
"regenerate"
>
Regenerate
</button>
<button
type=
"submit"
class=
"btn btn-default"
value=
"accn"
name=
"reindex"
>
Reindex
</button>
<button
type=
"submit"
class=
"btn btn-default"
value=
"accn"
name=
"regenerate"
>
Regenerate
</button>
</form>
<h3>
Content
</h3>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/elastic/action"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<select
name=
"type"
>
<option
value=
"All"
selected
>
All content entities
</option>
<c:forEach
items=
"
${
reindexTypes
}
"
var=
"type"
>
<option
value=
"${type.value}"
>
${type.key}
</option>
</c:forEach>
</select>
<button
type=
"submit"
class=
"btn btn-default"
value=
"content"
name=
"reindex"
>
Reindex
</button>
<button
type=
"submit"
class=
"btn btn-default"
value=
"content"
name=
"regenerate"
>
Regenerate
</button>
</form>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/elastic/action"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<button
type=
"submit"
class=
"btn btn-default"
name=
"clear-queues"
>
Clear ES update queues
</button>
</form>
<h3>
Index alias management
</h3>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/elastic/action"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
Alias:
<input
type=
"text"
name=
"aliasName"
/>
...
...
@@ -37,7 +45,7 @@
<button
type=
"submit"
class=
"btn btn-default"
value=
"delete-index"
name=
"action"
>
Delete index
</button>
</form>
<h3>
Indexes and
their
aliases
</h3>
<h3>
Indexes and aliases
</h3>
<c:if
test=
"
${
empty
indexes
}
"
>
<h5>
No indexes found.
</h5>
</c:if>
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment