Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Genesys Backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
45
Issues
45
List
Boards
Labels
Service Desk
Milestones
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Genesys Backend
Commits
c95547e0
Commit
c95547e0
authored
Sep 02, 2014
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ES institute.networks, taxonomy.genusSpecies
Drop index if mapping doesn't match (TODO re-index with scroll)
parent
7c325d1a
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
91 additions
and
47 deletions
+91
-47
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
...a/org/genesys2/server/model/elastic/AccessionDetails.java
+28
-14
src/main/java/org/genesys2/server/model/elastic/Alias.java
src/main/java/org/genesys2/server/model/elastic/Alias.java
+2
-0
src/main/java/org/genesys2/server/model/elastic/Collect.java
src/main/java/org/genesys2/server/model/elastic/Collect.java
+3
-2
src/main/java/org/genesys2/server/model/elastic/Geo.java
src/main/java/org/genesys2/server/model/elastic/Geo.java
+4
-0
src/main/java/org/genesys2/server/model/elastic/Institute.java
...ain/java/org/genesys2/server/model/elastic/Institute.java
+11
-0
src/main/java/org/genesys2/server/model/elastic/Taxonomy.java
...main/java/org/genesys2/server/model/elastic/Taxonomy.java
+12
-3
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
...2/server/service/impl/ElasticsearchSearchServiceImpl.java
+27
-10
src/main/java/org/genesys2/server/service/impl/FilterHandler.java
.../java/org/genesys2/server/service/impl/FilterHandler.java
+2
-3
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
.../org/genesys2/server/service/impl/GenesysServiceImpl.java
+1
-0
src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java
...va/org/genesys2/server/service/worker/ElasticUpdater.java
+1
-1
src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java
...nesys2/server/service/worker/ElasticUpdaterProcessor.java
+0
-14
No files found.
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
View file @
c95547e0
...
...
@@ -3,7 +3,9 @@ package org.genesys2.server.model.elastic;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Date
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Set
;
import
java.util.regex.Pattern
;
import
org.apache.commons.lang.StringUtils
;
...
...
@@ -17,6 +19,7 @@ import org.genesys2.server.model.genesys.AccessionGeo;
import
org.genesys2.server.model.genesys.AccessionRemark
;
import
org.genesys2.server.model.genesys.SvalbardData
;
import
org.genesys2.server.model.impl.Crop
;
import
org.genesys2.server.model.impl.Organization
;
import
org.springframework.data.annotation.Id
;
import
org.springframework.data.annotation.Version
;
import
org.springframework.data.elasticsearch.annotations.Document
;
...
...
@@ -45,9 +48,7 @@ public class AccessionDetails {
private
String
acqSrc
;
private
Boolean
available
;
private
Date
createdDate
;
// FIXME @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
// TODO Also handle ElasticsearchSearchServiceImpl and move to #isAnalyzed()
// section!
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
List
<
String
>
duplSite
;
private
Boolean
mlsStatus
;
private
Boolean
art15
;
...
...
@@ -63,8 +64,7 @@ public class AccessionDetails {
@Field
(
type
=
FieldType
.
Nested
)
private
ArrayList
<
Alias
>
aliases
;
private
Svalbard
sgsv
;
// FIXME TODO Enable inSgsv!
// private boolean inSgsv;
private
boolean
inSgsv
;
@Field
(
type
=
FieldType
.
Nested
,
store
=
false
)
private
ArrayList
<
Remark
>
remarks
;
...
...
@@ -124,8 +124,7 @@ public class AccessionDetails {
public
void
svalbard
(
SvalbardData
svalbardData
)
{
this
.
sgsv
=
Svalbard
.
from
(
svalbardData
);
// FIXME TODO inSgsv
// this.inSgsv = this.sgsv != null;
this
.
inSgsv
=
this
.
sgsv
!=
null
;
}
public
void
remarks
(
List
<
AccessionRemark
>
listAccessionRemarks
)
{
...
...
@@ -394,11 +393,26 @@ public class AccessionDetails {
this
.
art15
=
art15
;
}
// TODO FIXME enable #inSgsv
// public boolean isInSgsv() {
// return inSgsv;
// }
// public void setInSgsv(boolean inSgsv) {
// this.inSgsv = inSgsv;
// }
public
boolean
isInSgsv
()
{
return
inSgsv
;
}
public
void
setInSgsv
(
boolean
inSgsv
)
{
this
.
inSgsv
=
inSgsv
;
}
public
void
networks
(
List
<
Organization
>
organizations
)
{
if
(
organizations
==
null
||
organizations
.
isEmpty
())
return
;
Set
<
String
>
networks
=
new
HashSet
<
String
>();
for
(
Organization
org
:
organizations
)
{
networks
.
add
(
org
.
getSlug
());
}
if
(
institute
==
null
)
institute
=
new
Institute
();
institute
.
setNetworks
(
networks
);
}
}
src/main/java/org/genesys2/server/model/elastic/Alias.java
View file @
c95547e0
...
...
@@ -12,7 +12,9 @@ public class Alias {
private
String
name
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
String
lang
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
String
usedBy
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
AliasType
type
;
public
static
Alias
from
(
AccessionAlias
aa
)
{
...
...
src/main/java/org/genesys2/server/model/elastic/Collect.java
View file @
c95547e0
...
...
@@ -8,17 +8,18 @@ import java.util.Set;
import
org.apache.commons.lang.StringUtils
;
import
org.genesys2.server.model.genesys.AccessionCollect
;
import
org.springframework.data.elasticsearch.annotations.Field
;
import
org.springframework.data.elasticsearch.annotations.FieldIndex
;
import
org.springframework.data.elasticsearch.annotations.FieldType
;
public
class
Collect
{
public
Set
<
String
>
collCode
;
@Field
(
searchAnalyzer
=
"keyword"
,
type
=
FieldType
.
String
)
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
public
String
collDate
;
public
String
collInstAddr
;
public
String
collMissId
;
public
String
collName
;
@Field
(
searchAnalyzer
=
"keyword"
,
type
=
FieldType
.
String
)
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
public
String
collNumb
;
public
String
collSite
;
public
Integer
collSrc
;
...
...
src/main/java/org/genesys2/server/model/elastic/Geo.java
View file @
c95547e0
...
...
@@ -2,6 +2,9 @@ package org.genesys2.server.model.elastic;
import
org.apache.commons.lang.StringUtils
;
import
org.genesys2.server.model.genesys.AccessionGeo
;
import
org.springframework.data.elasticsearch.annotations.Field
;
import
org.springframework.data.elasticsearch.annotations.FieldIndex
;
import
org.springframework.data.elasticsearch.annotations.FieldType
;
public
class
Geo
{
...
...
@@ -11,6 +14,7 @@ public class Geo {
private
Double
longitude
;
private
Double
[]
coordinates
;
private
Double
elevation
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
String
datum
;
public
static
Geo
from
(
AccessionGeo
ag
)
{
...
...
src/main/java/org/genesys2/server/model/elastic/Institute.java
View file @
c95547e0
package
org.genesys2.server.model.elastic
;
import
java.util.Set
;
import
org.genesys2.server.model.impl.FaoInstitute
;
import
org.springframework.data.elasticsearch.annotations.Field
;
import
org.springframework.data.elasticsearch.annotations.FieldIndex
;
...
...
@@ -15,6 +17,8 @@ public class Institute {
private
Country
country
;
@Field
(
type
=
FieldType
.
String
)
private
String
fullName
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
Set
<
String
>
networks
;
public
static
Institute
from
(
FaoInstitute
institute
)
{
if
(
institute
==
null
)
...
...
@@ -59,4 +63,11 @@ public class Institute {
this
.
fullName
=
fullName
;
}
public
Set
<
String
>
getNetworks
()
{
return
networks
;
}
public
void
setNetworks
(
Set
<
String
>
networks
)
{
this
.
networks
=
networks
;
}
}
\ No newline at end of file
src/main/java/org/genesys2/server/model/elastic/Taxonomy.java
View file @
c95547e0
...
...
@@ -13,9 +13,9 @@ public class Taxonomy {
private
String
genus
;
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
String
species
;
//
TODO FIXME
Enable non-indexed access to genus+species
//
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
//
private String genusSpecies;
// Enable non-indexed access to genus+species
@Field
(
index
=
FieldIndex
.
not_analyzed
,
type
=
FieldType
.
String
)
private
String
genusSpecies
;
private
String
spAuthor
;
private
String
subtax
;
private
String
subtAuthor
;
...
...
@@ -28,6 +28,7 @@ public class Taxonomy {
t
.
sciName
=
taxonomy
.
getTaxonName
();
t
.
genus
=
taxonomy
.
getGenus
();
t
.
species
=
taxonomy
.
getSpecies
();
t
.
genusSpecies
=
taxonomy
.
getGenus
()
+
" "
+
taxonomy
.
getSpecies
();
t
.
spAuthor
=
StringUtils
.
defaultIfBlank
(
taxonomy
.
getSpAuthor
(),
null
);
t
.
subtax
=
StringUtils
.
defaultIfBlank
(
taxonomy
.
getSubtaxa
(),
null
);
t
.
subtAuthor
=
StringUtils
.
defaultIfBlank
(
taxonomy
.
getSubtAuthor
(),
null
);
...
...
@@ -58,6 +59,14 @@ public class Taxonomy {
this
.
species
=
species
;
}
public
void
setGenusSpecies
(
String
genusSpecies
)
{
this
.
genusSpecies
=
genusSpecies
;
}
public
String
getGenusSpecies
()
{
return
genusSpecies
;
}
public
String
getSpAuthor
()
{
return
spAuthor
;
}
...
...
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @
c95547e0
...
...
@@ -441,22 +441,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
LOG
.
info
(
"Initializing index"
);
elasticsearchTemplate
.
createIndex
(
AccessionDetails
.
class
);
LOG
.
info
(
"Putting mapping"
);
elasticsearchTemplate
.
putMapping
(
AccessionDetails
.
class
);
try
{
elasticsearchTemplate
.
putMapping
(
AccessionDetails
.
class
);
}
catch
(
Throwable
e
)
{
LOG
.
error
(
"Mapping mismatch. Need to reindex."
);
reindexExistingData
(
AccessionDetails
.
class
);
}
LOG
.
info
(
"Refreshing"
);
elasticsearchTemplate
.
refresh
(
AccessionDetails
.
class
,
true
);
// genesysarchive
Map
<?,
?>
indexMapping
=
elasticsearchTemplate
.
getMapping
(
AccessionDetails
.
class
);
// Ensure ES index 'genesysarchive'
if
(!
elasticsearchTemplate
.
indexExists
(
"genesysarchive"
))
{
LOG
.
info
(
"Initializing
index
"
);
LOG
.
info
(
"Initializing
genesysarchive
"
);
elasticsearchTemplate
.
createIndex
(
"genesysarchive"
);
LOG
.
info
(
"Copying mapping to genesysarchive"
);
elasticsearchTemplate
.
putMapping
(
"genesysarchive"
,
"mcpd"
,
indexMapping
);
}
Map
<?,
?>
indexMapping
=
elasticsearchTemplate
.
getMapping
(
AccessionDetails
.
class
);
// for (Object key : indexMapping.keySet()) {
// System.err.println("Mapping ke=" + key + " val=" +
// indexMapping.get(key));
// }
LOG
.
info
(
"Copying mapping"
);
elasticsearchTemplate
.
putMapping
(
"genesysarchive"
,
"mcpd"
,
indexMapping
);
}
// TODO FIXME Don't delete index; use aliases!
private
void
reindexExistingData
(
Class
<?>
clazz
)
{
LOG
.
warn
(
"Deleting index for class="
+
clazz
);
elasticsearchTemplate
.
deleteIndex
(
clazz
);
// Make new index
LOG
.
warn
(
"Creating new index for class="
+
clazz
);
elasticsearchTemplate
.
createIndex
(
clazz
);
elasticsearchTemplate
.
putMapping
(
clazz
);
// TODO Scan & Scroll
// TODO Re-alias
// But... For now, don't do anything
}
}
src/main/java/org/genesys2/server/service/impl/FilterHandler.java
View file @
c95547e0
...
...
@@ -36,7 +36,6 @@ import org.genesys2.server.model.filters.BasicFilter;
import
org.genesys2.server.model.filters.CodedMethodFilter
;
import
org.genesys2.server.model.filters.GenesysFilter
;
import
org.genesys2.server.model.filters.GenesysFilter.DataType
;
import
org.genesys2.server.model.filters.GenesysFilter.FilterType
;
import
org.genesys2.server.model.filters.I18nListFilter
;
import
org.genesys2.server.model.filters.MethodFilter
;
import
org.genesys2.server.model.filters.ValueName
;
...
...
@@ -94,8 +93,8 @@ public class FilterHandler {
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ORGANIZATION
,
DataType
.
STRING
));
this
.
availableFilters
.
add
(
new
AutocompleteFilter
(
FilterConstants
.
INSTCODE
,
"/explore/ac/instCode"
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ACCENUMB
,
DataType
.
STRING
,
FilterType
.
RANGE
).
setAnalyzed
(
true
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ALIAS
,
DataType
.
STRING
,
FilterType
.
RANGE
).
setAnalyzed
(
true
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ACCENUMB
,
DataType
.
STRING
).
setAnalyzed
(
true
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ALIAS
,
DataType
.
STRING
).
setAnalyzed
(
true
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
SGSV
,
DataType
.
BOOLEAN
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
MLSSTATUS
,
DataType
.
BOOLEAN
));
this
.
availableFilters
.
add
(
new
BasicFilter
(
FilterConstants
.
ART15
,
DataType
.
BOOLEAN
));
...
...
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @
c95547e0
...
...
@@ -317,6 +317,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
Accession
accession
=
getAccession
(
accessionId
);
AccessionDetails
ad
=
AccessionDetails
.
from
(
accession
);
ad
.
networks
(
organizationRepository
.
getOrganizations
(
accession
.
getInstitute
()));
ad
.
aliases
(
listAccessionAliases
(
accession
));
ad
.
exch
(
listAccessionExchange
(
accession
));
ad
.
collect
(
listAccessionCollect
(
accession
));
...
...
src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java
View file @
c95547e0
...
...
@@ -141,7 +141,7 @@ public class ElasticUpdater {
try
{
// Sleep to allow {@link ElasticUpdaterProcessor} to finish
Thread
.
sleep
(
5
00
);
Thread
.
sleep
(
10
00
);
}
catch
(
InterruptedException
e
)
{
}
...
...
src/main/java/org/genesys2/server/service/worker/ElasticUpdaterProcessor.java
View file @
c95547e0
...
...
@@ -71,15 +71,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
if
(
toRemove
!=
null
)
{
removeNode
(
toRemove
);
// updatedIndices.add(toRemove.getClassName());
}
if
(
i
%
100
==
0
)
{
LOG
.
info
(
"Queue size remove="
+
elasticRemoveQueue
.
size
()
+
" update="
+
elasticUpdateQueue
.
size
()
+
" deleted="
+
i
);
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
}
}
while
(
running
&&
toRemove
!=
null
);
...
...
@@ -95,15 +90,10 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
if
(
toUpdate
!=
null
)
{
updateNode
(
toUpdate
);
// updatedIndices.add(toUpdate.getClassName());
}
if
(
i
%
100
==
0
)
{
LOG
.
info
(
"Queue size remove="
+
elasticRemoveQueue
.
size
()
+
" update="
+
elasticUpdateQueue
.
size
()
+
" indexed="
+
i
);
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
}
}
while
(
running
&&
toUpdate
!=
null
);
...
...
@@ -122,10 +112,6 @@ class ElasticUpdaterProcessor implements Runnable, InitializingBean, DisposableB
buckets
.
clear
();
}
// if (!updatedIndices.isEmpty()) {
// updatedIndices.clear();
// }
}
catch
(
HazelcastInstanceNotActiveException
e
)
{
LOG
.
warn
(
"Hazelcast not active."
);
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment