Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Genesys PGR
Genesys Backend
Commits
804afc6b
Commit
804afc6b
authored
Oct 10, 2013
by
Matija Obreza
Browse files
Localized country names
parent
41d16322
Changes
21
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/genesys2/server/model/impl/Country.java
View file @
804afc6b
...
...
@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
@Entity
@Table
(
name
=
"country"
)
// TODO Fix indexer by i18n names!
@Indexed
public
class
Country
extends
BusinessModel
{
...
...
@@ -55,6 +56,7 @@ public class Country extends BusinessModel {
@Transient
private
JsonNode
nameJ
;
private
String
wikiLink
;
public
Country
()
{
}
...
...
@@ -133,25 +135,21 @@ public class Country extends BusinessModel {
try
{
this
.
nameJ
=
mapper
.
readTree
(
nameL
);
ObjectNode
newLang
=
mapper
.
createObjectNode
().
put
(
locale
.
getLanguage
(),
name
);
System
.
err
.
println
(
newLang
);
//
System.err.println(newLang);
}
catch
(
IOException
e
)
{
}
}
private
synchronized
String
getNameLocal
(
Locale
locale
)
{
if
(
this
.
nameJ
==
null
&&
this
.
nameL
!=
null
)
{
System
.
err
.
println
(
"Parsing nameL="
+
nameL
);
ObjectMapper
mapper
=
new
ObjectMapper
();
try
{
this
.
nameJ
=
mapper
.
readTree
(
nameL
);
if
(
this
.
nameJ
!=
null
)
{
System
.
err
.
println
(
">>> "
+
this
.
nameJ
+
" "
+
this
.
nameJ
);
}
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
return
this
.
nameJ
!=
null
&&
this
.
nameJ
.
has
(
locale
.
getLanguage
())
?
this
.
nameJ
.
get
(
locale
.
getLanguage
()).
textValue
()
:
this
.
name
;
}
...
...
@@ -159,4 +157,13 @@ public class Country extends BusinessModel {
public
String
toString
()
{
return
MessageFormat
.
format
(
"Country id={0} name={1} current={2}"
,
id
,
name
,
current
);
}
@Lob
public
String
getWikiLink
()
{
return
wikiLink
;
}
public
void
setWikiLink
(
String
wikiLink
)
{
this
.
wikiLink
=
wikiLink
;
}
}
src/main/java/org/genesys2/server/persistence/domain/CountryRepository.java
View file @
804afc6b
...
...
@@ -28,12 +28,12 @@ import org.springframework.data.jpa.repository.Query;
public
interface
CountryRepository
extends
JpaRepository
<
Country
,
Long
>
{
Country
findByName
(
String
name
);
//
@Query("from Country c where c.code3= ?1 and c.current=true")
//
@Query("from Country c where c.code3= ?1 and c.current=true")
Country
findByCode3
(
String
code3
);
Country
findByCode2
(
String
code2
);
//
List<Country> findByRegion(Region region);
//
List<Country> findByRegion(Region region);
@Query
(
"from Country c where c.current=false"
)
List
<
Country
>
findInactive
();
...
...
@@ -44,6 +44,11 @@ public interface CountryRepository extends JpaRepository<Country, Long> {
@Query
(
"update Country c set c.current=false"
)
void
deactivateAll
();
// @Query("select distinct c from Country c where c.region in ( ?1 )")
// List<Country> findByRegions(List<Region> regions);
Country
findByRefnameId
(
Long
refnameId
);
@Query
(
"select distinct c.refnameId from Country c where c.refnameId is not null"
)
List
<
Long
>
listRefnameIds
();
// @Query("select distinct c from Country c where c.region in ( ?1 )")
// List<Country> findByRegions(List<Region> regions);
}
src/main/java/org/genesys2/server/service/CountryNamesUpdater.java
0 → 100644
View file @
804afc6b
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package
org.genesys2.server.service
;
import
java.io.IOException
;
public
interface
CountryNamesUpdater
{
void
updateAlternateNames
()
throws
IOException
;
}
src/main/java/org/genesys2/server/service/GeoService.java
View file @
804afc6b
...
...
@@ -45,4 +45,17 @@ public interface GeoService {
List
<
Country
>
listAll
(
Locale
locale
);
/**
* Update alternate/translated names of countries
*/
void
updateCountryNames
()
throws
IOException
;
Country
getCountryByRefnameId
(
long
refnameId
);
List
<
Long
>
listCountryRefnameIds
();
void
updateCountryNames
(
String
isoCode3
,
String
jsonTranslations
);
Country
updateCountryWiki
(
String
code3
,
String
wiki
);
}
src/main/java/org/genesys2/server/service/impl/CSVReader2.java
0 → 100644
View file @
804afc6b
package
org.genesys2.server.service.impl
;
import
java.io.BufferedReader
;
import
java.io.IOException
;
import
java.io.InputStreamReader
;
public
class
CSVReader2
{
private
BufferedReader
reader
;
private
String
separator
;
public
CSVReader2
(
InputStreamReader
reader
,
String
separator
)
{
this
.
reader
=
new
BufferedReader
(
reader
);
this
.
separator
=
separator
;
}
public
String
[]
readNext
()
throws
IOException
{
String
l
=
reader
.
readLine
();
if
(
l
==
null
)
{
return
null
;
}
return
l
.
split
(
separator
);
}
}
src/main/java/org/genesys2/server/service/impl/CountryAlternateNamesUpdater.java
0 → 100644
View file @
804afc6b
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package
org.genesys2.server.service.impl
;
import
java.io.BufferedInputStream
;
import
java.io.FileInputStream
;
import
java.io.IOException
;
import
java.io.InputStream
;
import
java.io.InputStreamReader
;
import
java.io.UnsupportedEncodingException
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.concurrent.Executors
;
import
java.util.concurrent.ThreadPoolExecutor
;
import
java.util.concurrent.TimeUnit
;
import
java.util.zip.ZipEntry
;
import
java.util.zip.ZipInputStream
;
import
javax.annotation.PreDestroy
;
import
org.apache.commons.io.IOUtils
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.commons.logging.Log
;
import
org.apache.commons.logging.LogFactory
;
import
org.apache.http.HttpEntity
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.client.ClientProtocolException
;
import
org.apache.http.client.HttpClient
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.impl.client.DefaultHttpClient
;
import
org.genesys2.server.model.impl.Country
;
import
org.genesys2.server.service.CountryNamesUpdater
;
import
org.genesys2.server.service.GeoService
;
import
org.genesys2.server.service.InstituteService
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Component
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
@Component
public
class
CountryAlternateNamesUpdater
implements
CountryNamesUpdater
{
// TODO
public
static
String
ALTERNATE_NAMES_URL
=
"..."
;
private
static
String
ALTERNATE_NAMES_FILE
=
"/Users/mobreza/Downloads/alternateNames/alternateNames.txt"
;
public
static
final
Log
LOG
=
LogFactory
.
getLog
(
CountryAlternateNamesUpdater
.
class
);
@Autowired
private
InstituteService
instituteService
;
@Autowired
private
GeoService
geoService
;
private
static
final
int
nThreads
=
Runtime
.
getRuntime
().
availableProcessors
();
private
static
final
int
BATCH_SIZE
=
100
;
private
final
ThreadPoolExecutor
threadPool
=
(
ThreadPoolExecutor
)
Executors
.
newFixedThreadPool
(
nThreads
);
/**
* Update local FaoInstitute with data from WIEWS database
*
* @throws IOException
*/
@Override
public
void
updateAlternateNames
()
throws
IOException
{
InputStream
alternateNamesStream
=
null
;
final
HttpClient
httpclient
=
new
DefaultHttpClient
();
final
HttpGet
httpget
=
new
HttpGet
(
ALTERNATE_NAMES_URL
);
HttpResponse
response
=
null
;
try
{
response
=
httpclient
.
execute
(
httpget
);
// Get hold of the response entity
final
HttpEntity
entity
=
response
.
getEntity
();
LOG
.
debug
(
entity
.
getContentType
()
+
" "
+
entity
.
getContentLength
());
// If the response does not enclose an entity, there is no
// need
// to bother about connection release
if
(
entity
!=
null
)
{
ZipInputStream
instream
=
null
;
instream
=
new
ZipInputStream
(
entity
.
getContent
());
ZipEntry
zipEntry
=
null
;
do
{
zipEntry
=
instream
.
getNextEntry
();
LOG
.
debug
(
"Got entry: "
+
zipEntry
.
getName
());
if
(
zipEntry
.
getName
().
equals
(
"alternateNames.txt"
))
{
LOG
.
info
(
"Found alternateNames.zip"
);
alternateNamesStream
=
instream
;
break
;
}
}
while
(
zipEntry
!=
null
);
if
(
zipEntry
==
null
||
!
zipEntry
.
getName
().
equals
(
"alternateNames.zip"
))
{
LOG
.
warn
(
"Didn't find alternateNames.zip, stopping update."
);
IOUtils
.
closeQuietly
(
instream
);
return
;
}
}
}
catch
(
IllegalStateException
e
)
{
alternateNamesStream
=
new
FileInputStream
(
ALTERNATE_NAMES_FILE
);
}
catch
(
final
ClientProtocolException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
throw
new
IOException
(
e
);
}
finally
{
}
try
{
updateFromStream
(
alternateNamesStream
);
}
catch
(
IOException
e
)
{
IOUtils
.
closeQuietly
(
alternateNamesStream
);
}
}
private
void
updateFromStream
(
InputStream
instream
)
throws
IOException
{
final
CSVReader2
reader
=
new
CSVReader2
(
new
InputStreamReader
(
new
BufferedInputStream
(
instream
),
"UTF-8"
),
"\t"
);
try
{
final
List
<
String
[]>
batch
=
new
ArrayList
<
String
[]>(
BATCH_SIZE
);
String
prevRefnameId
=
null
;
String
[]
line
=
null
;
List
<
Long
>
countryRefnameIds
=
geoService
.
listCountryRefnameIds
();
LOG
.
info
(
"Got "
+
countryRefnameIds
.
size
()
+
" refnameIds"
);
int
counter
=
0
;
while
((
line
=
reader
.
readNext
())
!=
null
)
{
counter
++;
if
(
counter
%
10000
==
0
)
{
LOG
.
info
(
"Country alternate names @ line "
+
counter
);
}
for
(
int
i
=
0
;
i
<
line
.
length
;
i
++)
{
if
(
line
[
i
].
equals
(
"null"
)
||
StringUtils
.
isBlank
(
line
[
i
]))
{
line
[
i
]
=
null
;
}
}
final
String
refnameId
=
line
[
1
];
if
(
prevRefnameId
==
null
||
refnameId
.
equals
(
prevRefnameId
))
{
prevRefnameId
=
refnameId
;
batch
.
add
(
line
);
}
else
{
long
longRefnameId
=
Long
.
parseLong
(
prevRefnameId
);
if
(
countryRefnameIds
.
contains
(
longRefnameId
))
{
workIt
(
longRefnameId
,
batch
);
}
batch
.
clear
();
prevRefnameId
=
null
;
}
}
long
longRefnameId
=
Long
.
parseLong
(
prevRefnameId
);
if
(
countryRefnameIds
.
contains
(
longRefnameId
))
{
workIt
(
Long
.
parseLong
(
prevRefnameId
),
batch
);
batch
.
clear
();
}
LOG
.
info
(
"Done importing alternate geonames"
);
}
catch
(
UnsupportedEncodingException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
finally
{
IOUtils
.
closeQuietly
(
instream
);
}
}
private
void
workIt
(
final
long
refnameId
,
final
List
<
String
[]>
batch
)
{
System
.
gc
();
// Need copy!
final
List
<
String
[]>
batchCopy
=
new
ArrayList
<
String
[]>(
batch
);
// while (threadPool.getQueue().size() >= nThreads) {
// LOG.warn("Queue is too large, waiting...");
// try {
// Thread.sleep(100);
// } catch (final InterruptedException e) {
// LOG.warn(e.getMessage());
// }
// }
threadPool
.
execute
(
new
Runnable
()
{
@Override
public
void
run
()
{
// Fetch FaoInstitutes from DB
Country
country
=
geoService
.
getCountryByRefnameId
(
refnameId
);
if
(
country
==
null
)
{
if
(
LOG
.
isDebugEnabled
())
LOG
.
debug
(
"Nothing to check, no country with refnameId="
+
refnameId
);
return
;
}
LOG
.
info
(
"Processing country refnames "
+
batchCopy
.
size
());
String
wiki
=
country
.
getWikiLink
();
ObjectMapper
objectMapper
=
new
ObjectMapper
();
ObjectNode
jsonNames
=
objectMapper
.
createObjectNode
();
for
(
String
[]
line
:
batchCopy
)
{
if
(
line
[
2
]
!=
null
&&
line
[
2
].
length
()
==
2
&&
line
[
3
]
!=
null
)
{
boolean
preferred
=
line
.
length
>=
5
&&
"1"
.
equals
(
line
[
4
]);
if
(
preferred
||
jsonNames
.
get
(
line
[
2
])
==
null
)
{
jsonNames
.
put
(
line
[
2
],
line
[
3
]);
}
}
if
(
line
[
2
]
!=
null
&&
"link"
.
equals
(
line
[
2
]))
{
boolean
preferred
=
line
.
length
>=
5
&&
"1"
.
equals
(
line
[
4
]);
if
(
preferred
||
wiki
==
null
)
{
wiki
=
line
[
3
];
}
}
}
if
(
LOG
.
isDebugEnabled
())
LOG
.
debug
(
"We are at: "
+
country
+
" i18n = "
+
jsonNames
);
geoService
.
updateCountryNames
(
country
.
getCode3
(),
jsonNames
.
toString
());
geoService
.
updateCountryWiki
(
country
.
getCode3
(),
wiki
);
}
});
}
@PreDestroy
public
void
shutdownPool
()
{
threadPool
.
shutdown
();
LOG
.
info
(
"Waiting for all threads to terminate"
);
try
{
while
(!
threadPool
.
awaitTermination
(
1
,
TimeUnit
.
SECONDS
))
{
try
{
Thread
.
sleep
(
200
);
}
catch
(
final
InterruptedException
e
)
{
LOG
.
warn
(
e
.
getMessage
());
}
}
LOG
.
info
(
"All workers terminated."
);
}
catch
(
final
InterruptedException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
}
}
src/main/java/org/genesys2/server/service/impl/GeoServiceImpl.java
View file @
804afc6b
...
...
@@ -49,6 +49,9 @@ public class GeoServiceImpl implements GeoService {
@Autowired
ContentService
contentService
;
@Autowired
private
CountryAlternateNamesUpdater
alternateNamesUpdater
;
@Override
public
List
<
Country
>
listAll
()
{
return
countryRepository
.
findAll
(
new
Sort
(
"name"
,
"current"
));
...
...
@@ -77,6 +80,11 @@ public class GeoServiceImpl implements GeoService {
return
country
!=
null
?
country
:
countryRepository
.
findByName
(
countryString
);
}
@Override
public
Country
getCountryByRefnameId
(
long
refnameId
)
{
return
countryRepository
.
findByRefnameId
(
refnameId
);
}
@Override
@Transactional
(
readOnly
=
false
)
public
void
updateCountryData
()
throws
IOException
{
...
...
@@ -205,10 +213,40 @@ public class GeoServiceImpl implements GeoService {
}
}
@Override
public
void
updateCountryNames
()
throws
IOException
{
alternateNamesUpdater
.
updateAlternateNames
();
}
@Override
@PreAuthorize
(
"hasRole('ADMINISTRATOR')"
)
@Transactional
(
readOnly
=
false
)
public
void
updateBlurp
(
Country
country
,
String
blurp
,
Locale
locale
)
{
contentService
.
updateArticle
(
country
,
"blurp"
,
null
,
blurp
,
locale
);
}
@Override
public
List
<
Long
>
listCountryRefnameIds
()
{
return
countryRepository
.
listRefnameIds
();
}
@Override
// TODO Where do we autorize access?
// @PreAuthorize("hasRole('ADMINISTRATOR')")
@Transactional
(
readOnly
=
false
)
public
void
updateCountryNames
(
String
isoCode3
,
String
jsonTranslations
)
{
Country
country
=
countryRepository
.
findByCode3
(
isoCode3
);
country
.
setNameL
(
jsonTranslations
);
countryRepository
.
save
(
country
);
LOG
.
info
(
"Updated translations of "
+
country
);
}
@Override
@Transactional
(
readOnly
=
false
)
public
Country
updateCountryWiki
(
String
isoCode3
,
String
wiki
)
{
Country
country
=
countryRepository
.
findByCode3
(
isoCode3
);
country
.
setWikiLink
(
wiki
);
countryRepository
.
save
(
country
);
return
country
;
}
}
src/main/java/org/genesys2/server/servlet/controller/AdminController.java
View file @
804afc6b
...
...
@@ -20,6 +20,7 @@ import java.io.IOException;
import
org.apache.commons.logging.Log
;
import
org.apache.commons.logging.LogFactory
;
import
org.genesys2.server.service.CountryNamesUpdater
;
import
org.genesys2.server.service.GenesysService
;
import
org.genesys2.server.service.GeoService
;
import
org.genesys2.server.service.InstituteService
;
...
...
@@ -42,6 +43,9 @@ public class AdminController {
@Autowired
InstituteUpdater
instituteUpdater
;
@Autowired
CountryNamesUpdater
alternateNamesUpdater
;
@Autowired
LuceneIndexer
luceneIndexer
;
...
...
@@ -56,7 +60,7 @@ public class AdminController {
@Autowired
SGSVInsertMissing
sgsvImporter
;
@Autowired
ContentSanitizer
contentSanitizer
;
...
...
@@ -88,6 +92,16 @@ public class AdminController {
return
"redirect:/admin/"
;
}
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/refreshCountryNames"
)
public
String
refreshCountryNames
()
{
try
{
geoService
.
updateCountryNames
();
}
catch
(
final
IOException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
return
"redirect:/admin/"
;
}
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/reindexEverything"
)
public
String
reindexEverything
()
{
luceneIndexer
.
reindexEverything
();
...
...
@@ -105,13 +119,13 @@ public class AdminController {
genesysService
.
updateAccessionCountryRefs
();
return
"redirect:/admin/"
;
}
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/updateInstituteCountryRefs"
)
public
String
updateInstituteCountryRefs
()
{
instituteService
.
updateCountryRefs
();
return
"redirect:/admin/"
;
}
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/updateAccessionInstituteRefs"
)
public
String
updateAccessionInstituteRefs
()
{
genesysService
.
updateAccessionInstitueRefs
();
...
...
@@ -137,4 +151,16 @@ public class AdminController {
LOG
.
info
(
"Sanitizing content.. Done"
);
return
"redirect:/admin/"
;
}
@RequestMapping
(
method
=
RequestMethod
.
POST
,
value
=
"/updateAlternateNames"
)
public
String
updateAlternateNames
()
{
LOG
.
info
(
"Updating alternate GEO names"
);
try
{
alternateNamesUpdater
.
updateAlternateNames
();
}
catch
(
IOException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
LOG
.
info
(
"Updating alternate GEO names: done"
);
return
"redirect:/admin/"
;
}
}
src/main/resources/content/language.properties
View file @
804afc6b
...
...
@@ -113,6 +113,8 @@ country.stat.countByLocation={0} accessions at institutes in this country
country.stat.countByOrigin
=
{0} accessions registered in Genesys come from this country.
country.statistics
=
Country statistics
country.accessions.from
=
Accessions collected in {0}
country.more-information
=
More information:
faoInstitutes.page.list.title
=
WIEWS Institutes
faoInstitutes.page.profile.title
=
WIEWS {0}
...
...
src/main/resources/log4j.properties
View file @
804afc6b
...
...
@@ -15,16 +15,12 @@
#-------------------------------------------------------------------------------
### direct log messages to stdout ###
log4j.appender.stdout
=
org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target
=
System.out
log4j.appender.stdout.target
=
System.out
log4j.appender.stdout.encoding
=
UTF-8
log4j.appender.stdout.layout
=
org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern
=
%d{ABSOLUTE} %5p %c{1}:%L - %m%n
log4j.appender.stdout.layout.ConversionPattern
=
%d{ABSOLUTE}
%t
%5p %c{1}:%L - %m%n