Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Genesys PGR
validator.genesys-pgr.org
Commits
423ef5f0
Commit
423ef5f0
authored
May 01, 2018
by
Matija Obreza
Browse files
Source code cleanup
parent
f7559171
Changes
14
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/genesys/taxonomy/checker/web/config/ApplicationConfig.java
View file @
423ef5f0
...
...
@@ -53,7 +53,7 @@ import org.springframework.context.annotation.Configuration;
* The Class ApplicationConfig.
*/
@Configuration
@ComponentScan
({
"org.genesys.taxonomy.checker.web.util"
,
"org.genesys.taxonomy.checker.web.service"
})
@ComponentScan
({
"org.genesys.taxonomy.checker.web.util"
,
"org.genesys.taxonomy.checker.web.service"
})
public
class
ApplicationConfig
{
private
final
static
Logger
LOG
=
LoggerFactory
.
getLogger
(
ApplicationConfig
.
class
);
...
...
@@ -63,7 +63,7 @@ public class ApplicationConfig {
@Value
(
"${data.folder.path}"
)
public
String
dataFolderPath
;
/**
* Process service.
*
...
...
@@ -75,9 +75,8 @@ public class ApplicationConfig {
* @throws TaxonomyException the taxonomy exception
*/
@Bean
(
name
=
"taxonomyProcessService"
)
public
ProcessService
taxonomyProcessService
()
throws
UnsupportedEncodingException
,
FileNotFoundException
,
IOException
,
ParseException
,
TaxonomyException
{
TaxonomyProcessServiceImpl
taxonomyProcessService
=
new
TaxonomyProcessServiceImpl
();
public
ProcessService
taxonomyProcessService
()
throws
UnsupportedEncodingException
,
FileNotFoundException
,
IOException
,
ParseException
,
TaxonomyException
{
final
TaxonomyProcessServiceImpl
taxonomyProcessService
=
new
TaxonomyProcessServiceImpl
();
taxonomyProcessService
.
setTaxonomyChecker
(
taxonomyChecker
());
taxonomyProcessService
.
setTaxonomyDatabase
(
taxonomyDatabase
());
...
...
@@ -90,63 +89,62 @@ public class ApplicationConfig {
}
@Bean
(
name
=
"countryProcessService"
)
public
ProcessService
countryProcessService
(){
public
ProcessService
countryProcessService
()
{
return
new
CountryProcessServiceImpl
();
}
@Bean
(
name
=
"landOrSeaProcessService"
)
public
ProcessService
landOrSeaProcessService
(){
public
ProcessService
landOrSeaProcessService
()
{
return
new
LandOrSeaProcessServiceImpl
();
}
@Bean
(
name
=
"processService"
)
public
ProcessService
processService
(){
public
ProcessService
processService
()
{
return
new
ProcessServiceImpl
();
}
@Bean
public
LandOrSeaService
landOrSeaService
()
throws
MalformedURLException
,
IOException
{
LandOrSeaServiceImpl
s
=
new
LandOrSeaServiceImpl
();
final
LandOrSeaServiceImpl
s
=
new
LandOrSeaServiceImpl
();
s
.
setDataFolderPath
(
dataFolderPath
);
s
.
afterPropertiesSet
();
return
s
;
}
@Bean
public
CountryOfOriginService
countryOfOriginService
()
throws
MalformedURLException
,
IOException
{
CountryOfOriginServiceImpl
s
=
new
CountryOfOriginServiceImpl
();
final
CountryOfOriginServiceImpl
s
=
new
CountryOfOriginServiceImpl
();
s
.
setDataFolderPath
(
dataFolderPath
);
s
.
afterPropertiesSet
();
return
s
;
}
private
void
downloadDataIfNeeded
()
throws
IOException
{
if
(
StringUtils
.
isBlank
(
taxonomyDataPath
))
{
LOG
.
warn
(
"Taxonomy data directory not provided, creating new"
);
File
destinationDir
=
File
.
createTempFile
(
"grin-"
,
"-data"
);
final
File
destinationDir
=
File
.
createTempFile
(
"grin-"
,
"-data"
);
if
(
destinationDir
.
delete
()
&&
destinationDir
.
mkdir
())
{
LOG
.
warn
(
"Made temporary directory "
+
destinationDir
.
getAbsolutePath
());
}
taxonomyDataPath
=
destinationDir
.
getAbsolutePath
();
}
File
dataFolder
=
new
File
(
taxonomyDataPath
);
final
File
dataFolder
=
new
File
(
taxonomyDataPath
);
if
(!
dataFolder
.
exists
())
{
LOG
.
warn
(
"Making directory "
+
dataFolder
.
getAbsolutePath
());
dataFolder
.
mkdirs
();
}
// The two required files
File
genusFile
=
new
File
(
dataFolder
,
"TAXONOMY_GENUS.txt"
);
File
speciesFile
=
new
File
(
dataFolder
,
"TAXONOMY_SPECIES.txt"
);
final
File
genusFile
=
new
File
(
dataFolder
,
"TAXONOMY_GENUS.txt"
);
final
File
speciesFile
=
new
File
(
dataFolder
,
"TAXONOMY_SPECIES.txt"
);
if
(!
genusFile
.
exists
()
||
!
speciesFile
.
exists
())
{
LOG
.
warn
(
"Taxonomy data not provided in {}, starting download"
,
dataFolder
.
getAbsolutePath
());
TaxonomyDownloader
dl
=
new
TaxonomyDownloader
();
final
TaxonomyDownloader
dl
=
new
TaxonomyDownloader
();
LOG
.
warn
(
"Downloading GRIN-Taxonomy database to {}"
,
dataFolder
.
getAbsolutePath
());
File
downloadedCabFile
=
File
.
createTempFile
(
"grin-"
,
".cab"
);
final
File
downloadedCabFile
=
File
.
createTempFile
(
"grin-"
,
".cab"
);
dl
.
downloadCurrent
(
downloadedCabFile
);
TaxonomyDownloader
.
unpackCabinetFile
(
downloadedCabFile
,
dataFolder
,
false
);
...
...
src/main/java/org/genesys/taxonomy/checker/web/config/ServletInitializer.java
View file @
423ef5f0
...
...
@@ -41,8 +41,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
private
static
final
long
MAX_REQUEST_SIZE
=
MAX_FILE_SIZE
*
2
;
private
static
final
int
FILE_SIZE_THRESHOLD
=
0
;
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#createServletApplicationContext()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* createServletApplicationContext()
*/
@Override
protected
WebApplicationContext
createServletApplicationContext
()
{
...
...
@@ -51,24 +54,32 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
return
context
;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#getServletMappings()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* getServletMappings()
*/
@Override
protected
String
[]
getServletMappings
()
{
return
new
String
[]
{
"/"
};
}
/* (non-Javadoc)
* @see org.springframework.web.context.AbstractContextLoaderInitializer#createRootApplicationContext()
/*
* (non-Javadoc)
* @see org.springframework.web.context.AbstractContextLoaderInitializer#
* createRootApplicationContext()
*/
@Override
protected
WebApplicationContext
createRootApplicationContext
()
{
return
null
;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#onStartup(javax.servlet.ServletContext)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* onStartup(javax.servlet.ServletContext)
*/
@Override
public
void
onStartup
(
final
ServletContext
servletContext
)
throws
ServletException
{
...
...
@@ -76,8 +87,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
servletContext
.
setSessionTrackingModes
(
EnumSet
.
of
(
SessionTrackingMode
.
COOKIE
));
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#customizeRegistration(javax.servlet.ServletRegistration.Dynamic)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* customizeRegistration(javax.servlet.ServletRegistration.Dynamic)
*/
@Override
protected
void
customizeRegistration
(
final
ServletRegistration
.
Dynamic
registration
)
{
...
...
@@ -89,8 +103,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
return
multipartConfigElement
;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#getServletFilters()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* getServletFilters()
*/
@Override
protected
Filter
[]
getServletFilters
()
{
...
...
src/main/java/org/genesys/taxonomy/checker/web/config/WebConfiguration.java
View file @
423ef5f0
...
...
@@ -45,8 +45,12 @@ import org.springframework.web.servlet.view.JstlView;
@ComponentScan
({
"org.genesys.taxonomy.checker.web.controller"
})
public
class
WebConfiguration
extends
WebMvcConfigurerAdapter
{
/* (non-Javadoc)
* @see org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#addResourceHandlers(org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#
* addResourceHandlers(org.springframework.web.servlet.config.annotation.
* ResourceHandlerRegistry)
*/
@Override
public
void
addResourceHandlers
(
final
ResourceHandlerRegistry
registry
)
{
...
...
@@ -104,8 +108,12 @@ public class WebConfiguration extends WebMvcConfigurerAdapter {
return
cookieLocaleResolver
;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#addInterceptors(org.springframework.web.servlet.config.annotation.InterceptorRegistry)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#
* addInterceptors(org.springframework.web.servlet.config.annotation.
* InterceptorRegistry)
*/
@Override
public
void
addInterceptors
(
final
InterceptorRegistry
registry
)
{
...
...
src/main/java/org/genesys/taxonomy/checker/web/controller/ControllerExceptionHandler.java
View file @
423ef5f0
...
...
@@ -16,14 +16,14 @@ public class ControllerExceptionHandler {
private
final
static
Logger
LOG
=
LoggerFactory
.
getLogger
(
ControllerExceptionHandler
.
class
);
@ExceptionHandler
(
value
=
Throwable
.
class
)
public
ModelAndView
anyThrowable
(
Throwable
throwable
)
{
public
ModelAndView
anyThrowable
(
final
Throwable
throwable
)
{
LOG
.
error
(
"Application error"
,
throwable
);
ModelAndView
modelAndView
=
new
ModelAndView
(
"error"
);
final
ModelAndView
modelAndView
=
new
ModelAndView
(
"error"
);
modelAndView
.
setStatus
(
HttpStatus
.
INTERNAL_SERVER_ERROR
);
modelAndView
.
addObject
(
"errorText"
,
throwable
.
getMessage
());
return
modelAndView
;
}
}
src/main/java/org/genesys/taxonomy/checker/web/controller/ValidatorController.java
View file @
423ef5f0
...
...
@@ -81,17 +81,15 @@ public class ValidatorController {
public
String
processString
(
@RequestParam
(
name
=
"separator"
,
required
=
false
,
defaultValue
=
","
)
final
Character
separator
,
@RequestParam
(
name
=
"separatorOther"
,
required
=
false
,
defaultValue
=
""
)
final
Character
separatorOther
,
@RequestParam
(
name
=
"quoteChar"
,
required
=
false
,
defaultValue
=
"\""
)
final
Character
quoteChar
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
final
ModelMap
model
)
throws
IOException
,
ParseException
,
TaxonomyException
{
LOG
.
trace
(
"Processing uploaded CSV string:\n{}"
,
csvText
);
List
<
String
[]>
rows
=
processService
.
process
(
new
StringReader
(
StringUtils
.
defaultString
(
csvText
,
""
)),
(
separator
==
'O'
?
separatorOther
:
separator
)
,
quoteChar
,
escapeChar
,
validateType
,
toCurrentTaxa
,
decimalMark
);
final
List
<
String
[]>
rows
=
processService
.
process
(
new
StringReader
(
StringUtils
.
defaultString
(
csvText
,
""
)),
separator
==
'O'
?
separatorOther
:
separator
,
quoteChar
,
escapeChar
,
validateType
,
toCurrentTaxa
,
decimalMark
);
model
.
addAttribute
(
"result"
,
rows
);
model
.
addAttribute
(
"tableHeaders"
,
rows
.
get
(
0
));
model
.
addAttribute
(
"newLineSymbol"
,
"\n"
);
...
...
@@ -106,11 +104,9 @@ public class ValidatorController {
public
void
processStringToCSV1
(
@RequestParam
(
name
=
"separator"
,
required
=
false
,
defaultValue
=
","
)
final
Character
separator
,
@RequestParam
(
name
=
"separatorOther"
,
required
=
false
,
defaultValue
=
""
)
final
Character
separatorOther
,
@RequestParam
(
name
=
"quoteChar"
,
required
=
false
,
defaultValue
=
"\""
)
final
Character
quoteChar
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
final
HttpServletResponse
response
)
throws
IOException
,
ParseException
,
TaxonomyException
{
processStringToCSV
(
separator
,
separatorOther
,
quoteChar
,
escapeChar
,
csvText
,
toCurrentTaxa
,
validateType
,
decimalMark
,
response
);
...
...
@@ -134,26 +130,24 @@ public class ValidatorController {
public
void
processStringToCSV
(
@RequestParam
(
name
=
"separator"
,
required
=
false
,
defaultValue
=
","
)
final
Character
separator
,
@RequestParam
(
name
=
"separatorOther"
,
required
=
false
,
defaultValue
=
""
)
final
Character
separatorOther
,
@RequestParam
(
name
=
"quoteChar"
,
required
=
false
,
defaultValue
=
"\""
)
final
Character
quoteChar
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
@RequestParam
(
name
=
"escapeChar"
,
required
=
false
,
defaultValue
=
"\0"
)
final
Character
escapeChar
,
@RequestParam
(
name
=
"csvText"
,
required
=
true
)
final
String
csvText
,
@RequestParam
(
name
=
"toCurrentTaxa"
,
required
=
false
)
final
Boolean
toCurrentTaxa
,
@RequestParam
(
name
=
"validateType"
,
defaultValue
=
""
)
final
String
validateType
,
@RequestParam
(
name
=
"decimalMark"
,
defaultValue
=
"."
)
final
Character
decimalMark
,
final
HttpServletResponse
response
)
throws
IOException
,
ParseException
,
TaxonomyException
{
LOG
.
trace
(
"Processing uploaded CSV string:\n{}"
,
csvText
);
List
<
String
[]>
rows
=
processService
.
process
(
new
StringReader
(
StringUtils
.
defaultString
(
csvText
,
""
)),
(
separator
==
'O'
?
separatorOther
:
separator
)
,
quoteChar
,
escapeChar
,
validateType
,
toCurrentTaxa
,
decimalMark
);
final
List
<
String
[]>
rows
=
processService
.
process
(
new
StringReader
(
StringUtils
.
defaultString
(
csvText
,
""
)),
separator
==
'O'
?
separatorOther
:
separator
,
quoteChar
,
escapeChar
,
validateType
,
toCurrentTaxa
,
decimalMark
);
response
.
setContentType
(
"text/csv;charset=UTF-16LE"
);
response
.
addHeader
(
"Content-Disposition"
,
String
.
format
(
"attachment; filename=\"validator-results.csv\""
));
response
.
flushBuffer
();
// Write CSV to the stream.
final
OutputStreamWriter
osw
=
new
OutputStreamWriter
(
response
.
getOutputStream
(),
"UTF-16LE"
);
try
(
CSVWriter
csvWriter
=
new
CSVWriter
(
osw
,
(
separator
==
'O'
?
separatorOther
:
separator
)
,
quoteChar
,
escapeChar
,
"\n"
))
{
try
(
CSVWriter
csvWriter
=
new
CSVWriter
(
osw
,
separator
==
'O'
?
separatorOther
:
separator
,
quoteChar
,
escapeChar
,
"\n"
))
{
csvWriter
.
writeAll
(
rows
);
csvWriter
.
flush
();
}
finally
{
...
...
@@ -161,12 +155,12 @@ public class ValidatorController {
}
}
private
String
[]
getStyles
(
String
[]
strings
)
{
String
[]
styles
=
new
String
[
strings
.
length
];
private
String
[]
getStyles
(
final
String
[]
strings
)
{
final
String
[]
styles
=
new
String
[
strings
.
length
];
for
(
int
i
=
0
;
i
<
strings
.
length
;
i
++)
{
styles
[
i
]
=
""
;
if
(
"GENUS_check"
.
equals
(
strings
[
i
]))
{
//styles[i]=""
//
styles[i]=""
}
}
return
styles
;
...
...
src/main/java/org/genesys/taxonomy/checker/web/service/ProcessService.java
View file @
423ef5f0
...
...
@@ -47,15 +47,15 @@ public interface ProcessService {
List
<
String
[]>
process
(
Reader
reader
,
Character
separator
,
Character
quoteChar
,
Character
escapeChar
,
String
validateType
,
Boolean
toCurrentTaxa
,
Character
decimalMark
)
throws
ParseException
,
IOException
,
TaxonomyException
;
/**
* Create instance of DecimalFormat used to parse numbers from the CSV
*
* @param decimalMark
* @return DecimalFormat using the specified decimal mark
*/
default
DecimalFormat
createDecimalFormat
(
char
decimalMark
)
{
default
DecimalFormat
createDecimalFormat
(
final
char
decimalMark
)
{
final
DecimalFormat
decimalFormat
=
new
DecimalFormat
();
DecimalFormatSymbols
symbols
=
new
DecimalFormatSymbols
();
final
DecimalFormatSymbols
symbols
=
new
DecimalFormatSymbols
();
symbols
.
setDecimalSeparator
(
decimalMark
);
symbols
.
setGroupingSeparator
(
decimalMark
==
','
?
'.'
:
','
);
decimalFormat
.
setDecimalFormatSymbols
(
symbols
);
...
...
src/main/java/org/genesys/taxonomy/checker/web/service/impl/CachingInMemoryTaxonomyDatabase.java
View file @
423ef5f0
...
...
@@ -18,17 +18,18 @@ package org.genesys.taxonomy.checker.web.service.impl;
import
java.util.List
;
import
java.util.concurrent.ExecutionException
;
import
com.google.common.cache.Cache
;
import
com.google.common.cache.CacheBuilder
;
import
org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase
;
import
org.genesys.taxonomy.gringlobal.model.SpeciesRow
;
import
com.google.common.cache.Cache
;
import
com.google.common.cache.CacheBuilder
;
/**
* Guava cache supported in-memory taxonomy database.
*/
public
class
CachingInMemoryTaxonomyDatabase
extends
InMemoryTaxonomyDatabase
{
// private final static Logger LOG = LoggerFactory.getLogger(CachingInMemoryTaxonomyDatabase.class);
// private final static Logger LOG =
// LoggerFactory.getLogger(CachingInMemoryTaxonomyDatabase.class);
/** The cache genus species. */
private
final
Cache
<
String
,
List
<
SpeciesRow
>>
cacheGenusSpecies
=
CacheBuilder
.
newBuilder
().
maximumSize
(
100
).
build
();
...
...
@@ -47,8 +48,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
* @see org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#getAllGenusSpecies(java.lang.String)
* @see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#getAllGenusSpecies(java
* .lang.String)
*/
@Override
protected
List
<
SpeciesRow
>
getAllGenusSpecies
(
final
String
genus
)
{
...
...
@@ -68,8 +70,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
* @see org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarGenus(java.lang.String, int)
* @see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarGenus(java.
* lang.String, int)
*/
@Override
public
List
<
String
>
findSimilarGenus
(
final
String
genus
,
final
int
maxSize
)
{
...
...
@@ -90,9 +93,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
*
@see
org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarSpecies(java
.lang.String,
* java.lang.String, int)
*
@see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarSpecies(java
*
.lang.String,
java.lang.String, int)
*/
@Override
public
List
<
String
>
findSimilarSpecies
(
final
String
genus
,
final
String
species
,
final
int
maxSize
)
{
...
...
src/main/java/org/genesys/taxonomy/checker/web/service/impl/CountryProcessServiceImpl.java
View file @
423ef5f0
...
...
@@ -17,7 +17,6 @@ import java.util.concurrent.TimeUnit;
import
org.apache.commons.lang3.ArrayUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.concurrent.ConcurrentUtils
;
import
org.apache.commons.lang3.time.StopWatch
;
import
org.genesys.geotools.service.CountryOfOriginService
;
import
org.genesys.geotools.service.HeaderUtils
;
...
...
@@ -29,7 +28,6 @@ import org.slf4j.LoggerFactory;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Service
;
import
com.google.common.collect.Lists
;
import
com.opencsv.CSVReader
;
/**
...
...
@@ -42,51 +40,51 @@ public class CountryProcessServiceImpl implements ProcessService {
@Autowired
private
CountryOfOriginService
countryOfOriginService
;
@Override
public
List
<
String
[]>
process
(
Reader
reader
,
Character
separator
,
Character
quoteChar
,
Character
escapeChar
,
String
validateType
,
Boolean
toCurrentTaxa
,
Character
decimalMark
)
throws
ParseException
,
IOException
,
TaxonomyException
{
public
List
<
String
[]>
process
(
final
Reader
reader
,
final
Character
separator
,
final
Character
quoteChar
,
final
Character
escapeChar
,
final
String
validateType
,
final
Boolean
toCurrentTaxa
,
final
Character
decimalMark
)
throws
ParseException
,
IOException
,
TaxonomyException
{
return
execute
(
reader
,
separator
,
quoteChar
,
escapeChar
,
decimalMark
);
}
private
List
<
String
[]>
execute
(
Reader
readerr
,
char
separatorChar
,
char
quoteChar
,
char
escapeChar
,
Character
decimalMark
)
throws
IOException
{
private
List
<
String
[]>
execute
(
final
Reader
readerr
,
final
char
separatorChar
,
final
char
quoteChar
,
final
char
escapeChar
,
final
Character
decimalMark
)
throws
IOException
{
final
List
<
String
[]>
lines
=
Collections
.
synchronizedList
(
new
LinkedList
<>());
try
(
CSVReader
reader
=
new
CSVReader
(
readerr
,
separatorChar
,
quoteChar
,
escapeChar
,
0
,
false
))
{
String
[]
headers
=
reader
.
readNext
();
final
String
[]
headers
=
reader
.
readNext
();
LOG
.
debug
(
"Input CSV headers: {}"
,
Arrays
.
toString
(
headers
));
HeaderUtils
.
throwIfHeaderFound
(
headers
,
new
String
[]
{
ApplicationUtils
.
HEADER_ORIGCTY_CHECK
});
HeaderUtils
.
throwIfHeaderNotFound
(
headers
,
new
String
[]
{
ApplicationUtils
.
HEADER_LONGITUDE
,
ApplicationUtils
.
HEADER_LATITUDE
});
List
<
String
>
outputHeaders
=
new
ArrayList
<>(
Arrays
.
asList
(
headers
));
List
<
Integer
>
outputMapping
=
HeaderUtils
.
mapHeaderPositions
(
headers
);
final
List
<
String
>
outputHeaders
=
new
ArrayList
<>(
Arrays
.
asList
(
headers
));
final
List
<
Integer
>
outputMapping
=
HeaderUtils
.
mapHeaderPositions
(
headers
);
LOG
.
debug
(
"Source mapping indexes: {}"
,
outputMapping
);
LOG
.
debug
(
"Initial output headers: {}"
,
outputHeaders
);
Map
<
String
,
Integer
>
sourceMapping
=
HeaderUtils
.
makeSourceMapping
(
headers
,
new
String
[]
{
ApplicationUtils
.
HEADER_LONGITUDE
,
ApplicationUtils
.
HEADER_LATITUDE
,
ApplicationUtils
.
HEADER_ORIGCTY
});
final
Map
<
String
,
Integer
>
sourceMapping
=
HeaderUtils
.
makeSourceMapping
(
headers
,
new
String
[]
{
ApplicationUtils
.
HEADER_LONGITUDE
,
ApplicationUtils
.
HEADER_LATITUDE
,
ApplicationUtils
.
HEADER_ORIGCTY
});
if
(
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_ORIGCTY
)
>=
0
)
{
outputHeaders
.
add
(
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_ORIGCTY
)
+
1
,
ApplicationUtils
.
HEADER_ORIGCTY_CHECK
);
outputMapping
.
add
(
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_ORIGCTY
)
+
1
,
null
);
}
else
{
int
pos
=
Math
.
max
(
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LONGITUDE
),
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LATITUDE
));
final
int
pos
=
Math
.
max
(
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LONGITUDE
),
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LATITUDE
));
outputHeaders
.
add
(
pos
+
1
,
ApplicationUtils
.
HEADER_ORIGCTY_CHECK
);
outputMapping
.
add
(
pos
+
1
,
null
);
}
// Add extra header HEADER_DECLATITUDE_CHECK
{
int
pos
=
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LATITUDE
);
final
int
pos
=
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LATITUDE
);
outputHeaders
.
add
(
pos
+
1
,
ApplicationUtils
.
HEADER_DECLATITUDE_CHECK
);
outputMapping
.
add
(
pos
+
1
,
null
);
}
// Add extra header HEADER_DECLONGITUDE_CHECK
{
int
pos
=
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LONGITUDE
);
final
int
pos
=
outputHeaders
.
indexOf
(
ApplicationUtils
.
HEADER_LONGITUDE
);
outputHeaders
.
add
(
pos
+
1
,
ApplicationUtils
.
HEADER_DECLONGITUDE_CHECK
);
outputMapping
.
add
(
pos
+
1
,
null
);
}
...
...
@@ -97,14 +95,14 @@ public class CountryProcessServiceImpl implements ProcessService {
lines
.
add
(
outputHeaders
.
toArray
(
ArrayUtils
.
EMPTY_STRING_ARRAY
));
// Thread pool with nThreads concurrent threads
int
nThreads
=
Math
.
max
(
Runtime
.
getRuntime
().
availableProcessors
()
-
1
,
1
);
final
int
nThreads
=
Math
.
max
(
Runtime
.
getRuntime
().
availableProcessors
()
-
1
,
1
);
LOG
.
info
(
"Using {} threads for {} CPUs."
,
nThreads
,
Runtime
.
getRuntime
().
availableProcessors
());
BlockingQueue
<
Runnable
>
workQueue
=
new
ArrayBlockingQueue
<>(
nThreads
*
2
);
final
BlockingQueue
<
Runnable
>
workQueue
=
new
ArrayBlockingQueue
<>(
nThreads
*
2
);
ThreadPoolExecutor
threadPool
=
new
ThreadPoolExecutor
(
1
+
nThreads
/
2
,
nThreads
,
5
,
TimeUnit
.
SECONDS
,
workQueue
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
final
ThreadPoolExecutor
threadPool
=
new
ThreadPoolExecutor
(
1
+
nThreads
/
2
,
nThreads
,
5
,
TimeUnit
.
SECONDS
,
workQueue
,
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
StopWatch
stopWatch
=
new
StopWatch
();
final
StopWatch
stopWatch
=
new
StopWatch
();
int
lineCount
=
0
;
...
...
@@ -112,16 +110,16 @@ public class CountryProcessServiceImpl implements ProcessService {
stopWatch
.
start
();
do
{
final
ArrayList
<
String
[]>
inputs
=
new
ArrayList
<
String
[]
>();
final
ArrayList
<
String
[]>
inputs
=
new
ArrayList
<>();
for
(
int
j
=
0
;
j
<
100
;
j
++)
{
String
[]
nextLine
=
reader
.
readNext
();
final
String
[]
nextLine
=
reader
.
readNext
();
lineCount
++;
stopWatch
.
split
();
long
processingTimeSeconds
=
stopWatch
.
getSplitTime
()
/
1000
;
if
(
(
lineCount
%
1000
==
0
&&
processingTimeSeconds
>
0
)
||
(
lastLogTime
!=
processingTimeSeconds
&&
processingTimeSeconds
%
10
==
0
)
)
{
LOG
.
info
(
"FYI, {} entries have been processed in {}s = {} lines/second"
,
lineCount
,
processingTimeSeconds
,
(
(
float
)
lineCount
/
processingTimeSeconds
)
)
;
final
long
processingTimeSeconds
=
stopWatch
.
getSplitTime
()
/
1000
;
if
(
lineCount
%
1000
==
0
&&
processingTimeSeconds
>
0
||
lastLogTime
!=
processingTimeSeconds
&&
processingTimeSeconds
%
10
==
0
)
{
LOG
.
info
(
"FYI, {} entries have been processed in {}s = {} lines/second"
,
lineCount
,
processingTimeSeconds
,
(
float
)
lineCount
/
processingTimeSeconds
);
lastLogTime
=
processingTimeSeconds
;
}
...
...
@@ -132,18 +130,19 @@ public class CountryProcessServiceImpl implements ProcessService {
inputs
.
add
(
nextLine
);
}
if
(
inputs
.
size
()
==
0
)
if
(
inputs
.
size
()
==
0
)
{
break
;
}
threadPool
.
submit
(
new
Runnable
()
{
final
DecimalFormat
decimalFormat
=
createDecimalFormat
(
decimalMark
);
@Override
public
void
run
()
{
for
(
String
[]
nextLine
:
inputs
)
{
for
(
final
String
[]
nextLine
:
inputs
)
{
final
String
[]
outputLine
=
HeaderUtils
.
toOutputLine
(
nextLine
,
outputHeaders
,
outputMapping
);
String
[]
outputLine
=
HeaderUtils
.
toOutputLine
(
nextLine
,
outputHeaders
,
outputMapping
);
if
(
outputLine
==
null
)
{
LOG
.
warn
(
"Couldn't make outputline from {}"
,
Arrays
.
toString
(
nextLine
));
continue
;
...
...
@@ -153,32 +152,32 @@ public class CountryProcessServiceImpl implements ProcessService {
// if ("101346".equals(nextLine[0])) {
// LOG.info("!!! Got what we're looking for");
// }
String
declongitude
=
StringUtils
.
trimToNull
(
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_LONGITUDE
)]);
String
declatitude
=
StringUtils
.
trimToNull
(
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_LATITUDE
)]);
String
origCty
=
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_ORIGCTY
)].
trim
();
final
String
declongitude
=
StringUtils
.
trimToNull
(
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_LONGITUDE
)]);
final
String
declatitude
=
StringUtils
.
trimToNull
(
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_LATITUDE
)]);
final
String
origCty
=
nextLine
[
sourceMapping
.
get
(
ApplicationUtils
.
HEADER_ORIGCTY
)].
trim
();