Skip to content
GitLab
Commit 423ef5f0 authored by Matija Obreza's avatar Matija Obreza
Browse files

Source code cleanup

parent f7559171
Hide whitespace changes
Inline Side-by-side
src/main/java/org/genesys/taxonomy/checker/web/config/ApplicationConfig.java
View file @ 423ef5f0
......@@ -53,7 +53,7 @@ import org.springframework.context.annotation.Configuration;
* The Class ApplicationConfig.
*/
@Configuration
@ComponentScan({ "org.genesys.taxonomy.checker.web.util", "org.genesys.taxonomy.checker.web.service"})
@ComponentScan({ "org.genesys.taxonomy.checker.web.util", "org.genesys.taxonomy.checker.web.service" })
public class ApplicationConfig {
private final static Logger LOG = LoggerFactory.getLogger(ApplicationConfig.class);
......@@ -63,7 +63,7 @@ public class ApplicationConfig {
@Value("${data.folder.path}")
public String dataFolderPath;
/**
* Process service.
*
......@@ -75,9 +75,8 @@ public class ApplicationConfig {
* @throws TaxonomyException the taxonomy exception
*/
@Bean(name = "taxonomyProcessService")
public ProcessService taxonomyProcessService()
throws UnsupportedEncodingException, FileNotFoundException, IOException, ParseException, TaxonomyException {
TaxonomyProcessServiceImpl taxonomyProcessService = new TaxonomyProcessServiceImpl();
public ProcessService taxonomyProcessService() throws UnsupportedEncodingException, FileNotFoundException, IOException, ParseException, TaxonomyException {
final TaxonomyProcessServiceImpl taxonomyProcessService = new TaxonomyProcessServiceImpl();
taxonomyProcessService.setTaxonomyChecker(taxonomyChecker());
taxonomyProcessService.setTaxonomyDatabase(taxonomyDatabase());
......@@ -90,63 +89,62 @@ public class ApplicationConfig {
}
@Bean(name = "countryProcessService")
public ProcessService countryProcessService(){
public ProcessService countryProcessService() {
return new CountryProcessServiceImpl();
}
@Bean(name = "landOrSeaProcessService")
public ProcessService landOrSeaProcessService(){
public ProcessService landOrSeaProcessService() {
return new LandOrSeaProcessServiceImpl();
}
@Bean(name = "processService")
public ProcessService processService(){
public ProcessService processService() {
return new ProcessServiceImpl();
}
@Bean
public LandOrSeaService landOrSeaService() throws MalformedURLException, IOException {
LandOrSeaServiceImpl s = new LandOrSeaServiceImpl();
final LandOrSeaServiceImpl s = new LandOrSeaServiceImpl();
s.setDataFolderPath(dataFolderPath);
s.afterPropertiesSet();
return s;
}
@Bean
public CountryOfOriginService countryOfOriginService() throws MalformedURLException, IOException {
CountryOfOriginServiceImpl s = new CountryOfOriginServiceImpl();
final CountryOfOriginServiceImpl s = new CountryOfOriginServiceImpl();
s.setDataFolderPath(dataFolderPath);
s.afterPropertiesSet();
return s;
}
private void downloadDataIfNeeded() throws IOException {
if (StringUtils.isBlank(taxonomyDataPath)) {
LOG.warn("Taxonomy data directory not provided, creating new");
File destinationDir = File.createTempFile("grin-", "-data");
final File destinationDir = File.createTempFile("grin-", "-data");
if (destinationDir.delete() && destinationDir.mkdir()) {
LOG.warn("Made temporary directory " + destinationDir.getAbsolutePath());
}
taxonomyDataPath = destinationDir.getAbsolutePath();
}
File dataFolder = new File(taxonomyDataPath);
final File dataFolder = new File(taxonomyDataPath);
if (!dataFolder.exists()) {
LOG.warn("Making directory " + dataFolder.getAbsolutePath());
dataFolder.mkdirs();
}
// The two required files
File genusFile = new File(dataFolder, "TAXONOMY_GENUS.txt");
File speciesFile = new File(dataFolder, "TAXONOMY_SPECIES.txt");
final File genusFile = new File(dataFolder, "TAXONOMY_GENUS.txt");
final File speciesFile = new File(dataFolder, "TAXONOMY_SPECIES.txt");
if (!genusFile.exists() || !speciesFile.exists()) {
LOG.warn("Taxonomy data not provided in {}, starting download", dataFolder.getAbsolutePath());
TaxonomyDownloader dl = new TaxonomyDownloader();
final TaxonomyDownloader dl = new TaxonomyDownloader();
LOG.warn("Downloading GRIN-Taxonomy database to {}", dataFolder.getAbsolutePath());
File downloadedCabFile = File.createTempFile("grin-", ".cab");
final File downloadedCabFile = File.createTempFile("grin-", ".cab");
dl.downloadCurrent(downloadedCabFile);
TaxonomyDownloader.unpackCabinetFile(downloadedCabFile, dataFolder, false);
......
src/main/java/org/genesys/taxonomy/checker/web/config/ServletInitializer.java
View file @ 423ef5f0
......@@ -41,8 +41,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
private static final long MAX_REQUEST_SIZE = MAX_FILE_SIZE * 2;
private static final int FILE_SIZE_THRESHOLD = 0;
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#createServletApplicationContext()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* createServletApplicationContext()
*/
@Override
protected WebApplicationContext createServletApplicationContext() {
......@@ -51,24 +54,32 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
return context;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#getServletMappings()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* getServletMappings()
*/
@Override
protected String[] getServletMappings() {
return new String[] { "/" };
}
/* (non-Javadoc)
* @see org.springframework.web.context.AbstractContextLoaderInitializer#createRootApplicationContext()
/*
* (non-Javadoc)
* @see org.springframework.web.context.AbstractContextLoaderInitializer#
* createRootApplicationContext()
*/
@Override
protected WebApplicationContext createRootApplicationContext() {
return null;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#onStartup(javax.servlet.ServletContext)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* onStartup(javax.servlet.ServletContext)
*/
@Override
public void onStartup(final ServletContext servletContext) throws ServletException {
......@@ -76,8 +87,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
servletContext.setSessionTrackingModes(EnumSet.of(SessionTrackingMode.COOKIE));
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#customizeRegistration(javax.servlet.ServletRegistration.Dynamic)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* customizeRegistration(javax.servlet.ServletRegistration.Dynamic)
*/
@Override
protected void customizeRegistration(final ServletRegistration.Dynamic registration) {
......@@ -89,8 +103,11 @@ public class ServletInitializer extends AbstractDispatcherServletInitializer {
return multipartConfigElement;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#getServletFilters()
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.support.AbstractDispatcherServletInitializer#
* getServletFilters()
*/
@Override
protected Filter[] getServletFilters() {
......
src/main/java/org/genesys/taxonomy/checker/web/config/WebConfiguration.java
View file @ 423ef5f0
......@@ -45,8 +45,12 @@ import org.springframework.web.servlet.view.JstlView;
@ComponentScan({ "org.genesys.taxonomy.checker.web.controller" })
public class WebConfiguration extends WebMvcConfigurerAdapter {
/* (non-Javadoc)
* @see org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#addResourceHandlers(org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#
* addResourceHandlers(org.springframework.web.servlet.config.annotation.
* ResourceHandlerRegistry)
*/
@Override
public void addResourceHandlers(final ResourceHandlerRegistry registry) {
......@@ -104,8 +108,12 @@ public class WebConfiguration extends WebMvcConfigurerAdapter {
return cookieLocaleResolver;
}
/* (non-Javadoc)
* @see org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#addInterceptors(org.springframework.web.servlet.config.annotation.InterceptorRegistry)
/*
* (non-Javadoc)
* @see
* org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter#
* addInterceptors(org.springframework.web.servlet.config.annotation.
* InterceptorRegistry)
*/
@Override
public void addInterceptors(final InterceptorRegistry registry) {
......
src/main/java/org/genesys/taxonomy/checker/web/controller/ControllerExceptionHandler.java
View file @ 423ef5f0
......@@ -16,14 +16,14 @@ public class ControllerExceptionHandler {
private final static Logger LOG = LoggerFactory.getLogger(ControllerExceptionHandler.class);
@ExceptionHandler(value = Throwable.class)
public ModelAndView anyThrowable(Throwable throwable) {
public ModelAndView anyThrowable(final Throwable throwable) {
LOG.error("Application error", throwable);
ModelAndView modelAndView = new ModelAndView("error");
final ModelAndView modelAndView = new ModelAndView("error");
modelAndView.setStatus(HttpStatus.INTERNAL_SERVER_ERROR);
modelAndView.addObject("errorText", throwable.getMessage());
return modelAndView;
}
}
src/main/java/org/genesys/taxonomy/checker/web/controller/ValidatorController.java
View file @ 423ef5f0
......@@ -81,17 +81,15 @@ public class ValidatorController {
public String processString(@RequestParam(name = "separator", required = false, defaultValue = ",") final Character separator,
@RequestParam(name = "separatorOther", required = false, defaultValue = "") final Character separatorOther,
@RequestParam(name = "quoteChar", required = false, defaultValue = "\"") final Character quoteChar,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText,
@RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType,
@RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText, @RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType, @RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
final ModelMap model) throws IOException, ParseException, TaxonomyException {
LOG.trace("Processing uploaded CSV string:\n{}", csvText);
List<String[]> rows = processService.process(new StringReader(StringUtils.defaultString(csvText, "")), (separator == 'O' ? separatorOther : separator), quoteChar, escapeChar, validateType,
toCurrentTaxa, decimalMark);
final List<String[]> rows = processService.process(new StringReader(StringUtils.defaultString(csvText, "")), separator == 'O' ? separatorOther : separator, quoteChar,
escapeChar, validateType, toCurrentTaxa, decimalMark);
model.addAttribute("result", rows);
model.addAttribute("tableHeaders", rows.get(0));
model.addAttribute("newLineSymbol", "\n");
......@@ -106,11 +104,9 @@ public class ValidatorController {
public void processStringToCSV1(@RequestParam(name = "separator", required = false, defaultValue = ",") final Character separator,
@RequestParam(name = "separatorOther", required = false, defaultValue = "") final Character separatorOther,
@RequestParam(name = "quoteChar", required = false, defaultValue = "\"") final Character quoteChar,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText,
@RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType,
@RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText, @RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType, @RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
final HttpServletResponse response) throws IOException, ParseException, TaxonomyException {
processStringToCSV(separator, separatorOther, quoteChar, escapeChar, csvText, toCurrentTaxa, validateType, decimalMark, response);
......@@ -134,26 +130,24 @@ public class ValidatorController {
public void processStringToCSV(@RequestParam(name = "separator", required = false, defaultValue = ",") final Character separator,
@RequestParam(name = "separatorOther", required = false, defaultValue = "") final Character separatorOther,
@RequestParam(name = "quoteChar", required = false, defaultValue = "\"") final Character quoteChar,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText,
@RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType,
@RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
@RequestParam(name = "escapeChar", required = false, defaultValue = "\0") final Character escapeChar,
@RequestParam(name = "csvText", required = true) final String csvText, @RequestParam(name = "toCurrentTaxa", required = false) final Boolean toCurrentTaxa,
@RequestParam(name = "validateType", defaultValue = "") final String validateType, @RequestParam(name = "decimalMark", defaultValue = ".") final Character decimalMark,
final HttpServletResponse response) throws IOException, ParseException, TaxonomyException {
LOG.trace("Processing uploaded CSV string:\n{}", csvText);
List<String[]> rows = processService.process(new StringReader(StringUtils.defaultString(csvText, "")), (separator == 'O' ? separatorOther : separator), quoteChar, escapeChar, validateType,
toCurrentTaxa, decimalMark);
final List<String[]> rows = processService.process(new StringReader(StringUtils.defaultString(csvText, "")), separator == 'O' ? separatorOther : separator, quoteChar,
escapeChar, validateType, toCurrentTaxa, decimalMark);
response.setContentType("text/csv;charset=UTF-16LE");
response.addHeader("Content-Disposition", String.format("attachment; filename=\"validator-results.csv\""));
response.flushBuffer();
// Write CSV to the stream.
final OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream(), "UTF-16LE");
try (CSVWriter csvWriter=new CSVWriter(osw, (separator == 'O' ? separatorOther : separator), quoteChar, escapeChar, "\n")) {
try (CSVWriter csvWriter = new CSVWriter(osw, separator == 'O' ? separatorOther : separator, quoteChar, escapeChar, "\n")) {
csvWriter.writeAll(rows);
csvWriter.flush();
} finally {
......@@ -161,12 +155,12 @@ public class ValidatorController {
}
}
private String[] getStyles(String[] strings) {
String[] styles = new String[strings.length];
private String[] getStyles(final String[] strings) {
final String[] styles = new String[strings.length];
for (int i = 0; i < strings.length; i++) {
styles[i] = "";
if ("GENUS_check".equals(strings[i])) {
//styles[i]=""
// styles[i]=""
}
}
return styles;
......
src/main/java/org/genesys/taxonomy/checker/web/service/ProcessService.java
View file @ 423ef5f0
......@@ -47,15 +47,15 @@ public interface ProcessService {
List<String[]> process(Reader reader, Character separator, Character quoteChar, Character escapeChar, String validateType, Boolean toCurrentTaxa, Character decimalMark)
throws ParseException, IOException, TaxonomyException;
/**
* Create instance of DecimalFormat used to parse numbers from the CSV
*
* @param decimalMark
* @return DecimalFormat using the specified decimal mark
*/
default DecimalFormat createDecimalFormat(char decimalMark) {
default DecimalFormat createDecimalFormat(final char decimalMark) {
final DecimalFormat decimalFormat = new DecimalFormat();
DecimalFormatSymbols symbols = new DecimalFormatSymbols();
final DecimalFormatSymbols symbols = new DecimalFormatSymbols();
symbols.setDecimalSeparator(decimalMark);
symbols.setGroupingSeparator(decimalMark == ',' ? '.' : ',');
decimalFormat.setDecimalFormatSymbols(symbols);
......
src/main/java/org/genesys/taxonomy/checker/web/service/impl/CachingInMemoryTaxonomyDatabase.java
View file @ 423ef5f0
......@@ -18,17 +18,18 @@ package org.genesys.taxonomy.checker.web.service.impl;
import java.util.List;
import java.util.concurrent.ExecutionException;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase;
import org.genesys.taxonomy.gringlobal.model.SpeciesRow;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
/**
* Guava cache supported in-memory taxonomy database.
*/
public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
// private final static Logger LOG = LoggerFactory.getLogger(CachingInMemoryTaxonomyDatabase.class);
// private final static Logger LOG =
// LoggerFactory.getLogger(CachingInMemoryTaxonomyDatabase.class);
/** The cache genus species. */
private final Cache<String, List<SpeciesRow>> cacheGenusSpecies = CacheBuilder.newBuilder().maximumSize(100).build();
......@@ -47,8 +48,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
* @see org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#getAllGenusSpecies(java.lang.String)
* @see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#getAllGenusSpecies(java
* .lang.String)
*/
@Override
protected List<SpeciesRow> getAllGenusSpecies(final String genus) {
......@@ -68,8 +70,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
* @see org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarGenus(java.lang.String, int)
* @see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarGenus(java.
* lang.String, int)
*/
@Override
public List<String> findSimilarGenus(final String genus, final int maxSize) {
......@@ -90,9 +93,9 @@ public class CachingInMemoryTaxonomyDatabase extends InMemoryTaxonomyDatabase {
*/
/*
* (non-Javadoc)
*
* @see org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarSpecies(java.lang.String,
* java.lang.String, int)
* @see
* org.genesys.taxonomy.checker.InMemoryTaxonomyDatabase#findSimilarSpecies(java
* .lang.String, java.lang.String, int)
*/
@Override
public List<String> findSimilarSpecies(final String genus, final String species, final int maxSize) {
......
src/main/java/org/genesys/taxonomy/checker/web/service/impl/CountryProcessServiceImpl.java
View file @ 423ef5f0
......@@ -17,7 +17,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.ConcurrentUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.geotools.service.CountryOfOriginService;
import org.genesys.geotools.service.HeaderUtils;
......@@ -29,7 +28,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.collect.Lists;
import com.opencsv.CSVReader;
/**
......@@ -42,51 +40,51 @@ public class CountryProcessServiceImpl implements ProcessService {
@Autowired
private CountryOfOriginService countryOfOriginService;
@Override
public List<String[]> process(Reader reader, Character separator, Character quoteChar, Character escapeChar, String validateType, Boolean toCurrentTaxa, Character decimalMark)
throws ParseException, IOException, TaxonomyException {
public List<String[]> process(final Reader reader, final Character separator, final Character quoteChar, final Character escapeChar, final String validateType,
final Boolean toCurrentTaxa, final Character decimalMark) throws ParseException, IOException, TaxonomyException {
return execute(reader, separator, quoteChar, escapeChar, decimalMark);
}
private List<String[]> execute(Reader readerr, char separatorChar, char quoteChar, char escapeChar, Character decimalMark) throws IOException {
private List<String[]> execute(final Reader readerr, final char separatorChar, final char quoteChar, final char escapeChar, final Character decimalMark) throws IOException {
final List<String[]> lines = Collections.synchronizedList(new LinkedList<>());
try (CSVReader reader = new CSVReader(readerr, separatorChar, quoteChar, escapeChar, 0, false)) {
String[] headers = reader.readNext();
final String[] headers = reader.readNext();
LOG.debug("Input CSV headers: {}", Arrays.toString(headers));
HeaderUtils.throwIfHeaderFound(headers, new String[] { ApplicationUtils.HEADER_ORIGCTY_CHECK });
HeaderUtils.throwIfHeaderNotFound(headers, new String[] { ApplicationUtils.HEADER_LONGITUDE, ApplicationUtils.HEADER_LATITUDE });
List<String> outputHeaders = new ArrayList<>(Arrays.asList(headers));
List<Integer> outputMapping = HeaderUtils.mapHeaderPositions(headers);
final List<String> outputHeaders = new ArrayList<>(Arrays.asList(headers));
final List<Integer> outputMapping = HeaderUtils.mapHeaderPositions(headers);
LOG.debug("Source mapping indexes: {}", outputMapping);
LOG.debug("Initial output headers: {}", outputHeaders);
Map<String, Integer> sourceMapping = HeaderUtils
.makeSourceMapping(headers, new String[] { ApplicationUtils.HEADER_LONGITUDE, ApplicationUtils.HEADER_LATITUDE, ApplicationUtils.HEADER_ORIGCTY });
final Map<String, Integer> sourceMapping = HeaderUtils.makeSourceMapping(headers, new String[] { ApplicationUtils.HEADER_LONGITUDE, ApplicationUtils.HEADER_LATITUDE,
ApplicationUtils.HEADER_ORIGCTY });
if (outputHeaders.indexOf(ApplicationUtils.HEADER_ORIGCTY) >= 0) {
outputHeaders.add(outputHeaders.indexOf(ApplicationUtils.HEADER_ORIGCTY) + 1, ApplicationUtils.HEADER_ORIGCTY_CHECK);
outputMapping.add(outputHeaders.indexOf(ApplicationUtils.HEADER_ORIGCTY) + 1, null);
} else {
int pos = Math.max(outputHeaders.indexOf(ApplicationUtils.HEADER_LONGITUDE), outputHeaders.indexOf(ApplicationUtils.HEADER_LATITUDE));
final int pos = Math.max(outputHeaders.indexOf(ApplicationUtils.HEADER_LONGITUDE), outputHeaders.indexOf(ApplicationUtils.HEADER_LATITUDE));
outputHeaders.add(pos + 1, ApplicationUtils.HEADER_ORIGCTY_CHECK);
outputMapping.add(pos + 1, null);
}
// Add extra header HEADER_DECLATITUDE_CHECK
{
int pos = outputHeaders.indexOf(ApplicationUtils.HEADER_LATITUDE);
final int pos = outputHeaders.indexOf(ApplicationUtils.HEADER_LATITUDE);
outputHeaders.add(pos + 1, ApplicationUtils.HEADER_DECLATITUDE_CHECK);
outputMapping.add(pos + 1, null);
}
// Add extra header HEADER_DECLONGITUDE_CHECK
{
int pos = outputHeaders.indexOf(ApplicationUtils.HEADER_LONGITUDE);
final int pos = outputHeaders.indexOf(ApplicationUtils.HEADER_LONGITUDE);
outputHeaders.add(pos + 1, ApplicationUtils.HEADER_DECLONGITUDE_CHECK);
outputMapping.add(pos + 1, null);
}
......@@ -97,14 +95,14 @@ public class CountryProcessServiceImpl implements ProcessService {
lines.add(outputHeaders.toArray(ArrayUtils.EMPTY_STRING_ARRAY));
// Thread pool with nThreads concurrent threads
int nThreads = Math.max(Runtime.getRuntime().availableProcessors() - 1, 1);
final int nThreads = Math.max(Runtime.getRuntime().availableProcessors() - 1, 1);
LOG.info("Using {} threads for {} CPUs.", nThreads, Runtime.getRuntime().availableProcessors());
BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<>(nThreads * 2);
final BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue<>(nThreads * 2);
ThreadPoolExecutor threadPool = new ThreadPoolExecutor(1 + nThreads / 2, nThreads, 5, TimeUnit.SECONDS, workQueue, new ThreadPoolExecutor.CallerRunsPolicy());
final ThreadPoolExecutor threadPool = new ThreadPoolExecutor(1 + nThreads / 2, nThreads, 5, TimeUnit.SECONDS, workQueue, new ThreadPoolExecutor.CallerRunsPolicy());
StopWatch stopWatch = new StopWatch();
final StopWatch stopWatch = new StopWatch();
int lineCount = 0;
......@@ -112,16 +110,16 @@ public class CountryProcessServiceImpl implements ProcessService {
stopWatch.start();
do {
final ArrayList<String[]> inputs = new ArrayList<String[]>();
final ArrayList<String[]> inputs = new ArrayList<>();
for (int j = 0; j < 100; j++) {
String[] nextLine = reader.readNext();
final String[] nextLine = reader.readNext();
lineCount++;
stopWatch.split();
long processingTimeSeconds = stopWatch.getSplitTime() / 1000;
if ((lineCount % 1000 == 0 && processingTimeSeconds > 0) || (lastLogTime != processingTimeSeconds && processingTimeSeconds % 10 == 0)) {
LOG.info("FYI, {} entries have been processed in {}s = {} lines/second", lineCount, processingTimeSeconds, ((float)lineCount / processingTimeSeconds));
final long processingTimeSeconds = stopWatch.getSplitTime() / 1000;
if (lineCount % 1000 == 0 && processingTimeSeconds > 0 || lastLogTime != processingTimeSeconds && processingTimeSeconds % 10 == 0) {
LOG.info("FYI, {} entries have been processed in {}s = {} lines/second", lineCount, processingTimeSeconds, (float) lineCount / processingTimeSeconds);
lastLogTime = processingTimeSeconds;
}
......@@ -132,18 +130,19 @@ public class CountryProcessServiceImpl implements ProcessService {
inputs.add(nextLine);
}
if (inputs.size() == 0)
if (inputs.size() == 0) {
break;
}
threadPool.submit(new Runnable() {
final DecimalFormat decimalFormat = createDecimalFormat(decimalMark);
@Override
public void run() {
for (String[] nextLine : inputs) {
for (final String[] nextLine : inputs) {
final String[] outputLine = HeaderUtils.toOutputLine(nextLine, outputHeaders, outputMapping);
String[] outputLine = HeaderUtils.toOutputLine(nextLine, outputHeaders, outputMapping);
if (outputLine == null) {
LOG.warn("Couldn't make outputline from {}", Arrays.toString(nextLine));
continue;
......@@ -153,32 +152,32 @@ public class CountryProcessServiceImpl implements ProcessService {
// if ("101346".equals(nextLine[0])) {
// LOG.info("!!! Got what we're looking for");
// }
String declongitude = StringUtils.trimToNull(nextLine[sourceMapping.get(ApplicationUtils.HEADER_LONGITUDE)]);
String declatitude = StringUtils.trimToNull(nextLine[sourceMapping.get(ApplicationUtils.HEADER_LATITUDE)]);
String origCty = nextLine[sourceMapping.get(ApplicationUtils.HEADER_ORIGCTY)].trim();
final String declongitude = StringUtils.trimToNull(nextLine[sourceMapping.get(ApplicationUtils.HEADER_LONGITUDE)]);
final String declatitude = StringUtils.trimToNull(nextLine[sourceMapping.get(ApplicationUtils.HEADER_LATITUDE)]);
final String origCty = nextLine[sourceMapping.get(ApplicationUtils.HEADER_ORIGCTY)].trim();
if (StringUtils.isNotBlank(declatitude) && StringUtils.<