Commit 804afc6b authored by Matija Obreza's avatar Matija Obreza

Localized country names

parent 41d16322
......@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
@Entity
@Table(name = "country")
// TODO Fix indexer by i18n names!
@Indexed
public class Country extends BusinessModel {
......@@ -55,6 +56,7 @@ public class Country extends BusinessModel {
@Transient
private JsonNode nameJ;
private String wikiLink;
public Country() {
}
......@@ -133,25 +135,21 @@ public class Country extends BusinessModel {
try {
this.nameJ = mapper.readTree(nameL);
ObjectNode newLang = mapper.createObjectNode().put(locale.getLanguage(), name);
System.err.println(newLang);
// System.err.println(newLang);
} catch (IOException e) {
}
}
private synchronized String getNameLocal(Locale locale) {
if (this.nameJ == null && this.nameL != null) {
System.err.println("Parsing nameL=" + nameL);
ObjectMapper mapper = new ObjectMapper();
try {
this.nameJ = mapper.readTree(nameL);
if (this.nameJ != null) {
System.err.println(">>> " + this.nameJ + " " + this.nameJ);
}
} catch (IOException e) {
e.printStackTrace();
}
}
return this.nameJ != null && this.nameJ.has(locale.getLanguage()) ? this.nameJ.get(locale.getLanguage()).textValue() : this.name;
}
......@@ -159,4 +157,13 @@ public class Country extends BusinessModel {
public String toString() {
return MessageFormat.format("Country id={0} name={1} current={2}", id, name, current);
}
@Lob
public String getWikiLink() {
return wikiLink;
}
public void setWikiLink(String wikiLink) {
this.wikiLink = wikiLink;
}
}
......@@ -28,12 +28,12 @@ import org.springframework.data.jpa.repository.Query;
public interface CountryRepository extends JpaRepository<Country, Long> {
Country findByName(String name);
// @Query("from Country c where c.code3= ?1 and c.current=true")
// @Query("from Country c where c.code3= ?1 and c.current=true")
Country findByCode3(String code3);
Country findByCode2(String code2);
// List<Country> findByRegion(Region region);
// List<Country> findByRegion(Region region);
@Query("from Country c where c.current=false")
List<Country> findInactive();
......@@ -44,6 +44,11 @@ public interface CountryRepository extends JpaRepository<Country, Long> {
@Query("update Country c set c.current=false")
void deactivateAll();
// @Query("select distinct c from Country c where c.region in ( ?1 )")
// List<Country> findByRegions(List<Region> regions);
Country findByRefnameId(Long refnameId);
@Query("select distinct c.refnameId from Country c where c.refnameId is not null")
List<Long> listRefnameIds();
// @Query("select distinct c from Country c where c.region in ( ?1 )")
// List<Country> findByRegions(List<Region> regions);
}
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service;
import java.io.IOException;
public interface CountryNamesUpdater {
void updateAlternateNames() throws IOException;
}
......@@ -45,4 +45,17 @@ public interface GeoService {
List<Country> listAll(Locale locale);
/**
* Update alternate/translated names of countries
*/
void updateCountryNames() throws IOException;
Country getCountryByRefnameId(long refnameId);
List<Long> listCountryRefnameIds();
void updateCountryNames(String isoCode3, String jsonTranslations);
Country updateCountryWiki(String code3, String wiki);
}
package org.genesys2.server.service.impl;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
public class CSVReader2 {
private BufferedReader reader;
private String separator;
public CSVReader2(InputStreamReader reader, String separator) {
this.reader = new BufferedReader(reader);
this.separator = separator;
}
public String[] readNext() throws IOException {
String l = reader.readLine();
if (l==null) {
return null;
}
return l.split(separator);
}
}
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.impl;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.annotation.PreDestroy;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Component
public class CountryAlternateNamesUpdater implements CountryNamesUpdater {
// TODO
public static String ALTERNATE_NAMES_URL = "...";
private static String ALTERNATE_NAMES_FILE = "/Users/mobreza/Downloads/alternateNames/alternateNames.txt";
public static final Log LOG = LogFactory.getLog(CountryAlternateNamesUpdater.class);
@Autowired
private InstituteService instituteService;
@Autowired
private GeoService geoService;
private static final int nThreads = Runtime.getRuntime().availableProcessors();
private static final int BATCH_SIZE = 100;
private final ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(nThreads);
/**
* Update local FaoInstitute with data from WIEWS database
*
* @throws IOException
*/
@Override
public void updateAlternateNames() throws IOException {
InputStream alternateNamesStream = null;
final HttpClient httpclient = new DefaultHttpClient();
final HttpGet httpget = new HttpGet(ALTERNATE_NAMES_URL);
HttpResponse response = null;
try {
response = httpclient.execute(httpget);
// Get hold of the response entity
final HttpEntity entity = response.getEntity();
LOG.debug(entity.getContentType() + " " + entity.getContentLength());
// If the response does not enclose an entity, there is no
// need
// to bother about connection release
if (entity != null) {
ZipInputStream instream = null;
instream = new ZipInputStream(entity.getContent());
ZipEntry zipEntry = null;
do {
zipEntry = instream.getNextEntry();
LOG.debug("Got entry: " + zipEntry.getName());
if (zipEntry.getName().equals("alternateNames.txt")) {
LOG.info("Found alternateNames.zip");
alternateNamesStream = instream;
break;
}
} while (zipEntry != null);
if (zipEntry == null || !zipEntry.getName().equals("alternateNames.zip")) {
LOG.warn("Didn't find alternateNames.zip, stopping update.");
IOUtils.closeQuietly(instream);
return;
}
}
} catch (IllegalStateException e) {
alternateNamesStream = new FileInputStream(ALTERNATE_NAMES_FILE);
} catch (final ClientProtocolException e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
} finally {
}
try {
updateFromStream(alternateNamesStream);
} catch (IOException e) {
IOUtils.closeQuietly(alternateNamesStream);
}
}
private void updateFromStream(InputStream instream) throws IOException {
final CSVReader2 reader = new CSVReader2(new InputStreamReader(new BufferedInputStream(instream), "UTF-8"), "\t");
try {
final List<String[]> batch = new ArrayList<String[]>(BATCH_SIZE);
String prevRefnameId = null;
String[] line = null;
List<Long> countryRefnameIds = geoService.listCountryRefnameIds();
LOG.info("Got " + countryRefnameIds.size() + " refnameIds");
int counter = 0;
while ((line = reader.readNext()) != null) {
counter++;
if (counter % 10000 == 0) {
LOG.info("Country alternate names @ line " + counter);
}
for (int i = 0; i < line.length; i++) {
if (line[i].equals("null") || StringUtils.isBlank(line[i])) {
line[i] = null;
}
}
final String refnameId = line[1];
if (prevRefnameId == null || refnameId.equals(prevRefnameId)) {
prevRefnameId = refnameId;
batch.add(line);
} else {
long longRefnameId = Long.parseLong(prevRefnameId);
if (countryRefnameIds.contains(longRefnameId)) {
workIt(longRefnameId, batch);
}
batch.clear();
prevRefnameId = null;
}
}
long longRefnameId = Long.parseLong(prevRefnameId);
if (countryRefnameIds.contains(longRefnameId)) {
workIt(Long.parseLong(prevRefnameId), batch);
batch.clear();
}
LOG.info("Done importing alternate geonames");
} catch (UnsupportedEncodingException e) {
LOG.error(e.getMessage(), e);
} finally {
IOUtils.closeQuietly(instream);
}
}
private void workIt(final long refnameId, final List<String[]> batch) {
System.gc();
// Need copy!
final List<String[]> batchCopy = new ArrayList<String[]>(batch);
// while (threadPool.getQueue().size() >= nThreads) {
// LOG.warn("Queue is too large, waiting...");
// try {
// Thread.sleep(100);
// } catch (final InterruptedException e) {
// LOG.warn(e.getMessage());
// }
// }
threadPool.execute(new Runnable() {
@Override
public void run() {
// Fetch FaoInstitutes from DB
Country country = geoService.getCountryByRefnameId(refnameId);
if (country == null) {
if (LOG.isDebugEnabled())
LOG.debug("Nothing to check, no country with refnameId=" + refnameId);
return;
}
LOG.info("Processing country refnames " + batchCopy.size());
String wiki = country.getWikiLink();
ObjectMapper objectMapper = new ObjectMapper();
ObjectNode jsonNames = objectMapper.createObjectNode();
for (String[] line : batchCopy) {
if (line[2] != null && line[2].length() == 2 && line[3] != null) {
boolean preferred = line.length >= 5 && "1".equals(line[4]);
if (preferred || jsonNames.get(line[2]) == null) {
jsonNames.put(line[2], line[3]);
}
}
if (line[2] != null && "link".equals(line[2])) {
boolean preferred = line.length >= 5 && "1".equals(line[4]);
if (preferred || wiki == null) {
wiki = line[3];
}
}
}
if (LOG.isDebugEnabled())
LOG.debug("We are at: " + country + " i18n = " + jsonNames);
geoService.updateCountryNames(country.getCode3(), jsonNames.toString());
geoService.updateCountryWiki(country.getCode3(), wiki);
}
});
}
@PreDestroy
public void shutdownPool() {
threadPool.shutdown();
LOG.info("Waiting for all threads to terminate");
try {
while (!threadPool.awaitTermination(1, TimeUnit.SECONDS)) {
try {
Thread.sleep(200);
} catch (final InterruptedException e) {
LOG.warn(e.getMessage());
}
}
LOG.info("All workers terminated.");
} catch (final InterruptedException e) {
LOG.error(e.getMessage(), e);
}
}
}
......@@ -49,6 +49,9 @@ public class GeoServiceImpl implements GeoService {
@Autowired
ContentService contentService;
@Autowired
private CountryAlternateNamesUpdater alternateNamesUpdater;
@Override
public List<Country> listAll() {
return countryRepository.findAll(new Sort("name", "current"));
......@@ -77,6 +80,11 @@ public class GeoServiceImpl implements GeoService {
return country != null ? country : countryRepository.findByName(countryString);
}
@Override
public Country getCountryByRefnameId(long refnameId) {
return countryRepository.findByRefnameId(refnameId);
}
@Override
@Transactional(readOnly = false)
public void updateCountryData() throws IOException {
......@@ -205,10 +213,40 @@ public class GeoServiceImpl implements GeoService {
}
}
@Override
public void updateCountryNames() throws IOException {
alternateNamesUpdater.updateAlternateNames();
}
@Override
@PreAuthorize("hasRole('ADMINISTRATOR')")
@Transactional(readOnly = false)
public void updateBlurp(Country country, String blurp, Locale locale) {
contentService.updateArticle(country, "blurp", null, blurp, locale);
}
@Override
public List<Long> listCountryRefnameIds() {
return countryRepository.listRefnameIds();
}
@Override
// TODO Where do we autorize access?
// @PreAuthorize("hasRole('ADMINISTRATOR')")
@Transactional(readOnly = false)
public void updateCountryNames(String isoCode3, String jsonTranslations) {
Country country = countryRepository.findByCode3(isoCode3);
country.setNameL(jsonTranslations);
countryRepository.save(country);
LOG.info("Updated translations of " + country);
}
@Override
@Transactional(readOnly = false)
public Country updateCountryWiki(String isoCode3, String wiki) {
Country country = countryRepository.findByCode3(isoCode3);
country.setWikiLink(wiki);
countryRepository.save(country);
return country;
}
}
......@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.service.CountryNamesUpdater;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
......@@ -42,6 +43,9 @@ public class AdminController {
@Autowired
InstituteUpdater instituteUpdater;
@Autowired
CountryNamesUpdater alternateNamesUpdater;
@Autowired
LuceneIndexer luceneIndexer;
......@@ -56,7 +60,7 @@ public class AdminController {
@Autowired
SGSVInsertMissing sgsvImporter;
@Autowired
ContentSanitizer contentSanitizer;
......@@ -88,6 +92,16 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/refreshCountryNames")
public String refreshCountryNames() {
try {
geoService.updateCountryNames();
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/reindexEverything")
public String reindexEverything() {
luceneIndexer.reindexEverything();
......@@ -105,13 +119,13 @@ public class AdminController {
genesysService.updateAccessionCountryRefs();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateInstituteCountryRefs")
public String updateInstituteCountryRefs() {
instituteService.updateCountryRefs();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateAccessionInstituteRefs")
public String updateAccessionInstituteRefs() {
genesysService.updateAccessionInstitueRefs();
......@@ -137,4 +151,16 @@ public class AdminController {
LOG.info("Sanitizing content.. Done");
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateAlternateNames")
public String updateAlternateNames() {
LOG.info("Updating alternate GEO names");
try {
alternateNamesUpdater.updateAlternateNames();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
LOG.info("Updating alternate GEO names: done");
return "redirect:/admin/";
}
}
......@@ -113,6 +113,8 @@ country.stat.countByLocation={0} accessions at institutes in this country
country.stat.countByOrigin={0} accessions registered in Genesys come from this country.
country.statistics=Country statistics
country.accessions.from=Accessions collected in {0}
country.more-information=More information:
faoInstitutes.page.list.title=WIEWS Institutes
faoInstitutes.page.profile.title=WIEWS {0}
......
......@@ -15,16 +15,12 @@
#-------------------------------------------------------------------------------
### direct log messages to stdout ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.target=System.out
log4j.appender.stdout.encoding=UTF-8
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %t %5p %c{1}:%L - %m%n