Localized country names

src/main/java/org/genesys2/server/model/impl/Country.java

@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 
 @Entity
 @Table(name = "country")
+// TODO Fix indexer by i18n names!
 @Indexed
 public class Country extends BusinessModel {
 
@@ -55,6 +56,7 @@ public class Country extends BusinessModel {
 
 	@Transient
 	private JsonNode nameJ;
+	private String wikiLink;
 
 	public Country() {
 	}
@@ -133,25 +135,21 @@ public class Country extends BusinessModel {
 		try {
 			this.nameJ = mapper.readTree(nameL);
 			ObjectNode newLang = mapper.createObjectNode().put(locale.getLanguage(), name);
-			System.err.println(newLang);
+			// System.err.println(newLang);
 		} catch (IOException e) {
 		}
 	}
 
 	private synchronized String getNameLocal(Locale locale) {
 		if (this.nameJ == null && this.nameL != null) {
-			System.err.println("Parsing nameL=" + nameL);
 			ObjectMapper mapper = new ObjectMapper();
 			try {
 				this.nameJ = mapper.readTree(nameL);
-				if (this.nameJ != null) {
-					System.err.println(">>> " + this.nameJ + "  " + this.nameJ);
-				}
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
 		}
-		
+
 		return this.nameJ != null && this.nameJ.has(locale.getLanguage()) ? this.nameJ.get(locale.getLanguage()).textValue() : this.name;
 	}
 
@@ -159,4 +157,13 @@ public class Country extends BusinessModel {
 	public String toString() {
 		return MessageFormat.format("Country id={0} name={1} current={2}", id, name, current);
 	}
+
+	@Lob
+	public String getWikiLink() {
+		return wikiLink;
+	}
+
+	public void setWikiLink(String wikiLink) {
+		this.wikiLink = wikiLink;
+	}
 }

src/main/java/org/genesys2/server/persistence/domain/CountryRepository.java

@@ -28,12 +28,12 @@ import org.springframework.data.jpa.repository.Query;
 public interface CountryRepository extends JpaRepository<Country, Long> {
 	Country findByName(String name);
 
-//	@Query("from Country c where c.code3= ?1 and c.current=true")
+	// @Query("from Country c where c.code3= ?1 and c.current=true")
 	Country findByCode3(String code3);
 
 	Country findByCode2(String code2);
 
-//	List<Country> findByRegion(Region region);
+	// List<Country> findByRegion(Region region);
 
 	@Query("from Country c where c.current=false")
 	List<Country> findInactive();
@@ -44,6 +44,11 @@ public interface CountryRepository extends JpaRepository<Country, Long> {
 	@Query("update Country c set c.current=false")
 	void deactivateAll();
 
-//	@Query("select distinct c from Country c where c.region in ( ?1 )")
-//	List<Country> findByRegions(List<Region> regions);
+	Country findByRefnameId(Long refnameId);
+
+	@Query("select distinct c.refnameId from Country c where c.refnameId is not null")
+	List<Long> listRefnameIds();
+
+	// @Query("select distinct c from Country c where c.region in ( ?1 )")
+	// List<Country> findByRegions(List<Region> regions);
 }

src/main/java/org/genesys2/server/service/CountryNamesUpdater.java

+/**
+ * Copyright 2013 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service;
+
+import java.io.IOException;
+
+public interface CountryNamesUpdater {
+
+	void updateAlternateNames() throws IOException;
+
+}

src/main/java/org/genesys2/server/service/GeoService.java

@@ -45,4 +45,17 @@ public interface GeoService {
 
 	List<Country> listAll(Locale locale);
 
+	/**
+	 * Update alternate/translated names of countries
+	 */
+	void updateCountryNames() throws IOException;
+
+	Country getCountryByRefnameId(long refnameId);
+
+	List<Long> listCountryRefnameIds();
+
+	void updateCountryNames(String isoCode3, String jsonTranslations);
+
+	Country updateCountryWiki(String code3, String wiki);
+
 }

src/main/java/org/genesys2/server/service/impl/CSVReader2.java

+package org.genesys2.server.service.impl;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class CSVReader2 {
+
+	private BufferedReader reader;
+	private String separator;
+
+	public CSVReader2(InputStreamReader reader, String separator) {
+		this.reader = new BufferedReader(reader);
+		this.separator = separator;
+	}
+
+	public String[] readNext() throws IOException {
+		String l = reader.readLine();
+		if (l==null) {
+			return null;
+		}
+		return l.split(separator);
+	}
+
+}

src/main/java/org/genesys2/server/service/impl/CountryAlternateNamesUpdater.java

+/**
+ * Copyright 2013 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service.impl;
+
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import javax.annotation.PreDestroy;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.genesys2.server.model.impl.Country;
+import org.genesys2.server.service.CountryNamesUpdater;
+import org.genesys2.server.service.GeoService;
+import org.genesys2.server.service.InstituteService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+@Component
+public class CountryAlternateNamesUpdater implements CountryNamesUpdater {
+
+	// TODO
+	public static String ALTERNATE_NAMES_URL = "...";
+
+	private static String ALTERNATE_NAMES_FILE = "/Users/mobreza/Downloads/alternateNames/alternateNames.txt";
+
+	public static final Log LOG = LogFactory.getLog(CountryAlternateNamesUpdater.class);
+
+	@Autowired
+	private InstituteService instituteService;
+
+	@Autowired
+	private GeoService geoService;
+
+	private static final int nThreads = Runtime.getRuntime().availableProcessors();
+
+	private static final int BATCH_SIZE = 100;
+
+	private final ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(nThreads);
+
+	/**
+	 * Update local FaoInstitute with data from WIEWS database
+	 * 
+	 * @throws IOException
+	 */
+	@Override
+	public void updateAlternateNames() throws IOException {
+		InputStream alternateNamesStream = null;
+
+		final HttpClient httpclient = new DefaultHttpClient();
+		final HttpGet httpget = new HttpGet(ALTERNATE_NAMES_URL);
+		HttpResponse response = null;
+		try {
+			response = httpclient.execute(httpget);
+
+			// Get hold of the response entity
+			final HttpEntity entity = response.getEntity();
+			LOG.debug(entity.getContentType() + " " + entity.getContentLength());
+
+			// If the response does not enclose an entity, there is no
+			// need
+			// to bother about connection release
+			if (entity != null) {
+				ZipInputStream instream = null;
+
+				instream = new ZipInputStream(entity.getContent());
+				ZipEntry zipEntry = null;
+				do {
+					zipEntry = instream.getNextEntry();
+					LOG.debug("Got entry: " + zipEntry.getName());
+					if (zipEntry.getName().equals("alternateNames.txt")) {
+						LOG.info("Found alternateNames.zip");
+						alternateNamesStream = instream;
+						break;
+					}
+				} while (zipEntry != null);
+
+				if (zipEntry == null || !zipEntry.getName().equals("alternateNames.zip")) {
+					LOG.warn("Didn't find alternateNames.zip, stopping update.");
+					IOUtils.closeQuietly(instream);
+					return;
+				}
+			}
+		} catch (IllegalStateException e) {
+			alternateNamesStream = new FileInputStream(ALTERNATE_NAMES_FILE);
+		} catch (final ClientProtocolException e) {
+			LOG.error(e.getMessage(), e);
+			throw new IOException(e);
+		} finally {
+
+		}
+
+		try {
+			updateFromStream(alternateNamesStream);
+		} catch (IOException e) {
+			IOUtils.closeQuietly(alternateNamesStream);
+		}
+
+	}
+
+	private void updateFromStream(InputStream instream) throws IOException {
+		final CSVReader2 reader = new CSVReader2(new InputStreamReader(new BufferedInputStream(instream), "UTF-8"), "\t");
+
+		try {
+
+			final List<String[]> batch = new ArrayList<String[]>(BATCH_SIZE);
+
+			String prevRefnameId = null;
+
+			String[] line = null;
+			List<Long> countryRefnameIds = geoService.listCountryRefnameIds();
+
+			LOG.info("Got " + countryRefnameIds.size() + " refnameIds");
+			int counter = 0;
+
+			while ((line = reader.readNext()) != null) {
+				counter++;
+				if (counter % 10000 == 0) {
+					LOG.info("Country alternate names @ line " + counter);
+				}
+
+				for (int i = 0; i < line.length; i++) {
+					if (line[i].equals("null") || StringUtils.isBlank(line[i])) {
+						line[i] = null;
+					}
+				}
+
+				final String refnameId = line[1];
+
+				if (prevRefnameId == null || refnameId.equals(prevRefnameId)) {
+					prevRefnameId = refnameId;
+					batch.add(line);
+				} else {
+					long longRefnameId = Long.parseLong(prevRefnameId);
+					if (countryRefnameIds.contains(longRefnameId)) {
+						workIt(longRefnameId, batch);
+					}
+					batch.clear();
+					prevRefnameId = null;
+				}
+
+			}
+			long longRefnameId = Long.parseLong(prevRefnameId);
+			if (countryRefnameIds.contains(longRefnameId)) {
+				workIt(Long.parseLong(prevRefnameId), batch);
+				batch.clear();
+			}
+			LOG.info("Done importing alternate geonames");
+		} catch (UnsupportedEncodingException e) {
+			LOG.error(e.getMessage(), e);
+		} finally {
+			IOUtils.closeQuietly(instream);
+		}
+
+	}
+
+	private void workIt(final long refnameId, final List<String[]> batch) {
+
+		System.gc();
+
+		// Need copy!
+		final List<String[]> batchCopy = new ArrayList<String[]>(batch);
+
+		// while (threadPool.getQueue().size() >= nThreads) {
+		// LOG.warn("Queue is too large, waiting...");
+		// try {
+		// Thread.sleep(100);
+		// } catch (final InterruptedException e) {
+		// LOG.warn(e.getMessage());
+		// }
+		// }
+
+		threadPool.execute(new Runnable() {
+			@Override
+			public void run() {
+				// Fetch FaoInstitutes from DB
+				Country country = geoService.getCountryByRefnameId(refnameId);
+
+				if (country == null) {
+					if (LOG.isDebugEnabled())
+						LOG.debug("Nothing to check, no country with refnameId=" + refnameId);
+					return;
+				}
+
+				LOG.info("Processing country refnames " + batchCopy.size());
+				String wiki = country.getWikiLink();
+
+				ObjectMapper objectMapper = new ObjectMapper();
+				ObjectNode jsonNames = objectMapper.createObjectNode();
+
+				for (String[] line : batchCopy) {
+					if (line[2] != null && line[2].length() == 2 && line[3] != null) {
+						boolean preferred = line.length >= 5 && "1".equals(line[4]);
+						if (preferred || jsonNames.get(line[2]) == null) {
+							jsonNames.put(line[2], line[3]);
+						}
+					}
+					if (line[2] != null && "link".equals(line[2])) {
+						boolean preferred = line.length >= 5 && "1".equals(line[4]);
+						if (preferred || wiki == null) {
+							wiki = line[3];
+						}
+					}
+				}
+
+				if (LOG.isDebugEnabled())
+					LOG.debug("We are at: " + country + " i18n = " + jsonNames);
+				geoService.updateCountryNames(country.getCode3(), jsonNames.toString());
+				geoService.updateCountryWiki(country.getCode3(), wiki);
+			}
+		});
+	}
+
+	@PreDestroy
+	public void shutdownPool() {
+		threadPool.shutdown();
+		LOG.info("Waiting for all threads to terminate");
+		try {
+			while (!threadPool.awaitTermination(1, TimeUnit.SECONDS)) {
+				try {
+					Thread.sleep(200);
+				} catch (final InterruptedException e) {
+					LOG.warn(e.getMessage());
+				}
+			}
+			LOG.info("All workers terminated.");
+		} catch (final InterruptedException e) {
+			LOG.error(e.getMessage(), e);
+		}
+	}
+
+}

src/main/java/org/genesys2/server/service/impl/GeoServiceImpl.java

@@ -49,6 +49,9 @@ public class GeoServiceImpl implements GeoService {
 	@Autowired
 	ContentService contentService;
 
+	@Autowired
+	private CountryAlternateNamesUpdater alternateNamesUpdater;
+
 	@Override
 	public List<Country> listAll() {
 		return countryRepository.findAll(new Sort("name", "current"));
@@ -77,6 +80,11 @@ public class GeoServiceImpl implements GeoService {
 		return country != null ? country : countryRepository.findByName(countryString);
 	}
 
+	@Override
+	public Country getCountryByRefnameId(long refnameId) {
+		return countryRepository.findByRefnameId(refnameId);
+	}
+
 	@Override
 	@Transactional(readOnly = false)
 	public void updateCountryData() throws IOException {
@@ -205,10 +213,40 @@ public class GeoServiceImpl implements GeoService {
 		}
 	}
 
+	@Override
+	public void updateCountryNames() throws IOException {
+		alternateNamesUpdater.updateAlternateNames();
+	}
+
 	@Override
 	@PreAuthorize("hasRole('ADMINISTRATOR')")
 	@Transactional(readOnly = false)
 	public void updateBlurp(Country country, String blurp, Locale locale) {
 		contentService.updateArticle(country, "blurp", null, blurp, locale);
 	}
+
+	@Override
+	public List<Long> listCountryRefnameIds() {
+		return countryRepository.listRefnameIds();
+	}
+
+	@Override
+	// TODO Where do we autorize access?
+	// @PreAuthorize("hasRole('ADMINISTRATOR')")
+	@Transactional(readOnly = false)
+	public void updateCountryNames(String isoCode3, String jsonTranslations) {
+		Country country = countryRepository.findByCode3(isoCode3);
+		country.setNameL(jsonTranslations);
+		countryRepository.save(country);
+		LOG.info("Updated translations of " + country);
+	}
+
+	@Override
+	@Transactional(readOnly = false)
+	public Country updateCountryWiki(String isoCode3, String wiki) {
+		Country country = countryRepository.findByCode3(isoCode3);
+		country.setWikiLink(wiki);
+		countryRepository.save(country);
+		return country;
+	}
 }

src/main/java/org/genesys2/server/servlet/controller/AdminController.java

@@ -20,6 +20,7 @@ import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.genesys2.server.service.CountryNamesUpdater;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.GeoService;
 import org.genesys2.server.service.InstituteService;
@@ -42,6 +43,9 @@ public class AdminController {
 	@Autowired
 	InstituteUpdater instituteUpdater;
 
+	@Autowired
+	CountryNamesUpdater alternateNamesUpdater;
+
 	@Autowired
 	LuceneIndexer luceneIndexer;
 
@@ -56,7 +60,7 @@ public class AdminController {
 
 	@Autowired
 	SGSVInsertMissing sgsvImporter;
-	
+
 	@Autowired
 	ContentSanitizer contentSanitizer;
 
@@ -88,6 +92,16 @@ public class AdminController {
 		return "redirect:/admin/";
 	}
 
+	@RequestMapping(method = RequestMethod.POST, value = "/refreshCountryNames")
+	public String refreshCountryNames() {
+		try {
+			geoService.updateCountryNames();
+		} catch (final IOException e) {
+			LOG.error(e.getMessage(), e);
+		}
+		return "redirect:/admin/";
+	}
+
 	@RequestMapping(method = RequestMethod.POST, value = "/reindexEverything")
 	public String reindexEverything() {
 		luceneIndexer.reindexEverything();
@@ -105,13 +119,13 @@ public class AdminController {
 		genesysService.updateAccessionCountryRefs();
 		return "redirect:/admin/";
 	}
-	
+
 	@RequestMapping(method = RequestMethod.POST, value = "/updateInstituteCountryRefs")
 	public String updateInstituteCountryRefs() {
 		instituteService.updateCountryRefs();
 		return "redirect:/admin/";
 	}
-	
+
 	@RequestMapping(method = RequestMethod.POST, value = "/updateAccessionInstituteRefs")
 	public String updateAccessionInstituteRefs() {
 		genesysService.updateAccessionInstitueRefs();
@@ -137,4 +151,16 @@ public class AdminController {
 		LOG.info("Sanitizing content.. Done");
 		return "redirect:/admin/";
 	}
+
+	@RequestMapping(method = RequestMethod.POST, value = "/updateAlternateNames")
+	public String updateAlternateNames() {
+		LOG.info("Updating alternate GEO names");
+		try {
+			alternateNamesUpdater.updateAlternateNames();
+		} catch (IOException e) {
+			LOG.error(e.getMessage(), e);
+		}
+		LOG.info("Updating alternate GEO names: done");
+		return "redirect:/admin/";
+	}
 }

src/main/resources/content/language.properties

@@ -113,6 +113,8 @@ country.stat.countByLocation={0} accessions at institutes in this country
 country.stat.countByOrigin={0} accessions registered in Genesys come from this country.
 country.statistics=Country statistics
 country.accessions.from=Accessions collected in {0}
+country.more-information=More information:
+
 
 faoInstitutes.page.list.title=WIEWS Institutes
 faoInstitutes.page.profile.title=WIEWS {0}

src/main/resources/log4j.properties

@@ -15,16 +15,12 @@
 #-------------------------------------------------------------------------------
 ### direct log messages to stdout ###
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.target=System.out
+log4j.appender.stdout.encoding=UTF-8
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %t %5p %c{1}:%L - %m%n