Commit 419dbe0a authored by Matija Obreza's avatar Matija Obreza
Browse files

Automated updating of country ITPGFRA status

parent d5c6b130
......@@ -24,6 +24,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
public interface CountryRepository extends JpaRepository<Country, Long> {
Country findByName(String name);
......@@ -51,9 +52,12 @@ public interface CountryRepository extends JpaRepository<Country, Long> {
@Query("select distinct c.refnameId from Country c where c.refnameId is not null")
List<Long> listRefnameIds();
@Query("select distinct itpgrfa.country from ITPGRFAStatus itpgrfa where itpgrfa.country.current = true")
@Query("select distinct itpgrfa.country from ITPGRFAStatus itpgrfa where itpgrfa.contractingParty != 'No' and itpgrfa.country.current = true")
List<Country> findITPGRFA();
@Query("select distinct c from Country c where c.nameL like :pattern")
List<Country> findWithI18N(@Param("pattern") String pattern);
// @Query("select distinct c from Country c where c.region in ( ?1 )")
// List<Country> findByRegions(List<Region> regions);
}
......@@ -16,9 +16,12 @@
package org.genesys2.server.persistence.domain;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.ITPGRFAStatus;
import org.springframework.data.jpa.repository.JpaRepository;
public interface ITPGRFAStatusRepository extends JpaRepository<ITPGRFAStatus, Long> {
ITPGRFAStatus findByCountry(Country country);
}
......@@ -22,6 +22,7 @@ import java.util.Locale;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.ITPGRFAStatus;
import com.fasterxml.jackson.databind.node.ArrayNode;
......@@ -83,4 +84,16 @@ public interface GeoService {
*/
List<Country> listITPGRFA(Locale locale);
/**
* Upsert country ITPGRGA status
*
* @param country
* @param contractingParty
* @param membership
* @param membershipBy
* @param nameOfNFP
* @return
*/
ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy, String nameOfNFP);
}
......@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Set;
......@@ -32,7 +33,9 @@ import org.genesys2.geo.sources.DavrosCountrySource;
import org.genesys2.geo.sources.GeoNamesCountrySource;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.ITPGRFAStatus;
import org.genesys2.server.persistence.domain.CountryRepository;
import org.genesys2.server.persistence.domain.ITPGRFAStatusRepository;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.GeoService;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -41,6 +44,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
......@@ -56,6 +60,9 @@ public class GeoServiceImpl implements GeoService {
@Autowired
ContentService contentService;
@Autowired
ITPGRFAStatusRepository itpgrfaRepository;
@Override
public List<Country> listAll() {
return countryRepository.findAll(new Sort("name", "current"));
......@@ -79,7 +86,7 @@ public class GeoServiceImpl implements GeoService {
Country.sort(countries, locale);
return countries;
}
@Override
public List<Country> listITPGRFA(Locale locale) {
List<Country> countries = countryRepository.findITPGRFA();
......@@ -95,7 +102,48 @@ public class GeoServiceImpl implements GeoService {
} else if (countryString.length() == 2) {
country = countryRepository.findByCode2(countryString);
}
return country != null ? country : countryRepository.findByName(countryString);
// Let's try the name
if (country == null) {
country = countryRepository.findByName(countryString);
}
// Let's try translations
if (country == null) {
country = findCountryByName(countryString);
}
return country;
}
/**
* Check if we have a country that has
*
* @param name
* in i18n
* @param countryString
* @return
*/
private Country findCountryByName(String name) {
ObjectMapper mapper = new ObjectMapper();
List<Country> countries = countryRepository.findWithI18N("%" + name.trim() + "%");
System.err.println("Found " + countries.size() + " that have " + name);
for (Country c : countries) {
try {
JsonNode nameJ = mapper.readTree(c.getNameL());
Iterator<JsonNode> it = nameJ.elements();
while (it.hasNext()) {
JsonNode el = it.next();
if (name.equalsIgnoreCase(el.textValue())) {
LOG.debug("Found match for " + name + " in: " + c.getName());
return c;
}
}
} catch (IOException e) {
LOG.error(e);
}
}
return null;
}
/**
......@@ -313,4 +361,27 @@ public class GeoServiceImpl implements GeoService {
}
return jsonArray;
}
@Override
@Transactional(readOnly = false)
public ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy, String nameOfNFP) {
if (country == null) {
LOG.warn("Country is null, not updating ITPGRFA");
}
ITPGRFAStatus itpgrfaStatus = itpgrfaRepository.findByCountry(country);
if (itpgrfaStatus == null) {
LOG.info("New ITPGRFA entry for " + country.getName());
itpgrfaStatus = new ITPGRFAStatus();
itpgrfaStatus.setCountry(country);
} else {
LOG.info("Updating ITPGRFA entry for " + country.getName());
}
itpgrfaStatus.setContractingParty(contractingParty);
itpgrfaStatus.setMembership(membershipBy);
itpgrfaStatus.setMembershipBy(membershipBy);
itpgrfaStatus.setNameOfNFP(nameOfNFP);
return itpgrfaRepository.save(itpgrfaStatus);
}
}
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.worker;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.ITPGRFAStatus;
import org.genesys2.server.service.GeoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Component;
import au.com.bytecode.opencsv.CSVReader;
/**
* Update country ITPGRFA status by fetching data from
* {@link ITPGRFAStatusUpdater#ITPGRFA_STATUS_URL}.
*
* @author Matija Obreza
*/
@Component
public class ITPGRFAStatusUpdater {
public static String ITPGRFA_STATUS_URL = "http://www.planttreaty.org/list_countries/csv/countries";
public static final Log LOG = LogFactory.getLog(ITPGRFAStatusUpdater.class);
@Autowired
private GeoService geoService;
@Autowired
private TaskExecutor taskExecutor;
private static final int BATCH_SIZE = 20;
/**
* Update local {@link ITPGRFAStatus} entries with data from CSV
*
* @throws IOException
*/
public void downloadAndUpdate() throws IOException {
InputStream itpgrfaCSVStream = null;
final HttpClient httpclient = new DefaultHttpClient();
final HttpGet httpget = new HttpGet(ITPGRFA_STATUS_URL);
HttpResponse response = null;
try {
response = httpclient.execute(httpget);
// Get hold of the response entity
final HttpEntity entity = response.getEntity();
LOG.debug(entity.getContentType() + " " + entity.getContentLength());
// If the response does not enclose an entity, there is no
// need
// to bother about connection release
if (entity != null) {
itpgrfaCSVStream = new BufferedInputStream(entity.getContent());
}
} catch (final ClientProtocolException e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
} finally {
}
try {
updateFromStream(itpgrfaCSVStream);
} catch (IOException e) {
LOG.error(e);
throw e;
} finally {
IOUtils.closeQuietly(itpgrfaCSVStream);
}
}
private void updateFromStream(InputStream instream) throws IOException {
CSVReader reader = new CSVReader(new BufferedReader(new InputStreamReader(instream)), ',', '"', '\\', 1, true, true);
try {
final List<String[]> batch = new ArrayList<String[]>(BATCH_SIZE);
String[] line = null;
// Timer
StopWatch stopWatch = new StopWatch();
stopWatch.start();
while ((line = reader.readNext()) != null) {
for (int i = 0; i < line.length; i++) {
if (line[i].equals("null") || StringUtils.isBlank(line[i])) {
line[i] = null;
}
}
if (line[0] == null) {
LOG.warn("Country name is null, skipping line" + ArrayUtils.toString(line, "NULL"));
continue;
}
if (batch.size() < BATCH_SIZE) {
batch.add(line);
} else {
workIt(batch);
batch.clear();
}
}
if (batch.size() > 0) {
LOG.debug("Have items in the batch after loop.");
workIt(batch);
batch.clear();
}
stopWatch.stop();
LOG.info("Done importing ITPGRFA status in " + stopWatch.getTime() + "ms");
} catch (UnsupportedEncodingException e) {
LOG.error(e.getMessage(), e);
} finally {
IOUtils.closeQuietly(instream);
}
}
private void workIt(final List<String[]> batch) {
// Need copy!
final List<String[]> batchCopy = new ArrayList<String[]>(batch);
taskExecutor.execute(new Runnable() {
@Override
public void run() {
for (String[] line : batchCopy) {
if (LOG.isDebugEnabled()) {
LOG.debug("Working on " + ArrayUtils.toString(line, "NULL"));
}
updateCountry(line[0], line[3], line[4], line[5], line[6]);
}
}
});
}
protected void updateCountry(String countryName, String contractingParty, String nameOfNFP, String membership, String membershipBy) {
Country country = geoService.getCountry(countryName);
if (country == null) {
if (LOG.isDebugEnabled())
LOG.warn("No country with name=" + countryName);
return;
}
geoService.updateITPGRFA(country, contractingParty, membership, membershipBy, nameOfNFP);
}
}
......@@ -26,6 +26,7 @@ import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.LuceneIndexer;
import org.genesys2.server.service.impl.ContentSanitizer;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVInsertMissing;
import org.genesys2.server.service.worker.SGSVUpdate;
......@@ -64,6 +65,9 @@ public class AdminController {
@Autowired
ContentSanitizer contentSanitizer;
@Autowired
ITPGRFAStatusUpdater itpgrfaUpdater;
@Autowired
private InstituteService instituteService;
......@@ -153,4 +157,16 @@ public class AdminController {
LOG.info("Updating alternate GEO names: done");
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateITPGRFA")
public String updateITPGRFA() {
LOG.info("Updating country ITPGRFA status");
try {
itpgrfaUpdater.downloadAndUpdate();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
LOG.info("Updating done");
return "redirect:/admin/";
}
}
......@@ -19,6 +19,11 @@
<form method="post" action="<c:url value="/admin/updateAlternateNames" />">
<input type="submit" class="btn btn-default" value="Update alternate GEO names" />
</form>
<form method="post" action="<c:url value="/admin/updateITPGRFA" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Update country ITPGRFA status" />
</form>
<h3>WIEWS</h3>
<form method="post" action="<c:url value="/admin/refreshWiews" />">
......@@ -54,7 +59,6 @@
<input type="submit" class="btn btn-default" class="btn btn-default" value="Reindex search indexes" />
</form>
<form method="post" action="<c:url value="/admin/reindexEntity" />">
<select name="entity">
<option value="org.genesys2.rest.common.model.impl.Country">Countries</option>
......
......@@ -21,6 +21,9 @@
</c:forEach>
</div>
<div class="nav-header">
<spring:message code="paged.totalElements" arguments="${countries.size()}" />
</div>
<c:set value="" var="hoofdleter" />
<ul class="funny-list">
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment