Commit 75eafcd9 authored by Matija Obreza's avatar Matija Obreza

Update ITPGRFA membership from fao.org

- URL changed
- CSV file structure changed
- No name of NFP
parent 19128b8d
......@@ -118,7 +118,7 @@ public interface GeoService {
* @param nameOfNFP
* @return
*/
ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy, String nameOfNFP);
ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy);
ITPGRFAStatus getITPGRFAStatus(Country country);
......
......@@ -180,6 +180,16 @@ public class GeoServiceImpl implements GeoService {
if (country == null) {
country = findCountryByName(countryString);
}
// Let's sanitize input
if (country == null) {
country = findCountryByName(countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
if (country != null) {
LOG.warn("Found country {} for {}", country.getName(), countryString);
} else {
LOG.warn("No country for {} - queried by {}", countryString, countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
}
}
return country;
}
......@@ -488,7 +498,7 @@ public class GeoServiceImpl implements GeoService {
@Override
@Transactional(readOnly = false)
public ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy, String nameOfNFP) {
public ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy) {
if (country == null) {
LOG.warn("Country is null, not updating ITPGRFA");
return null;
......@@ -505,7 +515,6 @@ public class GeoServiceImpl implements GeoService {
itpgrfaStatus.setContractingParty(contractingParty);
itpgrfaStatus.setMembership(membership);
itpgrfaStatus.setMembershipBy(membershipBy);
itpgrfaStatus.setNameOfNFP(nameOfNFP);
return itpgrfaRepository.save(itpgrfaStatus);
}
......
/**
* Copyright 2014 Global Crop Diversity Trust
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,7 +12,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
*/
package org.genesys2.server.service.worker;
......@@ -24,8 +24,6 @@ import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import com.opencsv.CSVReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -45,6 +43,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Component;
import com.opencsv.CSVReader;
/**
* Update country ITPGRFA status by fetching data from
* {@link ITPGRFAStatusUpdater#ITPGRFA_STATUS_URL}.
......@@ -54,7 +54,7 @@ import org.springframework.stereotype.Component;
@Component
public class ITPGRFAStatusUpdater {
public static String ITPGRFA_STATUS_URL = "http://www.planttreaty.org/list_countries/csv/countries.csv";
public static String ITPGRFA_STATUS_URL = "http://www.fao.org/plant-treaty/countries/membership/export-membership/en/";
public static final Logger LOG = LoggerFactory.getLogger(ITPGRFAStatusUpdater.class);
......@@ -66,14 +66,13 @@ public class ITPGRFAStatusUpdater {
private static final int BATCH_SIZE = 20;
private static final String[] CSV_HEADERS = { "Country ISO CODE", "Country", "FAO Region 1", "FAO Region 2", "CP", "Name of NFP", "Membership", "by",
"Income", "Development" };
private static final String[] CSV_HEADERS = { "Country", "ISO3", "Region", "ITPGRFA Region", "Contracting Party", "Membership", "By", "Income", "Development", "Signature date",
"Accession date", "Ratification date", "Approval date", "Acceptance date", "Entry into Force" };
protected static final int COLUMN_ISOCODE3 = 0;
protected static final int COLUMN_COUNTRY_CODE3 = 1;
protected static final int COLUMN_CONTRACTING_PARTY = 4;
protected static final int COLUMN_NAME_OF_NFP = 5;
protected static final int COLUMN_MEMBERSHIP = 6;
protected static final int COLUMN_MEMBERSHIP_BY = 7;
protected static final int COLUMN_MEMBERSHIP = 5;
protected static final int COLUMN_MEMBERSHIP_BY = 6;
/**
* Update local {@link ITPGRFAStatus} entries with data from CSV
......@@ -104,7 +103,7 @@ public class ITPGRFAStatusUpdater {
}
updateFromStream(itpgrfaCSVStream);
} catch (final ClientProtocolException e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
......@@ -149,11 +148,6 @@ public class ITPGRFAStatusUpdater {
}
}
if (line[COLUMN_ISOCODE3] == null) {
LOG.warn("Country ISO CODE 3 is null, skipping line{}", ArrayUtils.toString(line, "NULL"));
continue;
}
if (LOG.isDebugEnabled()) {
LOG.debug(">>> {}", ArrayUtils.toString(line, "NULL"));
}
......@@ -193,21 +187,22 @@ public class ITPGRFAStatusUpdater {
if (LOG.isDebugEnabled()) {
LOG.debug("Working on {}", ArrayUtils.toString(line, "NULL"));
}
updateCountry(line[COLUMN_ISOCODE3], line[COLUMN_CONTRACTING_PARTY], line[COLUMN_NAME_OF_NFP], line[COLUMN_MEMBERSHIP],
line[COLUMN_MEMBERSHIP_BY]);
if (!StringUtils.isBlank(line[COLUMN_COUNTRY_CODE3])) {
updateCountry(line[COLUMN_COUNTRY_CODE3], line[COLUMN_CONTRACTING_PARTY], line[COLUMN_MEMBERSHIP], line[COLUMN_MEMBERSHIP_BY]);
}
}
}
});
}
protected void updateCountry(String countryIso3, String contractingParty, String nameOfNFP, String membership, String membershipBy) {
final Country country = geoService.getCountry(countryIso3);
protected void updateCountry(String countryCode, String contractingParty, String membership, String membershipBy) {
final Country country = geoService.findCountry(countryCode);
if (country == null) {
LOG.error("No country with name={}", countryIso3);
LOG.error("No country with name={}", countryCode);
return;
}
geoService.updateITPGRFA(country, contractingParty, membership, membershipBy, nameOfNFP);
geoService.updateITPGRFA(country, contractingParty, membership, membershipBy);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment