Commit 98969d32 authored by Alexander Basov's avatar Alexander Basov Committed by Matija Obreza
Browse files

Fix GeoRegion: Containment

parent c46860f1
......@@ -7,5 +7,5 @@ import org.springframework.data.jpa.repository.Query;
public interface GeoRegionRepository extends JpaRepository<GeoRegion, Long>{
@Query("select distinct gr from GeoRegion gr where gr.isoCode = ?1")
GeoRegion findeByIsoCode(int isoCode);
GeoRegion findeByIsoCode(String isoCode);
}
......@@ -10,7 +10,7 @@ import java.util.List;
public interface GeoRegionService {
GeoRegion find(int regionIsoCode);
GeoRegion find(String regionIsoCode);
void save(GeoRegion geoRegion);
......
......@@ -32,7 +32,7 @@ public class GeoRegionServiceImpl implements GeoRegionService {
GeoRegionDataCLDR geoRegionDataCLDR;
@Override
public GeoRegion find(int regionIsoCode) {
public GeoRegion find(String regionIsoCode) {
return geoRegionRepository.findeByIsoCode(regionIsoCode);
}
......@@ -78,9 +78,38 @@ public class GeoRegionServiceImpl implements GeoRegionService {
return;
}
List<GeoRegion> geoRegionList = geoRegionDataCLDR.getGeoRegionDataCLDR();
if(!geoRegionList.isEmpty()){
if (null == geoRegionRepository.findAll() || geoRegionRepository.findAll().isEmpty()) {
geoRegionRepository.save(geoRegionList);
for (GeoRegion geoRegion : geoRegionList) {
if (null != geoRegion.getCountries()) {
for (Country country : geoRegion.getCountries()) {
if (null != country) {
country.setRegion(geoRegion);
}
}
}
}
} else if (null != geoRegionRepository.findAll() || !geoRegionRepository.findAll().isEmpty()) {
List<GeoRegion> list = geoRegionRepository.findAll();
for (int i = 0; i < geoRegionList.size(); i++) {
GeoRegion geoRegionForUpdate = list.get(i);
geoRegionForUpdate.setName(geoRegionList.get(i).getName());
geoRegionForUpdate.setIsoCode(geoRegionList.get(i).getIsoCode());
geoRegionForUpdate.setNameL(geoRegionList.get(i).getNameL());
geoRegionForUpdate.setCountries(geoRegionList.get(i).getCountries());
geoRegionRepository.save(geoRegionForUpdate);
if(null != geoRegionForUpdate.getCountries()){
for (Country country : geoRegionForUpdate.getCountries()) {
if (null != country) {
country.setRegion(geoRegionForUpdate);
// countryRepository.save(country);
}
}
}
}
}
}
}
package org.genesys2.server.service.worker;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.annotation.Resource;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.GeoRegion;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
......@@ -31,230 +15,266 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import javax.annotation.Resource;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@Component
public class GeoRegionDataCLDR {
private static final String CLDR_URL = "http://unicode.org/Public/cldr/26.0.1/core.zip";
private static final String CLDR_URL = "http://unicode.org/Public/cldr/26.0.1/core.zip";
public static final Log LOG = LogFactory.getLog(GeoRegionDataCLDR.class);
public static final Log LOG = LogFactory.getLog(GeoRegionDataCLDR.class);
@Autowired
private GeoService geoService;
@Autowired
private GeoService geoService;
@Resource
private Set<String> supportedLocales;
@Autowired
private GeoRegionService geoRegionService;
public List<GeoRegion> getGeoRegionDataCLDR() throws IOException, ParserConfigurationException, SAXException {
@Resource
private Set<String> supportedLocales;
ZipFile zipFile = getCoreZip();
LOG.info("Obtained CLDR zip file: " + zipFile);
public List<GeoRegion> getGeoRegionDataCLDR() throws IOException, ParserConfigurationException, SAXException {
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
ZipFile zipFile = getCoreZip();
LOG.info("Obtained CLDR zip file: " + zipFile);
SAXParser parser = saxParserFactory.newSAXParser();
ParserForCLDR parserForCLDR = new ParserForCLDR();
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
List<GeoRegion> geoRegionList = null;
Map<String, Map<String, GeoRegion>> forNameJGeoRegions = new HashMap<>();
InputStream inputStream;
for (String language : supportedLocales) {
String fileName = "common/main/" + language + ".xml";
LOG.info("Examining " + fileName);
SAXParser parser = saxParserFactory.newSAXParser();
ParserForCLDR parserForCLDR = new ParserForCLDR();
ZipEntry zipEntry = zipFile.getEntry(fileName);
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, parserForCLDR);
List<GeoRegion> geoRegionList = null;
Map<String, Map<String, GeoRegion>> forNameJGeoRegions = new HashMap<>();
InputStream inputStream;
for (String language : supportedLocales) {
String fileName = "common/main/" + language + ".xml";
LOG.info("Examining " + fileName);
if (language.equals("en")) {
geoRegionList = assignParents(zipFile, parserForCLDR.getRegionMap());
} else {
forNameJGeoRegions.put(language, parserForCLDR.getRegionMap());
}
ZipEntry zipEntry = zipFile.getEntry(fileName);
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, parserForCLDR);
inputStream.close();
}
if (language.equals("en")) {
geoRegionList = assignParents(zipFile, parserForCLDR.getRegionMap());
} else {
forNameJGeoRegions.put(language, parserForCLDR.getRegionMap());
}
assignNameJ(geoRegionList, forNameJGeoRegions);
inputStream.close();
}
return geoRegionList;
}
assignNameJ(geoRegionList, forNameJGeoRegions);
private void assignNameJ(List<GeoRegion> geoRegionList, Map<String, Map<String, GeoRegion>> forNameJGeoRegions) throws JsonProcessingException {
return geoRegionList;
}
for (GeoRegion geoRegion : geoRegionList) {
private void assignNameJ(List<GeoRegion> geoRegionList, Map<String, Map<String, GeoRegion>> forNameJGeoRegions) throws JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
for (GeoRegion geoRegion : geoRegionList) {
Map<String, String> nameL = new HashMap<>();
nameL.put("en", geoRegion.getName());
for (Map.Entry<String, Map<String, GeoRegion>> entry : forNameJGeoRegions.entrySet()) {
nameL.put(entry.getKey(), entry.getValue().get(geoRegion.getIsoCode()).getName());
}
ObjectMapper objectMapper = new ObjectMapper();
String nameLJson = objectMapper.writeValueAsString(nameL);
Map<String, String> nameL = new HashMap<>();
nameL.put("en", geoRegion.getName());
for (Map.Entry<String, Map<String, GeoRegion>> entry : forNameJGeoRegions.entrySet()) {
nameL.put(entry.getKey(), entry.getValue().get(geoRegion.getIsoCode()).getName());
}
geoRegion.setNameL(nameLJson);
}
}
private List<GeoRegion> assignParents(ZipFile zipFile, Map<String, GeoRegion> geoRegions) throws SAXException, ParserConfigurationException, IOException {
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
String nameLJson = objectMapper.writeValueAsString(nameL);
SAXParser parser = saxParserFactory.newSAXParser();
SupplementalDataParser supplementalDataParser = new SupplementalDataParser();
geoRegion.setNameL(nameLJson);
}
}
String fileName = "common/supplemental/supplementalData.xml";
ZipEntry zipEntry = zipFile.getEntry(fileName);
InputStream inputStream;
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, supplementalDataParser);
private List<GeoRegion> assignParents(ZipFile zipFile, Map<String, GeoRegion> geoRegions) throws SAXException, ParserConfigurationException, IOException {
for (Map.Entry<String, String> entry : supplementalDataParser.getGeoregionsCodes().entrySet()) {
for (String isoCode : entry.getValue().split(" ")) {
if (isoCode.matches("[0-9]+")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(entry.getKey()));
}
}
}
for (Map.Entry<String, String> entry : supplementalDataParser.getCountryMap().entrySet()) {
List<Country> countries = new ArrayList<>();
for (String isoCode : entry.getValue().split(" ")) {
for (Country country : geoService.listAll()) {
if (country.getCode2().equals(isoCode)) {
countries.add(country);
}
}
geoRegions.get(entry.getKey()).setCountries(countries);
}
}
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
SAXParser parser = saxParserFactory.newSAXParser();
SupplementalDataParser supplementalDataParser = new SupplementalDataParser();
String fileName = "common/supplemental/supplementalData.xml";
ZipEntry zipEntry = zipFile.getEntry(fileName);
InputStream inputStream;
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, supplementalDataParser);
for (Map.Entry<String, String> entry : supplementalDataParser.getGeoregionsCodes().entrySet()) {
for (String isoCode : entry.getValue().split(" ")) {
if (isoCode.matches("[0-9]+")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(entry.getKey()));
}
}
}
for (Map.Entry<String, String> entry : supplementalDataParser.getCountryMap().entrySet()) {
List<Country> countries = new ArrayList<>();
for (String isoCode : entry.getValue().split(" ")) {
for (Country country : geoService.listAll()) {
if (country.getCode2().equals(isoCode)) {
countries.add(country);
}
}
return new ArrayList<>(geoRegions.values());
}
}
geoRegions.get(entry.getKey()).setCountries(countries);
}
private ZipFile getCoreZip() throws IOException {
return new ArrayList<>(geoRegions.values());
}
URL url = new URL(CLDR_URL);
LOG.info("Downloading " + url);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
File tempFileForXml = File.createTempFile("core", ".zip");
connection.setRequestMethod("GET");
InputStream in = connection.getInputStream();
FileOutputStream out = new FileOutputStream(tempFileForXml);
IOUtils.copy(in, out);
out.close();
LOG.info("Done copying CLDR data to file");
return new ZipFile(tempFileForXml);
}
public class SupplementalDataParser extends DefaultHandler {
Map<String, String> georegionsCodes;
Map<String, String> countryMap;
String xmlTag;
Attributes attributes;
public Map<String, String> getCountryMap() {
return countryMap;
}
public Map<String, String> getGeoregionsCodes() {
return georegionsCodes;
}
@Override
public void startDocument() throws SAXException {
georegionsCodes = new HashMap<>();
countryMap = new HashMap<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (this.xmlTag.equalsIgnoreCase("group") && attributes.getValue("type").matches("[0-9]+")) {
if (attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
georegionsCodes.put(attributes.getValue("type"), attributes.getValue("contains"));
} else if (!attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
countryMap.put(attributes.getValue("type"), attributes.getValue("contains"));
}
}
}
}
public class ParserForCLDR extends DefaultHandler {
private GeoRegion geoRegion;
private Map<String, GeoRegion> regionMap;
// needs for avoid duplicate geoRegions in regionList
private List<String> isoCodeList;
private String xmlTag = "";
private Attributes attributes;
public Map<String, GeoRegion> getRegionMap() {
return regionMap;
}
@Override
public void startDocument() throws SAXException {
regionMap = new HashMap<>();
isoCodeList = new ArrayList<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
geoRegion = new GeoRegion();
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
String attribute = attributes.getValue("type");
if (this.xmlTag.equalsIgnoreCase("territory") && attribute.matches("[0-9]+") && !isoCodeList.contains(attribute)) {
geoRegion.setIsoCode(attribute);
geoRegion.setName(new String(ch, start, length));
isoCodeList.add(attribute);
regionMap.put(geoRegion.getIsoCode(), geoRegion);
}
}
}
private ZipFile getCoreZip() throws IOException {
URL url = new URL(CLDR_URL);
LOG.info("Downloading " + url);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
File tempFileForXml = File.createTempFile("core", ".zip");
connection.setRequestMethod("GET");
InputStream in = connection.getInputStream();
FileOutputStream out = new FileOutputStream(tempFileForXml);
IOUtils.copy(in, out);
out.close();
LOG.info("Done copying CLDR data to file");
return new ZipFile(tempFileForXml);
}
public class SupplementalDataParser extends DefaultHandler {
Map<String, String> georegionsCodes;
Map<String, String> countryMap;
String xmlTag;
Attributes attributes;
public Map<String, String> getCountryMap() {
return countryMap;
}
public Map<String, String> getGeoregionsCodes() {
return georegionsCodes;
}
@Override
public void startDocument() throws SAXException {
georegionsCodes = new HashMap<>();
countryMap = new HashMap<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (this.xmlTag.equalsIgnoreCase("group") && attributes.getValue("type").matches("[0-9]+")) {
if (attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
georegionsCodes.put(attributes.getValue("type"), attributes.getValue("contains"));
} else if (!attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
// FIXME in supplementalData.xml <territoryContainment> has repeatable "<group type=" values and I read only the first one
if (!countryMap.containsKey(attributes.getValue("type"))) {
countryMap.put(attributes.getValue("type"), attributes.getValue("contains"));
// String countryCodes = countryMap.get(attributes.getValue("type")) + " " + attributes.getValue("contains");
// countryMap.remove(attributes.getValue("type"));
// countryMap.put(attributes.getValue("type"), countryCodes);
}
// else {
// countryMap.put(attributes.getValue("type"), attributes.getValue("contains"));
// }
}
}
}
}
public class ParserForCLDR extends DefaultHandler {
private GeoRegion geoRegion;
private Map<String, GeoRegion> regionMap;
// needs for avoid duplicate geoRegions in regionList
private List<String> isoCodeList;
private String xmlTag = "";
private Attributes attributes;
private int count = 0;
public Map<String, GeoRegion> getRegionMap() {
return regionMap;
}
@Override
public void startDocument() throws SAXException {
regionMap = new HashMap<>();
isoCodeList = new ArrayList<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
count = 0;
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
//
// if (qName.equalsIgnoreCase("territory") && attributes.getValue("type").matches("[0-9]+") && !isoCodeList.contains(attributes.getValue("type"))) {
//
// if(geoRegionService.find(attributes.getValue("type")) != null){
// geoRegion = geoRegionService.find(attributes.getValue("type"));
// }else {
geoRegion = new GeoRegion();
// }
// }
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
String attribute = attributes.getValue("type");
if (this.xmlTag.equalsIgnoreCase("territory") && attribute.matches("[0-9]+") && !isoCodeList.contains(attribute)) {
geoRegion.setIsoCode(attribute);
geoRegion.setName(new String(ch, start, length));
isoCodeList.add(attribute);
geoRegion.setNameL(null);
regionMap.put(geoRegion.getIsoCode(), geoRegion);
}
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment