Commit c46860f1 authored by Matija Obreza's avatar Matija Obreza
Browse files

Acceptable GeoRegion code

parent d32ef2ed
......@@ -72,18 +72,12 @@ public class GeoRegionServiceImpl implements GeoRegionService {
@Override
@Transactional(readOnly = false)
public void updateGeoRegionData() throws IOException, ParserConfigurationException, SAXException {
//update current geoRegions
updateGeoRegions();
}
private void updateGeoRegions() throws IOException, ParserConfigurationException, SAXException {
if (geoRegionDataCLDR == null) {
LOG.warn("unicode.org geoRegions source not available");
return;
}
List<GeoRegion> geoRegionList = geoRegionDataCLDR.getGeoRigionDataCLDR();
List<GeoRegion> geoRegionList = geoRegionDataCLDR.getGeoRegionDataCLDR();
if(!geoRegionList.isEmpty()){
geoRegionRepository.save(geoRegionList);
......
package org.genesys2.server.service.worker;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.annotation.Resource;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.GeoRegion;
import org.genesys2.server.service.GeoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import javax.annotation.Resource;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@Component
public class GeoRegionDataCLDR {
private static final String CLDR_URL = "http://unicode.org/Public/cldr/26.0.1/core.zip";
private static final String CLDR_URL = "http://unicode.org/Public/cldr/26.0.1/core.zip";
public static final Log LOG = LogFactory.getLog(GeoRegionDataCLDR.class);
@Autowired
private GeoService geoService;
@Resource
private Set<String> supportedLocales;
public List<GeoRegion> getGeoRegionDataCLDR() throws IOException, ParserConfigurationException, SAXException {
ZipFile zipFile = getCoreZip();
LOG.info("Obtained CLDR zip file: " + zipFile);
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
SAXParser parser = saxParserFactory.newSAXParser();
ParserForCLDR parserForCLDR = new ParserForCLDR();
List<GeoRegion> geoRegionList = null;
Map<String, Map<String, GeoRegion>> forNameJGeoRegions = new HashMap<>();
InputStream inputStream;
for (String language : supportedLocales) {
String fileName = "common/main/" + language + ".xml";
LOG.info("Examining " + fileName);
ZipEntry zipEntry = zipFile.getEntry(fileName);
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, parserForCLDR);
if (language.equals("en")) {
geoRegionList = assignParents(zipFile, parserForCLDR.getRegionMap());
} else {
forNameJGeoRegions.put(language, parserForCLDR.getRegionMap());
}
inputStream.close();
}
public static final Log LOG = LogFactory.getLog(GeoRegionDataCLDR.class);
@Resource
private Set<String> supportedLocales;
public List<GeoRegion> getGeoRigionDataCLDR() throws IOException, ParserConfigurationException, SAXException {
ZipFile zipFile = getCoreZip();
LOG.info("Obtained CLDR zip file: " + zipFile);
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
SAXParser parser = saxParserFactory.newSAXParser();
ParserForCLDR parserForCLDR = new ParserForCLDR();
assignNameJ(geoRegionList, forNameJGeoRegions);
ZipEntry zipEntry;
return geoRegionList;
}
List<GeoRegion> geoRegionList = null;
Map<String, Map<String, GeoRegion>> forNameJGeoRegions = new HashMap<>();
InputStream inputStream;
for (String language : supportedLocales) {
String fileName = "common/main/" + language + ".xml";
LOG.info("Examining " + fileName);
private void assignNameJ(List<GeoRegion> geoRegionList, Map<String, Map<String, GeoRegion>> forNameJGeoRegions) throws JsonProcessingException {
zipEntry = zipFile.getEntry(fileName);
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, parserForCLDR);
for (GeoRegion geoRegion : geoRegionList) {
if (language.equals("en")) {
geoRegionList = assignParents(parserForCLDR.getRegionMap());
} else {
forNameJGeoRegions.put(language, parserForCLDR.getRegionMap());
}
ObjectMapper objectMapper = new ObjectMapper();
inputStream.close();
}
Map<String, String> nameL = new HashMap<>();
nameL.put("en", geoRegion.getName());
for (Map.Entry<String, Map<String, GeoRegion>> entry : forNameJGeoRegions.entrySet()) {
nameL.put(entry.getKey(), entry.getValue().get(geoRegion.getIsoCode()).getName());
}
assignNameJ(geoRegionList, forNameJGeoRegions);
String nameLJson = objectMapper.writeValueAsString(nameL);
return geoRegionList;
}
geoRegion.setNameL(nameLJson);
}
}
private List<GeoRegion> assignParents(ZipFile zipFile, Map<String, GeoRegion> geoRegions) throws SAXException, ParserConfigurationException, IOException {
SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
private void assignNameJ(List<GeoRegion> geoRegionList, Map<String, Map<String, GeoRegion>> forNameJGeoRegions) throws JsonProcessingException {
SAXParser parser = saxParserFactory.newSAXParser();
SupplementalDataParser supplementalDataParser = new SupplementalDataParser();
for(GeoRegion geoRegion : geoRegionList){
String fileName = "common/supplemental/supplementalData.xml";
ZipEntry zipEntry = zipFile.getEntry(fileName);
InputStream inputStream;
inputStream = zipFile.getInputStream(zipEntry);
parser.parse(inputStream, supplementalDataParser);
ObjectMapper objectMapper = new ObjectMapper();
for (Map.Entry<String, String> entry : supplementalDataParser.getGeoregionsCodes().entrySet()) {
for (String isoCode : entry.getValue().split(" ")) {
if (isoCode.matches("[0-9]+")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(entry.getKey()));
}
}
}
for (Map.Entry<String, String> entry : supplementalDataParser.getCountryMap().entrySet()) {
List<Country> countries = new ArrayList<>();
for (String isoCode : entry.getValue().split(" ")) {
for (Country country : geoService.listAll()) {
if (country.getCode2().equals(isoCode)) {
countries.add(country);
}
}
geoRegions.get(entry.getKey()).setCountries(countries);
}
}
Map<String, String> nameL = new HashMap<>();
nameL.put("en", geoRegion.getName());
for(Map.Entry<String, Map<String, GeoRegion>> entry : forNameJGeoRegions.entrySet()){
nameL.put(entry.getKey(), entry.getValue().get(geoRegion.getIsoCode()).getName());
}
return new ArrayList<>(geoRegions.values());
}
String nameLJson = objectMapper.writeValueAsString(nameL);
private ZipFile getCoreZip() throws IOException {
geoRegion.setNameL(nameLJson);
}
}
private List<GeoRegion> assignParents(Map<String, GeoRegion> geoRegions) {
String world = "001";
String continents = "002,019,142,150,009";
String forAfrica = "015,014,017,018,011";
String forAsia = "143,030,034,035,145";
String forEurope = "151,154,039,155";
String forOceania = "053,054,057,061";
String forAmerica = "419,003,021";
String forLatinAmerica = "029,013,005";
for (String isoCode : continents.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(world));
}
for (String isoCode : forAfrica.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(continents.split(",")[0]));
}
for (String isoCode : forAmerica.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(continents.split(",")[1]));
}
for (String isoCode : forAsia.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(continents.split(",")[2]));
}
for (String isoCode : forEurope.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(continents.split(",")[3]));
}
for (String isoCode : forOceania.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(continents.split(",")[4]));
}
for (String isoCode : forLatinAmerica.split(",")) {
geoRegions.get(isoCode).setParentRegion(geoRegions.get(forAmerica.split(",")[0]));
}
return new ArrayList<>(geoRegions.values());
}
private ZipFile getCoreZip() throws IOException {
URL url = new URL(CLDR_URL);
LOG.info("Downloading " + url);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
File tempFileForXml = File.createTempFile("core", ".zip");
connection.setRequestMethod("GET");
InputStream in = connection.getInputStream();
FileOutputStream out = new FileOutputStream(tempFileForXml);
IOUtils.copy(in, out);
out.close();
LOG.info("Done copying CLDR data to file");
ZipFile zipFile = new ZipFile(tempFileForXml);
return zipFile;
}
public class ParserForCLDR extends DefaultHandler {
private GeoRegion geoRegion;
private Map<String, GeoRegion> regionMap;
// needs for avoid duplicate geoRegions in regionList
private List<String> isoCodeList;
private String xmlTag = "";
private Attributes attributes;
public Map<String, GeoRegion> getRegionMap() {
return regionMap;
}
@Override
public void startDocument() throws SAXException {
regionMap = new HashMap<>();
isoCodeList = new ArrayList<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
geoRegion = new GeoRegion();
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
String attribute = attributes.getValue("type");
if (this.xmlTag.equalsIgnoreCase("territory") && attribute.matches("[0-9]+") && !isoCodeList.contains(attribute)) {
geoRegion.setIsoCode(attribute);
geoRegion.setName(new String(ch, start, length));
isoCodeList.add(attribute);
regionMap.put(geoRegion.getIsoCode(), geoRegion);
}
}
}
URL url = new URL(CLDR_URL);
LOG.info("Downloading " + url);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
File tempFileForXml = File.createTempFile("core", ".zip");
connection.setRequestMethod("GET");
InputStream in = connection.getInputStream();
FileOutputStream out = new FileOutputStream(tempFileForXml);
IOUtils.copy(in, out);
out.close();
LOG.info("Done copying CLDR data to file");
return new ZipFile(tempFileForXml);
}
public class SupplementalDataParser extends DefaultHandler {
Map<String, String> georegionsCodes;
Map<String, String> countryMap;
String xmlTag;
Attributes attributes;
public Map<String, String> getCountryMap() {
return countryMap;
}
public Map<String, String> getGeoregionsCodes() {
return georegionsCodes;
}
@Override
public void startDocument() throws SAXException {
georegionsCodes = new HashMap<>();
countryMap = new HashMap<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (this.xmlTag.equalsIgnoreCase("group") && attributes.getValue("type").matches("[0-9]+")) {
if (attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
georegionsCodes.put(attributes.getValue("type"), attributes.getValue("contains"));
} else if (!attributes.getValue("contains").split(" ")[0].matches("[0-9]+")) {
countryMap.put(attributes.getValue("type"), attributes.getValue("contains"));
}
}
}
}
public class ParserForCLDR extends DefaultHandler {
private GeoRegion geoRegion;
private Map<String, GeoRegion> regionMap;
// needs for avoid duplicate geoRegions in regionList
private List<String> isoCodeList;
private String xmlTag = "";
private Attributes attributes;
public Map<String, GeoRegion> getRegionMap() {
return regionMap;
}
@Override
public void startDocument() throws SAXException {
regionMap = new HashMap<>();
isoCodeList = new ArrayList<>();
}
@Override
public void endDocument() throws SAXException {
super.endDocument();
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
geoRegion = new GeoRegion();
this.xmlTag = qName;
this.attributes = attributes;
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
super.endElement(uri, localName, qName);
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
String attribute = attributes.getValue("type");
if (this.xmlTag.equalsIgnoreCase("territory") && attribute.matches("[0-9]+") && !isoCodeList.contains(attribute)) {
geoRegion.setIsoCode(attribute);
geoRegion.setName(new String(ch, start, length));
isoCodeList.add(attribute);
regionMap.put(geoRegion.getIsoCode(), geoRegion);
}
}
}
}
......@@ -369,9 +369,7 @@ public class AdminController {
@RequestMapping(value = "/georegion", method = RequestMethod.POST)
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
if (geoRegionService.findAll().isEmpty()) {
geoRegionService.updateGeoRegionData();
}
geoRegionService.updateGeoRegionData();
return "redirect:/admin/";
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment