Commit 5af68cd0 authored by Maxym Borodenko's avatar Maxym Borodenko

Merge branch '419-sitemap-instcode-xml' into 'master'

Resolve "sitemap-INSTCODE.xml"

Closes #419

See merge request genesys-pgr/genesys-server!353
parents beafa94b 7500bfb2
......@@ -298,30 +298,32 @@ public class AccessionController {
final ObjectNode geoJson = mapper.createObjectNode();
final ArrayNode featuresArray = geoJson.arrayNode();
accessionProcessor.process(filter, (accession) -> {
final ObjectNode feature = featuresArray.objectNode();
feature.put("type", "Feature");
feature.put("id", accession.getId());
ObjectNode geometry;
feature.set("geometry", geometry = feature.objectNode());
geometry.put("type", "Point");
ArrayNode coordArray;
geometry.set("coordinates", coordArray = geometry.arrayNode());
coordArray.add(accession.getAccessionId().getGeo().getLongitude());
coordArray.add(accession.getAccessionId().getGeo().getLatitude());
ObjectNode properties;
feature.set("properties", properties = feature.objectNode());
properties.put("uuid", accession.getAccessionId().getUuid().toString());
properties.put("accessionNumber", accession.getAccessionNumber());
properties.put("instCode", accession.getInstCode());
properties.put("datum", accession.getAccessionId().getGeo().getDatum());
properties.put("uncertainty", accession.getAccessionId().getGeo().getUncertainty());
featuresArray.add(feature);
return accession;
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession: accessions) {
final ObjectNode feature = featuresArray.objectNode();
feature.put("type", "Feature");
feature.put("id", accession.getId());
ObjectNode geometry;
feature.set("geometry", geometry = feature.objectNode());
geometry.put("type", "Point");
ArrayNode coordArray;
geometry.set("coordinates", coordArray = geometry.arrayNode());
coordArray.add(accession.getAccessionId().getGeo().getLongitude());
coordArray.add(accession.getAccessionId().getGeo().getLatitude());
ObjectNode properties;
feature.set("properties", properties = feature.objectNode());
properties.put("uuid", accession.getAccessionId().getUuid().toString());
properties.put("accessionNumber", accession.getAccessionNumber());
properties.put("instCode", accession.getInstCode());
properties.put("datum", accession.getAccessionId().getGeo().getDatum());
properties.put("uncertainty", accession.getAccessionId().getGeo().getUncertainty());
featuresArray.add(feature);
}
return accessions;
}, limit);
geoJson.set("geoJson", featuresArray);
......@@ -429,25 +431,27 @@ public class AccessionController {
writer.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n");
writer.write("<Document>\n");
try {
accessionProcessor.process(filterInfo.filter, (accession) -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null && geo.getLongitude() != null && geo.getLatitude() != null) {
writer.append("<Placemark>");
writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");
writer.append("<description><![CDATA[\n");
writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
writer.append("<p><a href=\"").append(frontendUrl).append("/a/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
writer.append("\n]]></description>");
writer.append("<Point><coordinates>");
writer.append(decimalFormat.format(geo.getLongitude())).append(",").append(decimalFormat.format(geo.getLatitude()));
writer.append("</coordinates></Point>");
writer.append("</Placemark>\n");
writer.flush();
accessionProcessor.process(filterInfo.filter, (accessions) -> {
for (Accession accession: accessions) {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null && geo.getLongitude() != null && geo.getLatitude() != null) {
writer.append("<Placemark>");
writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");
writer.append("<description><![CDATA[\n");
writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
writer.append("<p><a href=\"").append(frontendUrl).append("/a/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
writer.append("\n]]></description>");
writer.append("<Point><coordinates>");
writer.append(decimalFormat.format(geo.getLongitude())).append(",").append(decimalFormat.format(geo.getLatitude()));
writer.append("</coordinates></Point>");
writer.append("</Placemark>\n");
writer.flush();
}
}
return accession;
return accessions;
});
writer.write("</Document>\n</kml>\n");
......
/**
* Copyright 2014 Global Crop Diversity Trust
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,26 +12,38 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
*/
package org.genesys2.server.mvc;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import com.google.common.collect.Sets;
import org.genesys2.server.exception.NotFoundElement;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
......@@ -44,6 +56,8 @@ import org.springframework.web.bind.annotation.ResponseBody;
@Controller
public class SitemapXMLController {
private static final Logger LOG = LoggerFactory.getLogger(SitemapXMLController.class);
@Value("${frontend.url}")
private String frontendUrl;
......@@ -56,6 +70,9 @@ public class SitemapXMLController {
@Autowired
private CropService cropService;
@Autowired
private AccessionProcessor accessionProcessor;
private static class SitemapPage {
String url;
String freq = "always";
......@@ -120,7 +137,7 @@ public class SitemapXMLController {
List<Country> activeCountries = geoService.listActive(LocaleContextHolder.getLocale());
for (Country country : activeCountries) {
writePage(response, sb, new SitemapPage("/geo/" + country.getCode3().toUpperCase()));
writePage(response, sb, "/iso3166/" + country.getCode3().toUpperCase(), null, null);
}
sb.append("</urlset>");
......@@ -128,20 +145,61 @@ public class SitemapXMLController {
}
@RequestMapping(value = "/sitemap-wiews.xml", method = RequestMethod.GET, produces = { MediaType.TEXT_XML_VALUE })
public @ResponseBody
String sitemapWiewsXml(HttpServletResponse response) {
public void sitemapWiewsXml(HttpServletResponse response) throws IOException {
response.setContentType("text/xml");
final StringBuffer sb = new StringBuffer();
final BufferedWriter sb = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()), 2048);
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sb.append("<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
for (FaoInstitute institute : instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
sb.append(" <sitemap>");
sb.append(" <loc>").append(frontendUrl).append("/sitemap-").append(institute.getCode().toUpperCase()).append(".xml").append("</loc>");
sb.append(" </sitemap>");
}
sb.append("</sitemapindex>");
sb.flush();
response.flushBuffer();
}
@RequestMapping(value = "/sitemap-{instCode}.xml", method = RequestMethod.GET, produces = { MediaType.TEXT_XML_VALUE })
public
void sitemapWiewAndAccessionsXml(@PathVariable(value = "instCode") String instCode, HttpServletResponse response) throws IOException {
FaoInstitute faoInstitute = instituteService.getInstitute(instCode);
if (faoInstitute == null) {
throw new NotFoundElement("No Institute with specified code.");
}
response.setContentType("text/xml");
final BufferedWriter sb = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()), 2048);
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sb.append("<urlset xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
for (FaoInstitute institute : instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
writePage(response, sb, new SitemapPage("/wiews/" + institute.getCode().toUpperCase()));
writePage(response, sb, "/wiews/" + faoInstitute.getCode(), null, null);
// Create JSON filter
AccessionFilter instituteFilter = new AccessionFilter();
instituteFilter.holder().code = Sets.newHashSet(faoInstitute.getCode());
try {
accessionProcessor.process(instituteFilter, (accessions) -> {
for (Accession accession: accessions) {
if (accession.getDoi() != null) {
writePage(response, sb, "/" + accession.getDoi(), null, null);
} else {
writePage(response, sb, "/a/" + accession.getUuid(), null, null);
}
}
sb.flush();
return accessions;
});
} catch (Exception e) {
LOG.warn(e.getMessage());
}
sb.append("</urlset>");
return sb.toString();
sb.flush();
response.flushBuffer();
}
@RequestMapping(value = "/sitemap-crop.xml", method = RequestMethod.GET, produces = { MediaType.TEXT_XML_VALUE })
......@@ -154,7 +212,7 @@ public class SitemapXMLController {
sb.append("<urlset xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
for (Crop crop : cropService.listCrops()) {
writePage(response, sb, new SitemapPage("/c/" + crop.getShortName() + "/data"));
writePage(response, sb, "/c/" + crop.getShortName() + "/data", null, null);
}
sb.append("</urlset>");
......@@ -168,20 +226,32 @@ public class SitemapXMLController {
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sb.append("<urlset xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
for (final SitemapPage page : pages) {
writePage(response, sb, page);
writePage(response, sb, page.url, page.freq, page.priority);
}
sb.append("</urlset>");
return sb.toString();
}
private void writePage(HttpServletResponse response, final StringBuffer sb, final SitemapPage page) {
private void writePage(HttpServletResponse response, final StringBuffer sb, final String url, final String frequency, final Double priority) {
sb.append(" <url>");
sb.append(" <loc>").append(frontendUrl).append(response.encodeURL(url)).append("</loc>");
if (frequency != null) {
sb.append(" <changefreq>").append(frequency).append("</changefreq>");
}
if (priority != null) {
sb.append(" <priority>").append(priority).append("</priority>");
}
sb.append(" </url>");
}
private void writePage(HttpServletResponse response, final Writer sb, final String url, final String frequency, final Double priority) throws IOException {
sb.append(" <url>");
sb.append(" <loc>").append(frontendUrl).append(response.encodeURL(page.url)).append("</loc>");
if (page.freq != null) {
sb.append(" <changefreq>").append(page.freq).append("</changefreq>");
sb.append(" <loc>").append(frontendUrl).append(response.encodeURL(url)).append("</loc>");
if (frequency != null) {
sb.append(" <changefreq>").append(frequency).append("</changefreq>");
}
if (page.priority != null) {
sb.append(" <priority>").append(page.priority).append("</priority>");
if (priority != null) {
sb.append(" <priority>").append(priority.toString()).append("</priority>");
}
sb.append(" </url>");
}
......
......@@ -136,16 +136,6 @@ public interface AccessionService {
*/
List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action);
public static interface IAccessionAction {
/**
* Run action on Accession
*
* @param a the accession
* @return must return the resulting {@link Accession}
*/
Accession apply(Accession a) throws Exception;
}
public static interface IAccessionBatchAction {
/**
* Run action on batch of Accessions
......
......@@ -239,9 +239,11 @@ public class DownloadServiceImpl implements DownloadService {
// Write accession information
final Writer writer = new Writer();
try {
accessionProcessor.process(predicate, (accession) -> {
writer.processMCPD(sheet, wbStyles, accession);
return accession;
accessionProcessor.process(predicate, (accessions) -> {
for (Accession accession: accessions) {
writer.processMCPD(sheet, wbStyles, accession);
}
return accessions;
});
LOG.warn("Done streaming MCPD rows");
} catch (Exception e) {
......@@ -706,9 +708,11 @@ public class DownloadServiceImpl implements DownloadService {
// Write accession information
try {
accessionProcessor.process(filter, (accession) -> {
writer.processPDCI(sheet, accession);
return accession;
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession : accessions) {
writer.processPDCI(sheet, accession);
}
return accessions;
});
} catch (Exception e) {
LOG.warn("Error generating: {}", e.getMessage());
......
......@@ -1213,9 +1213,11 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
// Write accession information
try {
accessionProcessor.process(filter, (accession) -> {
writer.processCore(csv, accession);
return accession;
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession: accessions) {
writer.processCore(csv, accession);
}
return accessions;
});
} catch (Exception e) {
LOG.error(e.getMessage(), e);
......@@ -1233,12 +1235,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
// Write accession geo information
try {
accessionProcessor.process(filter, (accession) -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
writer.processGeo(csv, geo);
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession: accessions) {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
writer.processGeo(csv, geo);
}
}
return accession;
return accessions;
});
} catch (Exception e) {
LOG.error(e.getMessage(), e);
......@@ -1256,12 +1260,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
// Write accession aliases information
try {
accessionProcessor.process(filter, (accession) -> {
List<AccessionAlias> accessionAliases = accession.getAccessionId().getAliases();
if (accessionAliases != null && accessionAliases.size() > 0) {
writer.processAliases(csv, accessionAliases);
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession: accessions) {
List<AccessionAlias> accessionAliases = accession.getAccessionId().getAliases();
if (accessionAliases != null && accessionAliases.size() > 0) {
writer.processAliases(csv, accessionAliases);
}
}
return accession;
return accessions;
});
} catch (Exception e) {
LOG.error(e.getMessage(), e);
......@@ -1280,12 +1286,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
// Write accession collections information
try {
accessionProcessor.process(filter, (accession) -> {
AccessionCollect accessionCollect = accession.getAccessionId().getColl();
if (accessionCollect != null) {
writer.processCollect(csv, accessionCollect);
accessionProcessor.process(filter, (accessions) -> {
for (Accession accession: accessions) {
AccessionCollect accessionCollect = accession.getAccessionId().getColl();
if (accessionCollect != null) {
writer.processCollect(csv, accessionCollect);
}
}
return accession;
return accessions;
});
} catch (Exception e) {
LOG.error(e.getMessage(), e);
......
......@@ -25,7 +25,6 @@ import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.AccessionService.IAccessionAction;
import org.genesys2.server.service.AccessionService.IAccessionBatchAction;
import org.genesys2.server.service.filter.AccessionFilter;
import org.slf4j.Logger;
......@@ -66,22 +65,22 @@ public class AccessionProcessor {
private int batchSize = 1000;
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(AccessionFilter filter, IAccessionAction action) throws Exception {
public void process(AccessionFilter filter, IAccessionBatchAction action) throws Exception {
process(filter.buildPredicate(), action, null);
}
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(AccessionFilter filter, IAccessionAction action, Long maxSize) throws Exception {
public void process(AccessionFilter filter, IAccessionBatchAction action, Long maxSize) throws Exception {
process(filter.buildPredicate(), action, maxSize);
}
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(Predicate predicate, IAccessionAction action) throws Exception {
public void process(Predicate predicate, IAccessionBatchAction action) throws Exception {
process(predicate, action, null);
}
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(Predicate predicate, IAccessionAction action, Long maxSize) throws Exception {
public void process(Predicate predicate, IAccessionBatchAction action, Long maxSize) throws Exception {
long count = accessionRepository.count(predicate);
......@@ -130,11 +129,9 @@ public class AccessionProcessor {
LOG.info("Processing Accessions took {}ms", stopWatch.getTime());
}
private void loadAndProcess(List<Long> accessionIds, IAccessionAction action) throws Exception {
private void loadAndProcess(List<Long> accessionIds, IAccessionBatchAction action) throws Exception {
List<Accession> accessions = accessionRepository.findAll(accessionIds);
for (Accession a : accessions) {
action.apply(a);
}
action.apply(accessions);
}
/**
* Apply action on accessions matching the provided filter.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment