Commit e574ae3b authored by Matija Obreza's avatar Matija Obreza
Browse files

Download KML for selected filters



- Added geo.referenced = true to AccessionGeoFilter
- Using AccessionProcessor#process to generate KML
Signed-off-by: Matija Obreza's avatarMatija Obreza <matija.obreza@croptrust.org>
parent ba9da0b3
......@@ -16,12 +16,19 @@
package org.genesys2.server.api.v1;
import java.io.BufferedWriter;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import javax.servlet.http.HttpServletResponse;
import org.genesys.blocks.model.JsonViews;
import org.genesys.catalog.api.FilteredPage;
import org.genesys.catalog.model.dataset.Dataset;
......@@ -30,6 +37,7 @@ import org.genesys.catalog.service.ShortFilterService;
import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.Subset;
......@@ -38,9 +46,14 @@ import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.TermResult;
import org.genesys2.server.service.SubsetService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.filter.AccessionGeoFilter;
import org.genesys2.server.service.impl.SearchException;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.genesys2.spring.ResourceNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
......@@ -52,6 +65,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.annotation.JsonView;
......@@ -73,9 +87,13 @@ import io.swagger.annotations.Api;
@Api(tags = { "accession" })
public class AccessionController {
private static final Logger LOG = LoggerFactory.getLogger(AccessionController.class);
/** The Constant API_BASE. */
public static final String API_BASE = ApiBaseController.APIv1_BASE + "/acn";
private static final int DOWNLOAD_LIMIT = 200000;
@Autowired
private AccessionService accessionService;
......@@ -92,11 +110,18 @@ public class AccessionController {
@Autowired
private SubsetService subsetService;
@Autowired
private AccessionProcessor accessionProcessor;
@Value("${base.url}")
private String baseUrl;
private final Set<String> terms = Sets.newHashSet("institute.code", "institute.country.code3", "cropName", "crop.shortName", "taxonomy.genus", "taxonomy.species",
"taxonomy.genusSpecies", "countryOfOrigin.code3", "sampStat", "available", "mlsStatus", "donorCode", "sgsv", "storage", "duplSite", "breederCode");
private ObjectMapper objectMapper = new ObjectMapper();
/**
* Gets the accession
*
......@@ -275,6 +300,73 @@ public class AccessionController {
return objectMapper.readValue(objectMapper.writeValueAsString(filter), AccessionFilter.class);
}
@RequestMapping(value = "/downloadKml", produces = "application/vnd.google-earth.kml+xml", method = RequestMethod.POST)
@ResponseBody
public void kml(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, HttpServletResponse response) throws IOException {
// get AccessionFilter from filterCode
AccessionFilter filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
AccessionGeoFilter geoFilter = filter.geo;
if (geoFilter == null) {
filter.geo = geoFilter = new AccessionGeoFilter();
}
geoFilter.referenced = true;
final long countFiltered = accessionService.countAccessions(filter);
// LOG.info("Attempting to download KML for {} accessions", countFiltered);
if (countFiltered > DOWNLOAD_LIMIT) {
throw new RuntimeException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
}
response.setContentType("application/vnd.google-earth.kml+xml");
response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-kml-" + filterCode + ".kml\""));
DecimalFormatSymbols dfs=new DecimalFormatSymbols();
dfs.setDecimalSeparator('.');
DecimalFormat decimalFormat=new DecimalFormat("0.#", dfs);
decimalFormat.setMinimumIntegerDigits(1);
decimalFormat.setMinimumFractionDigits(6);
decimalFormat.setGroupingUsed(false);
// Write KML to the stream.
final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()));
writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
writer.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n");
writer.write("<Document>\n");
try {
accessionProcessor.process(filter, (accession) -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null && geo.getLongitude() != null && geo.getLatitude() != null) {
writer.append("<Placemark>");
writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");
writer.append("<description><![CDATA[\n");
writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
writer.append("<p><a href=\"").append(baseUrl).append("/acn/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
writer.append("\n]]></description>");
writer.append("<Point><coordinates>");
writer.append(decimalFormat.format(geo.getLongitude())).append(",").append(decimalFormat.format(geo.getLatitude()));
writer.append("</coordinates></Point>");
writer.append("</Placemark>\n");
writer.flush();
}
return accession;
});
writer.write("</Document>\n</kml>\n");
} catch (EOFException e) {
LOG.warn("Download was aborted: {}", e.getMessage());
} catch (Exception e) {
LOG.warn("Error generating KML: {}", e.getMessage());
} finally {
writer.flush();
writer.close();
response.flushBuffer();
}
}
public static class AccessionDetailsJson {
public Accession details;
public PDCI pdci;
......
......@@ -43,6 +43,7 @@ import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.service.worker.SGSVUpdate;
......
......@@ -27,6 +27,14 @@ import org.springframework.data.domain.Pageable;
public interface AccessionService {
/**
* Count accessions. Uses Elasticsearch, but counts from database when number is small enough.
*
* @param filter the filter
* @return the count
*/
long countAccessions(AccessionFilter filter);
/**
* List accessions by filter
*
......@@ -76,6 +84,6 @@ public interface AccessionService {
* @param a the accession
* @return must return the resulting {@link Accession}
*/
Accession apply(Accession a);
Accession apply(Accession a) throws Exception;
}
}
......@@ -38,6 +38,8 @@ public class AccessionGeoFilter extends BasicModelFilter<AccessionGeoFilter, Acc
/** The elevation. */
public NumberFilter<Double> elevation;
public Boolean referenced = true;
/**
* Builds the query.
......@@ -58,6 +60,15 @@ public class AccessionGeoFilter extends BasicModelFilter<AccessionGeoFilter, Acc
final BooleanBuilder and = new BooleanBuilder();
super.buildQuery(accessiongeo, accessiongeo._super, and);
if (referenced != null) {
if (referenced.booleanValue()) {
and.and(accessiongeo.longitude.isNotNull());
and.and(accessiongeo.latitude.isNotNull());
} else {
and.and(accessiongeo.longitude.isNull());
and.and(accessiongeo.latitude.isNull());
}
}
if (longitude != null) {
and.and(longitude.buildQuery(accessiongeo.longitude));
}
......
......@@ -129,6 +129,18 @@ public class AccessionServiceImpl implements AccessionService {
return res;
}
@Override
public long countAccessions(AccessionFilter filter) {
long total = elasticsearchService.count(Accession.class, filter);
if (total < 10000) {
// If total is below 10K, use actual count
total = accessionRepository.count(filter.buildQuery());
}
return total;
}
/*
* (non-Javadoc)
* @see
......@@ -154,7 +166,14 @@ public class AccessionServiceImpl implements AccessionService {
List<Accession> accessions = accessionRepository.findAll(accessionIds);
accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute));
LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size());
accessions.forEach(accession -> action.apply(accession));
accessions.forEach(accession -> {
try {
action.apply(accession);
} catch (Exception e) {
LOG.info("Error processing accession: {}", e.getMessage());
}
});
return accessions;
}
}
......@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.mvc.admin;
package org.genesys2.server.service.worker;
import java.util.ArrayList;
import java.util.List;
......@@ -64,6 +64,56 @@ public class AccessionProcessor {
/// Size of database batch scan for IDs
private int batchSize = 1000;
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(AccessionFilter filter, IAccessionAction action) throws Exception {
Predicate predicate = filter.buildQuery();
long count = accessionRepository.count(predicate);
PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
Querydsl querydsl = new Querydsl(em, builder);
JPQLQuery<Long> query = querydsl.createQuery(QAccession.accession)
// select id only
.select(QAccession.accession.id)
// order by id
.orderBy(QAccession.accession.id.asc());
// apply filter
query.where(predicate);
int startPosition = 0;
query.offset(startPosition);
query.limit(batchSize);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
List<Long> results;
do {
stopWatch.split();
LOG.debug("Reading Accessions. Stopwatch={}s {}+{} of {}. Processing at {} accessions/s", stopWatch.getSplitTime() / 1000, startPosition, batchSize, count, (double)(startPosition+batchSize)/(stopWatch.getSplitTime() / 1000));
results = query.fetch();
loadAndProcess(results, action);
// Next page
query.offset(startPosition += results.size());
// Clear anything cached in the entity manager
em.clear();
} while (results.size() > 0);
stopWatch.stop();
LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime());
}
private void loadAndProcess(List<Long> accessionIds, IAccessionAction action) throws Exception {
List<Accession> accessions = accessionRepository.findAll(accessionIds);
for (Accession a : accessions) {
action.apply(a);
}
}
/**
* Apply action on accessions matching the provided filter.
*
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment