Merge branch '329-accession-api-v1-download-kml' into 'master'

Resolve "Accession API v1: Download KML"

Closes #329

See merge request genesys-pgr/genesys-server!222

src/main/java/org/genesys2/server/api/v1/AccessionController.java

@@ -16,12 +16,19 @@
 
 package org.genesys2.server.api.v1;
 
+import java.io.BufferedWriter;
+import java.io.EOFException;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
+import javax.servlet.http.HttpServletResponse;
+
 import org.genesys.blocks.model.JsonViews;
 import org.genesys.catalog.api.FilteredPage;
 import org.genesys.catalog.model.dataset.Dataset;
@@ -30,6 +37,7 @@ import org.genesys.catalog.service.ShortFilterService;
 import org.genesys2.server.api.ApiBaseController;
 import org.genesys2.server.api.model.AccessionHeaderJson;
 import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.genesys.AccessionGeo;
 import org.genesys2.server.model.genesys.PDCI;
 import org.genesys2.server.model.impl.AccessionIdentifier3;
 import org.genesys2.server.model.impl.Subset;
@@ -38,9 +46,14 @@ import org.genesys2.server.service.ElasticsearchService;
 import org.genesys2.server.service.ElasticsearchService.TermResult;
 import org.genesys2.server.service.SubsetService;
 import org.genesys2.server.service.filter.AccessionFilter;
+import org.genesys2.server.service.filter.AccessionGeoFilter;
 import org.genesys2.server.service.impl.SearchException;
+import org.genesys2.server.service.worker.AccessionProcessor;
 import org.genesys2.spring.ResourceNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.http.MediaType;
@@ -52,6 +65,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
 
 import com.fasterxml.jackson.annotation.JsonView;
@@ -73,9 +87,13 @@ import io.swagger.annotations.Api;
 @Api(tags = { "accession" })
 public class AccessionController {
 
+    private static final Logger LOG = LoggerFactory.getLogger(AccessionController.class);
+
 	/** The Constant API_BASE. */
 	public static final String API_BASE = ApiBaseController.APIv1_BASE + "/acn";
 
+	private static final int DOWNLOAD_LIMIT = 200000;
+
 	@Autowired
 	private AccessionService accessionService;
 
@@ -92,11 +110,18 @@ public class AccessionController {
 	@Autowired
 	private SubsetService subsetService;
 
+	@Autowired
+	private AccessionProcessor accessionProcessor;
+
+	@Value("${base.url}")
+	private String baseUrl;
+	
 	private final Set<String> terms = Sets.newHashSet("institute.code", "institute.country.code3", "cropName", "crop.shortName", "taxonomy.genus", "taxonomy.species",
 		"taxonomy.genusSpecies", "countryOfOrigin.code3", "sampStat", "available", "mlsStatus", "donorCode", "sgsv", "storage", "duplSite", "breederCode");
 
 	private ObjectMapper objectMapper = new ObjectMapper();
 
+
 	/**
 	 * Gets the accession
 	 *
@@ -275,6 +300,73 @@ public class AccessionController {
 		return objectMapper.readValue(objectMapper.writeValueAsString(filter), AccessionFilter.class);
 	}
 
+	@RequestMapping(value = "/downloadKml", produces = "application/vnd.google-earth.kml+xml", method = RequestMethod.POST)
+	@ResponseBody
+	public void kml(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, HttpServletResponse response) throws IOException {
+
+		// get AccessionFilter from filterCode
+		AccessionFilter filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
+		AccessionGeoFilter geoFilter = filter.geo;
+		if (geoFilter == null) {
+			filter.geo = geoFilter = new AccessionGeoFilter();
+		}
+		geoFilter.referenced = true;
+
+		final long countFiltered = accessionService.countAccessions(filter);
+		// LOG.info("Attempting to download KML for {} accessions", countFiltered);
+		if (countFiltered > DOWNLOAD_LIMIT) {
+			throw new RuntimeException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
+		}
+
+		response.setContentType("application/vnd.google-earth.kml+xml");
+		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-kml-" + filterCode + ".kml\""));
+
+		DecimalFormatSymbols dfs=new DecimalFormatSymbols();
+		dfs.setDecimalSeparator('.');
+		DecimalFormat decimalFormat=new DecimalFormat("0.#", dfs);
+		decimalFormat.setMinimumIntegerDigits(1);
+		decimalFormat.setMinimumFractionDigits(6);
+		decimalFormat.setGroupingUsed(false);
+
+		// Write KML to the stream.
+		final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()));
+		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+		writer.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n");
+		writer.write("<Document>\n");
+		try {
+			accessionProcessor.process(filter, (accession) -> {
+				AccessionGeo geo = accession.getAccessionId().getGeo();
+				if (geo != null && geo.getLongitude() != null && geo.getLatitude() != null) {
+					writer.append("<Placemark>");
+					writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");
+					
+					writer.append("<description><![CDATA[\n");
+					writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
+					writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
+					writer.append("<p><a href=\"").append(baseUrl).append("/acn/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
+					writer.append("\n]]></description>");
+					
+					writer.append("<Point><coordinates>");
+					writer.append(decimalFormat.format(geo.getLongitude())).append(",").append(decimalFormat.format(geo.getLatitude()));
+					writer.append("</coordinates></Point>");
+					writer.append("</Placemark>\n");
+					writer.flush();
+				}
+				return accession;
+			});
+			writer.write("</Document>\n</kml>\n");
+
+		} catch (EOFException e) {
+			 LOG.warn("Download was aborted: {}", e.getMessage());
+		} catch (Exception e) {
+			 LOG.warn("Error generating KML: {}", e.getMessage());
+		} finally {
+			writer.flush();
+			writer.close();
+			response.flushBuffer();			
+		}
+	}
+
 	public static class AccessionDetailsJson {
 		public Accession details;
 		public PDCI pdci;

src/main/java/org/genesys2/server/mvc/admin/AdminController.java

@@ -43,6 +43,7 @@ import org.genesys2.server.service.GeoService;
 import org.genesys2.server.service.InstituteService;
 import org.genesys2.server.service.TaxonomyService;
 import org.genesys2.server.service.filter.AccessionFilter;
+import org.genesys2.server.service.worker.AccessionProcessor;
 import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
 import org.genesys2.server.service.worker.InstituteUpdater;
 import org.genesys2.server.service.worker.SGSVUpdate;

src/main/java/org/genesys2/server/service/AccessionService.java

@@ -27,6 +27,14 @@ import org.springframework.data.domain.Pageable;
 
 public interface AccessionService {
 
+	/**
+	 * Count accessions. Uses Elasticsearch, but counts from database when number is small enough.
+	 *
+	 * @param filter the filter
+	 * @return the count 
+	 */
+	long countAccessions(AccessionFilter filter);
+
 	/**
 	 * List accessions by filter
 	 *
@@ -76,6 +84,6 @@ public interface AccessionService {
 		 * @param a the accession
 		 * @return must return the resulting {@link Accession}
 		 */
-		Accession apply(Accession a);
+		Accession apply(Accession a) throws Exception;
 	}
 }

src/main/java/org/genesys2/server/service/filter/AccessionGeoFilter.java

@@ -38,6 +38,8 @@ public class AccessionGeoFilter extends BasicModelFilter<AccessionGeoFilter, Acc
 	
 	/** The elevation. */
 	public NumberFilter<Double> elevation;
+	
+	public Boolean referenced = true;
 
 	/**
 	 * Builds the query.
@@ -58,6 +60,15 @@ public class AccessionGeoFilter extends BasicModelFilter<AccessionGeoFilter, Acc
 		final BooleanBuilder and = new BooleanBuilder();
 		super.buildQuery(accessiongeo, accessiongeo._super, and);
 
+		if (referenced != null) {
+			if (referenced.booleanValue()) {
+				and.and(accessiongeo.longitude.isNotNull());
+				and.and(accessiongeo.latitude.isNotNull());
+			} else {
+				and.and(accessiongeo.longitude.isNull());
+				and.and(accessiongeo.latitude.isNull());
+			}
+		}
 		if (longitude != null) {
 			and.and(longitude.buildQuery(accessiongeo.longitude));
 		}

src/main/java/org/genesys2/server/service/impl/AccessionServiceImpl.java

@@ -129,6 +129,18 @@ public class AccessionServiceImpl implements AccessionService {
 		return res;
 	}
 
+	@Override
+	public long countAccessions(AccessionFilter filter) {
+		long total = elasticsearchService.count(Accession.class, filter);
+
+		if (total < 10000) {
+			// If total is below 10K, use actual count
+			total = accessionRepository.count(filter.buildQuery());
+		}
+		
+		return total;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see
@@ -154,7 +166,14 @@ public class AccessionServiceImpl implements AccessionService {
 		List<Accession> accessions = accessionRepository.findAll(accessionIds);
 		accessions.stream().map(a -> a.getInstitute()).distinct().forEach(institute -> accessionCounter.recountInstitute(institute));
 		LOG.debug("Processing {} accessions of {} IDs provided", accessions.size(), accessionIds.size());
-		accessions.forEach(accession -> action.apply(accession));
+		accessions.forEach(accession -> {
+			try {
+				action.apply(accession);
+			} catch (Exception e) {
+				LOG.info("Error processing accession: {}", e.getMessage());
+			}
+		});
 		return accessions;
 	}
+
 }

src/main/java/org/genesys2/server/mvc/admin/AccessionProcessor.java → src/main/java/org/genesys2/server/service/worker/AccessionProcessor.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.genesys2.server.mvc.admin;
+package org.genesys2.server.service.worker;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -64,6 +64,56 @@ public class AccessionProcessor {
 	/// Size of database batch scan for IDs
 	private int batchSize = 1000;
 
+	@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
+	public void process(AccessionFilter filter, IAccessionAction action) throws Exception {
+
+		Predicate predicate = filter.buildQuery();
+		long count = accessionRepository.count(predicate);
+
+		PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
+		Querydsl querydsl = new Querydsl(em, builder);
+		JPQLQuery<Long> query = querydsl.createQuery(QAccession.accession)
+			// select id only
+			.select(QAccession.accession.id)
+			// order by id
+			.orderBy(QAccession.accession.id.asc());
+
+		// apply filter
+		query.where(predicate);
+
+		int startPosition = 0;
+		query.offset(startPosition);
+		query.limit(batchSize);
+
+		StopWatch stopWatch = new StopWatch();
+		stopWatch.start();
+
+		List<Long> results;
+
+		do {
+			stopWatch.split();
+			LOG.debug("Reading Accessions. Stopwatch={}s {}+{} of {}. Processing at {} accessions/s", stopWatch.getSplitTime() / 1000, startPosition, batchSize, count, (double)(startPosition+batchSize)/(stopWatch.getSplitTime() / 1000));
+			results = query.fetch();
+			loadAndProcess(results, action);
+
+			// Next page
+			query.offset(startPosition += results.size());
+			
+			// Clear anything cached in the entity manager
+			em.clear();
+		} while (results.size() > 0);
+
+		stopWatch.stop();
+		LOG.info("Processing Accessions for filter {} took {}ms", filter, stopWatch.getTime());
+	}
+	
+	private void loadAndProcess(List<Long> accessionIds, IAccessionAction action) throws Exception {
+		List<Accession> accessions = accessionRepository.findAll(accessionIds);
+		for (Accession a : accessions) {
+			action.apply(a);
+		}
+	}
+
 	/**
 	 * Apply action on accessions matching the provided filter.
 	 *