Added BatchRESTService to process upsert batches

doc/ddl/ind002.sql

+
+update accession set ACC_Numb_HI=concat('ICG ', ACC_Numb_HI) where Institute='IND002' and Genuss='Arachis';
+update accession set ACC_Numb_HI=concat('ICP ', ACC_Numb_HI) where Institute='IND002' and Genuss='Cajanus';
+update accession set ACC_Numb_HI=concat('ICC ', ACC_Numb_HI) where Institute='IND002' and Genuss='Cicer';
+update accession set ACC_Numb_HI=concat('IEc ', ACC_Numb_HI) where Institute='IND002' and Genuss='Echinochloa';
+update accession set ACC_Numb_HI=concat('IE ', ACC_Numb_HI) where Institute='IND002' and Genuss='Eleusine';
+update accession set ACC_Numb_HI=concat('IPs ', ACC_Numb_HI) where Institute='IND002' and Genuss='Paspalum';
+update accession set ACC_Numb_HI=concat('IP ', ACC_Numb_HI) where Institute='IND002' and Genuss='Pennisetum';
+update accession set ACC_Numb_HI=concat('ISe ', ACC_Numb_HI) where Institute='IND002' and Genuss='Setaria';
+update accession set ACC_Numb_HI=concat('IS ', ACC_Numb_HI) where Institute='IND002' and Genuss='Sorghum';
+update accession set ACC_Numb_HI=concat('IPmr ', ACC_Numb_HI) where Institute='IND002' and Taxon_Code in (select Taxon_Code from taxonomy where genus='Panicum' and species='sumatrense');
+update accession set ACC_Numb_HI=concat('IPm ', ACC_Numb_HI) where Institute='IND002' and Taxon_Code in (select Taxon_Code from taxonomy where genus='Panicum' and species='miliaceum');

pom.xml

@@ -67,10 +67,10 @@
 			<name>ibiblio.mirrors</name>
 			<url>http://mirrors.ibiblio.org/pub/mirrors/maven2</url>
 		</repository>
-		<repository>
+<!-- 		<repository>
 			<id>sonatype mirror</id>
 			<url>http://search.maven.org/remotecontent?filepath=</url>
-		</repository>
+		</repository> -->
 	</repositories>
 
 	<dependencies>

src/main/java/org/genesys2/server/model/genesys/Taxonomy.java

@@ -114,4 +114,8 @@ public class Taxonomy implements java.io.Serializable {
 		return MessageFormat.format("Tax id={0} genus={1} sp={2}, full={3}", id, genus, species, taxonName);
 	}
 
+	public boolean sameAs(Taxonomy taxonomy) {
+		return taxonomy == null ? false : taxonomy.getId().equals(id);
+	}
+
 }

src/main/java/org/genesys2/server/service/BatchRESTService.java

+/**
+ * Copyright 2013 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service;
+
+import java.text.MessageFormat;
+import java.util.Map;
+
+import org.genesys2.server.model.impl.AccessionIdentifier3;
+import org.genesys2.server.model.impl.FaoInstitute;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public interface BatchRESTService {
+
+	boolean upsertAccessionData(FaoInstitute institute, Map<DataJson, ObjectNode> batch);
+
+	public static class DataJson implements AccessionIdentifier3 {
+		public String instCode;
+		public String acceNumb;
+		public String genus;
+
+		@Override
+		public String getHoldingInstitute() {
+			return instCode;
+		}
+
+		@Override
+		public String getAccessionName() {
+			return acceNumb;
+		}
+
+		@Override
+		public String getGenus() {
+			return genus;
+		}
+
+		@Override
+		public String toString() {
+			return MessageFormat.format("AID3 instCode={0} acceNumb={1} genus={2}", instCode, acceNumb, genus);
+		}
+	}
+
+}

src/main/java/org/genesys2/server/service/impl/BatchRESTServiceImpl.java

+/**
+ * Copyright 2013 Global Crop Diversity Trust
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.genesys.AccessionGeo;
+import org.genesys2.server.model.genesys.AllAcqCollect;
+import org.genesys2.server.model.genesys.Taxonomy;
+import org.genesys2.server.model.impl.Country;
+import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.service.BatchRESTService;
+import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.GeoService;
+import org.genesys2.server.service.TaxonomyService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+@Service
+@Transactional(readOnly = true)
+public class BatchRESTServiceImpl implements BatchRESTService {
+
+	private final Log LOG = LogFactory.getLog(getClass());
+
+	@Autowired
+	GenesysService genesysService;
+
+	@Autowired
+	GeoService geoService;
+
+	@Autowired
+	TaxonomyService taxonomyService;
+
+	@Override
+	@Transactional
+	public boolean upsertAccessionData(FaoInstitute institute, Map<BatchRESTService.DataJson, ObjectNode> batch) {
+		LOG.info("Batch processing " + batch.size() + " entries for " + institute);
+		List<Accession> toSave = new ArrayList<Accession>();
+		List<AllAcqCollect> toSaveColl = new ArrayList<AllAcqCollect>();
+		List<AccessionGeo> toSaveGeo = new ArrayList<AccessionGeo>();
+
+		for (BatchRESTService.DataJson dataJson : batch.keySet()) {
+			if (LOG.isDebugEnabled())
+				LOG.debug("Loading accession " + dataJson);
+
+			if (!institute.getCode().equals(dataJson.instCode)) {
+				throw new RuntimeException("Accession does not belong to instCode=" + institute.getCode() + " acn=" + dataJson);
+			}
+
+			Accession accession = genesysService.getAccession(dataJson.instCode, dataJson.genus, dataJson.acceNumb);
+
+			boolean updated = false;
+
+			if (accession == null) {
+				LOG.warn("New accession " + dataJson);
+
+				accession = new Accession();
+				accession.setInstituteCode(dataJson.instCode);
+				accession.setInstitute(institute);
+				accession.setGenus(dataJson.genus);
+				accession.setAccessionName(dataJson.acceNumb);
+
+				updated = true;
+			}
+
+			ObjectNode accnJson = batch.get(dataJson);
+
+			JsonNode value = accnJson.get("orgCty");
+			if (value != null) {
+				String orgCty = value.textValue();
+				if (!StringUtils.equals(orgCty, accession.getOrigin())) {
+					Country country = geoService.getCountry(orgCty);
+					accession.setOrigin(orgCty);
+					accession.setCountryOfOrigin(country);
+					updated = true;
+				}
+			}
+			value = accnJson.get("acqDate");
+			if (value != null) {
+				String acqDate = value.isNull() ? null : value.textValue();
+				if (!StringUtils.equals(acqDate, accession.getAcquisitionDate())) {
+					accession.setAcquisitionDate(acqDate);
+					updated = true;
+				}
+			}
+			value = accnJson.get("mlsStat");
+			if (value != null) {
+				Boolean inMls = value.isNull() ? null : value.asBoolean();
+
+				if (inMls != accession.getMlsStatus()) {
+					accession.setMlsStatus(inMls);
+					updated = true;
+				}
+			}
+			value = accnJson.get("species");
+			if (value != null) {
+				String species = value.isNull() ? null : value.textValue();
+				Taxonomy taxonomy = taxonomyService.ensureTaxonomy(accession.getGenus(), species);
+				if (!taxonomy.sameAs(accession.getTaxonomy())) {
+					accession.setTaxonomy(taxonomy);
+					updated = true;
+				}
+			}
+
+			value = accnJson.get("storage");
+			if (value != null) {
+				if (!StringUtils.equals(value.textValue(), accession.getStorage())) {
+					accession.setStorage(StringUtils.defaultIfBlank(value.textValue(), null));
+					updated = true;
+				}
+			}
+
+			value = accnJson.get("sampStat");
+			if (value != null) {
+				Integer sampStat = value.isNull() || !value.isNumber() ? null : value.asInt();
+				if (!StringUtils.equals("" + sampStat, accession.getSampleStatus())) {
+					accession.setSampleStatus(StringUtils.defaultIfBlank("" + sampStat, null));
+					updated = true;
+				}
+			}
+
+			value = accnJson.get("dublInst");
+			if (value != null) {
+				String dublInst = value.isNull() ? null : value.textValue();
+				if (!StringUtils.equals(dublInst, accession.getDublInst())) {
+					accession.setDublInst(StringUtils.defaultIfBlank(dublInst, null));
+					updated = true;
+				}
+			}
+
+			if (accnJson.has("coll")) {
+				ObjectNode collecting = (ObjectNode) accnJson.get("coll");
+				AllAcqCollect accnColl = genesysService.listAccessionCollect(accession);
+				if (accnColl == null) {
+					accnColl = new AllAcqCollect();
+					accnColl.setAccession(accession);
+				}
+				value = collecting.get("date");
+				if (value != null) {
+					accnColl.setCollectDate(StringUtils.defaultIfBlank(value.textValue(), null));
+				}
+				value = collecting.get("site");
+				if (value != null) {
+					accnColl.setCollectSite(StringUtils.defaultIfBlank(value.textValue(), null));
+				}
+				toSaveColl.add(accnColl);
+			}
+
+			if (accnJson.has("geo")) {
+				ObjectNode geo = (ObjectNode) accnJson.get("geo");
+				AccessionGeo accnGeo = genesysService.listAccessionGeo(accession);
+				if (accnGeo == null) {
+					accnGeo = new AccessionGeo();
+					accnGeo.setAccession(accession);
+				}
+				value = geo.get("lat");
+				if (value != null) {
+					accnGeo.setLatitude(value.isNumber() ? value.asDouble() : null);
+				}
+				value = geo.get("lng");
+				if (value != null) {
+					accnGeo.setLongitude(value.isNumber() ? value.asDouble() : null);
+				}
+				value = geo.get("alt");
+				if (value != null) {
+					accnGeo.setAltitude(value.isNumber() ? value.asDouble() : null);
+				}
+				toSaveGeo.add(accnGeo);
+			}
+
+			if (updated) {
+				toSave.add(accession);
+			}
+		}
+
+		if (toSave.size() > 0) {
+			LOG.info("Storing " + toSave.size() + " accessions.");
+			genesysService.saveAccessions(toSave);
+			genesysService.updateAccessionCount(institute);
+		}
+
+		if (toSaveColl.size() > 0) {
+			genesysService.saveCollecting(toSaveColl);
+		}
+
+		if (toSaveGeo.size() > 0) {
+			genesysService.saveGeo(toSaveGeo);
+		}
+
+		return toSave.size() > 0 || toSaveColl.size() > 0 || toSaveGeo.size() > 0;
+	}
+}

src/main/java/org/genesys2/server/servlet/controller/rest/AccessionController.java

@@ -17,20 +17,14 @@
 package org.genesys2.server.servlet.controller.rest;
 
 import java.io.IOException;
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.lang.StringUtils;
 import org.genesys2.server.model.genesys.Accession;
-import org.genesys2.server.model.genesys.AccessionGeo;
-import org.genesys2.server.model.genesys.AllAcqCollect;
-import org.genesys2.server.model.genesys.Taxonomy;
-import org.genesys2.server.model.impl.AccessionIdentifier3;
-import org.genesys2.server.model.impl.Country;
 import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.service.BatchRESTService;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.GeoService;
 import org.genesys2.server.service.InstituteService;
@@ -62,6 +56,9 @@ public class AccessionController extends RestController {
 
 	@Autowired
 	GenesysService genesysService;
+	
+	@Autowired
+	BatchRESTService batchRESTService;
 
 	@Autowired
 	InstituteService instituteService;
@@ -104,18 +101,18 @@ public class AccessionController extends RestController {
 	JsonNode check(@PathVariable("instCode") String instCode, @RequestBody String content) throws JsonProcessingException, IOException {
 		// TODO Check user's permissions to update this WIEWS institute.
 		JsonNode json = mapper.readTree(content);
-		Map<DataJson, ObjectNode> batch = new HashMap<DataJson, ObjectNode>();
+		Map<BatchRESTService.DataJson, ObjectNode> batch = new HashMap<BatchRESTService.DataJson, ObjectNode>();
 
 		if (json.isArray()) {
 			for (JsonNode j : json) {
-				DataJson dataJson = readAid3(j);
+				BatchRESTService.DataJson dataJson = readAid3(j);
 				if (!instCode.equals(dataJson.instCode)) {
 					throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 				}
 				batch.put(dataJson, (ObjectNode) j);
 			}
 		} else {
-			DataJson dataJson = readAid3(json);
+			BatchRESTService.DataJson dataJson = readAid3(json);
 			if (!instCode.equals(dataJson.instCode)) {
 				throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 			}
@@ -125,7 +122,7 @@ public class AccessionController extends RestController {
 		LOG.info("Batch processing " + batch.size() + " entries");
 		ArrayNode rets = mapper.createArrayNode();
 
-		for (DataJson dataJson : batch.keySet()) {
+		for (BatchRESTService.DataJson dataJson : batch.keySet()) {
 			if (LOG.isDebugEnabled())
 				LOG.debug("Loading accession " + dataJson);
 
@@ -161,28 +158,28 @@ public class AccessionController extends RestController {
 		}
 
 		JsonNode json = mapper.readTree(content);
-		Map<DataJson, ObjectNode> batch = new HashMap<DataJson, ObjectNode>();
+		Map<BatchRESTService.DataJson, ObjectNode> batch = new HashMap<BatchRESTService.DataJson, ObjectNode>();
 
 		if (json.isArray()) {
 			for (JsonNode j : json) {
-				DataJson dataJson = readAid3(j);
+				BatchRESTService.DataJson dataJson = readAid3(j);
 				if (!instCode.equals(dataJson.instCode)) {
 					throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 				}
 				batch.put(dataJson, (ObjectNode) j);
 			}
 		} else {
-			DataJson dataJson = readAid3(json);
+			BatchRESTService.DataJson dataJson = readAid3(json);
 			if (!instCode.equals(dataJson.instCode)) {
 				throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 			}
 			batch.put(dataJson, (ObjectNode) json);
 		}
 
-		LOG.info("Batch processing " + batch.size() + " entries");
+		LOG.info("Batch processing " + batch.size() + " entries for " + institute);
 		List<Accession> toSave = new ArrayList<Accession>();
 
-		for (DataJson dataJson : batch.keySet()) {
+		for (BatchRESTService.DataJson dataJson : batch.keySet()) {
 			if (LOG.isDebugEnabled())
 				LOG.debug("Loading accession " + dataJson);
 
@@ -250,167 +247,29 @@ public class AccessionController extends RestController {
 		}
 
 		JsonNode json = mapper.readTree(content);
-		Map<DataJson, ObjectNode> batch = new HashMap<DataJson, ObjectNode>();
+		Map<BatchRESTService.DataJson, ObjectNode> batch = new HashMap<BatchRESTService.DataJson, ObjectNode>();
 
 		if (json.isArray()) {
 			for (JsonNode j : json) {
-				DataJson dataJson = readAid3(j);
+				BatchRESTService.DataJson dataJson = readAid3(j);
 				if (!instCode.equals(dataJson.instCode)) {
 					throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 				}
 				batch.put(dataJson, (ObjectNode) j);
 			}
 		} else {
-			DataJson dataJson = readAid3(json);
+			BatchRESTService.DataJson dataJson = readAid3(json);
 			if (!instCode.equals(dataJson.instCode)) {
 				throw new RuntimeException("Accession does not belong to instCode=" + instCode + " acn=" + dataJson);
 			}
 			batch.put(dataJson, (ObjectNode) json);
 		}
 
-		LOG.info("Batch processing " + batch.size() + " entries");
-		List<Accession> toSave = new ArrayList<Accession>();
-		List<AllAcqCollect> toSaveColl = new ArrayList<AllAcqCollect>();
-		List<AccessionGeo> toSaveGeo = new ArrayList<AccessionGeo>();
-
-		for (DataJson dataJson : batch.keySet()) {
-			if (LOG.isDebugEnabled())
-				LOG.debug("Loading accession " + dataJson);
-
-			Accession accession = genesysService.getAccession(dataJson.instCode, dataJson.genus, dataJson.acceNumb);
-
-			boolean updated = false;
-
-			if (accession == null) {
-				LOG.warn("New accession " + dataJson);
-
-				accession = new Accession();
-				accession.setInstituteCode(dataJson.instCode);
-				accession.setInstitute(institute);
-				accession.setGenus(dataJson.genus);
-				accession.setAccessionName(dataJson.acceNumb);
-
-				updated = true;
-			}
-
-			ObjectNode accnJson = batch.get(dataJson);
-
-			JsonNode value = accnJson.get("orgCty");
-			if (value != null) {
-				String orgCty = value.textValue();
-				// Validate!!!
-				Country country = geoService.getCountry(orgCty);
-				accession.setOrigin(orgCty);
-				accession.setCountryOfOrigin(country);
-				updated = true;
-			}
-			value = accnJson.get("acqDate");
-			if (value != null) {
-				String acqDate = value.isNull() ? null : value.textValue();
-				accession.setAcquisitionDate(acqDate);
-				updated = true;
-			}
-			value = accnJson.get("mlsStat");
-			if (value != null) {
-				Boolean inMls = value.isNull() ? null : value.asBoolean();
-
-				if (inMls != accession.getMlsStatus()) {
-					accession.setMlsStatus(inMls);
-					updated = true;
-				}
-			}
-			value = accnJson.get("species");
-			if (value != null) {
-				String species = value.isNull() ? null : value.textValue();
-				Taxonomy taxonomy = taxonomyService.ensureTaxonomy(accession.getGenus(), species);
-				accession.setTaxonomy(taxonomy);
-				updated = true;
-			}
-
-			value = accnJson.get("storage");
-			if (value != null) {
-				accession.setStorage(StringUtils.defaultIfBlank(value.textValue(), null));
-				updated = true;
-			}
-
-			value = accnJson.get("sampStat");
-			if (value != null) {
-				Integer sampStat = value.isNull() ? null : value.asInt();
-				accession.setSampleStatus(StringUtils.defaultIfBlank("" + sampStat, null));
-				updated = true;
-			}
-
-			value = accnJson.get("dublInst");
-			if (value != null) {
-				String dublInst = value.isNull() ? null : value.textValue();
-				accession.setDublInst(StringUtils.defaultIfBlank(dublInst, null));
-				updated = true;
-			}
-
-			if (accnJson.has("coll")) {
-				ObjectNode collecting = (ObjectNode) accnJson.get("coll");
-				AllAcqCollect accnColl = genesysService.listAccessionCollect(accession);
-				if (accnColl == null) {
-					accnColl = new AllAcqCollect();
-					accnColl.setAccession(accession);
-				}
-				value = collecting.get("date");
-				if (value != null) {
-					accnColl.setCollectDate(StringUtils.defaultIfBlank(value.textValue(), null));
-				}
-				value = collecting.get("site");
-				if (value != null) {
-					accnColl.setCollectSite(StringUtils.defaultIfBlank(value.textValue(), null));
-				}
-				toSaveColl.add(accnColl);
-			}
-
-			if (accnJson.has("geo")) {
-				ObjectNode geo = (ObjectNode) accnJson.get("geo");
-				AccessionGeo accnGeo = genesysService.listAccessionGeo(accession);
-				if (accnGeo == null) {
-					accnGeo = new AccessionGeo();
-					accnGeo.setAccession(accession);
-				}
-				value = geo.get("lat");
-				if (value != null) {
-					accnGeo.setLatitude(value.isNumber() ? value.asDouble() : null);
-				}
-				value = geo.get("lng");
-				if (value != null) {
-					accnGeo.setLongitude(value.isNumber() ? value.asDouble() : null);
-				}
-				value = geo.get("alt");
-				if (value != null) {
-					accnGeo.setAltitude(value.isNumber() ? value.asDouble() : null);
-				}
-				toSaveGeo.add(accnGeo);
-			}
-
-			if (updated) {
-				toSave.add(accession);
-			}
-		}
-
-		if (toSave.size() > 0) {
-			LOG.info("Storing " + toSave.size() + " accessions.");
-			genesysService.saveAccessions(toSave);
-			genesysService.updateAccessionCount(institute);
-		}
-
-		if (toSaveColl.size() > 0) {
-			genesysService.saveCollecting(toSaveColl);
-		}