Improved SGSVUpdater

src/main/java/org/genesys2/server/service/impl/TaxonomyServiceImpl.java

@@ -59,14 +59,13 @@ public class TaxonomyServiceImpl implements TaxonomyService {
 			taxonomy.setGenus(genus);
 			taxonomy.setSpecies(species);
 			taxonomy.setTaxonName((genus + " " + StringUtils.defaultIfEmpty(species, "")).trim());
-			// try {
-			// Thread.sleep(2000);
-			// } catch (InterruptedException e) {
-			// e.printStackTrace();
-			// }
+
 			try {
 				taxonomyRepository.save(taxonomy);
 				LOG.warn("Inserted new:" + taxonomy);
+				
+				// TODO Should update crop taxonomy lists?
+				
 			} catch (Exception e) {
 				LOG.warn("Error " + e.getMessage() + " :" + taxonomy);
 			}

src/main/java/org/genesys2/server/service/worker/SGSVUpdate.java

@@ -43,10 +43,13 @@ import org.genesys2.server.model.genesys.SvalbardData;
 import org.genesys2.server.model.impl.AccessionIdentifier3;
 import org.genesys2.server.persistence.domain.SvalbardRepository;
 import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.GeoService;
+import org.genesys2.server.service.TaxonomyService;
 import org.hibernate.exception.ConstraintViolationException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.task.TaskExecutor;
+import org.springframework.orm.hibernate3.HibernateOptimisticLockingFailureException;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Component;
 
@@ -67,7 +70,13 @@ public class SGSVUpdate {
 
 	@Autowired
 	private GenesysService genesysService;
-	
+
+	@Autowired
+	private TaxonomyService taxonomyService;
+
+	@Autowired
+	private GeoService geoService;
+
 	@Autowired
 	private SvalbardRepository svalbardRepository;
 
@@ -219,75 +228,110 @@ public class SGSVUpdate {
 			@Override
 			public void run() {
 				List<SGSVEntry> accns = new ArrayList<SGSVEntry>(bulkCopy.size());
-
 				// Extract INSTCODE and ACCENUMB
 				for (String[] entry : bulkCopy) {
-					accns.add(new SGSVEntry(entry));
+					try {
+						accns.add(new SGSVEntry(entry));
+					} catch (ArrayIndexOutOfBoundsException | NumberFormatException e) {
+						LOG.warn("Invalid entry: " + ArrayUtils.toString(entry, "NULL"));
+					} 
 				}
 
 				LOG.trace("Got " + accns.size() + " entries");
 
-				// Find accessions based on holdingInstitute and accession name
-				List<Accession> accessions = genesysService.listAccessions(accns);
-
-				// Must be same order
-				if (accessions.size() != accns.size()) {
-					LOG.error("Accession lists don't match");
-					return;
-				}
-
-				// Check if they match (Genus, OrigCty) for now
-				List<Accession> matching = new ArrayList<Accession>(accessions.size());
-				List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
+				for (int retry = 5; retry > 0; retry--) {
+					if (retry < 5) {
+						LOG.warn("Persistence attempt " + (5 - retry));
+					}
 
-				for (int i = accns.size() - 1; i >= 0; i--) {
-					Accession accn = accessions.get(i);
-					SGSVEntry entry = accns.get(i);
+					try {
+						// Find accessions based on holdingInstitute and
+						// accession
+						// name
+						List<Accession> accessions = genesysService.listAccessions(accns);
+
+						// Must be same order
+						if (accessions.size() != accns.size()) {
+							LOG.error("Accession lists don't match");
+							return;
+						}
 
-					if (accn == null || entry == null)
-						continue;
+						// Check if they match (Genus, OrigCty) for now
+						List<Accession> matching = new ArrayList<Accession>(accessions.size());
+						List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
+
+						for (int i = accns.size() - 1; i >= 0; i--) {
+							boolean updateAccession = false;
+							Accession accn = accessions.get(i);
+							SGSVEntry entry = accns.get(i);
+
+							if (accn == null || entry == null)
+								continue;
+
+							if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
+
+								if (!Boolean.TRUE.equals(accn.getInSvalbard())) {
+									accn.setInSvalbard(true);
+									updateAccession = true;
+								}
+
+								// Doesn't have taxonomy due to no species
+								if (accn.getTaxonomy() == null && StringUtils.isNotBlank(entry.species)) {
+									accn.setTaxonomy(taxonomyService.ensureTaxonomy(entry.genus, entry.species));
+									updateAccession = true;
+								}
+
+								if (entry.origCty != null && StringUtils.isBlank(accn.getOrigin()) && !StringUtils.equals(accn.getOrigin(), entry.origCty)) {
+									accn.setOrigin(entry.origCty);
+									accn.setCountryOfOrigin(geoService.getCountry(entry.origCty));
+									updateAccession = true;
+								}
+
+								SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
+								if (svalbardData == null) {
+									svalbardData = new SvalbardData();
+								}
+								svalbardData.setId(accn.getId());
+								svalbardData.setBoxNumber(entry.boxNo);
+								svalbardData.setDepositDate(entry.depositDate);
+								svalbardData.setQuantity(entry.quantity);
+								svalbardData.setTaxonomy(entry.fullTaxa);
+
+								svalbards.add(svalbardData);
+								if (updateAccession)
+									matching.add(accn);
+							} else {
+								LOG.warn("This should not be");
+							}
 
-					if (accn.getOrigin().equalsIgnoreCase(entry.origCty) && accn.getTaxonomy().getGenus().equals(entry.genus)) {
-						if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
+						}
 
-							if (accn.getInSvalbard() == null || accn.getInSvalbard() == false) {
-								LOG.info("Found one in SGSV: " + accn);
-							}
-							accn.setInSvalbard(true);
-							SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
-							if (svalbardData == null) {
-								svalbardData = new SvalbardData();
+						// Save data
+						if (matching.size() > 0) {
+							LOG.info("Setting inSGSV=true for accessions size=" + matching.size());
+							genesysService.saveAccessions(matching);
+						}
+						if (svalbards.size() > 0) {
+							try {
+								genesysService.saveSvalbards(svalbards);
+							} catch (ConstraintViolationException e) {
+								LOG.warn(e.getMessage());
+								for (Accession a : matching) {
+									LOG.warn("\t" + a);
+								}
 							}
-							svalbardData.setId(accn.getId());
-							svalbardData.setBoxNumber(entry.boxNo);
-							svalbardData.setDepositDate(entry.depositDate);
-							svalbardData.setQuantity(entry.quantity);
-							svalbardData.setTaxonomy(entry.fullTaxa);
-
-							svalbards.add(svalbardData);
-							matching.add(accn);
-						} else {
-							LOG.warn("This should not be");
 						}
-					} else {
-						LOG.debug("Ignoring " + entry + ": " + accn.getOrigin() + "!=" + entry.origCty + ", " + accn.getGenus() + "!=" + entry.genus);
-					}
-				}
 
-				// Save data
-				if (matching.size() > 0) {
-					genesysService.saveAccessions(matching);
-				}
-				if (svalbards.size() > 0) {
-					try {
-						genesysService.saveSvalbards(svalbards);
-					} catch (ConstraintViolationException e) {
-						LOG.warn(e.getMessage());
-						for (Accession a : matching) {
-							LOG.warn("\t" + a);
-						}
+						// EXIT
+						return;
+
+					} catch (HibernateOptimisticLockingFailureException e) {
+						LOG.warn("Failed to save data, will retry. " + e.getMessage());
+						continue;
 					}
 				}
+
+				LOG.warn("Persistence retries failed!");
 			}
 		});
 	}
@@ -297,6 +341,7 @@ public class SGSVUpdate {
 		String acceNumb;
 		String origCty;
 		String genus;
+		String species;
 		String fullTaxa;
 		String boxNo;
 		String depositDate;
@@ -308,6 +353,7 @@ public class SGSVUpdate {
 			fullTaxa = entry[5];
 			origCty = entry[12];
 			genus = entry[16];
+			species = entry[17];
 			depositDate = entry[20];
 			if (entry[7] != null) {
 				quantity = Float.parseFloat(entry[7]);