SGSVUpdate tests

src/main/java/org/genesys2/server/persistence/domain/FaoInstituteSettingRepository.java

@@ -18,8 +18,14 @@ package org.genesys2.server.persistence.domain;
 
 import org.genesys2.server.model.impl.FaoInstituteSetting;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
 public interface FaoInstituteSettingRepository extends JpaRepository<FaoInstituteSetting, Long> {
 
 	FaoInstituteSetting findByInstCodeAndSetting(String instCode, String setting);
+
+	@Modifying
+	@Query("DELETE FROM FaoInstituteSetting fis WHERE fis.instCode = ?1")
+	void deleteFor(String instCode);
 }

src/main/java/org/genesys2/server/service/InstituteService.java

@@ -59,4 +59,6 @@ public interface InstituteService {
 
 	void setUniqueAcceNumbs(FaoInstitute faoInstitute, boolean uniqueAcceNumbs);
 
+	void delete(String instCode);
+
 }

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -213,10 +213,10 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 			// Including null's
 			result.add(accn);
 			if (accn == null) {
-				FaoInstitute inst = instituteRepository.findByCode(aid3.getHoldingInstitute());
-				if (inst == null)
+				// FaoInstitute inst = instituteRepository.findByCode(aid3.getHoldingInstitute());
+				if (LOG.isDebugEnabled())
 					// Only log full miss
-					LOG.debug("No accession " + aid3 + " in " + inst);
+					LOG.debug("No accession " + aid3);
 			}
 		}
 		return result;

src/main/java/org/genesys2/server/service/impl/InstituteServiceImpl.java

@@ -179,4 +179,14 @@ public class InstituteServiceImpl implements InstituteService {
 		return organizationRepository.getOrganizations(faoInstitute);
 	}
 
+	@Override
+	@Transactional
+	@PreAuthorize("hasRole('ADMINISTRATOR')")
+	public void delete(String instCode) {
+		FaoInstitute institute = getInstitute(instCode);
+		if (institute!=null) {
+			instituteSettingRepository.deleteFor(institute.getCode());
+			instituteRepository.delete(institute);
+		}
+	}
 }

src/main/java/org/genesys2/server/service/worker/SGSVEntry.java

@@ -15,7 +15,10 @@ class SGSVEntry implements AccessionIdentifier3 {
 	String boxNo;
 	String depositDate;
 	Float quantity;
-
+	
+	public SGSVEntry() {
+	}
+	
 	public SGSVEntry(String[] entry) {
 		instCode = entry[1];
 		acceNumb = entry[4];

src/main/java/org/genesys2/server/service/worker/SGSVUpdate.java

@@ -54,11 +54,12 @@ import au.com.bytecode.opencsv.CSVReader;
 public class SGSVUpdate {
 	private static final String SGSV_DOWNLOAD_URL = "http://www.nordgen.org/sgsv/download.php?file=/scope/sgsv/files/sgsv_templates.tab";
 
-	private static final String[] SGSV_HEADERS = { "sgsv_id", "institute_code", "deposit_box_number", "collection_name", "accession_number",
-			"full_scientific_name", "country_of_collection_or_source", "number_of_seeds", "regeneration_month_and_year", "other_accession_designations",
-			"provider_institute_code", "accession_url", "country_code", "country_name", "continent_name", "seeds", "genus", "species_epithet", "species",
-			"taxon_name", "date_of_deposit", "date_of_dataset", "sgsv_template_id", "box_id", "sgsv_taxon_id", "taxon_authority", "infraspesific_epithet",
-			"vernacular_name", "itis_tsn", "sgsv_genus_id", "accession_name" };
+	static final String[] SGSV_HEADERS = { "sgsv_id", "institute_code", "deposit_box_number", "collection_name", "accession_number", "full_scientific_name",
+			"country_of_collection_or_source", "number_of_seeds", "regeneration_month_and_year", "other_accession_designations", "provider_institute_code",
+			"accession_url", "country_code", "country_name", "continent_name", "seeds", "genus", "species_epithet", "species", "taxon_name", "date_of_deposit",
+			"date_of_dataset", "sgsv_template_id", "box_id", "sgsv_taxon_id", "taxon_authority", "infraspesific_epithet", "vernacular_name", "itis_tsn",
+			"sgsv_genus_id", "accession_name" };
+
 	public static final Log LOG = LogFactory.getLog(SGSVUpdate.class);
 	private static final int BATCH_SIZE = 50;
 
@@ -111,7 +112,36 @@ public class SGSVUpdate {
 		}
 	}
 
-	private void importSGSVStream(final InputStream str, final String source) throws IOException {
+	InputStream getDownloadStream() throws IllegalStateException, IOException {
+		LOG.warn("Downloading SGSV data from " + SGSV_DOWNLOAD_URL);
+
+		final HttpClient httpclient = new DefaultHttpClient();
+		final HttpGet httpget = new HttpGet(SGSV_DOWNLOAD_URL);
+
+		HttpResponse response = null;
+		try {
+			response = httpclient.execute(httpget);
+		} catch (final ClientProtocolException e) {
+			LOG.error(e.getMessage(), e);
+		} catch (final IOException e) {
+			LOG.error(e.getMessage(), e);
+		}
+
+		LOG.debug(response.getStatusLine());
+
+		// Get hold of the response entity
+		final HttpEntity entity = response.getEntity();
+
+		for (final Header header : response.getAllHeaders()) {
+			LOG.debug(header);
+		}
+
+		LOG.debug(entity.getContentType() + " " + entity.getContentLength());
+
+		return entity.getContent();
+	}
+
+	void importSGSVStream(final InputStream str, final String source) throws IOException {
 
 		int counter = 0;
 
@@ -120,18 +150,9 @@ public class SGSVUpdate {
 			reader = new CSVReader(new BufferedReader(new InputStreamReader(str)), '\t', '"', false);
 
 			final String[] headers = reader.readNext();
-			LOG.debug("Headers: " + headers.length);
-			if (headers.length != SGSV_HEADERS.length) {
-				LOG.warn("Expected 30 headers, got " + headers.length);
+			if (!validHeaders(headers)) {
 				return;
 			}
-			for (int i = 0; i < SGSV_HEADERS.length; i++) {
-				if (!StringUtils.equals(headers[i], SGSV_HEADERS[i])) {
-					LOG.warn("SGSV template header mismatch pos=" + i + " expected=" + SGSV_HEADERS[i] + " found=" + headers[i]);
-					return;
-				}
-
-			}
 
 			final List<String[]> bulk = new ArrayList<String[]>(BATCH_SIZE);
 
@@ -172,24 +193,33 @@ public class SGSVUpdate {
 		LOG.info("Done importing SGSV data. Imported: " + counter);
 	}
 
-	private void workIt(final List<String[]> bulk) {
-		// Need copy!
-		final ArrayList<String[]> bulkCopy = new ArrayList<String[]>(bulk);
-		bulk.clear();
+	boolean validHeaders(String[] headers) {
+		if (headers == null) {
+			LOG.warn("null headers received");
+			return false;
+		}
+		LOG.debug("Headers: " + headers.length);
+		if (headers.length != SGSV_HEADERS.length) {
+			LOG.warn("Expected 30 headers, got " + headers.length);
+			return false;
+		}
+		for (int i = 0; i < SGSV_HEADERS.length; i++) {
+			if (!StringUtils.equals(headers[i], SGSV_HEADERS[i])) {
+				LOG.warn("SGSV template header mismatch pos=" + i + " expected=" + SGSV_HEADERS[i] + " found=" + headers[i]);
+				return false;
+			}
+		}
+		return true;
+	}
 
+	void workIt(final List<String[]> bulk) {
 		taskExecutor.execute(new Runnable() {
+			// Need copy!
+			final ArrayList<String[]> bulkCopy = new ArrayList<String[]>(bulk);
+
 			@Override
 			public void run() {
-				List<SGSVEntry> accns = new ArrayList<SGSVEntry>(bulkCopy.size());
-				// Extract INSTCODE and ACCENUMB
-				for (String[] entry : bulkCopy) {
-					try {
-						accns.add(new SGSVEntry(entry));
-					} catch (ArrayIndexOutOfBoundsException | NumberFormatException e) {
-						LOG.warn("Invalid entry: " + ArrayUtils.toString(entry, "NULL"));
-					}
-				}
-
+				List<SGSVEntry> accns = bulkToList(bulkCopy);
 				LOG.trace("Got " + accns.size() + " entries");
 
 				for (int retry = 5; retry > 0; retry--) {
@@ -198,87 +228,7 @@ public class SGSVUpdate {
 					}
 
 					try {
-						// Find accessions based on holdingInstitute and
-						// accession
-						// name
-						List<Accession> accessions = genesysService.listAccessions(accns);
-
-						// Must be same order
-						if (accessions.size() != accns.size()) {
-							LOG.error("Accession lists don't match");
-							return;
-						}
-
-						// Check if they match (Genus, OrigCty) for now
-						List<Accession> matching = new ArrayList<Accession>(accessions.size());
-						List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
-
-						for (int i = accns.size() - 1; i >= 0; i--) {
-							boolean updateAccession = false;
-							Accession accn = accessions.get(i);
-							SGSVEntry entry = accns.get(i);
-
-							if (accn == null || entry == null)
-								continue;
-
-							if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
-
-								if (!Boolean.TRUE.equals(accn.getInSvalbard())) {
-									accn.setInSvalbard(true);
-									updateAccession = true;
-								}
-
-								// // Doesn't have taxonomy due to no species
-								// if (accn.getTaxonomy() == null &&
-								// StringUtils.isNotBlank(entry.species)) {
-								// accn.setTaxonomy(taxonomyService.ensureTaxonomy(entry.genus,
-								// entry.species));
-								// updateAccession = true;
-								// }
-								//
-								// if (entry.origCty != null &&
-								// StringUtils.isBlank(accn.getOrigin()) &&
-								// !StringUtils.equals(accn.getOrigin(),
-								// entry.origCty)) {
-								// accn.setOrigin(entry.origCty);
-								// accn.setCountryOfOrigin(geoService.getCountry(entry.origCty));
-								// updateAccession = true;
-								// }
-
-								SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
-								if (svalbardData == null) {
-									svalbardData = new SvalbardData();
-								}
-								svalbardData.setId(accn.getId());
-								svalbardData.setBoxNumber(entry.boxNo);
-								svalbardData.setDepositDate(entry.depositDate);
-								svalbardData.setQuantity(entry.quantity);
-								svalbardData.setTaxonomy(entry.fullTaxa);
-
-								svalbards.add(svalbardData);
-								if (updateAccession)
-									matching.add(accn);
-							} else {
-								LOG.warn("This should not be");
-							}
-
-						}
-
-						// Save data
-						if (matching.size() > 0) {
-							LOG.info("Setting inSGSV=true for accessions size=" + matching.size());
-							genesysService.setInSvalbard(matching);
-						}
-						if (svalbards.size() > 0) {
-							try {
-								genesysService.saveSvalbards(svalbards);
-							} catch (ConstraintViolationException e) {
-								LOG.warn(e.getMessage());
-								for (Accession a : matching) {
-									LOG.warn("\t" + a);
-								}
-							}
-						}
+						updateSvalbards(accns);
 
 						// EXIT
 						return;
@@ -292,5 +242,107 @@ public class SGSVUpdate {
 				LOG.warn("Persistence retries failed!");
 			}
 		});
+
+		bulk.clear();
+	}
+
+	List<SGSVEntry> bulkToList(ArrayList<String[]> bulkCopy) {
+		List<SGSVEntry> entries = new ArrayList<SGSVEntry>(bulkCopy.size());
+		// Extract INSTCODE and ACCENUMB
+		for (String[] entry : bulkCopy) {
+			try {
+				entries.add(new SGSVEntry(entry));
+			} catch (ArrayIndexOutOfBoundsException | NumberFormatException e) {
+				LOG.warn("Invalid entry: " + ArrayUtils.toString(entry, "NULL"));
+			}
+		}
+		return entries;
+	}
+
+	void updateSvalbards(List<SGSVEntry> accns) {
+		// Find accessions based on holdingInstitute and
+		// accession
+		// name
+		List<Accession> accessions = genesysService.listAccessions(accns);
+
+		// Must be same order
+		if (accessions.size() != accns.size()) {
+			LOG.error("Accession lists don't match");
+			return;
+		}
+
+		// Check if they match (Genus, OrigCty) for now
+		List<Accession> matching = new ArrayList<Accession>(accessions.size());
+		List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
+
+		for (int i = accns.size() - 1; i >= 0; i--) {
+			boolean updateAccession = false;
+			Accession accn = accessions.get(i);
+			SGSVEntry entry = accns.get(i);
+
+			if (accn == null || entry == null) {
+				if (LOG.isDebugEnabled()) 
+					LOG.debug("No such accession in the database. Skipping");
+				continue;
+			}
+
+			if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
+
+				if (!Boolean.TRUE.equals(accn.getInSvalbard())) {
+					accn.setInSvalbard(true);
+					updateAccession = true;
+				}
+
+				// // Doesn't have taxonomy due to no species
+				// if (accn.getTaxonomy() == null &&
+				// StringUtils.isNotBlank(entry.species)) {
+				// accn.setTaxonomy(taxonomyService.ensureTaxonomy(entry.genus,
+				// entry.species));
+				// updateAccession = true;
+				// }
+				//
+				// if (entry.origCty != null &&
+				// StringUtils.isBlank(accn.getOrigin()) &&
+				// !StringUtils.equals(accn.getOrigin(),
+				// entry.origCty)) {
+				// accn.setOrigin(entry.origCty);
+				// accn.setCountryOfOrigin(geoService.getCountry(entry.origCty));
+				// updateAccession = true;
+				// }
+
+				SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
+				if (svalbardData == null) {
+					svalbardData = new SvalbardData();
+				}
+				svalbardData.setId(accn.getId());
+				svalbardData.setBoxNumber(entry.boxNo);
+				svalbardData.setDepositDate(entry.depositDate);
+				svalbardData.setQuantity(entry.quantity);
+				svalbardData.setTaxonomy(entry.fullTaxa);
+
+				svalbards.add(svalbardData);
+				if (updateAccession)
+					matching.add(accn);
+			} else {
+				LOG.warn("This should not be");
+			}
+
+		}
+
+		// Save data
+		if (matching.size() > 0) {
+			LOG.info("Setting inSGSV=true for accessions size=" + matching.size());
+			genesysService.setInSvalbard(matching);
+		}
+		if (svalbards.size() > 0) {
+			try {
+				genesysService.saveSvalbards(svalbards);
+			} catch (ConstraintViolationException e) {
+				LOG.warn(e.getMessage());
+				for (Accession a : matching) {
+					LOG.warn("\t" + a);
+				}
+			}
+		}
 	}
 }

src/test/java/org/genesys2/server/service/worker/SGSVUpdateTest.java

+package org.genesys2.server.service.worker;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.exception.VelocityException;
+import org.genesys2.server.aspect.AsAdminAspect;
+import org.genesys2.server.model.genesys.Accession;
+import org.genesys2.server.model.impl.FaoInstitute;
+import org.genesys2.server.persistence.domain.SvalbardRepository;
+import org.genesys2.server.service.AclService;
+import org.genesys2.server.service.BatchRESTService;
+import org.genesys2.server.service.ContentService;
+import org.genesys2.server.service.GenesysService;
+import org.genesys2.server.service.GeoService;
+import org.genesys2.server.service.HtmlSanitizer;
+import org.genesys2.server.service.InstituteService;
+import org.genesys2.server.service.OrganizationService;
+import org.genesys2.server.service.TaxonomyService;
+import org.genesys2.server.service.UserService;
+import org.genesys2.server.service.impl.AclServiceImpl;
+import org.genesys2.server.service.impl.BatchRESTServiceImpl;
+import org.genesys2.server.service.impl.ContentServiceImpl;
+import org.genesys2.server.service.impl.GenesysServiceImpl;
+import org.genesys2.server.service.impl.GeoServiceImpl;
+import org.genesys2.server.service.impl.InstituteServiceImpl;
+import org.genesys2.server.service.impl.OWASPSanitizer;
+import org.genesys2.server.service.impl.OrganizationServiceImpl;
+import org.genesys2.server.service.impl.TaxonomyServiceImpl;
+import org.genesys2.server.service.impl.UserServiceImpl;
+import org.genesys2.server.test.JpaDataConfig;
+import org.genesys2.server.test.PropertyPlacholderInitializer;
+import org.hsqldb.lib.StringInputStream;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.support.NoOpCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.task.TaskExecutor;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.ui.velocity.VelocityEngineFactoryBean;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = SGSVUpdateTest.Config.class, initializers = PropertyPlacholderInitializer.class)
+public class SGSVUpdateTest {
+
+	private static final Log LOG = LogFactory.getLog(SGSVUpdateTest.class);
+
+	@Import(JpaDataConfig.class)
+	@ComponentScan(basePackages = { "org.genesys2.server.persistence.domain" })
+	public static class Config {
+		@Bean
+		public AsAdminAspect asAdminAspect() {
+			return new AsAdminAspect();
+		}
+
+		@Bean
+		public UserService userService() {
+			return new UserServiceImpl();
+		}
+
+		@Bean
+		public AclService aclService() {
+			return new AclServiceImpl();
+		}
+
+		@Bean
+		public TaxonomyService taxonomyService() {
+			return new TaxonomyServiceImpl();
+		}
+
+		@Bean
+		public BatchRESTService batchRESTService() {
+			return new BatchRESTServiceImpl();
+		}
+
+		@Bean
+		public GenesysService genesysService() {
+			return new GenesysServiceImpl();
+		}
+
+		@Bean
+		public CacheManager cacheManager() {
+			return new NoOpCacheManager();
+		}
+
+		@Bean
+		public HtmlSanitizer htmlSanitizer() {
+			return new OWASPSanitizer();
+		}
+
+		@Bean
+		public GeoService geoService() {
+			return new GeoServiceImpl();
+		}
+
+		@Bean
+		public ContentService contentService() {
+			return new ContentServiceImpl();
+		}
+
+		@Bean
+		public VelocityEngine velocityEngine() throws VelocityException, IOException {
+			VelocityEngineFactoryBean vf = new VelocityEngineFactoryBean();
+			return vf.createVelocityEngine();
+		}
+
+		@Bean
+		public OrganizationService organizationService() {
+			return new OrganizationServiceImpl();
+		}
+
+		@Bean
+		public InstituteService instituteService() {
+			return new InstituteServiceImpl();
+		}
+
+		@Bean
+		public SGSVUpdate sgsvUpdate() {
+			return new SGSVUpdate();
+		}
+
+		@Bean
+		public TaskExecutor taskExecutor() {
+			return new ThreadPoolTaskExecutor();
+		}
+
+	}
+
+	@Autowired
+	private SGSVUpdate sgsvUpdate;
+
+	@Autowired
+	private InstituteService instituteService;
+
+	@Autowired
+	private GenesysService genesysService;
+
+	@Autowired
+	private SvalbardRepository svalbardRepository;
+
+	@Autowired
+	private TaxonomyService taxonomyService;
+
+	@Autowired
+	private TaskExecutor taskExecutor;
+
+	@Before
+	public void setup() {
+		LOG.info("Setting up");
+		Collection<FaoInstitute> institutes = new ArrayList<FaoInstitute>();
+		for (String instCode : new String[] { "INS001", "INS002" }) {
+			FaoInstitute institute = new FaoInstitute();
+			institute.setFullName(instCode + " institute");
+			institute.setCode(instCode);
+			institute.setUniqueAcceNumbs(true);
+			institutes.add(institute);
+		}
+		instituteService.update(institutes);
+	}
+
+	@After
+	public void afterEachTest() {
+		LOG.info("Deleting institutes");
+		instituteService.delete("INS001");
+		instituteService.delete("INS002");
+		assertTrue(instituteService.list(new PageRequest(0, 1)).getNumber()==0);
+	}
+
+	private void makeAccession(String instCode, String acceNumb, String genus, String species) {
+		FaoInstitute institute = instituteService.getInstitute(instCode);
+		List<Accession> accessions = new ArrayList<Accession>();
+
+		Accession a = new Accession();
+		a.setInstitute(institute);