Commit 54cd8a2f authored by Matija Obreza's avatar Matija Obreza
Browse files

SGSVUpdate tests

parent 1362dae5
......@@ -18,8 +18,14 @@ package org.genesys2.server.persistence.domain;
import org.genesys2.server.model.impl.FaoInstituteSetting;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface FaoInstituteSettingRepository extends JpaRepository<FaoInstituteSetting, Long> {
FaoInstituteSetting findByInstCodeAndSetting(String instCode, String setting);
@Modifying
@Query("DELETE FROM FaoInstituteSetting fis WHERE fis.instCode = ?1")
void deleteFor(String instCode);
}
......@@ -59,4 +59,6 @@ public interface InstituteService {
void setUniqueAcceNumbs(FaoInstitute faoInstitute, boolean uniqueAcceNumbs);
void delete(String instCode);
}
......@@ -213,10 +213,10 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
// Including null's
result.add(accn);
if (accn == null) {
FaoInstitute inst = instituteRepository.findByCode(aid3.getHoldingInstitute());
if (inst == null)
// FaoInstitute inst = instituteRepository.findByCode(aid3.getHoldingInstitute());
if (LOG.isDebugEnabled())
// Only log full miss
LOG.debug("No accession " + aid3 + " in " + inst);
LOG.debug("No accession " + aid3);
}
}
return result;
......
......@@ -179,4 +179,14 @@ public class InstituteServiceImpl implements InstituteService {
return organizationRepository.getOrganizations(faoInstitute);
}
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void delete(String instCode) {
FaoInstitute institute = getInstitute(instCode);
if (institute!=null) {
instituteSettingRepository.deleteFor(institute.getCode());
instituteRepository.delete(institute);
}
}
}
......@@ -15,7 +15,10 @@ class SGSVEntry implements AccessionIdentifier3 {
String boxNo;
String depositDate;
Float quantity;
public SGSVEntry() {
}
public SGSVEntry(String[] entry) {
instCode = entry[1];
acceNumb = entry[4];
......
......@@ -54,11 +54,12 @@ import au.com.bytecode.opencsv.CSVReader;
public class SGSVUpdate {
private static final String SGSV_DOWNLOAD_URL = "http://www.nordgen.org/sgsv/download.php?file=/scope/sgsv/files/sgsv_templates.tab";
private static final String[] SGSV_HEADERS = { "sgsv_id", "institute_code", "deposit_box_number", "collection_name", "accession_number",
"full_scientific_name", "country_of_collection_or_source", "number_of_seeds", "regeneration_month_and_year", "other_accession_designations",
"provider_institute_code", "accession_url", "country_code", "country_name", "continent_name", "seeds", "genus", "species_epithet", "species",
"taxon_name", "date_of_deposit", "date_of_dataset", "sgsv_template_id", "box_id", "sgsv_taxon_id", "taxon_authority", "infraspesific_epithet",
"vernacular_name", "itis_tsn", "sgsv_genus_id", "accession_name" };
static final String[] SGSV_HEADERS = { "sgsv_id", "institute_code", "deposit_box_number", "collection_name", "accession_number", "full_scientific_name",
"country_of_collection_or_source", "number_of_seeds", "regeneration_month_and_year", "other_accession_designations", "provider_institute_code",
"accession_url", "country_code", "country_name", "continent_name", "seeds", "genus", "species_epithet", "species", "taxon_name", "date_of_deposit",
"date_of_dataset", "sgsv_template_id", "box_id", "sgsv_taxon_id", "taxon_authority", "infraspesific_epithet", "vernacular_name", "itis_tsn",
"sgsv_genus_id", "accession_name" };
public static final Log LOG = LogFactory.getLog(SGSVUpdate.class);
private static final int BATCH_SIZE = 50;
......@@ -111,7 +112,36 @@ public class SGSVUpdate {
}
}
private void importSGSVStream(final InputStream str, final String source) throws IOException {
InputStream getDownloadStream() throws IllegalStateException, IOException {
LOG.warn("Downloading SGSV data from " + SGSV_DOWNLOAD_URL);
final HttpClient httpclient = new DefaultHttpClient();
final HttpGet httpget = new HttpGet(SGSV_DOWNLOAD_URL);
HttpResponse response = null;
try {
response = httpclient.execute(httpget);
} catch (final ClientProtocolException e) {
LOG.error(e.getMessage(), e);
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
LOG.debug(response.getStatusLine());
// Get hold of the response entity
final HttpEntity entity = response.getEntity();
for (final Header header : response.getAllHeaders()) {
LOG.debug(header);
}
LOG.debug(entity.getContentType() + " " + entity.getContentLength());
return entity.getContent();
}
void importSGSVStream(final InputStream str, final String source) throws IOException {
int counter = 0;
......@@ -120,18 +150,9 @@ public class SGSVUpdate {
reader = new CSVReader(new BufferedReader(new InputStreamReader(str)), '\t', '"', false);
final String[] headers = reader.readNext();
LOG.debug("Headers: " + headers.length);
if (headers.length != SGSV_HEADERS.length) {
LOG.warn("Expected 30 headers, got " + headers.length);
if (!validHeaders(headers)) {
return;
}
for (int i = 0; i < SGSV_HEADERS.length; i++) {
if (!StringUtils.equals(headers[i], SGSV_HEADERS[i])) {
LOG.warn("SGSV template header mismatch pos=" + i + " expected=" + SGSV_HEADERS[i] + " found=" + headers[i]);
return;
}
}
final List<String[]> bulk = new ArrayList<String[]>(BATCH_SIZE);
......@@ -172,24 +193,33 @@ public class SGSVUpdate {
LOG.info("Done importing SGSV data. Imported: " + counter);
}
private void workIt(final List<String[]> bulk) {
// Need copy!
final ArrayList<String[]> bulkCopy = new ArrayList<String[]>(bulk);
bulk.clear();
boolean validHeaders(String[] headers) {
if (headers == null) {
LOG.warn("null headers received");
return false;
}
LOG.debug("Headers: " + headers.length);
if (headers.length != SGSV_HEADERS.length) {
LOG.warn("Expected 30 headers, got " + headers.length);
return false;
}
for (int i = 0; i < SGSV_HEADERS.length; i++) {
if (!StringUtils.equals(headers[i], SGSV_HEADERS[i])) {
LOG.warn("SGSV template header mismatch pos=" + i + " expected=" + SGSV_HEADERS[i] + " found=" + headers[i]);
return false;
}
}
return true;
}
void workIt(final List<String[]> bulk) {
taskExecutor.execute(new Runnable() {
// Need copy!
final ArrayList<String[]> bulkCopy = new ArrayList<String[]>(bulk);
@Override
public void run() {
List<SGSVEntry> accns = new ArrayList<SGSVEntry>(bulkCopy.size());
// Extract INSTCODE and ACCENUMB
for (String[] entry : bulkCopy) {
try {
accns.add(new SGSVEntry(entry));
} catch (ArrayIndexOutOfBoundsException | NumberFormatException e) {
LOG.warn("Invalid entry: " + ArrayUtils.toString(entry, "NULL"));
}
}
List<SGSVEntry> accns = bulkToList(bulkCopy);
LOG.trace("Got " + accns.size() + " entries");
for (int retry = 5; retry > 0; retry--) {
......@@ -198,87 +228,7 @@ public class SGSVUpdate {
}
try {
// Find accessions based on holdingInstitute and
// accession
// name
List<Accession> accessions = genesysService.listAccessions(accns);
// Must be same order
if (accessions.size() != accns.size()) {
LOG.error("Accession lists don't match");
return;
}
// Check if they match (Genus, OrigCty) for now
List<Accession> matching = new ArrayList<Accession>(accessions.size());
List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
for (int i = accns.size() - 1; i >= 0; i--) {
boolean updateAccession = false;
Accession accn = accessions.get(i);
SGSVEntry entry = accns.get(i);
if (accn == null || entry == null)
continue;
if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
if (!Boolean.TRUE.equals(accn.getInSvalbard())) {
accn.setInSvalbard(true);
updateAccession = true;
}
// // Doesn't have taxonomy due to no species
// if (accn.getTaxonomy() == null &&
// StringUtils.isNotBlank(entry.species)) {
// accn.setTaxonomy(taxonomyService.ensureTaxonomy(entry.genus,
// entry.species));
// updateAccession = true;
// }
//
// if (entry.origCty != null &&
// StringUtils.isBlank(accn.getOrigin()) &&
// !StringUtils.equals(accn.getOrigin(),
// entry.origCty)) {
// accn.setOrigin(entry.origCty);
// accn.setCountryOfOrigin(geoService.getCountry(entry.origCty));
// updateAccession = true;
// }
SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
if (svalbardData == null) {
svalbardData = new SvalbardData();
}
svalbardData.setId(accn.getId());
svalbardData.setBoxNumber(entry.boxNo);
svalbardData.setDepositDate(entry.depositDate);
svalbardData.setQuantity(entry.quantity);
svalbardData.setTaxonomy(entry.fullTaxa);
svalbards.add(svalbardData);
if (updateAccession)
matching.add(accn);
} else {
LOG.warn("This should not be");
}
}
// Save data
if (matching.size() > 0) {
LOG.info("Setting inSGSV=true for accessions size=" + matching.size());
genesysService.setInSvalbard(matching);
}
if (svalbards.size() > 0) {
try {
genesysService.saveSvalbards(svalbards);
} catch (ConstraintViolationException e) {
LOG.warn(e.getMessage());
for (Accession a : matching) {
LOG.warn("\t" + a);
}
}
}
updateSvalbards(accns);
// EXIT
return;
......@@ -292,5 +242,107 @@ public class SGSVUpdate {
LOG.warn("Persistence retries failed!");
}
});
bulk.clear();
}
List<SGSVEntry> bulkToList(ArrayList<String[]> bulkCopy) {
List<SGSVEntry> entries = new ArrayList<SGSVEntry>(bulkCopy.size());
// Extract INSTCODE and ACCENUMB
for (String[] entry : bulkCopy) {
try {
entries.add(new SGSVEntry(entry));
} catch (ArrayIndexOutOfBoundsException | NumberFormatException e) {
LOG.warn("Invalid entry: " + ArrayUtils.toString(entry, "NULL"));
}
}
return entries;
}
void updateSvalbards(List<SGSVEntry> accns) {
// Find accessions based on holdingInstitute and
// accession
// name
List<Accession> accessions = genesysService.listAccessions(accns);
// Must be same order
if (accessions.size() != accns.size()) {
LOG.error("Accession lists don't match");
return;
}
// Check if they match (Genus, OrigCty) for now
List<Accession> matching = new ArrayList<Accession>(accessions.size());
List<SvalbardData> svalbards = new ArrayList<SvalbardData>(accessions.size());
for (int i = accns.size() - 1; i >= 0; i--) {
boolean updateAccession = false;
Accession accn = accessions.get(i);
SGSVEntry entry = accns.get(i);
if (accn == null || entry == null) {
if (LOG.isDebugEnabled())
LOG.debug("No such accession in the database. Skipping");
continue;
}
if (accn.getAccessionName().equalsIgnoreCase(entry.acceNumb) && accn.getInstituteCode().equals(entry.instCode)) {
if (!Boolean.TRUE.equals(accn.getInSvalbard())) {
accn.setInSvalbard(true);
updateAccession = true;
}
// // Doesn't have taxonomy due to no species
// if (accn.getTaxonomy() == null &&
// StringUtils.isNotBlank(entry.species)) {
// accn.setTaxonomy(taxonomyService.ensureTaxonomy(entry.genus,
// entry.species));
// updateAccession = true;
// }
//
// if (entry.origCty != null &&
// StringUtils.isBlank(accn.getOrigin()) &&
// !StringUtils.equals(accn.getOrigin(),
// entry.origCty)) {
// accn.setOrigin(entry.origCty);
// accn.setCountryOfOrigin(geoService.getCountry(entry.origCty));
// updateAccession = true;
// }
SvalbardData svalbardData = svalbardRepository.findOne(accn.getId());
if (svalbardData == null) {
svalbardData = new SvalbardData();
}
svalbardData.setId(accn.getId());
svalbardData.setBoxNumber(entry.boxNo);
svalbardData.setDepositDate(entry.depositDate);
svalbardData.setQuantity(entry.quantity);
svalbardData.setTaxonomy(entry.fullTaxa);
svalbards.add(svalbardData);
if (updateAccession)
matching.add(accn);
} else {
LOG.warn("This should not be");
}
}
// Save data
if (matching.size() > 0) {
LOG.info("Setting inSGSV=true for accessions size=" + matching.size());
genesysService.setInSvalbard(matching);
}
if (svalbards.size() > 0) {
try {
genesysService.saveSvalbards(svalbards);
} catch (ConstraintViolationException e) {
LOG.warn(e.getMessage());
for (Accession a : matching) {
LOG.warn("\t" + a);
}
}
}
}
}
package org.genesys2.server.service.worker;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.exception.VelocityException;
import org.genesys2.server.aspect.AsAdminAspect;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.domain.SvalbardRepository;
import org.genesys2.server.service.AclService;
import org.genesys2.server.service.BatchRESTService;
import org.genesys2.server.service.ContentService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.UserService;
import org.genesys2.server.service.impl.AclServiceImpl;
import org.genesys2.server.service.impl.BatchRESTServiceImpl;
import org.genesys2.server.service.impl.ContentServiceImpl;
import org.genesys2.server.service.impl.GenesysServiceImpl;
import org.genesys2.server.service.impl.GeoServiceImpl;
import org.genesys2.server.service.impl.InstituteServiceImpl;
import org.genesys2.server.service.impl.OWASPSanitizer;
import org.genesys2.server.service.impl.OrganizationServiceImpl;
import org.genesys2.server.service.impl.TaxonomyServiceImpl;
import org.genesys2.server.service.impl.UserServiceImpl;
import org.genesys2.server.test.JpaDataConfig;
import org.genesys2.server.test.PropertyPlacholderInitializer;
import org.hsqldb.lib.StringInputStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.CacheManager;
import org.springframework.cache.support.NoOpCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Import;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.PageRequest;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.ui.velocity.VelocityEngineFactoryBean;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = SGSVUpdateTest.Config.class, initializers = PropertyPlacholderInitializer.class)
public class SGSVUpdateTest {
private static final Log LOG = LogFactory.getLog(SGSVUpdateTest.class);
@Import(JpaDataConfig.class)
@ComponentScan(basePackages = { "org.genesys2.server.persistence.domain" })
public static class Config {
@Bean
public AsAdminAspect asAdminAspect() {
return new AsAdminAspect();
}
@Bean
public UserService userService() {
return new UserServiceImpl();
}
@Bean
public AclService aclService() {
return new AclServiceImpl();
}
@Bean
public TaxonomyService taxonomyService() {
return new TaxonomyServiceImpl();
}
@Bean
public BatchRESTService batchRESTService() {
return new BatchRESTServiceImpl();
}
@Bean
public GenesysService genesysService() {
return new GenesysServiceImpl();
}
@Bean
public CacheManager cacheManager() {
return new NoOpCacheManager();
}
@Bean
public HtmlSanitizer htmlSanitizer() {
return new OWASPSanitizer();
}
@Bean
public GeoService geoService() {
return new GeoServiceImpl();
}
@Bean
public ContentService contentService() {
return new ContentServiceImpl();
}
@Bean
public VelocityEngine velocityEngine() throws VelocityException, IOException {
VelocityEngineFactoryBean vf = new VelocityEngineFactoryBean();
return vf.createVelocityEngine();
}
@Bean
public OrganizationService organizationService() {
return new OrganizationServiceImpl();
}
@Bean
public InstituteService instituteService() {
return new InstituteServiceImpl();
}
@Bean
public SGSVUpdate sgsvUpdate() {
return new SGSVUpdate();
}
@Bean
public TaskExecutor taskExecutor() {
return new ThreadPoolTaskExecutor();
}
}
@Autowired
private SGSVUpdate sgsvUpdate;
@Autowired
private InstituteService instituteService;
@Autowired
private GenesysService genesysService;
@Autowired
private SvalbardRepository svalbardRepository;
@Autowired
private TaxonomyService taxonomyService;
@Autowired
private TaskExecutor taskExecutor;
@Before
public void setup() {
LOG.info("Setting up");
Collection<FaoInstitute> institutes = new ArrayList<FaoInstitute>();
for (String instCode : new String[] { "INS001", "INS002" }) {
FaoInstitute institute = new FaoInstitute();
institute.setFullName(instCode + " institute");
institute.setCode(instCode);
institute.setUniqueAcceNumbs(true);
institutes.add(institute);
}
instituteService.update(institutes);
}
@After
public void afterEachTest() {
LOG.info("Deleting institutes");
instituteService.delete("INS001");
instituteService.delete("INS002");
assertTrue(instituteService.list(new PageRequest(0, 1)).getNumber()==0);
}
private void makeAccession(String instCode, String acceNumb, String genus, String species) {
FaoInstitute institute = instituteService.getInstitute(instCode);
List<Accession> accessions = new ArrayList<Accession>();
Accession a = new Accession();
a.setInstitute(institute);