Commit b2e4975f authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch 'tests-to-migrate-to-head'

* tests-to-migrate-to-head:
  Removed old runners
  SGSV data import
  Added SvalbardData entity
  Multiple "crops" possible
  Update In_Svalbard status from SGSV data

Conflicts:
	src/main/java/org/genesys2/rest/common/persistence/domain/SvalbardRepository.java
	src/main/java/org/genesys2/server/service/GenesysService.java
	src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
	src/main/java/org/genesys2/server/service/impl/TaxonomyServiceImpl.java
	src/main/java/org/genesys2/server/servlet/controller/AdminController.java
	src/main/webapp/WEB-INF/jsp/admin/index.jsp
	src/test/java/org/crophub/rest/test/RestTest.java
	src/test/java/org/genesys2/server/test/RestTest.java
	src/test/java/org/genesys2/test/RestTest.java
parents 0ba0402c b93886bc
......@@ -6,3 +6,4 @@ target
logs
/bin
/lucene
/data
......@@ -20,6 +20,7 @@ package org.genesys2.server.model.genesys;
import java.text.MessageFormat;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
......@@ -60,11 +61,13 @@ public class Accession implements java.io.Serializable {
private Boolean availability;
private Boolean mlsStatus;
private String genus;
private String dataSource;
private AllAccnames accessionNames;
private AllAcqBreeding accessionBreeding;
private AllAcqCollect accessionCollection;
private AllAcqExchange accessionExchange;
private AllEnvironment accessionEnvironment;
private SvalbardData svalbardData;
public Accession() {
}
......@@ -278,6 +281,25 @@ public class Accession implements java.io.Serializable {
this.accessionEnvironment = accessionEnvironment;
}
@OneToOne(orphanRemoval = true, cascade = CascadeType.REMOVE, optional = true)
@JoinColumn(name = "id", referencedColumnName = "id")
public SvalbardData getSvalbardData() {
return svalbardData;
}
public void setSvalbardData(SvalbardData svalbardData) {
this.svalbardData = svalbardData;
}
@Column(name = "dataSource", length = 45)
public String getDataSource() {
return dataSource;
}
public void setDataSource(String dataSource) {
this.dataSource = dataSource;
}
@Override
public String toString() {
return MessageFormat.format("Accession id={0,number,#} A={3} inst={1} genus={2}", id, institute.getCode(), genus, accNumbHi);
......
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.model.genesys;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Version;
@Entity
@Table(name = "svalbard")
public class SvalbardData implements java.io.Serializable {
private static final long serialVersionUID = -5015395025122055617L;
private Long id;
private Float quantity;
private String depositDate;
private String boxNo;
private String taxonomy;
private int version;
@Id
@Column(name = "id")
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public Float getQuantity() {
return quantity;
}
public void setQuantity(Float quantity) {
this.quantity = quantity;
}
@Column(length = 20)
public String getDepositDate() {
return depositDate;
}
public void setDepositDate(String depositDate) {
this.depositDate = depositDate;
}
@Column(length = 20)
public String getBoxNumber() {
return boxNo;
}
public void setBoxNumber(String boxNo) {
this.boxNo = boxNo;
}
@Column(length = 200)
public String getTaxonomy() {
return taxonomy;
}
public void setTaxonomy(String taxonomy) {
this.taxonomy = taxonomy;
}
@Version
public int getVersion() {
return version;
}
public void setVersion(int version) {
this.version = version;
}
}
package org.genesys2.server.model.impl;
public interface AccessionIdentifier3 {
String getHoldingInstitute();
String getAccessionName();
String getGenus();
}
......@@ -84,4 +84,8 @@ public interface AccessionRepository extends JpaRepository<Accession, Long> {
@Query("select a from Accession a where a.taxonomy in ( ?1 )")
Page<Accession> findByTaxonomy(Collection<Taxonomy> taxonomies, Pageable pageable);
// Accession findByInstituteCodeAndAccessionName(String holdingInstitute, String accessionName);
Accession findByInstituteCodeAndAccessionNameAndGenus(String holdingInstitute, String accessionName, String genus);
}
......@@ -42,8 +42,8 @@ public interface CropTaxonomyRepository extends JpaRepository<CropTaxonomy, Long
@Query("select distinct t from CropTaxonomy ct inner join ct.taxonomy t where ct.crop = ?1")
List<Taxonomy> findTaxonomiesByCrop(Crop crop);
@Query("select ct.crop from CropTaxonomy ct where ct.taxonomy= ?1 and ct.crop.language = ?2")
Crop findByTaxonomyAndLanguage(Taxonomy taxonomy, String language);
@Query("select distinct ct.crop from CropTaxonomy ct where ct.taxonomy= ?1 and ct.crop.language = ?2")
List<Crop> findByTaxonomyAndLanguage(Taxonomy taxonomy, String language);
@Modifying
@Query("delete from CropTaxonomy ct where ct.crop = ?1")
......
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.persistence.domain;
import org.genesys2.server.model.genesys.SvalbardData;
import org.springframework.data.jpa.repository.JpaRepository;
public interface SvalbardRepository extends JpaRepository<SvalbardData, Long> {
}
......@@ -25,4 +25,6 @@ public interface TaxonomyRepository extends JpaRepository<Taxonomy, Long> {
Taxonomy getByGenusAndSpecies(String genus, String species);
List<Taxonomy> findByGenus(String genus);
Taxonomy getByTaxonName(String fullTaxa);
}
......@@ -32,7 +32,7 @@ public interface CropService {
List<Crop> list(Locale locale);
Crop getCrop(Locale locale, Taxonomy taxonomy);
List<Crop> getCrop(Locale locale, Taxonomy taxonomy);
void rebuildTaxonomies();
......
......@@ -29,7 +29,9 @@ import org.genesys2.server.model.genesys.AllEnvironment;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.SvalbardData;
import org.genesys2.server.model.genesys.Taxonomy;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
......@@ -82,6 +84,8 @@ public interface GenesysService {
Page<Accession> listAccessions(Collection<Long> accessionIds, Pageable pageable);
List<Accession> listAccessions(List<? extends AccessionIdentifier3> accns);
Page<Object[]> statisticsGenusByInstitute(FaoInstitute faoInstitute, Pageable pageable);
Page<Object[]> statisticsTaxonomyByInstitute(FaoInstitute faoInstitute, Pageable pageable);
......@@ -100,4 +104,8 @@ public interface GenesysService {
void updateAccessionInstitueRefs();
void saveAccessions(List<Accession> matching);
void saveSvalbards(List<SvalbardData> svalbards);
}
......@@ -23,4 +23,6 @@ public interface TaxonomyService {
Taxonomy get(String genus, String species);
Taxonomy ensureTaxonomy(String genus, String species, String fullTaxa);
}
......@@ -70,7 +70,7 @@ public class CropServiceImpl implements CropService {
}
@Override
public Crop getCrop(Locale locale, Taxonomy taxonomy) {
public List<Crop> getCrop(Locale locale, Taxonomy taxonomy) {
return cropTaxonomyRepository.findByTaxonomyAndLanguage(taxonomy, locale.getLanguage());
}
......
......@@ -16,6 +16,7 @@
package org.genesys2.server.service.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
......@@ -32,7 +33,9 @@ import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.genesys.Parameter;
import org.genesys2.server.model.genesys.SvalbardData;
import org.genesys2.server.model.genesys.Taxonomy;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
......@@ -45,12 +48,14 @@ import org.genesys2.server.persistence.domain.AccessionNameRepository;
import org.genesys2.server.persistence.domain.AccessionRepository;
import org.genesys2.server.persistence.domain.AccessionTraitRepository;
import org.genesys2.server.persistence.domain.CropTaxonomyRepository;
import org.genesys2.server.persistence.domain.FaoInstituteRepository;
import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import org.genesys2.server.persistence.domain.MetadataMethodRepository;
import org.genesys2.server.persistence.domain.MetadataRepository;
import org.genesys2.server.persistence.domain.MethodRepository;
import org.genesys2.server.persistence.domain.OrganizationRepository;
import org.genesys2.server.persistence.domain.ParameterRepository;
import org.genesys2.server.persistence.domain.SvalbardRepository;
import org.genesys2.server.persistence.domain.TraitValueRepository;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.TraitService;
......@@ -101,9 +106,15 @@ public class GenesysServiceImpl implements GenesysService, TraitService {
@Autowired
private OrganizationRepository organizationRepository;
@Autowired
private FaoInstituteRepository instituteRepository;
@Autowired
private GenesysLowlevelRepository genesysLowlevelRepository;
@Autowired
private SvalbardRepository svalbardRepository;
@Override
public long countByInstitute(FaoInstitute institute) {
return accessionRepository.countByInstitute(institute);
......@@ -137,6 +148,24 @@ public class GenesysServiceImpl implements GenesysService, TraitService {
return accessionRepository.findById(accessionIds, pageable);
}
@Override
public List<Accession> listAccessions(List<? extends AccessionIdentifier3> accns) {
List<Accession> result = new ArrayList<Accession>(accns.size());
for (AccessionIdentifier3 aid3 : accns) {
Accession accn = accessionRepository.findByInstituteCodeAndAccessionNameAndGenus(aid3.getHoldingInstitute(), aid3.getAccessionName(),
aid3.getGenus());
// Including null's
result.add(accn);
if (accn == null) {
FaoInstitute inst = instituteRepository.findByCode(aid3.getHoldingInstitute());
if (inst == null)
// Only log full miss
LOG.debug("No accession " + aid3 + " in " + inst);
}
}
return result;
}
@Override
public Accession getAccession(long accessionId) {
return accessionRepository.findOne(accessionId);
......@@ -291,4 +320,20 @@ public class GenesysServiceImpl implements GenesysService, TraitService {
public void updateAccessionInstitueRefs() {
genesysLowlevelRepository.updateFaoInstituteRefs();
}
@Override
@Transactional(readOnly = false)
public void saveAccessions(List<Accession> accessions) {
if (LOG.isDebugEnabled()) {
LOG.debug("Saving " + accessions.size() + " accessions");
}
accessionRepository.save(accessions);
}
@Override
@Transactional(readOnly = false)
public void saveSvalbards(List<SvalbardData> svalbards) {
svalbardRepository.save(svalbards);
}
}
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.impl;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.annotation.PreDestroy;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
import au.com.bytecode.opencsv.CSVReader;
@Component
public class SGSVInsertMissing {
private static final String SGSV_DOWNLOAD_URL = "http://www.nordgen.org/sgsv/download.php?file=/scope/sgsv/files/sgsv_templates.tab";
public static final Log LOG = LogFactory.getLog(SGSVInsertMissing.class);
private static final int BATCH_SIZE = 50;
private static final int nThreads = Runtime.getRuntime().availableProcessors();
private final ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(nThreads);
private static final String[] institutes = { "AUS039", "BDI003", "BDI004", "BDI005", "BRA001", "BRA008", "CAN004", "CHL002", "CIV039", "CRI001", "ECU076",
"GBR072", "GEO028", "GRC035", "IDN179", "ISR003", "ITA411", "KEN015", "KEN023", "KEN045", "KEN053", "KEN055", "KOR043", "MLI002", "MLI003",
"MLI219", "MMR003", "MNG030", "NGA010", "PAK001", "PER002", "PHL129", "PRK013", "SDN034", "THA214", "TJK027", "TWN006", "UGA031", "UKR001",
"UZB006", "ZMB030" };
@Value("${download.files.dir}")
String filesPath;
@Autowired
private GenesysService genesysService;
@Autowired
private InstituteService instituteService;
@Autowired
private TaxonomyService taxonomyService;
private Map<String, FaoInstitute> faoInstituteCache = new HashMap<String, FaoInstitute>();
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void importMissingSGSV() {
LOG.warn("Importing SGSV data from " + SGSV_DOWNLOAD_URL);
final HttpClient httpclient = new DefaultHttpClient();
final HttpGet httpget = new HttpGet(SGSV_DOWNLOAD_URL);
HttpResponse response = null;
try {
response = httpclient.execute(httpget);
} catch (final ClientProtocolException e) {
LOG.error(e.getMessage(), e);
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
}
LOG.debug(response.getStatusLine());
// Get hold of the response entity
final HttpEntity entity = response.getEntity();
for (final Header header : response.getAllHeaders()) {
LOG.debug(header);
}
LOG.debug(entity.getContentType() + " " + entity.getContentLength());
try {
importSGSVStream(entity.getContent(), SGSV_DOWNLOAD_URL);
} catch (final Throwable e) {
httpget.abort();
}
}
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void importMissingSGSVFile() {
// sgsv_templates_20130610.tab
// sgsv_templates_20130502.tab
final File file = new File(filesPath, "sgsv_templates_20130610.tab");
FileInputStream fis = null;
try {
fis = new FileInputStream(file);
LOG.warn("Importing SGSV data from " + file.getAbsolutePath());
importSGSVStream(fis, file.getAbsolutePath());
} catch (final FileNotFoundException e) {
LOG.error(e.getMessage(), e);
} catch (final IOException e) {
LOG.error(e.getMessage(), e);
} finally {
IOUtils.closeQuietly(fis);
}
}
private void importSGSVStream(final InputStream str, final String source) throws IOException {
int counter = 0;
CSVReader reader = null;
try {
reader = new CSVReader(new BufferedReader(new InputStreamReader(str)), '\t', '"', false);
final String[] headers = reader.readNext();
LOG.debug("Headers: " + headers.length);
if (headers.length != 30) {
LOG.warn("Expected 30 headers, got " + headers.length);
return;
}
// descriptors[0] = descriptorRepository.findByCode("SGSV_ID");
// descriptors[1] = descriptorRepository.findByCode("INSTCODE");
// descriptors[2] = descriptorRepository.findByCode("SGSV_BOXNO");
// descriptors[3] = descriptorRepository.findByCode("COLLNAME");
// descriptors[4] = descriptorRepository.findByCode("ACCENUMB");
// descriptors[5] = descriptorRepository.findByCode("FULL_SCINAME");
// descriptors[7] = descriptorRepository.findByCode("QTY");
// descriptors[8] =
// descriptorRepository.findByCode("regeneration_month_and_year");
// descriptors[9] = descriptorRepository.findByCode("OTHERNUMB");
// descriptors[10] =
// descriptorRepository.findByCode("provider_institute_code");
// descriptors[12] = descriptorRepository.findByCode("ORIGCTY");
// descriptors[16] = descriptorRepository.findByCode("GENUS");
// descriptors[17] =
// descriptorRepository.findByCode("species_epithet");
// descriptors[17] = descriptorRepository.findByCode("SPECIES");
// descriptors[19] = descriptorRepository.findByCode("TAXON_NAME");
// descriptors[20] =
// descriptorRepository.findByCode("SGSV_DEPOSIT_DATE");
// descriptors[23] = descriptorRepository.findByCode("SGSV_BOXID");
// descriptors[24] =
// descriptorRepository.findByCode("SGSV_TAXONID");
// descriptors[25] =
// descriptorRepository.findByCode("taxon_authority");
// descriptors[26] =
// descriptorRepository.findByCode("infraspesific_epithet");
// descriptors[27] =
// descriptorRepository.findByCode("VERNACULAR_NAME");
// descriptors[29] =
// descriptorRepository.findByCode("SGSV_GENUSID");
final List<String[]> bulk = new ArrayList<String[]>(BATCH_SIZE);
String[] line = null;
while ((line = reader.readNext()) != null) {
if (counter % 1000 == 0) {
LOG.info(counter + ": " + ArrayUtils.toString(line));
}
// Clean up
for (int i = 0; i < line.length; i++) {
line[i] = line[i].trim();
if (line[i].equals("null") || line[i].equals("<null>") || StringUtils.isBlank(line[i])) {
line[i] = null;
}
}
bulk.add(line);
counter++;
if (counter % BATCH_SIZE == 0) {
workIt(bulk);
bulk.clear();
}
}
workIt(bulk);
bulk.clear();
} catch (final Throwable e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
} finally {