Commit 82c8bf50 authored by Matija Obreza's avatar Matija Obreza

Genesys model update

- Flipped relationships to AccessionId
- Property names corrected
- Migrate database to updated model
parent b0e44778
...@@ -18,39 +18,34 @@ package org.genesys2.server.listener.sample; ...@@ -18,39 +18,34 @@ package org.genesys2.server.listener.sample;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.oauth.model.OAuthClient; import org.genesys.blocks.oauth.model.OAuthClient;
import org.genesys.blocks.oauth.model.OAuthRole; import org.genesys.blocks.oauth.model.OAuthRole;
import org.genesys.blocks.oauth.persistence.OAuthClientRepository; import org.genesys.blocks.oauth.persistence.OAuthClientRepository;
import org.genesys2.server.listener.RunAsAdminListener; import org.genesys2.server.listener.RunAsAdminListener;
import org.genesys2.server.model.impl.FaoInstitute; import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.domain.AccessionRepository; import org.genesys2.server.persistence.domain.AccessionRepository;
import org.genesys2.server.service.BatchRESTService;
import org.genesys2.server.service.CropService; import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GenesysService; import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService; import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService; import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService; import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.worker.AccessionUploader;
import org.genesys2.server.service.worker.InstituteUpdater; import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.servlet.controller.rest.model.AccessionHeaderJson;
import org.genesys2.util.InvalidDOIException;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
/** /**
* Import country data and WIEWS institutes if none exist at startup * Import country data and WIEWS institutes if none exist at startup
* *
...@@ -70,31 +65,34 @@ public class FirstRunListener extends RunAsAdminListener { ...@@ -70,31 +65,34 @@ public class FirstRunListener extends RunAsAdminListener {
private String defaultOAuthClientSecret; private String defaultOAuthClientSecret;
@Autowired @Autowired
GeoService geoService; private GeoService geoService;
@Autowired @Autowired
InstituteService instituteService; private InstituteService instituteService;
@Autowired @Autowired
InstituteUpdater instituteUpdater; private InstituteUpdater instituteUpdater;
@Autowired @Autowired
GeoRegionService geoRegionService; private GeoRegionService geoRegionService;
@Autowired @Autowired
CropService cropService; private CropService cropService;
@Autowired @Autowired
OAuthClientRepository oauthClientRepository; private OAuthClientRepository oauthClientRepository;
@Autowired @Autowired
AccessionRepository accessionRepository; private AccessionRepository accessionRepository;
@Autowired @Autowired
BatchRESTService batchService; private AccessionUploader uploader;
@Autowired @Autowired
private GenesysService genesysService; private GenesysService genesysService;
@Autowired
private PasswordEncoder passwordEncoder;
@Override @Override
protected void init() throws Exception { protected void init() throws Exception {
...@@ -150,7 +148,7 @@ public class FirstRunListener extends RunAsAdminListener { ...@@ -150,7 +148,7 @@ public class FirstRunListener extends RunAsAdminListener {
LOG.warn("Creating default OAuth client id={} secret={}", defaultOAuthClientId, defaultOAuthClientSecret); LOG.warn("Creating default OAuth client id={} secret={}", defaultOAuthClientId, defaultOAuthClientSecret);
final OAuthClient client = new OAuthClient(); final OAuthClient client = new OAuthClient();
client.setClientId(defaultOAuthClientId); client.setClientId(defaultOAuthClientId);
client.setClientSecret(defaultOAuthClientSecret); client.setClientSecret(passwordEncoder.encode(defaultOAuthClientSecret));
client.setTitle("Default OAuth client"); client.setTitle("Default OAuth client");
client.setDescription("This OAuth client was automatically created by the system."); client.setDescription("This OAuth client was automatically created by the system.");
client.getAuthorizedGrantTypes().add("authorization_code"); client.getAuthorizedGrantTypes().add("authorization_code");
...@@ -177,33 +175,23 @@ public class FirstRunListener extends RunAsAdminListener { ...@@ -177,33 +175,23 @@ public class FirstRunListener extends RunAsAdminListener {
final ObjectMapper mapper = new ObjectMapper(); final ObjectMapper mapper = new ObjectMapper();
try (InputStream stream = r.getInputStream()) { try (InputStream stream = r.getInputStream()) {
final JsonNode json = mapper.readTree(stream); final ArrayNode updates = (ArrayNode) mapper.readTree(stream);
FaoInstitute institute = instituteService.getInstitute(instCode); FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) { if (institute == null) {
throw new Exception("No Institute with instCode=" + instCode); throw new Exception("No Institute with instCode=" + instCode);
} }
final Map<AccessionHeaderJson, ObjectNode> batch = new HashMap<>(); if (updates.size() > 0) {
((ArrayNode) json).forEach(node -> { LOG.warn("Upserting {} accession records for {}", updates.size(), instCode);
try { uploader.upsertAccessions(institute, updates);
batch.put(AccessionHeaderJson.fromJson(node), (ObjectNode) node);
} catch (InvalidDOIException e) {
LOG.warn("Invalid DOI found in accession dummy data: {}", r.getFilename(), e);
}
});
if (batch.size() > 0) {
LOG.warn("Upserting {} accession records for {}", batch.size(), instCode);
batchService.ensureTaxonomies(institute, batch);
batchService.upsertAccessionData(institute, batch);
genesysService.updateAccessionCount(institute); genesysService.updateAccessionCount(institute);
} }
} catch (JsonParseException e) { } catch (JsonParseException e) {
LOG.error("Could not read dummy data for {} from {}. JSON is not valid.", instCode, r.getFilename(), e); LOG.error("Could not read dummy data for {} from {}. JSON is not valid.", instCode, r.getFilename(), e);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Could not create dummy data. {}", e.getMessage()); LOG.error("Could not create dummy data. {}", e.getMessage(), e);
} }
} }
} }
......
...@@ -11,18 +11,15 @@ import java.util.UUID; ...@@ -11,18 +11,15 @@ import java.util.UUID;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.genesys.AccessionAlias; import org.genesys2.server.model.genesys.AccessionAlias;
import org.genesys2.server.model.genesys.AccessionAlias.AliasType; import org.genesys2.server.model.genesys.AccessionAlias.AliasType;
import org.genesys2.server.model.genesys.AccessionBreeding;
import org.genesys2.server.model.genesys.AccessionCollect; import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.server.model.genesys.AccessionData; import org.genesys2.server.model.genesys.AccessionData;
import org.genesys2.server.model.genesys.AccessionExchange;
import org.genesys2.server.model.genesys.AccessionGeo; import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionList;
import org.genesys2.server.model.genesys.AccessionRemark; import org.genesys2.server.model.genesys.AccessionRemark;
import org.genesys2.server.model.genesys.SvalbardDeposit; import org.genesys2.server.model.genesys.SvalbardDeposit;
import org.genesys2.server.model.impl.AccessionList;
import org.genesys2.server.model.impl.Crop; import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.Organization; import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.IndexAliasConstants; import org.genesys2.server.service.IndexAliasConstants;
import org.genesys2.util.MCPDUtil;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Id;
...@@ -56,7 +53,7 @@ public class AccessionDetails { ...@@ -56,7 +53,7 @@ public class AccessionDetails {
private Boolean available; private Boolean available;
private Date createdDate; private Date createdDate;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private List<String> duplSite; private Set<String> duplSite;
private Boolean mlsStatus; private Boolean mlsStatus;
private Boolean art15; private Boolean art15;
private Integer sampStat; private Integer sampStat;
...@@ -87,7 +84,7 @@ public class AccessionDetails { ...@@ -87,7 +84,7 @@ public class AccessionDetails {
private Geo geo; private Geo geo;
@Field(type = FieldType.Object) @Field(type = FieldType.Object)
private Collect coll; private Collect coll;
private String pedigree; private String ancest;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String donorCode; private String donorCode;
private String donorName; private String donorName;
...@@ -100,7 +97,7 @@ public class AccessionDetails { ...@@ -100,7 +97,7 @@ public class AccessionDetails {
private boolean historic; private boolean historic;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String bredCode; private Set<String> bredCode;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String acceUrl; private String acceUrl;
...@@ -117,24 +114,29 @@ public class AccessionDetails { ...@@ -117,24 +114,29 @@ public class AccessionDetails {
ad.uuid = accession.getUuid(); ad.uuid = accession.getUuid();
ad.createdDate = accession.getCreatedDate(); ad.createdDate = accession.getCreatedDate();
ad.lastModifiedDate = accession.getLastModifiedDate(); ad.lastModifiedDate = accession.getLastModifiedDate();
ad.acceNumb = accession.getAccessionName(); ad.acceNumb = accession.getAccessionNumber();
ad.seqNo = accession.getSeqNo(); ad.seqNo = accession.getSeqNo();
ad.acqDate = accession.getAcquisitionDate(); ad.acqDate = accession.getAcquisitionDate();
ad.acqSrc = accession.getAcquisitionSource(); ad.acqSrc = accession.getAcquisitionSource();
ad.available = accession.getAvailability(); ad.available = accession.isAvailable();
ad.historic = accession.getHistoric(); ad.historic = accession.isHistoric();
ad.orgCty = Country.from(accession.getCountryOfOrigin()); ad.orgCty = Country.from(accession.getCountryOfOrigin());
ad.duplSite = MCPDUtil.toList(accession.getDuplSite()); ad.duplSite = accession.getAccessionId().getDuplSite();
ad.institute = Institute.from(accession.getInstitute()); ad.institute = Institute.from(accession.getInstitute());
ad.mlsStatus = accession.getMlsStatus(); ad.mlsStatus = accession.getMlsStatus();
ad.art15 = accession.getInTrust(); ad.art15 = accession.getInTrust();
ad.sampStat = accession.getSampleStatus(); ad.sampStat = accession.getSampStat();
ad.storage = new ArrayList<Integer>(accession.getStoRage()); ad.storage = new ArrayList<Integer>(accession.getAccessionId().getStorage());
ad.lists = new HashSet<String>(); ad.lists = new HashSet<String>();
ad.cropName=accession.getCropName(); ad.cropName=accession.getCropName();
ad.inSgsv = accession.getInSvalbard() != null && accession.getInSvalbard() == true ? true : false; ad.inSgsv = accession.getInSvalbard() != null && accession.getInSvalbard() == true ? true : false;
ad.cropName = accession.getCropName(); ad.cropName = accession.getCropName();
ad.doi = accession.getDoi(); ad.doi = accession.getDoi();
ad.donorCode = accession.getDonorCode();
ad.donorName = accession.getDonorName();
ad.donorNumb = accession.getDonorNumb();
ad.ancest = accession.getAncest();
ad.bredCode = accession.getAccessionId().getBreederCode();
if (accession.getAccessionId().getPdci() != null) { if (accession.getAccessionId().getPdci() != null) {
ad.pdciScore = accession.getAccessionId().getPdci().getScore(); ad.pdciScore = accession.getAccessionId().getPdci().getScore();
...@@ -216,21 +218,6 @@ public class AccessionDetails { ...@@ -216,21 +218,6 @@ public class AccessionDetails {
this.coll = Collect.from(collect); this.coll = Collect.from(collect);
} }
public void breeding(AccessionBreeding breeding) {
if (breeding == null)
return;
this.pedigree = StringUtils.defaultIfBlank(breeding.getPedigree(), null);
this.bredCode = StringUtils.defaultIfBlank(breeding.getBreederCode(), null);
}
public void exch(AccessionExchange exch) {
if (exch == null)
return;
this.donorCode = StringUtils.defaultIfBlank(exch.getDonorInstitute(), null);
this.donorName = StringUtils.defaultIfBlank(exch.getDonorName(), null);
this.donorNumb = StringUtils.defaultIfBlank(exch.getAccNumbDonor(), null);
}
public Long getVersion() { public Long getVersion() {
return version; return version;
} }
...@@ -303,11 +290,11 @@ public class AccessionDetails { ...@@ -303,11 +290,11 @@ public class AccessionDetails {
this.createdDate = createdDate; this.createdDate = createdDate;
} }
public List<String> getDuplSite() { public Set<String> getDuplSite() {
return duplSite; return duplSite;
} }
public void setDuplSite(List<String> duplSite) { public void setDuplSite(Set<String> duplSite) {
this.duplSite = duplSite; this.duplSite = duplSite;
} }
...@@ -415,20 +402,20 @@ public class AccessionDetails { ...@@ -415,20 +402,20 @@ public class AccessionDetails {
this.coll = coll; this.coll = coll;
} }
public String getBredCode() { public Set<String> getBredCode() {
return bredCode; return bredCode;
} }
public void setBredCode(String bredCode) { public void setBredCode(Set<String> bredCode) {
this.bredCode = bredCode; this.bredCode = bredCode;
} }
public String getPedigree() { public String getPedigree() {
return pedigree; return ancest;
} }
public void setPedigree(String pedigree) { public void setPedigree(String pedigree) {
this.pedigree = pedigree; this.ancest = pedigree;
} }
public String getDonorCode() { public String getDonorCode() {
......
package org.genesys2.server.model.elastic; package org.genesys2.server.model.elastic;
import java.util.Date; import java.util.Date;
import java.util.HashSet;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.genesys.AccessionCollect; import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.util.MCPDUtil;
import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex; import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.FieldType;
...@@ -16,9 +14,9 @@ public class Collect { ...@@ -16,9 +14,9 @@ public class Collect {
private Set<String> collCode; private Set<String> collCode;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String collDate; private String collDate;
private String collInstAddr; private Set<String> collInstAddr;
private String collMissId; private String collMissId;
private String collName; private Set<String> collName;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String) @Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String collNumb; private String collNumb;
private String collSite; private String collSite;
...@@ -30,14 +28,13 @@ public class Collect { ...@@ -30,14 +28,13 @@ public class Collect {
return null; return null;
Collect c = new Collect(); Collect c = new Collect();
if (StringUtils.isNotBlank(collect.getCollCode())) c.collCode = collect.getCollCode();
c.collCode = new HashSet<String>(MCPDUtil.toList(collect.getCollCode()));
c.collDate = StringUtils.defaultIfBlank(collect.getCollDate(), null); c.collDate = StringUtils.defaultIfBlank(collect.getCollDate(), null);
c.collInstAddr = StringUtils.defaultIfBlank(collect.getCollInstAddress(), null); c.collInstAddr = collect.getCollInstAddress();
c.collMissId = StringUtils.defaultIfBlank(collect.getCollMissId(), null); c.collMissId = collect.getCollMissId();
c.collName = StringUtils.defaultIfBlank(collect.getCollName(), null); c.collName = collect.getCollName();
c.collNumb = StringUtils.defaultIfBlank(collect.getCollNumb(), null); c.collNumb = collect.getCollNumb();
c.collSite = StringUtils.defaultIfBlank(collect.getCollSite(), null); c.collSite = collect.getCollSite();
c.collSrc = collect.getCollSrc(); c.collSrc = collect.getCollSrc();
return c; return c;
} }
...@@ -58,11 +55,11 @@ public class Collect { ...@@ -58,11 +55,11 @@ public class Collect {
this.collDate = collDate; this.collDate = collDate;
} }
public String getCollInstAddr() { public Set<String> getCollInstAddr() {
return collInstAddr; return collInstAddr;
} }
public void setCollInstAddr(String collInstAddr) { public void setCollInstAddr(Set<String> collInstAddr) {
this.collInstAddr = collInstAddr; this.collInstAddr = collInstAddr;
} }
...@@ -74,11 +71,11 @@ public class Collect { ...@@ -74,11 +71,11 @@ public class Collect {
this.collMissId = collMissId; this.collMissId = collMissId;
} }
public String getCollName() { public Set<String> getCollName() {
return collName; return collName;
} }
public void setCollName(String collName) { public void setCollName(Set<String> collName) {
this.collName = collName; this.collName = collName;
} }
......
...@@ -16,56 +16,33 @@ ...@@ -16,56 +16,33 @@
package org.genesys2.server.model.genesys; package org.genesys2.server.model.genesys;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Index; import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.OrderBy;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.UniqueConstraint; import javax.persistence.UniqueConstraint;
import org.genesys.blocks.auditlog.annotations.Audited; import org.genesys.blocks.auditlog.annotations.Audited;
import org.springframework.data.elasticsearch.annotations.Document;
/** /**
* Active accession records with all possible constraints. * Active accession records with all possible constraints.
* *
* @author mobreza * @author mobreza
* *
*/ */
@Entity @Entity
@Table(name = "accession", @Table(name = "accession",
// Unique constraints // Unique constraints
uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "taxGenus", "acceNumb" }), @UniqueConstraint(name = "UQ_accession_doi", columnNames = {"doi"})}, uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "genus", "acceNumb" }),
// Indexes @UniqueConstraint(name = "UQ_accession_doi", columnNames = { "doi" }) },
indexes = { @Index(name = "IX_seqNo", columnList = "seqNo"), @Index(name = "IX_accession_lastModifiedDate", columnList = "lastModifiedDate") }) // Indexes
indexes = { @Index(name = "IX_acceNumb", columnList = "acceNumb"), @Index(name = "IX_seqNo", columnList = "seqNo"), @Index(name = "IX_accession_lastModifiedDate", columnList = "lastModifiedDate") })
@Audited @Audited
@Document(indexName = "genesys")
public class Accession extends AccessionData { public class Accession extends AccessionData {
/** /**
* *
*/ */
private static final long serialVersionUID = -4847875217506974494L; private static final long serialVersionUID = -4847875217506974494L;
public static final List<Accession> EMPTY_LIST = Collections.unmodifiableList(new ArrayList<Accession>());
@Column(name = "storage", nullable = false)
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "accessionstorage", joinColumns = @JoinColumn(name = "accessionId", referencedColumnName = "id"))
@OrderBy("storage")
private List<Integer> stoRage = new ArrayList<Integer>();
public List<Integer> getStoRage() {
return stoRage;
}
public void setStoRage(List<Integer> stoRage) {
this.stoRage = stoRage;
}
} }
...@@ -16,38 +16,47 @@ ...@@ -16,38 +16,47 @@
package org.genesys2.server.model.genesys; package org.genesys2.server.model.genesys;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.FetchType; import javax.persistence.FetchType;
import javax.persistence.JoinColumn; import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne; import javax.persistence.ManyToOne;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Version; import javax.persistence.Version;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.auditlog.annotations.Audited; import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.BasicModel; import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.SelfCleaning;
/** /**
* Accession "alias" * Accession "alias"
*/ */
@Entity @Entity
@Table(name = "accessionalias") //@Table(uniqueConstraints = { @UniqueConstraint(columnNames = { "accessionId", "aliasType", "name" }) })
@Table(name = "accession_alias")
@Audited @Audited
public class AccessionAlias extends BasicModel implements AccessionRelated { public class AccessionAlias extends BasicModel implements AccessionRelated, SelfCleaning {
private static final long serialVersionUID = 4990299133164025782L; private static final long serialVersionUID = 4990299133164025782L;
private static Pattern USED_BY_PATTERN = Pattern.compile("^((\\p{Alnum}+):(.+))?$");
public static enum AliasType { public static enum AliasType {
ACCENAME(0), DONORNUMB(1), BREDNUMB(2), DATABASEID