Skip to content

GitLab

Commit b6ddc4ed authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch '250-data-model-cleanup' into 'master' 

Resolve "Data model cleanup"

Closes #250

See merge request genesys-pgr/genesys-server!147
parents b0e44778 f2f17819
Hide whitespace changes
Inline Side-by-side
Showing with 716 additions and 613 deletions
pom.xml
View file @ b6ddc4ed
......@@ -645,7 +645,10 @@
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
<configuration>
<stopPort>8888</stopPort>
<httpConnector>
<port>8080</port>
</httpConnector>
<stopPort>8998</stopPort>
<stopKey>stop</stopKey>
<jvmArgs>-Dspring.profiles.active=dev -Xmx2048M -Xms1024M -XX:MaxPermSize=128M -Djava.awt.headless=true -server -Dorg.eclipse.jetty.server.Request.maxFormContentSize=5000000</jvmArgs>
</configuration>
......
src/main/java/org/genesys2/server/listener/sample/FirstRunListener.java
View file @ b6ddc4ed
......@@ -18,39 +18,34 @@ package org.genesys2.server.listener.sample;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.oauth.model.OAuthClient;
import org.genesys.blocks.oauth.model.OAuthRole;
import org.genesys.blocks.oauth.persistence.OAuthClientRepository;
import org.genesys2.server.listener.RunAsAdminListener;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.domain.AccessionRepository;
import org.genesys2.server.service.BatchRESTService;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.worker.AccessionUploader;
import org.genesys2.server.service.worker.InstituteUpdater;
import org.genesys2.server.servlet.controller.rest.model.AccessionHeaderJson;
import org.genesys2.util.InvalidDOIException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
/**
* Import country data and WIEWS institutes if none exist at startup
*
......@@ -70,31 +65,34 @@ public class FirstRunListener extends RunAsAdminListener {
private String defaultOAuthClientSecret;
@Autowired
GeoService geoService;
private GeoService geoService;
@Autowired
InstituteService instituteService;
private InstituteService instituteService;
@Autowired
InstituteUpdater instituteUpdater;
private InstituteUpdater instituteUpdater;
@Autowired
GeoRegionService geoRegionService;
private GeoRegionService geoRegionService;
@Autowired
CropService cropService;
private CropService cropService;
@Autowired
OAuthClientRepository oauthClientRepository;
private OAuthClientRepository oauthClientRepository;
@Autowired
AccessionRepository accessionRepository;
private AccessionRepository accessionRepository;
@Autowired
BatchRESTService batchService;
private AccessionUploader uploader;
@Autowired
private GenesysService genesysService;
@Autowired
private PasswordEncoder passwordEncoder;
@Override
protected void init() throws Exception {
......@@ -150,7 +148,7 @@ public class FirstRunListener extends RunAsAdminListener {
LOG.warn("Creating default OAuth client id={} secret={}", defaultOAuthClientId, defaultOAuthClientSecret);
final OAuthClient client = new OAuthClient();
client.setClientId(defaultOAuthClientId);
client.setClientSecret(defaultOAuthClientSecret);
client.setClientSecret(passwordEncoder.encode(defaultOAuthClientSecret));
client.setTitle("Default OAuth client");
client.setDescription("This OAuth client was automatically created by the system.");
client.getAuthorizedGrantTypes().add("authorization_code");
......@@ -177,33 +175,23 @@ public class FirstRunListener extends RunAsAdminListener {
final ObjectMapper mapper = new ObjectMapper();
try (InputStream stream = r.getInputStream()) {
final JsonNode json = mapper.readTree(stream);
final ArrayNode updates = (ArrayNode) mapper.readTree(stream);
FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
throw new Exception("No Institute with instCode=" + instCode);
}
final Map<AccessionHeaderJson, ObjectNode> batch = new HashMap<>();
((ArrayNode) json).forEach(node -> {
try {
batch.put(AccessionHeaderJson.fromJson(node), (ObjectNode) node);
} catch (InvalidDOIException e) {
LOG.warn("Invalid DOI found in accession dummy data: {}", r.getFilename(), e);
}
});
if (batch.size() > 0) {
LOG.warn("Upserting {} accession records for {}", batch.size(), instCode);
batchService.ensureTaxonomies(institute, batch);
batchService.upsertAccessionData(institute, batch);
if (updates.size() > 0) {
LOG.warn("Upserting {} accession records for {}", updates.size(), instCode);
uploader.upsertAccessions(institute, updates);
genesysService.updateAccessionCount(institute);
}
} catch (JsonParseException e) {
LOG.error("Could not read dummy data for {} from {}. JSON is not valid.", instCode, r.getFilename(), e);
} catch (Exception e) {
LOG.error("Could not create dummy data. {}", e.getMessage());
LOG.error("Could not create dummy data. {}", e.getMessage(), e);
}
}
}
......
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
View file @ b6ddc4ed
......@@ -11,18 +11,15 @@ import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.genesys.AccessionAlias;
import org.genesys2.server.model.genesys.AccessionAlias.AliasType;
import org.genesys2.server.model.genesys.AccessionBreeding;
import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.server.model.genesys.AccessionData;
import org.genesys2.server.model.genesys.AccessionExchange;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionList;
import org.genesys2.server.model.genesys.AccessionRemark;
import org.genesys2.server.model.genesys.SvalbardDeposit;
import org.genesys2.server.model.impl.AccessionList;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.IndexAliasConstants;
import org.genesys2.util.MCPDUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.annotation.Id;
......@@ -56,7 +53,7 @@ public class AccessionDetails {
private Boolean available;
private Date createdDate;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private List<String> duplSite;
private Set<String> duplSite;
private Boolean mlsStatus;
private Boolean art15;
private Integer sampStat;
......@@ -87,7 +84,7 @@ public class AccessionDetails {
private Geo geo;
@Field(type = FieldType.Object)
private Collect coll;
private String pedigree;
private String ancest;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String donorCode;
private String donorName;
......@@ -100,7 +97,7 @@ public class AccessionDetails {
private boolean historic;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String bredCode;
private Set<String> bredCode;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String acceUrl;
......@@ -117,24 +114,29 @@ public class AccessionDetails {
ad.uuid = accession.getUuid();
ad.createdDate = accession.getCreatedDate();
ad.lastModifiedDate = accession.getLastModifiedDate();
ad.acceNumb = accession.getAccessionName();
ad.acceNumb = accession.getAccessionNumber();
ad.seqNo = accession.getSeqNo();
ad.acqDate = accession.getAcquisitionDate();
ad.acqSrc = accession.getAcquisitionSource();
ad.available = accession.getAvailability();
ad.historic = accession.getHistoric();
ad.available = accession.isAvailable();
ad.historic = accession.isHistoric();
ad.orgCty = Country.from(accession.getCountryOfOrigin());
ad.duplSite = MCPDUtil.toList(accession.getDuplSite());
ad.duplSite = accession.getAccessionId().getDuplSite();
ad.institute = Institute.from(accession.getInstitute());
ad.mlsStatus = accession.getMlsStatus();
ad.art15 = accession.getInTrust();
ad.sampStat = accession.getSampleStatus();
ad.storage = new ArrayList<Integer>(accession.getStoRage());
ad.sampStat = accession.getSampStat();
ad.storage = new ArrayList<Integer>(accession.getAccessionId().getStorage());
ad.lists = new HashSet<String>();
ad.cropName=accession.getCropName();
ad.inSgsv = accession.getInSvalbard() != null && accession.getInSvalbard() == true ? true : false;
ad.cropName = accession.getCropName();
ad.doi = accession.getDoi();
ad.donorCode = accession.getDonorCode();
ad.donorName = accession.getDonorName();
ad.donorNumb = accession.getDonorNumb();
ad.ancest = accession.getAncest();
ad.bredCode = accession.getAccessionId().getBreederCode();
if (accession.getAccessionId().getPdci() != null) {
ad.pdciScore = accession.getAccessionId().getPdci().getScore();
......@@ -216,21 +218,6 @@ public class AccessionDetails {
this.coll = Collect.from(collect);
}
public void breeding(AccessionBreeding breeding) {
if (breeding == null)
return;
this.pedigree = StringUtils.defaultIfBlank(breeding.getPedigree(), null);
this.bredCode = StringUtils.defaultIfBlank(breeding.getBreederCode(), null);
}
public void exch(AccessionExchange exch) {
if (exch == null)
return;
this.donorCode = StringUtils.defaultIfBlank(exch.getDonorInstitute(), null);
this.donorName = StringUtils.defaultIfBlank(exch.getDonorName(), null);
this.donorNumb = StringUtils.defaultIfBlank(exch.getAccNumbDonor(), null);
}
public Long getVersion() {
return version;
}
......@@ -303,11 +290,11 @@ public class AccessionDetails {
this.createdDate = createdDate;
}
public List<String> getDuplSite() {
public Set<String> getDuplSite() {
return duplSite;
}
public void setDuplSite(List<String> duplSite) {
public void setDuplSite(Set<String> duplSite) {
this.duplSite = duplSite;
}
......@@ -415,20 +402,20 @@ public class AccessionDetails {
this.coll = coll;
}
public String getBredCode() {
public Set<String> getBredCode() {
return bredCode;
}
public void setBredCode(String bredCode) {
public void setBredCode(Set<String> bredCode) {
this.bredCode = bredCode;
}
public String getPedigree() {
return pedigree;
return ancest;
}
public void setPedigree(String pedigree) {
this.pedigree = pedigree;
this.ancest = pedigree;
}
public String getDonorCode() {
......
src/main/java/org/genesys2/server/model/elastic/Collect.java
View file @ b6ddc4ed
package org.genesys2.server.model.elastic;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.genesys.AccessionCollect;
import org.genesys2.util.MCPDUtil;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
......@@ -16,9 +14,9 @@ public class Collect {
private Set<String> collCode;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String collDate;
private String collInstAddr;
private Set<String> collInstAddr;
private String collMissId;
private String collName;
private Set<String> collName;
@Field(index = FieldIndex.not_analyzed, type = FieldType.String)
private String collNumb;
private String collSite;
......@@ -30,14 +28,13 @@ public class Collect {
return null;
Collect c = new Collect();
if (StringUtils.isNotBlank(collect.getCollCode()))
c.collCode = new HashSet<String>(MCPDUtil.toList(collect.getCollCode()));
c.collCode = collect.getCollCode();
c.collDate = StringUtils.defaultIfBlank(collect.getCollDate(), null);
c.collInstAddr = StringUtils.defaultIfBlank(collect.getCollInstAddress(), null);
c.collMissId = StringUtils.defaultIfBlank(collect.getCollMissId(), null);
c.collName = StringUtils.defaultIfBlank(collect.getCollName(), null);
c.collNumb = StringUtils.defaultIfBlank(collect.getCollNumb(), null);
c.collSite = StringUtils.defaultIfBlank(collect.getCollSite(), null);
c.collInstAddr = collect.getCollInstAddress();
c.collMissId = collect.getCollMissId();
c.collName = collect.getCollName();
c.collNumb = collect.getCollNumb();
c.collSite = collect.getCollSite();
c.collSrc = collect.getCollSrc();
return c;
}
......@@ -58,11 +55,11 @@ public class Collect {
this.collDate = collDate;
}
public String getCollInstAddr() {
public Set<String> getCollInstAddr() {
return collInstAddr;
}
public void setCollInstAddr(String collInstAddr) {
public void setCollInstAddr(Set<String> collInstAddr) {
this.collInstAddr = collInstAddr;
}
......@@ -74,11 +71,11 @@ public class Collect {
this.collMissId = collMissId;
}
public String getCollName() {
public Set<String> getCollName() {
return collName;
}
public void setCollName(String collName) {
public void setCollName(Set<String> collName) {
this.collName = collName;
}
......
src/main/java/org/genesys2/server/model/genesys/Accession.java
View file @ b6ddc4ed
......@@ -16,56 +16,39 @@
package org.genesys2.server.model.genesys;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.OrderBy;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import org.genesys.blocks.auditlog.annotations.Audited;
import org.springframework.data.elasticsearch.annotations.Document;
/**
* Active accession records with all possible constraints.
*
*
* @author mobreza
*
*/
@Entity
@Table(name = "accession",
// Unique constraints
uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "taxGenus", "acceNumb" }), @UniqueConstraint(name = "UQ_accession_doi", columnNames = {"doi"})},
// Indexes
indexes = { @Index(name = "IX_seqNo", columnList = "seqNo"), @Index(name = "IX_accession_lastModifiedDate", columnList = "lastModifiedDate") })
// Unique constraints
uniqueConstraints = { @UniqueConstraint(name = "UQ_accession_genus_inst", columnNames = { "instituteId", "genus", "acceNumb" }),
@UniqueConstraint(name = "UQ_accession_doi", columnNames = { "doi" }) },
// Indexes
indexes = {
@Index(name = "IX_acceNumb", columnList = "acceNumb"),
@Index(name = "IX_inst_seqNo", columnList = "instituteId,seqNo"),
@Index(name = "IX_origcty_seqNo", columnList = "orgCtyId,seqNo"),
@Index(name = "IX_taxa_seq", columnList = "taxonomyId2,seqNo"),
@Index(name = "IX_seqNo", columnList = "seqNo"),
@Index(name = "IX_accession_lastModifiedDate", columnList = "lastModifiedDate") })
@Audited
@Document(indexName = "genesys")
public class Accession extends AccessionData {
/**
*
*
*/
private static final long serialVersionUID = -4847875217506974494L;
public static final List<Accession> EMPTY_LIST = Collections.unmodifiableList(new ArrayList<Accession>());
@Column(name = "storage", nullable = false)
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "accessionstorage", joinColumns = @JoinColumn(name = "accessionId", referencedColumnName = "id"))
@OrderBy("storage")
private List<Integer> stoRage = new ArrayList<Integer>();
public List<Integer> getStoRage() {
return stoRage;
}
public void setStoRage(List<Integer> stoRage) {
this.stoRage = stoRage;
}
}
src/main/java/org/genesys2/server/model/genesys/AccessionAlias.java
View file @ b6ddc4ed
......@@ -16,38 +16,47 @@
package org.genesys2.server.model.genesys;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table;
import javax.persistence.Version;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.SelfCleaning;
/**
* Accession "alias"
*/
@Entity
@Table(name = "accessionalias")
//@Table(uniqueConstraints = { @UniqueConstraint(columnNames = { "accessionId", "aliasType", "name" }) })
@Table(name = "accession_alias")
@Audited
public class AccessionAlias extends BasicModel implements AccessionRelated {
public class AccessionAlias extends BasicModel implements AccessionRelated, SelfCleaning {
private static final long serialVersionUID = 4990299133164025782L;
private static Pattern USED_BY_PATTERN = Pattern.compile("^((\\p{Alnum}+):(.+))?$");
public static enum AliasType {
ACCENAME(0), DONORNUMB(1), BREDNUMB(2), DATABASEID(3), LOCALNAME(4), OTHERNUMB(5), COLLNUMB(6);
private int id;
private AliasType(int id) {
private AliasType(final int id) {
this.id = id;
}
public static AliasType getType(Integer id) {
public static AliasType getType(final Integer id) {
if (id == null) {
return null;
}
......@@ -76,11 +85,8 @@ public class AccessionAlias extends BasicModel implements AccessionRelated {
@Column(name = "name", length = 150)
private String name;
@Column(length = 10)
private String instCode;
@Column
private int aliasType = AliasType.ACCENAME.id;
private AliasType aliasType = AliasType.ACCENAME;
@Column(length = 2)
private String lang;
......@@ -91,11 +97,31 @@ public class AccessionAlias extends BasicModel implements AccessionRelated {
public AccessionAlias() {
}
public AccessionAlias(AliasType aliasType, String name) {
this.aliasType = aliasType;
Matcher m = USED_BY_PATTERN.matcher(name);
if (m.matches()) {
this.usedBy = StringUtils.trimToNull(m.group(2));
this.name = StringUtils.trimToNull(m.group(3));
} else {
if (name.startsWith(":")) {
name = StringUtils.trimToNull(name.substring(1));
}
this.name = name;
}
}
@PrePersist
@PreUpdate
private void prePersist() {
trimStringsToNull();
}
public long getVersion() {
return version;
}
public void setVersion(long version) {
public void setVersion(final long version) {
this.version = version;
}
......@@ -104,7 +130,7 @@ public class AccessionAlias extends BasicModel implements AccessionRelated {
return accession;
}
public void setAccession(AccessionId accession) {
public void setAccession(final AccessionId accession) {
this.accession = accession;
}
......@@ -112,31 +138,23 @@ public class AccessionAlias extends BasicModel implements AccessionRelated {
return name;
}
public void setName(String name) {
public void setName(final String name) {
this.name = name;
}
public String getInstCode() {
return instCode;
}
public void setInstCode(String instCode) {
this.instCode = instCode;
}
public AliasType getAliasType() {
return AliasType.getType(this.aliasType);
return this.aliasType;
}
public void setAliasType(AliasType aliasType) {
this.aliasType = aliasType.getId();
public void setAliasType(final AliasType aliasType) {
this.aliasType = aliasType;
}
public String getLang() {
return lang;
}
public void setLang(String lang) {
public void setLang(final String lang) {
this.lang = lang;
}
......@@ -144,12 +162,27 @@ public class AccessionAlias extends BasicModel implements AccessionRelated {
return usedBy;