Commit f205f233 authored by Matija Obreza's avatar Matija Obreza
Browse files

Accession updater cleanup

- GLIS client is optional
- PDCI entity is not audited, PDCI score is audited on Accession. Remove existing auditlogs
- Fixed testing if entities related to Accession are persisted
- Database: Replace existing blank strings with NULL in Accession#acceUrl, #acqDate and Alias#usedBy
parent 9e625c35
...@@ -29,12 +29,10 @@ import javax.persistence.PreUpdate; ...@@ -29,12 +29,10 @@ import javax.persistence.PreUpdate;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Transient; import javax.persistence.Transient;
import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.VersionedModel; import org.genesys.blocks.model.VersionedModel;
@Entity @Entity
@Table(name = "pdci", indexes = { @Index(columnList = "score") }) @Table(name = "pdci", indexes = { @Index(columnList = "score") })
@Audited
public class PDCI extends VersionedModel implements AccessionRelated { public class PDCI extends VersionedModel implements AccessionRelated {
private static final long serialVersionUID = -1312366054528702261L; private static final long serialVersionUID = -1312366054528702261L;
......
...@@ -83,8 +83,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository, ...@@ -83,8 +83,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository,
if (uniqueAcceNumbs) { if (uniqueAcceNumbs) {
restrictions.add(root.get("accessionName").in(acceNumbs)); restrictions.add(root.get("accessionName").in(acceNumbs));
if (LOG.isDebugEnabled()) LOG.trace("*** {} accenumbs={}", institute.getCode(), acceNumbs);
LOG.debug("*** {} accenumbs={}", institute.getCode(), acceNumbs);
} else { } else {
// A lot of .. (acceNumb=? and genus=?) // A lot of .. (acceNumb=? and genus=?)
for (AccessionIdentifier3 ah : accessionIds) { for (AccessionIdentifier3 ah : accessionIds) {
...@@ -95,8 +94,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository, ...@@ -95,8 +94,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository,
List<Accession> res = em.createQuery(cq).getResultList(); List<Accession> res = em.createQuery(cq).getResultList();
if (LOG.isDebugEnabled()) LOG.trace("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
LOG.debug("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
// Check for duplicate names if institute is using unique acceNumbs // Check for duplicate names if institute is using unique acceNumbs
if (uniqueAcceNumbs) { if (uniqueAcceNumbs) {
......
...@@ -103,7 +103,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -103,7 +103,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
@Autowired @Autowired
private TaxonomyManager taxonomyManager; private TaxonomyManager taxonomyManager;
@Autowired @Autowired(required = false)
private GenesysApi glisGenesysApi; private GenesysApi glisGenesysApi;
@Override @Override
...@@ -133,9 +133,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -133,9 +133,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Sanitize // Sanitize
sanitizeTaxonomy(current); sanitizeTaxonomy(current);
if (LOG.isDebugEnabled()) { LOG.trace("Ensuring {}", current);
LOG.debug("Ensuring {}", current);
}
Taxonomy2 ensuredTaxonomy = null; Taxonomy2 ensuredTaxonomy = null;
try { try {
...@@ -211,9 +209,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -211,9 +209,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
} }
for (final AccessionHeaderJson dataJson : batch.keySet()) { for (final AccessionHeaderJson dataJson : batch.keySet()) {
if (LOG.isDebugEnabled()) { LOG.trace("Loading accession {}", dataJson);
LOG.debug("Loading accession {}", dataJson);
}
AccessionOpResponse upsertResponse = new AccessionOpResponse(dataJson.instCode, dataJson.acceNumb, dataJson.genus); AccessionOpResponse upsertResponse = new AccessionOpResponse(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
upsertResponses.add(upsertResponse); upsertResponses.add(upsertResponse);
...@@ -402,7 +398,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -402,7 +398,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
if (!accnColl.isEmpty()) { if (!accnColl.isEmpty()) {
toSaveColl.add(accnColl); toSaveColl.add(accnColl);
} else if (accnColl.getId() != null) { } else if (accnColl.isPersisted()) {
toRemoveColl.add(accnColl); toRemoveColl.add(accnColl);
} }
} }
...@@ -452,7 +448,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -452,7 +448,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
} }
if (!accnGeo.isEmpty()) { if (!accnGeo.isEmpty()) {
toSaveGeo.add(accnGeo); toSaveGeo.add(accnGeo);
} else if (accnGeo.getId() != null) { } else if (accnGeo.isPersisted()) {
toRemoveGeo.add(accnGeo); toRemoveGeo.add(accnGeo);
} }
} }
...@@ -473,7 +469,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -473,7 +469,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
} }
if (!accnBred.isEmpty()) { if (!accnBred.isEmpty()) {
toSaveBreed.add(accnBred); toSaveBreed.add(accnBred);
} else if (accnBred.getId() != null) { } else if (accnBred.isPersisted()) {
toRemoveBreed.add(accnBred); toRemoveBreed.add(accnBred);
} }
} }
...@@ -512,7 +508,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -512,7 +508,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
if (!accnExch.isEmpty()) { if (!accnExch.isEmpty()) {
toSaveExch.add(accnExch); toSaveExch.add(accnExch);
} else if (accnExch.getId() != null) { } else if (accnExch.isPersisted()) {
toRemoveExch.add(accnExch); toRemoveExch.add(accnExch);
} }
} }
...@@ -592,24 +588,28 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -592,24 +588,28 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Update PDCI // Update PDCI
genesysService.updatePDCI(savedData.stream().map(a -> a.getId()).collect(Collectors.toSet())); genesysService.updatePDCI(savedData.stream().map(a -> a.getId()).collect(Collectors.toSet()));
LOG.debug("Done updating PDCI");
// Update GLIS // Update GLIS
UpdateTargets targets = new UpdateTargets(); if (glisGenesysApi != null) {
targets.addKwsItem(TermConstants.PASSPORT_DATA); UpdateTargets targets = new UpdateTargets();
savedData.stream().map(a -> a.getDoi()).filter(doi -> doi != null).forEach(doi -> targets.addDoisItem(doi)); targets.addKwsItem(TermConstants.PASSPORT_DATA);
if (targets.getDois() != null && targets.getDois().size() > 0) { savedData.stream().map(a -> a.getDoi()).filter(doi -> doi != null).forEach(doi -> targets.addDoisItem(doi));
try {
LOG.debug("Updating GLIS for {} accessions with DOIs", targets.getDois().size()); if (targets.getDois() != null && targets.getDois().size() > 0) {
List<UpdatedTarget> glisResponse = glisGenesysApi.registerGenesysAsTarget(targets); try {
glisResponse.forEach(updatedTarget -> { LOG.debug("Updating GLIS for {} accessions with DOIs", targets.getDois().size());
if (StringUtils.equals("OK", updatedTarget.getResult())) { List<UpdatedTarget> glisResponse = glisGenesysApi.registerGenesysAsTarget(targets);
LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult()); glisResponse.forEach(updatedTarget -> {
} else if (StringUtils.equals("KO", updatedTarget.getResult())) { if (StringUtils.equals("OK", updatedTarget.getResult())) {
LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult()); LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult());
} } else if (StringUtils.equals("KO", updatedTarget.getResult())) {
}); LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult());
} catch (Throwable e) { }
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e); });
} catch (Throwable e) {
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e);
}
} }
} }
...@@ -774,7 +774,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -774,7 +774,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
continue; continue;
} }
LOG.debug("fieldName={} remark={}", fieldName, remark); LOG.trace("fieldName={} remark={}", fieldName, remark);
AccessionRemark ar = findRemark(existingRemarks, fieldName, remark); AccessionRemark ar = findRemark(existingRemarks, fieldName, remark);
if (ar == null) { if (ar == null) {
...@@ -1003,10 +1003,10 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -1003,10 +1003,10 @@ public class BatchRESTServiceImpl implements BatchRESTService {
final String val = item.textValue().trim(); final String val = item.textValue().trim();
if (splitInstCode && val.contains(":")) { if (splitInstCode && val.contains(":")) {
final String[] s = val.split(":", 2); final String[] s = val.split(":", 2);
alias.usedBy = s[0].trim(); alias.usedBy = StringUtils.trimToNull(s[0]);
alias.name = s[1].trim(); alias.name = StringUtils.trimToNull(s[1]);
} else { } else {
alias.name = val; alias.name = StringUtils.trimToNull(val);
} }
alias.type = aliasType.getId(); alias.type = aliasType.getId();
aliases.add(alias); aliases.add(alias);
...@@ -1091,7 +1091,7 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -1091,7 +1091,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Allows us to focus only on a particular alias type // Allows us to focus only on a particular alias type
if (aliasType != null) { if (aliasType != null) {
LOG.debug("Filtering accession aliases by {}", aliasType); LOG.trace("Filtering accession aliases by {}", aliasType);
CollectionUtils.filter(existingAliases, new Predicate<AccessionAlias>() { CollectionUtils.filter(existingAliases, new Predicate<AccessionAlias>() {
@Override @Override
public boolean evaluate(AccessionAlias alias) { public boolean evaluate(AccessionAlias alias) {
...@@ -1127,8 +1127,8 @@ public class BatchRESTServiceImpl implements BatchRESTService { ...@@ -1127,8 +1127,8 @@ public class BatchRESTServiceImpl implements BatchRESTService {
} }
accessionAlias.setName(aa.name); accessionAlias.setName(aa.name);
accessionAlias.setInstCode(aa.instCode); accessionAlias.setInstCode(StringUtils.trimToNull(aa.instCode));
accessionAlias.setUsedBy(aa.usedBy); accessionAlias.setUsedBy(StringUtils.trimToNull(aa.usedBy));
accessionAlias.setAliasType(aliasType != null ? aliasType : AliasType.getType(aa.type)); accessionAlias.setAliasType(aliasType != null ? aliasType : AliasType.getType(aa.type));
toSave.add(accessionAlias); toSave.add(accessionAlias);
......
...@@ -365,8 +365,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -365,8 +365,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
return; return;
} }
if (LOG.isDebugEnabled()) LOG.debug("Updating {} bulk_size={}", className, ids.size());
LOG.debug("Updating {} bulk_size={}", className, ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>(); List<IndexQuery> queries = new ArrayList<IndexQuery>();
...@@ -386,9 +385,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ ...@@ -386,9 +385,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
} }
if (!queries.isEmpty()) { if (!queries.isEmpty()) {
if (LOG.isDebugEnabled()) { LOG.debug("Indexing {} count={} of provided objects count={}", className, queries.size(), ids.size());
LOG.debug("Indexing {} count={} of provided objects count={}", className, queries.size(), ids.size());
}
elasticsearchTemplate.bulkIndex(queries); elasticsearchTemplate.bulkIndex(queries);
} }
} }
......
...@@ -1760,7 +1760,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { ...@@ -1760,7 +1760,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
pdcis.add(pdci); pdcis.add(pdci);
}); });
LOG.info("Generated {} new PDCIs", pdcis.size()); LOG.info("Updated {} PDCIs", pdcis.size());
repoPdci.save(pdcis); repoPdci.save(pdcis);
return pdcis; return pdcis;
} }
......
...@@ -197,7 +197,7 @@ public class AccessionController extends RestController { ...@@ -197,7 +197,7 @@ public class AccessionController extends RestController {
public @ResponseBody Object upsertInstituteAccession(@PathVariable("instCode") String instCode, @RequestBody JsonNode json) throws JsonProcessingException, IOException, public @ResponseBody Object upsertInstituteAccession(@PathVariable("instCode") String instCode, @RequestBody JsonNode json) throws JsonProcessingException, IOException,
RESTApiException { RESTApiException {
LOG.debug("Received:\n {}", json); LOG.trace("Received:\n {}", json);
// User's permission to WRITE to this WIEWS institute are checked in // User's permission to WRITE to this WIEWS institute are checked in
// BatchRESTService. // BatchRESTService.
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
*/ */
package org.genesys2.spring.config; package org.genesys2.spring.config;
import org.elasticsearch.common.lang3.StringUtils;
import org.genesys.glis.v1.GlisRateLimiter; import org.genesys.glis.v1.GlisRateLimiter;
import org.genesys.glis.v1.invoker.ApiClient; import org.genesys.glis.v1.invoker.ApiClient;
import org.genesys.glis.v1.invoker.auth.HttpBasicAuth; import org.genesys.glis.v1.invoker.auth.HttpBasicAuth;
...@@ -41,22 +42,29 @@ public class GLISConfig { ...@@ -41,22 +42,29 @@ public class GLISConfig {
private double glisRateLimit; private double glisRateLimit;
@Bean @Bean
public GlisRateLimiter glisRateLimiter() { public GlisRateLimiter glisRateLimiter(ApiClient client) {
return new GlisRateLimiter(glisRateLimit); if (client != null)
return new GlisRateLimiter(glisRateLimit);
else
return null;
} }
@Bean @Bean
public ApiClient glisClient() { public ApiClient glisClient() {
ApiClient client = new ApiClient(); if (StringUtils.isNotBlank(username)) {
client.setDebugging("true".equals(System.getenv("GLIS_DEBUG"))); ApiClient client = new ApiClient();
client.setBasePath(basePath); client.setDebugging("true".equals(System.getenv("GLIS_DEBUG")));
client.setBasePath(basePath);
// Configure HTTP basic authorization: glis_auth
HttpBasicAuth easySmtaAuth = (HttpBasicAuth) client.getAuthentication("easySmta"); // Configure HTTP basic authorization: glis_auth
easySmtaAuth.setUsername(username); HttpBasicAuth easySmtaAuth = (HttpBasicAuth) client.getAuthentication("easySmta");
easySmtaAuth.setPassword(password); easySmtaAuth.setUsername(username);
easySmtaAuth.setPassword(password);
System.err.println("GLIS at " + client.getBasePath() + " with username=" + username);
return client; System.err.println("GLIS at " + client.getBasePath() + " with username=" + username);
return client;
} else {
return null;
}
} }
} }
...@@ -1179,3 +1179,49 @@ databaseChangeLog: ...@@ -1179,3 +1179,49 @@ databaseChangeLog:
name: lastLogin name: lastLogin
type: datetime(6) type: datetime(6)
tableName: user tableName: user
- changeSet:
id: 1530443149000-2
author: mobreza
comment: Replace blanks with null and delete empty records
changes:
- sql:
sql: update accession set acceUrl = null where acceUrl = '';
- sql:
sql: update accession set acqDate = null where acqDate = '';
- sql:
sql: update accessionalias set usedBy = null where usedBy = '';
- sql:
sql: delete from accessionalias where name = '' and (usedBy is null or instCode is null);
- sql:
sql: update accessionbreeding set breederCode = NULL where breederCode = '';
- sql:
sql: update accessionbreeding set pedigree = NULL where pedigree = '';
- sql:
sql: delete from accessionbreeding where pedigree is null and breederCode is null;
- sql:
sql: update accessioncollect set collDate = null where collDate = '';
- sql:
sql: update accessioncollect set collSite = null where collSite = '';
- sql:
sql: update accessioncollect set collCode = null where collCode = '';
- sql:
sql: update accessioncollect set collNumb = null where collNumb = '';
- sql:
sql: update accessioncollect set collMissId = null where collMissId = '';
- sql:
sql: update accessioncollect set collName = null where collName = '';
- sql:
sql: update accessioncollect set collInstAddress = null where collInstAddress = '';
- sql:
sql: delete from accessioncollect where collDate = null and collSite = null and collCode = null and collNumb = null and collMissId = null and collName = null and collInstAddress = null and collSrc = null;
- sql:
sql: update accessionexchange set acceNumb = null where acceNumb = '';
- sql:
sql: update accessionexchange set donorInst = null where donorInst = '';
- sql:
sql: update accessionexchange set donorName = null where donorName = '';
- sql:
sql: delete from accessionexchange where acceNumb is null and donorInst is null and donorName is null;
- sql:
sql: delete from auditlog where classPk = (select id from classpk where classname='org.genesys2.server.model.genesys.PDCI');
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment