Commit f205f233 authored by Matija Obreza's avatar Matija Obreza

Accession updater cleanup

- GLIS client is optional
- PDCI entity is not audited, PDCI score is audited on Accession. Remove existing auditlogs
- Fixed testing if entities related to Accession are persisted
- Database: Replace existing blank strings with NULL in Accession#acceUrl, #acqDate and Alias#usedBy
parent 9e625c35
......@@ -29,12 +29,10 @@ import javax.persistence.PreUpdate;
import javax.persistence.Table;
import javax.persistence.Transient;
import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.VersionedModel;
@Entity
@Table(name = "pdci", indexes = { @Index(columnList = "score") })
@Audited
public class PDCI extends VersionedModel implements AccessionRelated {
private static final long serialVersionUID = -1312366054528702261L;
......
......@@ -83,8 +83,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository,
if (uniqueAcceNumbs) {
restrictions.add(root.get("accessionName").in(acceNumbs));
if (LOG.isDebugEnabled())
LOG.debug("*** {} accenumbs={}", institute.getCode(), acceNumbs);
LOG.trace("*** {} accenumbs={}", institute.getCode(), acceNumbs);
} else {
// A lot of .. (acceNumb=? and genus=?)
for (AccessionIdentifier3 ah : accessionIds) {
......@@ -95,8 +94,7 @@ public class AccessionRepositoryCustomImpl implements AccessionCustomRepository,
List<Accession> res = em.createQuery(cq).getResultList();
if (LOG.isDebugEnabled())
LOG.debug("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
LOG.trace("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
// Check for duplicate names if institute is using unique acceNumbs
if (uniqueAcceNumbs) {
......
......@@ -103,7 +103,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
@Autowired
private TaxonomyManager taxonomyManager;
@Autowired
@Autowired(required = false)
private GenesysApi glisGenesysApi;
@Override
......@@ -133,9 +133,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Sanitize
sanitizeTaxonomy(current);
if (LOG.isDebugEnabled()) {
LOG.debug("Ensuring {}", current);
}
LOG.trace("Ensuring {}", current);
Taxonomy2 ensuredTaxonomy = null;
try {
......@@ -211,9 +209,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
for (final AccessionHeaderJson dataJson : batch.keySet()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Loading accession {}", dataJson);
}
LOG.trace("Loading accession {}", dataJson);
AccessionOpResponse upsertResponse = new AccessionOpResponse(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
upsertResponses.add(upsertResponse);
......@@ -402,7 +398,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
if (!accnColl.isEmpty()) {
toSaveColl.add(accnColl);
} else if (accnColl.getId() != null) {
} else if (accnColl.isPersisted()) {
toRemoveColl.add(accnColl);
}
}
......@@ -452,7 +448,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
if (!accnGeo.isEmpty()) {
toSaveGeo.add(accnGeo);
} else if (accnGeo.getId() != null) {
} else if (accnGeo.isPersisted()) {
toRemoveGeo.add(accnGeo);
}
}
......@@ -473,7 +469,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
if (!accnBred.isEmpty()) {
toSaveBreed.add(accnBred);
} else if (accnBred.getId() != null) {
} else if (accnBred.isPersisted()) {
toRemoveBreed.add(accnBred);
}
}
......@@ -512,7 +508,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
if (!accnExch.isEmpty()) {
toSaveExch.add(accnExch);
} else if (accnExch.getId() != null) {
} else if (accnExch.isPersisted()) {
toRemoveExch.add(accnExch);
}
}
......@@ -592,24 +588,28 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Update PDCI
genesysService.updatePDCI(savedData.stream().map(a -> a.getId()).collect(Collectors.toSet()));
LOG.debug("Done updating PDCI");
// Update GLIS
UpdateTargets targets = new UpdateTargets();
targets.addKwsItem(TermConstants.PASSPORT_DATA);
savedData.stream().map(a -> a.getDoi()).filter(doi -> doi != null).forEach(doi -> targets.addDoisItem(doi));
if (targets.getDois() != null && targets.getDois().size() > 0) {
try {
LOG.debug("Updating GLIS for {} accessions with DOIs", targets.getDois().size());
List<UpdatedTarget> glisResponse = glisGenesysApi.registerGenesysAsTarget(targets);
glisResponse.forEach(updatedTarget -> {
if (StringUtils.equals("OK", updatedTarget.getResult())) {
LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult());
} else if (StringUtils.equals("KO", updatedTarget.getResult())) {
LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult());
}
});
} catch (Throwable e) {
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e);
if (glisGenesysApi != null) {
UpdateTargets targets = new UpdateTargets();
targets.addKwsItem(TermConstants.PASSPORT_DATA);
savedData.stream().map(a -> a.getDoi()).filter(doi -> doi != null).forEach(doi -> targets.addDoisItem(doi));
if (targets.getDois() != null && targets.getDois().size() > 0) {
try {
LOG.debug("Updating GLIS for {} accessions with DOIs", targets.getDois().size());
List<UpdatedTarget> glisResponse = glisGenesysApi.registerGenesysAsTarget(targets);
glisResponse.forEach(updatedTarget -> {
if (StringUtils.equals("OK", updatedTarget.getResult())) {
LOG.trace("GLIS for {} result={}", updatedTarget.getDoi(), updatedTarget.getResult());
} else if (StringUtils.equals("KO", updatedTarget.getResult())) {
LOG.info("GLIS for {} result={} msg={}", updatedTarget.getDoi(), updatedTarget.getResult());
}
});
} catch (Throwable e) {
LOG.error("Error updating GLIS targets: {}", e.getMessage(), e);
}
}
}
......@@ -774,7 +774,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
continue;
}
LOG.debug("fieldName={} remark={}", fieldName, remark);
LOG.trace("fieldName={} remark={}", fieldName, remark);
AccessionRemark ar = findRemark(existingRemarks, fieldName, remark);
if (ar == null) {
......@@ -1003,10 +1003,10 @@ public class BatchRESTServiceImpl implements BatchRESTService {
final String val = item.textValue().trim();
if (splitInstCode && val.contains(":")) {
final String[] s = val.split(":", 2);
alias.usedBy = s[0].trim();
alias.name = s[1].trim();
alias.usedBy = StringUtils.trimToNull(s[0]);
alias.name = StringUtils.trimToNull(s[1]);
} else {
alias.name = val;
alias.name = StringUtils.trimToNull(val);
}
alias.type = aliasType.getId();
aliases.add(alias);
......@@ -1091,7 +1091,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
// Allows us to focus only on a particular alias type
if (aliasType != null) {
LOG.debug("Filtering accession aliases by {}", aliasType);
LOG.trace("Filtering accession aliases by {}", aliasType);
CollectionUtils.filter(existingAliases, new Predicate<AccessionAlias>() {
@Override
public boolean evaluate(AccessionAlias alias) {
......@@ -1127,8 +1127,8 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
accessionAlias.setName(aa.name);
accessionAlias.setInstCode(aa.instCode);
accessionAlias.setUsedBy(aa.usedBy);
accessionAlias.setInstCode(StringUtils.trimToNull(aa.instCode));
accessionAlias.setUsedBy(StringUtils.trimToNull(aa.usedBy));
accessionAlias.setAliasType(aliasType != null ? aliasType : AliasType.getType(aa.type));
toSave.add(accessionAlias);
......
......@@ -365,8 +365,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
return;
}
if (LOG.isDebugEnabled())
LOG.debug("Updating {} bulk_size={}", className, ids.size());
LOG.debug("Updating {} bulk_size={}", className, ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>();
......@@ -386,9 +385,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
if (!queries.isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Indexing {} count={} of provided objects count={}", className, queries.size(), ids.size());
}
LOG.debug("Indexing {} count={} of provided objects count={}", className, queries.size(), ids.size());
elasticsearchTemplate.bulkIndex(queries);
}
}
......
......@@ -1760,7 +1760,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
pdcis.add(pdci);
});
LOG.info("Generated {} new PDCIs", pdcis.size());
LOG.info("Updated {} PDCIs", pdcis.size());
repoPdci.save(pdcis);
return pdcis;
}
......
......@@ -197,7 +197,7 @@ public class AccessionController extends RestController {
public @ResponseBody Object upsertInstituteAccession(@PathVariable("instCode") String instCode, @RequestBody JsonNode json) throws JsonProcessingException, IOException,
RESTApiException {
LOG.debug("Received:\n {}", json);
LOG.trace("Received:\n {}", json);
// User's permission to WRITE to this WIEWS institute are checked in
// BatchRESTService.
......
......@@ -15,6 +15,7 @@
*/
package org.genesys2.spring.config;
import org.elasticsearch.common.lang3.StringUtils;
import org.genesys.glis.v1.GlisRateLimiter;
import org.genesys.glis.v1.invoker.ApiClient;
import org.genesys.glis.v1.invoker.auth.HttpBasicAuth;
......@@ -41,22 +42,29 @@ public class GLISConfig {
private double glisRateLimit;
@Bean
public GlisRateLimiter glisRateLimiter() {
return new GlisRateLimiter(glisRateLimit);
public GlisRateLimiter glisRateLimiter(ApiClient client) {
if (client != null)
return new GlisRateLimiter(glisRateLimit);
else
return null;
}
@Bean
public ApiClient glisClient() {
ApiClient client = new ApiClient();
client.setDebugging("true".equals(System.getenv("GLIS_DEBUG")));
client.setBasePath(basePath);
// Configure HTTP basic authorization: glis_auth
HttpBasicAuth easySmtaAuth = (HttpBasicAuth) client.getAuthentication("easySmta");
easySmtaAuth.setUsername(username);
easySmtaAuth.setPassword(password);
System.err.println("GLIS at " + client.getBasePath() + " with username=" + username);
return client;
if (StringUtils.isNotBlank(username)) {
ApiClient client = new ApiClient();
client.setDebugging("true".equals(System.getenv("GLIS_DEBUG")));
client.setBasePath(basePath);
// Configure HTTP basic authorization: glis_auth
HttpBasicAuth easySmtaAuth = (HttpBasicAuth) client.getAuthentication("easySmta");
easySmtaAuth.setUsername(username);
easySmtaAuth.setPassword(password);
System.err.println("GLIS at " + client.getBasePath() + " with username=" + username);
return client;
} else {
return null;
}
}
}
......@@ -1179,3 +1179,49 @@ databaseChangeLog:
name: lastLogin
type: datetime(6)
tableName: user
- changeSet:
id: 1530443149000-2
author: mobreza
comment: Replace blanks with null and delete empty records
changes:
- sql:
sql: update accession set acceUrl = null where acceUrl = '';
- sql:
sql: update accession set acqDate = null where acqDate = '';
- sql:
sql: update accessionalias set usedBy = null where usedBy = '';
- sql:
sql: delete from accessionalias where name = '' and (usedBy is null or instCode is null);
- sql:
sql: update accessionbreeding set breederCode = NULL where breederCode = '';
- sql:
sql: update accessionbreeding set pedigree = NULL where pedigree = '';
- sql:
sql: delete from accessionbreeding where pedigree is null and breederCode is null;
- sql:
sql: update accessioncollect set collDate = null where collDate = '';
- sql:
sql: update accessioncollect set collSite = null where collSite = '';
- sql:
sql: update accessioncollect set collCode = null where collCode = '';
- sql:
sql: update accessioncollect set collNumb = null where collNumb = '';
- sql:
sql: update accessioncollect set collMissId = null where collMissId = '';
- sql:
sql: update accessioncollect set collName = null where collName = '';
- sql:
sql: update accessioncollect set collInstAddress = null where collInstAddress = '';
- sql:
sql: delete from accessioncollect where collDate = null and collSite = null and collCode = null and collNumb = null and collMissId = null and collName = null and collInstAddress = null and collSrc = null;
- sql:
sql: update accessionexchange set acceNumb = null where acceNumb = '';
- sql:
sql: update accessionexchange set donorInst = null where donorInst = '';
- sql:
sql: update accessionexchange set donorName = null where donorName = '';
- sql:
sql: delete from accessionexchange where acceNumb is null and donorInst is null and donorName is null;
- sql:
sql: delete from auditlog where classPk = (select id from classpk where classname='org.genesys2.server.model.genesys.PDCI');
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment