Skip to content

GitLab

Commit 6e69fe21 authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch 'accessionref-updating' into 'master' 

AccessionRefs can be manually re-linked, but are automatically cleared

See merge request genesys-pgr/genesys-server!261
parents 3aa5a897 4ee8ce1e
Hide whitespace changes
Inline Side-by-side
Showing with 425 additions and 354 deletions
src/main/java/org/genesys/catalog/service/DatasetService.java
View file @ 6e69fe21
......@@ -15,9 +15,13 @@
*/
package org.genesys.catalog.service;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.genesys.catalog.exceptions.NotFoundElement;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionRef;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.dataset.DatasetCreator;
import org.genesys.catalog.model.dataset.DatasetLocation;
......@@ -27,18 +31,14 @@ import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.UUID;
/**
* The Interface DatasetService.
*/
......@@ -262,8 +262,19 @@ public interface DatasetService {
*/
List<Object[]> lastPublished();
/**
* Rematch dataset accessions for all datasets.
*/
void rematchDatasetAccessions();
/**
* Rematch dataset accessions.
*
* @param dataset the dataset
* @return the dataset
*/
Dataset rematchDatasetAccessions(Dataset dataset);
/**
* Create DatasetCreator for Dataset.
*
......@@ -415,4 +426,12 @@ public interface DatasetService {
*/
DatasetLocation updateLocation(Dataset dataset, DatasetLocation input) throws NotFoundElement;
/**
* Clear accession references across all Datasets for specified accessions.
*
* @param accessions the accessions
* @return the int
*/
int clearAccessionRefs(Collection<Accession> accessions);
}
src/main/java/org/genesys/catalog/service/impl/DatasetServiceImpl.java
View file @ 6e69fe21
......@@ -23,23 +23,25 @@ import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import com.querydsl.core.types.Predicate;
import org.apache.commons.lang.StringUtils;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.catalog.exceptions.InvalidApiUsageException;
import org.genesys.catalog.exceptions.NotFoundElement;
import org.genesys.catalog.model.Partner;
import org.genesys2.server.model.genesys.AccessionRef;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.dataset.DatasetCreator;
import org.genesys.catalog.model.dataset.DatasetLocation;
import org.genesys.catalog.model.dataset.DatasetVersions;
import org.genesys.catalog.model.dataset.QDataset;
import org.genesys.catalog.model.dataset.*;
import org.genesys.catalog.model.filters.DatasetFilter;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.persistence.dataset.DatasetCreatorRepository;
......@@ -55,7 +57,7 @@ import org.genesys.filerepository.service.RepositoryService;
import org.genesys2.server.model.PublishState;
import org.genesys2.server.model.UserRole;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.AccessionRef;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.security.SecurityUtils;
import org.genesys2.server.service.DownloadService;
......@@ -64,16 +66,23 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.data.domain.*;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.acls.domain.BasePermission;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.collect.Lists;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate;
/**
......@@ -145,7 +154,7 @@ public class DatasetServiceImpl implements DatasetService {
dataset.setCrops(source.getCrops());
if (source.getAccessionRefs() != null && !source.getAccessionRefs().isEmpty()) {
dataset.setAccessionRefs(lookupMatchingAccessions(new HashSet<>(source.getAccessionRefs())));
dataset.setAccessionRefs(lookupMatchingAccessions(source.getAccessionRefs()));
}
copyDescriptors(dataset, source.getDescriptors());
......@@ -193,7 +202,7 @@ public class DatasetServiceImpl implements DatasetService {
LOG.info("Update AccessionIdentifiers. Input dataset: {}, input accessionRefs {}", dataset, accessionRefs);
dataset = loadDataset(dataset);
dataset.getAccessionRefs().clear();
dataset.setAccessionRefs(lookupMatchingAccessions(new HashSet<>(accessionRefs)));
dataset.setAccessionRefs(lookupMatchingAccessions(accessionRefs));
return lazyLoad(datasetRepository.save(dataset));
}
......@@ -466,7 +475,7 @@ public class DatasetServiceImpl implements DatasetService {
throw new NotFoundElement("Dataset doesn't exist");
}
loadedDataset.setState(PublishState.DRAFT);
loadedDataset.setAccessionRefs(lookupMatchingAccessions(new HashSet<>(accessionRefs)));
loadedDataset.setAccessionRefs(lookupMatchingAccessions(accessionRefs));
return lazyLoad(datasetRepository.save(loadedDataset));
}
......@@ -768,6 +777,7 @@ public class DatasetServiceImpl implements DatasetService {
* Looking for matching Accession and then sets that to AccessionRef
*
* @param accessionRefs the accessionRefs
*
* @return accessionRefs with matching accessions from Genesys
*/
private Set<AccessionRef> lookupMatchingAccessions(final Set<AccessionRef> accessionRefs) {
......@@ -776,20 +786,20 @@ public class DatasetServiceImpl implements DatasetService {
LOG.info("Found {} matches for {} identifiers", foundAccessions.size(), accessionRefs.size());
accessionRefs.forEach(accei -> {
accessionRefs.forEach(ref -> {
Accession foundAccession = foundAccessions.stream().filter(a -> {
return StringUtils.equalsIgnoreCase(a.getInstCode(), accei.getInstCode())
return StringUtils.equalsIgnoreCase(a.getInstCode(), ref.getInstCode())
&& (
// when DOI provided
(accei.getDoi() != null && a.getDoi() != null && StringUtils.equals(a.getDoi(), accei.getDoi()))
(ref.getDoi() != null && a.getDoi() != null && StringUtils.equals(a.getDoi(), ref.getDoi()))
// OR when without DOI
||
(StringUtils.equalsIgnoreCase(a.getAccessionNumber(), accei.getAcceNumb()) && StringUtils.equalsIgnoreCase(a.getGenus(), accei.getGenus()))
(StringUtils.equalsIgnoreCase(a.getAccessionNumber(), ref.getAcceNumb()) && StringUtils.equalsIgnoreCase(a.getGenus(), ref.getGenus()))
);
}).findFirst().orElse(null);
accei.setAccession(foundAccession);
ref.setAccession(foundAccession);
if (foundAccession == null) {
LOG.debug("No match for {}", accei);
LOG.debug("No match for {}", ref);
}
});
......@@ -799,14 +809,43 @@ public class DatasetServiceImpl implements DatasetService {
@Override
@Transactional(readOnly = false)
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void rematchDatasetAccessions() {
datasetRepository.findAll().forEach(dataset -> {
LOG.warn("Linking {} accessions with dataset {}", dataset.getAccessionCount(), dataset.getId());
dataset.setAccessionRefs(lookupMatchingAccessions(new HashSet<>(dataset.getAccessionRefs())));
datasetRepository.save(dataset);
rematchDatasetAccessions(dataset);
});
}
@Override
@Transactional(readOnly = false)
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#dataset, 'write')")
public Dataset rematchDatasetAccessions(Dataset dataset) {
dataset = datasetRepository.findByUuid(dataset.getUuid());
LOG.warn("Linking {} accessions with dataset {}", dataset.getAccessionCount(), dataset.getId());
dataset.setAccessionRefs(lookupMatchingAccessions(dataset.getAccessionRefs()));
return datasetRepository.save(dataset);
}
@Override
@Transactional(readOnly = false, propagation = Propagation.REQUIRED, isolation = Isolation.READ_UNCOMMITTED)
public int clearAccessionRefs(Collection<Accession> accessions) {
Set<UUID> uuids = accessions.stream().map(a -> a.getUuid()).collect(Collectors.toSet());
final Iterable<Dataset> datasets = datasetRepository.findAll(dataset.accessionRefs.any().accession.in(accessions));
AtomicInteger updates = new AtomicInteger(0);
datasets.forEach(dataset -> {
dataset.getAccessionRefs().stream()
// filter for matching accessions
.filter(ref -> ref.getAccession() != null && uuids.contains(ref.getAccession().getUuid()))
// clear accession reference
.forEach(ref -> {
updates.incrementAndGet();
ref.setAccession(null);
});
});
datasetRepository.save(datasets);
return updates.get();
}
/**
* {@inheritDoc}
......
src/main/java/org/genesys2/server/component/elastic/ElasticJPAListener.java
View file @ 6e69fe21
......@@ -45,7 +45,7 @@ import org.springframework.stereotype.Component;
* {@link ElasticReindexProcessor}.
*/
@Aspect
@Component
@Component("elasticJpaListener")
public class ElasticJPAListener {
private final static Logger LOG = LoggerFactory.getLogger(ElasticJPAListener.class);
......@@ -149,7 +149,7 @@ public class ElasticJPAListener {
} else {
Class<?> clazz = toReindex.getClass();
if (isIndexed(clazz)) {
LOG.trace("Reindexing {} {}", clazz.getName(), toReindex);
LOG.trace("Scheduling reindexing of {} {}", clazz.getName(), toReindex);
if (toReindex instanceof BasicModel) {
BasicModel entity = (BasicModel) toReindex;
elasticReindexQueue.add(new ElasticReindex(clazz.getName(), entity.getId()));
......
src/main/java/org/genesys2/server/component/elastic/ElasticReindexProcessor.java
View file @ 6e69fe21
......@@ -40,7 +40,7 @@ import org.springframework.stereotype.Component;
* with 2000ms delay measured from the completion time of each preceding
* invocation.
*/
@Component
@Component("elasticReindexProcessor")
public class ElasticReindexProcessor {
/** The Constant LOG. */
......@@ -54,6 +54,10 @@ public class ElasticReindexProcessor {
@Autowired
private ElasticsearchService elasticsearch;
public ElasticReindexProcessor() {
System.err.println("Making ElasticReindexProcessor");
}
/**
* Process queues.
*/
......
src/main/java/org/genesys2/server/model/genesys/AccessionData.java
View file @ 6e69fe21
......@@ -501,23 +501,9 @@ public abstract class AccessionData extends AuditedVersionedModel implements IdU
public String getInstCode() {
return this.instituteCode;
}
// FIXME This seems a bit of a hack
public boolean getSgsv() {
return this.duplSiteStr != null && this.duplSiteStr.contains("NOR051");
}
// TODO Check if needed
public AccessionRef toAccessionRef(){
AccessionRef res = new AccessionRef();
res.setInstCode(getInstCode());
res.setGenus(getGenus());
res.setDoi(getDoi());
res.setAcceNumb(getAccessionNumber());
res.setSpecies(getTaxonomy().getSpecies());
res.setAccession((Accession) this);
return res;
}
}
src/main/java/org/genesys2/server/model/genesys/AccessionRef.java
View file @ 6e69fe21
......@@ -68,6 +68,23 @@ public class AccessionRef implements AccessionIdentifier3, Serializable {
@JsonView({ JsonViews.Public.class })
private Accession accession;
public AccessionRef() {
}
public AccessionRef(Accession accession) {
this.instCode = accession.getInstitute().getCode();
this.acceNumb = accession.getAccessionNumber();
this.genus = accession.getTaxonomy().getGenus();
this.doi = accession.getDoi();
this.accession = accession;
}
public AccessionRef(String instCode, String acceNumb, String genus, String doi) {
this.instCode = instCode;
this.acceNumb = acceNumb;
this.genus = genus;
this.doi = doi;
}
/**
* Gets the doi.
......@@ -271,7 +288,7 @@ public class AccessionRef implements AccessionIdentifier3, Serializable {
}
return true;
}
@Override
public String toString() {
return "AccessionRef " + instCode + " " + acceNumb + " " + genus;
......
src/main/java/org/genesys2/server/model/impl/Subset.java
View file @ 6e69fe21
......@@ -49,7 +49,6 @@ import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonView;
// TODO: Auto-generated Javadoc
/**
* The Class Subset.
*/
......@@ -99,7 +98,7 @@ public class Subset extends UuidModel implements AclAwareModel, SelfCleaning {
indexes = { @Index(columnList = "subsetId, instCode, acceNumb"), @Index(columnList = "subsetId, genus") })
@Field(type = FieldType.Object)
@JsonView({ JsonViews.Public.class })
private List<AccessionRef> accessionRefs;
private Set<AccessionRef> accessionRefs;
/** The accession count. */
@Column(name = "accession_count", nullable = false)
......@@ -276,7 +275,7 @@ public class Subset extends UuidModel implements AclAwareModel, SelfCleaning {
*
* @return the accessionRefs
*/
public List<AccessionRef> getAccessionRefs() {
public Set<AccessionRef> getAccessionRefs() {
return accessionRefs;
}
......@@ -285,7 +284,7 @@ public class Subset extends UuidModel implements AclAwareModel, SelfCleaning {
*
* @param accessionRefs the new accessionRefs
*/
public void setAccessionRefs(List<AccessionRef> accessions) {
public void setAccessionRefs(Set<AccessionRef> accessions) {
this.accessionRefs = accessions;
}
......
src/main/java/org/genesys2/server/mvc/admin/AdminController.java
View file @ 6e69fe21
......@@ -28,11 +28,15 @@ import javax.persistence.PersistenceContext;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.lang3.ObjectUtils;
import org.genesys.catalog.service.DatasetService;
import org.genesys.worldclim.WorldClimUtil;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionGeo;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.genesys.QAccessionId;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.persistence.PDCIRepository;
......@@ -45,6 +49,7 @@ import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.worker.AccessionCounter;
import org.genesys2.server.service.worker.AccessionProcessor;
import org.genesys2.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys2.server.service.worker.InstituteUpdater;
......@@ -67,12 +72,13 @@ import org.xml.sax.SAXException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.querydsl.jpa.impl.JPAQueryFactory;
@Controller
@RequestMapping("/admin")
@PreAuthorize("hasRole('ADMINISTRATOR')")
public class AdminController {
public static final Logger LOG = LoggerFactory.getLogger(AdminController.class);
@Autowired
......@@ -114,6 +120,12 @@ public class AdminController {
@PersistenceContext
private EntityManager entityManager;
@Autowired
private JPAQueryFactory jpaQueryFactory;
@Autowired
private AccessionCounter accessionCounter;
ObjectMapper mapper = new ObjectMapper();
@Autowired
......@@ -124,10 +136,10 @@ public class AdminController {
@Autowired
private AccessionProcessor accessionProcessor;
@Autowired
private PDCIRepository pdciRepository;
@RequestMapping("/")
public String root(Model model) {
return "/admin/index";
......@@ -231,9 +243,9 @@ public class AdminController {
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "action=institute-pdci")
public String updatePDCI() {
for (FaoInstitute institute: instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
for (FaoInstitute institute : instituteService.listActive(new PageRequest(0, Integer.MAX_VALUE))) {
LOG.info("Updating PDCI for {}", institute.getCode());
genesysService.updatePDCI(institute);
accessionCounter.recountInstitute(institute);
}
return "redirect:/admin/";
......@@ -241,12 +253,12 @@ public class AdminController {
@RequestMapping(value = "/pdci", method = RequestMethod.POST, params = "action=filtered-pdci")
public String updateFilteredPDCI(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter);
accessionProcessor.apply(filter, (accessions) -> {
// Everything here is executed within a @Transaction(readOnly = false) context
if (accessions == null) {
return accessions;
......@@ -257,28 +269,37 @@ public class AdminController {
PDCI pdci = accessionId.getPdci();
// create new PDCI if missing
PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession);
// if PDCI was missing, link it with accession
if (pdci == null) {
LOG.trace("Assigning new PDCI for {}", accession);
resultingPdci.setAccession(accessionId);
accessionId.setPdci(resultingPdci);
} else {
LOG.trace("Updating PDCI for {}", accession);
accessionId.setPdci(pdciRepository.save(resultingPdci));
}
updateAccessionPDCI(accession, resultingPdci);
accessionCounter.recountInstitute(accession.getInstitute());
});
LOG.debug("Updated {} PDCI entries", accessions.size());
return accessionRepository.save(accessions);
return accessions;
});
return "redirect:/admin/";
}
private void updateAccessionPDCI(Accession accession, PDCI pdci) {
AccessionId accessionId = accession.getAccessionId();
pdci.setAccession(accessionId);
pdciRepository.save(pdci);
if (accessionId.getPdci() == null) {
accessionId.setPdci(pdci);
LOG.trace("Assigning new PDCI for {}", accession);
jpaQueryFactory.update(QAccessionId.accessionId).where(QAccessionId.accessionId.eq(accessionId)).set(QAccessionId.accessionId.pdci, pdci).execute();
// jpaQueryFactory.update(QAccession.accession).where(QAccession.accession.eq(accession)).set(QAccession.accession.pdci,
// pdci).execute();
}
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "action=tile-index")
public String updateTileIndex(@RequestParam(name = "filter") String filters) throws JsonParseException, JsonMappingException, IOException {
AccessionFilter filter = mapper.readValue(filters, AccessionFilter.class);
LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter);
......@@ -288,19 +309,35 @@ public class AdminController {
}
accessions.forEach(accession -> {
AccessionGeo geo = accession.getAccessionId().getGeo();
if (geo != null) {
accession.setTileIndex(WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude()));
Long index = WorldClimUtil.getTileIndex(5, geo.getLongitude(), geo.getLatitude());
if (ObjectUtils.compare(index, accession.getTileIndex()) != 0) {
updateAccessionTileIndex(accession, index);
}
} else {
if (accession.getTileIndex() != null) {
updateAccessionTileIndex(accession, null);
}
}
});
LOG.debug("Regenerated {} tileIndexes", accessions.size());
return accessionRepository.save(accessions);
return accessions;
});
return "redirect:/admin/";
}
private void updateAccessionTileIndex(Accession accession, Long index) {
if (index == null) {
jpaQueryFactory.update(QAccession.accession).where(QAccession.accession.eq(accession)).setNull(QAccession.accession.tileIndex).execute();
} else {
jpaQueryFactory.update(QAccession.accession).where(QAccession.accession.eq(accession)).set(QAccession.accession.tileIndex, index).execute();
}
}
@RequestMapping(value = "/admin-action", method = RequestMethod.POST, params = "georegion")
public String updateGeoReg() throws IOException, ParserConfigurationException, SAXException {
geoRegionService.updateGeoRegionData();
......@@ -308,7 +345,8 @@ public class AdminController {
}
/**
* Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract the number from the ACCNUMB)
* Scan AccessionData table and convert ACCENUMB to ACCENUMBNUMB (extract the
* number from the ACCNUMB)
*
* @return
*/
......@@ -364,7 +402,7 @@ public class AdminController {
}
@PostMapping(value = "/clear-dois")
public String clearDois(){
public String clearDois() {
LOG.info("Clear DOIs");
genesysService.removeDOIs();
return "redirect:/admin/";
......
src/main/java/org/genesys2/server/persistence/AccessionRepositoryCustom.java
View file @ 6e69fe21
......@@ -33,6 +33,8 @@ public interface AccessionRepositoryCustom {
List<AccessionData> findActiveAndHistoric(Collection<UUID> accessionUuids);
List<Accession> find(Collection<UUID> accessionUuids);
List<Accession> find(List<Accession> accessions);
List<Accession> findById(List<? extends AccessionIdentifier3> identifiers);
......
src/main/java/org/genesys2/server/persistence/AccessionRepositoryCustomImpl.java
View file @ 6e69fe21
......@@ -45,6 +45,7 @@ import org.genesys2.server.service.impl.NonUniqueAccessionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.data.domain.Pageable;