Commit 46cf8694 authored by Matija Obreza's avatar Matija Obreza

Merge branch 'accessionref-performance' into 'master'

Accessionref performance

See merge request genesys-pgr/genesys-server!311
parents 665b44af 0b9757c6
......@@ -15,14 +15,32 @@
*/
package org.genesys.catalog.model.dataset;
import javax.persistence.*;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonView;
import javax.persistence.Cacheable;
import javax.persistence.CascadeType;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.JoinTable;
import javax.persistence.Lob;
import javax.persistence.ManyToMany;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.OrderColumn;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.persistence.Table;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.genesys.blocks.auditlog.annotations.Audited;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.model.Publishable;
......@@ -35,13 +53,15 @@ import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.service.PublishValidationInterface;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys2.server.model.PublishState;
import org.genesys2.server.model.genesys.AccessionRef;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.util.MCPDUtil;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonView;
/**
* {@link Dataset} captures the metadata information as a snapshot. Multiple
* versions of the versions may exist.
......@@ -76,13 +96,10 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
private Partner owner;
/** The accession identifiers. */
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "dataset_accessions", joinColumns = @JoinColumn(name = "datasetId"),
// index
indexes = { @Index(columnList = "datasetId, instCode, acceNumb"), @Index(columnList = "datasetId, genus") })
@OneToMany(mappedBy = "dataset", cascade = { CascadeType.REFRESH, CascadeType.DETACH, CascadeType.REMOVE }, orphanRemoval = true, fetch = FetchType.LAZY)
@Field(type = FieldType.Object)
@JsonIgnore
private Set<AccessionRef> accessionRefs;
private List<DatasetAccessionRef> accessionRefs;
/** The descriptors. */
@ManyToMany(fetch = FetchType.LAZY, cascade = {})
......@@ -127,7 +144,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
/** The accession count. */
// Number of accessions in the {@link #accessionRefs} list
@Column(name = "accessions")
private Integer accessionCount;
private int accessionCount = 0;
/** The descriptor count. */
// Number of descriptors in the {@link #descriptors} list
......@@ -195,9 +212,9 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
@PreUpdate
@PrePersist
private void preupdate() {
if (accessionRefs != null) {
this.accessionCount = accessionRefs.size();
}
// if (accessionRefs != null) {
// this.accessionCount = accessionRefs.size();
// }
if (descriptors != null) {
this.descriptorCount = descriptors.size();
}
......@@ -524,7 +541,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
*
* @return the accessionRefs
*/
public final Set<AccessionRef> getAccessionRefs() {
public final List<DatasetAccessionRef> getAccessionRefs() {
return accessionRefs;
}
......@@ -533,7 +550,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
*
* @param accessionRefs the accessionRefs to set
*/
public final void setAccessionRefs(final Set<AccessionRef> accessionRefs) {
protected final void setAccessionRefs(final List<DatasetAccessionRef> accessionRefs) {
this.accessionRefs = accessionRefs;
}
......@@ -578,9 +595,14 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
*
* @return the accession count
*/
public Integer getAccessionCount() {
public int getAccessionCount() {
return accessionCount;
}
public void setAccessionCount(int count) {
this.accessionCount = count;
}
/**
* Gets the descriptor count.
......@@ -608,4 +630,5 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
public void setCrops(final Set<String> crops) {
this.crops = crops;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.model.dataset;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import org.genesys2.server.model.genesys.AccessionRef;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* The Class DatasetAccessionRef.
*/
@Entity
@Table(name = "dataset_accessions", indexes = { @Index(columnList = "datasetId, instCode, acceNumb"), @Index(columnList = "datasetId, genus") })
public class DatasetAccessionRef extends AccessionRef {
private static final long serialVersionUID = -8155179462312813571L;
@Id
@ManyToOne(cascade= {}, optional=false, fetch=FetchType.LAZY)
@JoinColumn(name = "datasetId")
@JsonIgnore
private Dataset dataset;
/**
* Instantiates a new dataset accession ref.
*/
public DatasetAccessionRef() {
}
/**
* Instantiates a new dataset accession ref.
*
* @param ref the ref
*/
public DatasetAccessionRef(AccessionRef ref) {
this.instCode=ref.getInstCode();
this.acceNumb=ref.getAcceNumb();
this.genus=ref.getGenus();
this.species=ref.getSpecies();
this.doi=ref.getDoi();
this.accession = ref.getAccession();
}
/**
* Gets the dataset.
*
* @return the dataset
*/
public Dataset getDataset() {
return dataset;
}
/**
* Sets the dataset.
*
* @param dataset the new dataset
*/
public void setDataset(Dataset dataset) {
this.dataset = dataset;
}
/* (non-Javadoc)
* @see org.genesys2.server.model.genesys.AccessionRef#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!super.equals(obj))
return false;
if (getClass() != obj.getClass())
return false;
DatasetAccessionRef other = (DatasetAccessionRef) obj;
if (dataset == null) {
if (other.dataset != null)
return false;
} else if (!dataset.getUuid().equals(other.dataset.getUuid()))
return false;
return true;
}
}
......@@ -19,13 +19,13 @@ import java.util.HashSet;
import java.util.Set;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys2.server.model.genesys.AccessionRef;
import org.genesys.catalog.model.dataset.DatasetAccessionRef;
import org.genesys.catalog.model.dataset.QDatasetAccessionRef;
import org.genesys2.server.service.filter.AccessionFilter;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.dsl.SetPath;
import org.genesys2.server.model.genesys.QAccessionRef;
import org.genesys2.server.service.filter.AccessionFilter;
import com.querydsl.core.types.dsl.ListPath;
/**
* The Class AccessionRefFilter.
......@@ -59,10 +59,10 @@ public class AccessionRefFilter {
* @param accessionRefs the accession identifiers
* @return the predicate
*/
public Predicate buildQuery(final SetPath<AccessionRef, QAccessionRef> accessionRefs) {
public Predicate buildQuery(final ListPath<DatasetAccessionRef, QDatasetAccessionRef> accessionRefs) {
final BooleanBuilder and = new BooleanBuilder();
final QAccessionRef accessionRef = accessionRefs.any();
final QDatasetAccessionRef accessionRef = accessionRefs.any();
if (doi != null && !doi.isEmpty()) {
and.and(accessionRef.doi.in(doi));
}
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.persistence.dataset;
import java.util.List;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.dataset.DatasetAccessionRef;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
/**
* The DatasetAccessionRefRepository.
*/
@Repository
public interface DatasetAccessionRefRepository {
/**
* Delete all.
*/
void deleteAll();
/**
* Save D.
*
* @param entities the entities
* @return the iterable
*/
Iterable<DatasetAccessionRef> save(Iterable<DatasetAccessionRef> entities);
/**
* Update records.
*
* @param entities the existing DatasetAccessionRefs to update
*/
void update(Iterable<DatasetAccessionRef> entities);
/**
* Delete for dataset.
*
* @param dataset the dataset
* @return the long
*/
long deleteForDataset(Dataset dataset);
/**
* @param dataset
* @return
*/
List<DatasetAccessionRef> findAll(Dataset dataset);
/**
* Find all.
*
* @param dataset the dataset
* @param page the page
* @return the page
*/
Page<DatasetAccessionRef> findAll(Dataset dataset, Pageable page);
/**
* Count by dataset.
*
* @param dataset the dataset
* @return the long
*/
long countByDataset(Dataset dataset);
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.persistence.dataset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.dataset.DatasetAccessionRef;
import org.genesys.catalog.model.dataset.QDatasetAccessionRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Assert;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
/**
* The Class DatasetAccessionRefRepositoryCustomImpl.
*/
@Repository
public class DatasetAccessionRefRepositoryCustomImpl implements DatasetAccessionRefRepository {
private final static Logger LOG = LoggerFactory.getLogger(DatasetAccessionRefRepository.class);
private static final Predicate[] EMPTY_PREDICATE_ARRAY = new Predicate[] {};
@Autowired
private JPAQueryFactory jpaQueryFactory;
@PersistenceContext
private EntityManager em;
@Override
@Transactional(readOnly = true)
public List<DatasetAccessionRef> findAll(Dataset dataset) {
return jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset)).fetch();
}
@Override
@Transactional
public void update(Iterable<DatasetAccessionRef> entities) {
for (DatasetAccessionRef entity: entities) {
em.merge(entity);
}
}
/* (non-Javadoc)
* @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#save(java.lang.Iterable)
*/
@Override
@Transactional
public Iterable<DatasetAccessionRef> save(Iterable<DatasetAccessionRef> entities) {
final CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
final AtomicInteger batchCounter = new AtomicInteger(0);
Iterables.partition(entities, 1000).forEach(batch -> {
LOG.info("Batch {} with size {}", batchCounter.incrementAndGet(), batch.size());
CriteriaQuery<DatasetAccessionRef> cq = criteriaBuilder.createQuery(DatasetAccessionRef.class);
Root<DatasetAccessionRef> root = cq.from(DatasetAccessionRef.class);
cq.distinct(true);
// cq.select(root);
// Path<Object> theDoi = root.get("doi");
Path<Object> theDataset = root.get("dataset");
Path<Object> theInstCode = root.get("instCode");
Path<Object> theAcceNumb = root.get("acceNumb");
Path<Object> theGenus = root.get("genus");
List<Predicate> restrictions = new ArrayList<Predicate>();
// A lot of .. (instCode=? and acceNumb=? and genus=?)
for (DatasetAccessionRef ref : batch) {
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(theDataset, ref.getDataset()), criteriaBuilder.equal(theInstCode, ref
.getInstCode()), criteriaBuilder.equal(theAcceNumb, ref
.getAcceNumb()), criteriaBuilder.equal(theGenus, ref.getGenus())));
}
cq.where(criteriaBuilder.or(restrictions.toArray(EMPTY_PREDICATE_ARRAY)));
List<DatasetAccessionRef> existing = em.createQuery(cq).getResultList();
LOG.debug("Got {} existing records", existing.size());
for (DatasetAccessionRef entity : batch) {
if (existing.contains(entity)) {
em.merge(entity);
} else {
em.persist(entity);
}
}
});
LOG.info("Done persisting {} AccessionRef batches", batchCounter.get());
return entities;
}
/* (non-Javadoc)
* @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#deleteAll()
*/
@Override
@Transactional
public void deleteAll() {
for (DatasetAccessionRef entity : jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).fetch()) {
delete(entity);
}
}
/**
* Delete.
*
* @param entity the entity
*/
@Transactional
private void delete(Object entity) {
Assert.notNull(entity, "The entity must not be null!");
em.remove(em.contains(entity) ? entity : em.merge(entity));
}
/* (non-Javadoc)
* @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#deleteForDataset(org.genesys.catalog.model.dataset.Dataset)
*/
@Override
@Transactional
public long deleteForDataset(Dataset dataset) {
return jpaQueryFactory.delete(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset)).execute();
}
/* (non-Javadoc)
* @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#findAll(org.genesys.catalog.model.dataset.Dataset, org.springframework.data.domain.Pageable)
*/
@Override
public Page<DatasetAccessionRef> findAll(Dataset dataset, Pageable page) {
JPAQuery<DatasetAccessionRef> q = jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset));
long total = q.fetchCount();
List<DatasetAccessionRef> list = q.orderBy(QDatasetAccessionRef.datasetAccessionRef.accession.seqNo.asc(), QDatasetAccessionRef.datasetAccessionRef.acceNumb.asc()).leftJoin(QDatasetAccessionRef.datasetAccessionRef.accession).fetchJoin().offset(page.getOffset()).limit(page.getPageSize()).fetch();
return new PageImpl<>(list, page, total);
}
/* (non-Javadoc)
* @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#countByDataset(org.genesys.catalog.model.dataset.Dataset)
*/
@Override
public long countByDataset(Dataset dataset) {
return jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset)).fetchCount();
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.server.rest.model;
/**
* The Class PermissionJson.
*/
public class PermissionJson {
private Long sid;
private String authority;
private long oid;
private String clazz;
private boolean create;
private boolean read;
private boolean write;
private boolean delete;
private boolean manage;
@Override
public String toString() {
return "PJ oid=" + oid + " class=" + clazz + " (sid=" + sid + " OR authority=" + authority + ")";
}
/**
* Sets the sid.
*
* @param sid the new sid
*/
public void setSid(final Long sid) {
this.sid = sid;
}
/**
* Gets the sid.
*
* @return the sid
*/
public Long getSid() {
return sid;
}
/**
* Sets the authority.
*
* @param authority the new authority
*/
public void setAuthority(final String authority) {
this.authority = authority;
}