Reworked AccessionRefs for subsets and datasets

- No longer an embedded collection
- Don't deepLoad when starting work in @Transactional

src/main/java/org/genesys/catalog/model/dataset/Dataset.java

@@ -15,14 +15,32 @@
  */
 package org.genesys.catalog.model.dataset;
 
-import javax.persistence.*;
-import javax.validation.constraints.NotNull;
-import javax.validation.constraints.Size;
 import java.util.List;
 import java.util.Set;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonView;
+import javax.persistence.Cacheable;
+import javax.persistence.CascadeType;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.FetchType;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.JoinTable;
+import javax.persistence.Lob;
+import javax.persistence.ManyToMany;
+import javax.persistence.ManyToOne;
+import javax.persistence.OneToMany;
+import javax.persistence.OrderColumn;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Table;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Size;
+
 import org.genesys.blocks.auditlog.annotations.Audited;
 import org.genesys.blocks.model.JsonViews;
 import org.genesys.blocks.model.Publishable;
@@ -35,13 +53,15 @@ import org.genesys.catalog.model.traits.Descriptor;
 import org.genesys.catalog.service.PublishValidationInterface;
 import org.genesys.filerepository.model.RepositoryFile;
 import org.genesys2.server.model.PublishState;
-import org.genesys2.server.model.genesys.AccessionRef;
 import org.genesys2.server.model.impl.Crop;
 import org.genesys2.util.MCPDUtil;
 import org.springframework.data.elasticsearch.annotations.Document;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonView;
+
 /**
  * {@link Dataset} captures the metadata information as a snapshot. Multiple
  * versions of the versions may exist.
@@ -76,13 +96,10 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	private Partner owner;
 
 	/** The accession identifiers. */
-	@ElementCollection(fetch = FetchType.LAZY)
-	@CollectionTable(name = "dataset_accessions", joinColumns = @JoinColumn(name = "datasetId"),
-			// index
-			indexes = { @Index(columnList = "datasetId, instCode, acceNumb"), @Index(columnList = "datasetId, genus") })
+	@OneToMany(mappedBy = "dataset", cascade = { CascadeType.REFRESH, CascadeType.DETACH, CascadeType.REMOVE }, orphanRemoval = true, fetch = FetchType.LAZY)
 	@Field(type = FieldType.Object)
 	@JsonIgnore
-	private Set<AccessionRef> accessionRefs;
+	private List<DatasetAccessionRef> accessionRefs;
 
 	/** The descriptors. */
 	@ManyToMany(fetch = FetchType.LAZY, cascade = {})
@@ -127,7 +144,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	/** The accession count. */
 	// Number of accessions in the {@link #accessionRefs} list
 	@Column(name = "accessions")
-	private Integer accessionCount;
+	private int accessionCount = 0;
 
 	/** The descriptor count. */
 	// Number of descriptors in the {@link #descriptors} list
@@ -195,9 +212,9 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	@PreUpdate
 	@PrePersist
 	private void preupdate() {
-		if (accessionRefs != null) {
-			this.accessionCount = accessionRefs.size();
-		}
+//		if (accessionRefs != null) {
+//			this.accessionCount = accessionRefs.size();
+//		}
 		if (descriptors != null) {
 			this.descriptorCount = descriptors.size();
 		}
@@ -524,7 +541,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	 *
 	 * @return the accessionRefs
 	 */
-	public final Set<AccessionRef> getAccessionRefs() {
+	public final List<DatasetAccessionRef> getAccessionRefs() {
 		return accessionRefs;
 	}
 
@@ -533,7 +550,7 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	 *
 	 * @param accessionRefs the accessionRefs to set
 	 */
-	public final void setAccessionRefs(final Set<AccessionRef> accessionRefs) {
+	protected final void setAccessionRefs(final List<DatasetAccessionRef> accessionRefs) {
 		this.accessionRefs = accessionRefs;
 	}
 
@@ -578,9 +595,14 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	 *
 	 * @return the accession count
 	 */
-	public Integer getAccessionCount() {
+	public int getAccessionCount() {
 		return accessionCount;
 	}
+	
+
+	public void setAccessionCount(int count) {
+		this.accessionCount = count;
+	}
 
 	/**
 	 * Gets the descriptor count.
@@ -608,4 +630,5 @@ public class Dataset extends UuidModel implements Publishable, SelfCleaning, Pub
 	public void setCrops(final Set<String> crops) {
 		this.crops = crops;
 	}
+
 }

src/main/java/org/genesys/catalog/model/dataset/DatasetAccessionRef.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.genesys.catalog.model.dataset;
+
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
+
+import org.genesys2.server.model.genesys.AccessionRef;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+/**
+ * The Class DatasetAccessionRef.
+ */
+@Entity
+@Table(name = "dataset_accessions", indexes = { @Index(columnList = "datasetId, instCode, acceNumb"), @Index(columnList = "datasetId, genus") })
+public class DatasetAccessionRef extends AccessionRef {
+
+	private static final long serialVersionUID = -8155179462312813571L;
+
+	@Id
+	@ManyToOne(cascade= {}, optional=false, fetch=FetchType.LAZY)
+	@JoinColumn(name = "datasetId")
+	@JsonIgnore
+	private Dataset dataset;
+	
+	/**
+	 * Instantiates a new dataset accession ref.
+	 */
+	public DatasetAccessionRef() {
+	}
+	
+	/**
+	 * Instantiates a new dataset accession ref.
+	 *
+	 * @param ref the ref
+	 */
+	public DatasetAccessionRef(AccessionRef ref) {
+		this.instCode=ref.getInstCode();
+		this.acceNumb=ref.getAcceNumb();
+		this.genus=ref.getGenus();
+		this.species=ref.getSpecies();
+		this.doi=ref.getDoi();
+		this.accession = ref.getAccession();
+	}
+
+	/**
+	 * Gets the dataset.
+	 *
+	 * @return the dataset
+	 */
+	public Dataset getDataset() {
+		return dataset;
+	}
+
+	/**
+	 * Sets the dataset.
+	 *
+	 * @param dataset the new dataset
+	 */
+	public void setDataset(Dataset dataset) {
+		this.dataset = dataset;
+	}
+	
+
+	/* (non-Javadoc)
+	 * @see org.genesys2.server.model.genesys.AccessionRef#equals(java.lang.Object)
+	 */
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		DatasetAccessionRef other = (DatasetAccessionRef) obj;
+		if (dataset == null) {
+			if (other.dataset != null)
+				return false;
+		} else if (!dataset.getUuid().equals(other.dataset.getUuid()))
+			return false;
+		return true;
+	}
+}

src/main/java/org/genesys/catalog/model/filters/AccessionRefFilter.java

@@ -19,13 +19,13 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.genesys.blocks.model.filters.StringFilter;
-import org.genesys2.server.model.genesys.AccessionRef;
+import org.genesys.catalog.model.dataset.DatasetAccessionRef;
+import org.genesys.catalog.model.dataset.QDatasetAccessionRef;
+import org.genesys2.server.service.filter.AccessionFilter;
 
 import com.querydsl.core.BooleanBuilder;
 import com.querydsl.core.types.Predicate;
-import com.querydsl.core.types.dsl.SetPath;
-import org.genesys2.server.model.genesys.QAccessionRef;
-import org.genesys2.server.service.filter.AccessionFilter;
+import com.querydsl.core.types.dsl.ListPath;
 
 /**
  * The Class AccessionRefFilter.
@@ -59,10 +59,10 @@ public class AccessionRefFilter {
 	 * @param accessionRefs the accession identifiers
 	 * @return the predicate
 	 */
-	public Predicate buildQuery(final SetPath<AccessionRef, QAccessionRef> accessionRefs) {
+	public Predicate buildQuery(final ListPath<DatasetAccessionRef, QDatasetAccessionRef> accessionRefs) {
 		final BooleanBuilder and = new BooleanBuilder();
 
-		final QAccessionRef accessionRef = accessionRefs.any();
+		final QDatasetAccessionRef accessionRef = accessionRefs.any();
 		if (doi != null && !doi.isEmpty()) {
 			and.and(accessionRef.doi.in(doi));
 		}

src/main/java/org/genesys/catalog/persistence/dataset/DatasetAccessionRefRepository.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.genesys.catalog.persistence.dataset;
+
+import org.genesys.catalog.model.dataset.Dataset;
+import org.genesys.catalog.model.dataset.DatasetAccessionRef;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.stereotype.Repository;
+
+/**
+ * The DatasetAccessionRefRepository.
+ */
+@Repository
+public interface DatasetAccessionRefRepository {
+
+	/**
+	 * Delete all.
+	 */
+	void deleteAll();
+	
+	/**
+	 * Save D.
+	 *
+	 * @param entities the entities
+	 * @return the iterable
+	 */
+	Iterable<DatasetAccessionRef> save(Iterable<DatasetAccessionRef> entities);
+
+	/**
+	 * Delete for dataset.
+	 *
+	 * @param dataset the dataset
+	 * @return the long
+	 */
+	long deleteForDataset(Dataset dataset);
+	
+	/**
+	 * Find all.
+	 *
+	 * @param dataset the dataset
+	 * @param page the page
+	 * @return the page
+	 */
+	Page<DatasetAccessionRef> findAll(Dataset dataset, Pageable page);
+
+	/**
+	 * Count by dataset.
+	 *
+	 * @param dataset the dataset
+	 * @return the long
+	 */
+	long countByDataset(Dataset dataset);
+
+}

src/main/java/org/genesys/catalog/persistence/dataset/DatasetAccessionRefRepositoryCustomImpl.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.genesys.catalog.persistence.dataset;
+
+import java.util.List;
+
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+
+import org.genesys.catalog.model.dataset.Dataset;
+import org.genesys.catalog.model.dataset.DatasetAccessionRef;
+import org.genesys.catalog.model.dataset.QDatasetAccessionRef;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.Assert;
+
+import com.google.common.collect.Iterables;
+import com.querydsl.core.types.dsl.BooleanExpression;
+import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
+
+/**
+ * The Class DatasetAccessionRefRepositoryCustomImpl.
+ */
+@Repository
+public class DatasetAccessionRefRepositoryCustomImpl implements DatasetAccessionRefRepository {
+
+	@Autowired
+	private JPAQueryFactory jpaQueryFactory;
+
+	@PersistenceContext
+	private EntityManager em;
+
+	/* (non-Javadoc)
+	 * @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#save(java.lang.Iterable)
+	 */
+	@Override
+	public Iterable<DatasetAccessionRef> save(Iterable<DatasetAccessionRef> entities) {
+		Iterables.partition(entities, 50).forEach(batch -> {
+			System.err.println("Batch size " + batch.size());
+			JPAQuery<DatasetAccessionRef> q = jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef);
+			for (DatasetAccessionRef ref : entities) {
+				BooleanExpression p = QDatasetAccessionRef.datasetAccessionRef.dataset.eq(ref.getDataset()).and(
+					QDatasetAccessionRef.datasetAccessionRef.instCode.eq(ref.getInstCode())).and(
+						QDatasetAccessionRef.datasetAccessionRef.acceNumb.eq(ref.getAcceNumb())).and(
+							QDatasetAccessionRef.datasetAccessionRef.genus.eq(ref.getGenus()));
+				q.where(p);
+			}
+			List<DatasetAccessionRef> existing = q.fetch();
+			for (DatasetAccessionRef entity : entities) {
+				if (existing.contains(entity)) {
+					em.merge(entity);
+				} else {
+					em.persist(entity);
+				}
+			}
+		});
+		return entities;
+	}
+	
+	
+	/* (non-Javadoc)
+	 * @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#deleteAll()
+	 */
+	@Override
+	@Transactional
+	public void deleteAll() {
+		for (DatasetAccessionRef entity : jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).fetch()) {
+			delete(entity);
+		}
+	}
+	
+	/**
+	 * Delete.
+	 *
+	 * @param entity the entity
+	 */
+	@Transactional
+	private void delete(Object entity) {
+		Assert.notNull(entity, "The entity must not be null!");
+		em.remove(em.contains(entity) ? entity : em.merge(entity));
+	}
+
+	/* (non-Javadoc)
+	 * @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#deleteForDataset(org.genesys.catalog.model.dataset.Dataset)
+	 */
+	@Override
+	@Transactional
+	public long deleteForDataset(Dataset dataset) {
+		return jpaQueryFactory.delete(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset)).execute();
+	}
+
+	/* (non-Javadoc)
+	 * @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#findAll(org.genesys.catalog.model.dataset.Dataset, org.springframework.data.domain.Pageable)
+	 */
+	@Override
+	public Page<DatasetAccessionRef> findAll(Dataset dataset, Pageable page) {
+		JPAQuery<DatasetAccessionRef> q = jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset));
+		long total = q.fetchCount();
+		List<DatasetAccessionRef> list = q.orderBy(QDatasetAccessionRef.datasetAccessionRef.accession.seqNo.asc(), QDatasetAccessionRef.datasetAccessionRef.acceNumb.asc()).leftJoin(QDatasetAccessionRef.datasetAccessionRef.accession).fetchJoin().offset(page.getOffset()).limit(page.getPageSize()).fetch();
+		return new PageImpl<>(list, page, total);
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository#countByDataset(org.genesys.catalog.model.dataset.Dataset)
+	 */
+	@Override
+	public long countByDataset(Dataset dataset) {
+		return jpaQueryFactory.selectFrom(QDatasetAccessionRef.datasetAccessionRef).where(QDatasetAccessionRef.datasetAccessionRef.dataset.eq(dataset)).fetchCount();
+	}
+	
+}

src/main/java/org/genesys/catalog/service/DatasetService.java

@@ -73,7 +73,7 @@ public interface DatasetService {
 	 * @param accessionRefs new accessionRefs
 	 * @return updated Dataset in db.
 	 */
-	Dataset updateAccessionRefs(Dataset dataset, @Valid Set<AccessionRef> accessionRefs);
+	Dataset setAccessionRefs(Dataset dataset, @Valid Set<AccessionRef> accessionRefs);
 
 	/**
 	 * Method adding new descriptor to Dataset.
@@ -126,7 +126,7 @@ public interface DatasetService {
 	 * @param page Pageable
 	 * @return PageImpl of AccessionRef
 	 */
-	Page<AccessionRef> listAccessions(Dataset dataset, Pageable page);
+	Page<? extends AccessionRef> listAccessions(Dataset dataset, Pageable page);
 
 	/**
 	 * List published datasets by accession.
@@ -221,7 +221,7 @@ public interface DatasetService {
 	 * @return updated Dataset
 	 * @throws NotFoundElement the not found element
 	 */
-	Dataset upsertAccessions(Dataset dataset, @Valid Set<AccessionRef> accessionRefs) throws NotFoundElement;
+	Dataset addAccessionRefs(Dataset dataset, @Valid Set<AccessionRef> accessionRefs) throws NotFoundElement;
 
 	/**
 	 * Puts the dataset into the Review state to be reviewed by admin.
@@ -444,4 +444,14 @@ public interface DatasetService {
 	 */
 	Path getDatasetRepositoryFolder(Dataset dataset);
 
+	
+	/**
+	 * Gets the dataset, nothing lazyloaded.
+	 *
+	 * @param uuid the uuid
+	 * @param version the version
+	 * @return the dataset
+	 */
+	Dataset getDataset(UUID uuid, Integer version);
+
 }

src/main/java/org/genesys/catalog/service/DescriptorService.java

@@ -75,7 +75,7 @@ public interface DescriptorService {
 	 * @return loaded descriptor
 	 * @throws NotFoundElement if descriptor is not found
 	 */
-	Descriptor getDescriptor(UUID uuid) throws NotFoundElement;
+	Descriptor loadDescriptor(UUID uuid) throws NotFoundElement;
 
 	/**
 	 * Gets the descriptor by uuid and version.
@@ -192,4 +192,12 @@ public interface DescriptorService {
 	 */
 	long countDescriptors(DescriptorFilter filter);
 
+	/**
+	 * Get descriptor for UUID. No lazy-loading.
+	 * 
+	 * @param uuid
+	 * @return
+	 */
+	Descriptor getDescriptor(UUID uuid);
+
 }

src/main/java/org/genesys/catalog/service/impl/DatasetServiceImpl.java

@@ -25,7 +25,6 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -36,18 +35,21 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import org.apache.commons.collections.CollectionUtils;
+import javax.validation.Valid;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.time.StopWatch;
 import org.genesys.blocks.security.service.CustomAclService;
 import org.genesys.catalog.model.Partner;
 import org.genesys.catalog.model.dataset.Dataset;
+import org.genesys.catalog.model.dataset.DatasetAccessionRef;
 import org.genesys.catalog.model.dataset.DatasetCreator;
 import org.genesys.catalog.model.dataset.DatasetLocation;
 import org.genesys.catalog.model.dataset.DatasetVersions;
 import org.genesys.catalog.model.dataset.QDataset;
 import org.genesys.catalog.model.filters.DatasetFilter;
 import org.genesys.catalog.model.traits.Descriptor;
+import org.genesys.catalog.persistence.dataset.DatasetAccessionRefRepository;
 import org.genesys.catalog.persistence.dataset.DatasetCreatorRepository;
 import org.genesys.catalog.persistence.dataset.DatasetLocationRepository;
 import org.genesys.catalog.persistence.dataset.DatasetRepository;
@@ -74,7 +76,6 @@ import org.genesys2.util.JPAUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.dao.ConcurrencyFailureException;
-import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.PageRequest;
@@ -94,8 +95,6 @@ import com.google.common.collect.Lists;
 import com.querydsl.core.BooleanBuilder;
 import com.querydsl.core.types.Predicate;
 
-import javax.validation.Valid;
-
 /**
  * The Class DatasetServiceImpl.
  */
@@ -119,6 +118,9 @@ public class DatasetServiceImpl implements DatasetService {
 	/** The dataset repository. */
 	@Autowired
 	private DatasetRepository datasetRepository;
+	
+	@Autowired
+	private DatasetAccessionRefRepository accessionRefRepository;
 
 	/** The file repo service. */
 	@Autowired
@@ -228,18 +230,16 @@ public class DatasetServiceImpl implements DatasetService {
 	@Override