Commit 3ca2c119 authored by Matija Obreza's avatar Matija Obreza
Browse files

Faster access to datasets through MetadataAccession

parent 932dc613
......@@ -21,11 +21,12 @@ import javax.persistence.Entity;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import org.genesys2.server.model.BusinessModel;
@Entity
@Table(name = "accessiontrait")
@Table(name = "accessiontrait", uniqueConstraints = { @UniqueConstraint(name="UQ_accessiontrait_all", columnNames = { "metadataId", "accessionId", "methodId" }) })
public class AccessionTrait extends BusinessModel {
private static final long serialVersionUID = -240056837800843686L;
......
......@@ -173,6 +173,7 @@ public class Metadata extends VersionedAuditedModel implements AclAwareModel, Ge
this.uuid = uuid;
}
@Override
public String toString() {
return MessageFormat.format("Metadata id={0,number,#} inst={2} title={1}", id, title, instituteCode);
......
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.model.genesys;
import javax.persistence.Entity;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import org.genesys2.server.model.BusinessModel;
@Entity
@Table(name = "metadataaccession", uniqueConstraints = { @UniqueConstraint(name="UI_metadataaccession_all", columnNames = { "metadataId", "accessionId" }) })
public class MetadataAccession extends BusinessModel {
private static final long serialVersionUID = -240056837800843686L;
@ManyToOne(cascade = {}, optional = false)
@JoinColumn(name = "metadataId")
private Metadata metadata;
private long accessionId;
public Metadata getMetadata() {
return metadata;
}
public void setMetadata(Metadata metadata) {
this.metadata = metadata;
}
public long getAccessionId() {
return accessionId;
}
public void setAccessionId(long accessionId) {
this.accessionId = accessionId;
}
}
......@@ -20,11 +20,12 @@ import javax.persistence.Entity;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import org.genesys2.server.model.BusinessModel;
@Entity
@Table(name = "metadatamethod")
@Table(name = "metadatamethod", uniqueConstraints = { @UniqueConstraint(name = "UI_metadatamethod_all", columnNames = { "metadataId", "methodId" }) })
public class MetadataMethod extends BusinessModel {
private static final long serialVersionUID = -240056837800843686L;
......
......@@ -23,6 +23,7 @@ import org.genesys2.server.model.genesys.AccessionTrait;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface AccessionTraitRepository extends JpaRepository<AccessionTrait, Long> {
......@@ -35,9 +36,17 @@ public interface AccessionTraitRepository extends JpaRepository<AccessionTrait,
@Query("select distinct at.methodId from AccessionTrait at where at.accession = ?1")
List<Long> listMethodIds(Accession accession);
@Query("select distinct a from AccessionTrait at inner join at.accession a where at.metadataId = ?1")
@Query(value = "select distinct a from AccessionTrait at inner join at.accession a where at.metadataId = ?1", countQuery = "select count(ma.accessionId) from MetadataAccession ma where ma.metadata.id = ?1")
Page<Accession> listMetadataAccessions(long metadataId, Pageable pageable);
List<AccessionTrait> findByMetadataIdAndAccession(Long id, Accession accession);
@Query("select count(id) from AccessionTrait at where at.metadataId = ?1 and at.accession = ?2")
long countByMetadataAndAccessionId(long metadataId, Accession accession);
@Modifying
@Query("delete from AccessionTrait where accession = ?1 and metadataId= ?2 and methodId = ?3")
void deleteByAccessionAndMetadataIdAndMethodId(Accession accession, long metadataId, long methodId);
}
......@@ -43,4 +43,6 @@ public interface GenesysLowlevelRepository {
void listAccessionsGeo(String code, RowCallbackHandler rowCallbackHandler);
void listAccessionsColl(String code, RowCallbackHandler rowCallbackHandler);
void listMetadataAccessions(long id, RowCallbackHandler rowCallbackHandler);
}
......@@ -247,4 +247,19 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
}
}, rowCallbackHandler);
}
@Override
public void listMetadataAccessions(final long id, RowCallbackHandler rowCallbackHandler) {
jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
PreparedStatement stmt = con
.prepareStatement("select distinct a.id, a.instCode, a.acceNumb, a.genus from accessiontrait at inner join accession a on a.id=at.accessionId where at.metadataId = ?");
stmt.setLong(1, id);
// Set mysql JConnector to stream results
stmt.setFetchSize(Integer.MIN_VALUE);
return stmt;
}
}, rowCallbackHandler);
}
}
/**
* Copyright 2013 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.persistence.domain;
import java.util.List;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataAccession;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface MetadataAccessionRepository extends JpaRepository<MetadataAccession, Long> {
List<MetadataMethod> findByMetadata(Metadata metadata);
@Query("select distinct ma.accessionId from MetadataAccession ma where ma.metadata = ?1")
List<Long> listMetadataAccessions(Metadata metadata);
@Query("select distinct ma.metadata from MetadataAccession ma where ma.accessionId = ?1")
List<Metadata> listMetadataByAccessionId(long accessionId);
@Modifying
@Query("delete from MetadataAccession ma where ma.metadata = ?1 and ma.accessionId = ?2")
void deleteByMetadataAndAccessionId(Metadata metadata, Long id);
}
......@@ -21,6 +21,7 @@ import java.util.List;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
public interface MetadataMethodRepository extends JpaRepository<MetadataMethod, Long> {
......@@ -33,4 +34,10 @@ public interface MetadataMethodRepository extends JpaRepository<MetadataMethod,
@Query("select distinct mm.metadata from MetadataMethod mm where mm.methodId = ?1")
List<Metadata> listMetadataByMethodId(long methodId);
@Modifying
@Query("delete from MetadataMethod mm where mm.metadata = ?1 and mm.methodId = ?2")
int deleteByMetadataAndMethodId(Metadata metadata, long methodId);
MetadataMethod findByMetadataAndMethodId(Metadata metadata, long methodId);
}
......@@ -18,14 +18,12 @@ package org.genesys2.server.persistence.domain;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.ExperimentAccessionTrait;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.genesys2.server.model.genesys.Method;
public interface TraitValueRepository {
......@@ -40,36 +38,9 @@ public interface TraitValueRepository {
Map<Long, Map<Long, List<Object>>> getValues(Metadata metadata, List<Method> methods, List<Accession> accessions);
/**
* @return list of non-null trait values
*/
List<Object> insert(Metadata metadata, Accession accession, Method method, List<Object> accessionValues);
void delete(Metadata metadata, Accession accession, Method method);
/**
* @return list of non-null trait values
*/
List<Object> upsert(Metadata metadata, Accession accession, Method method, List<Object> list);
/**
* Rescan and fix all {@link AccessionTrait} data for various methods
* recorded for the accession in all metadatas
*
* @param metadata
* @param accession
* @param methodIds
* newly inserted methodIds
*/
void updateAccessionTraits(Metadata metadata, Accession accession, Set<Long> methodIds);
/**
* Rescan and fix all {@link MetadataMethod} data
*
* @param metadata
* @param methodIds
* newly inserted methodIds
*/
void updateMetadataMethods(Metadata metadata, Set<Long> methodIds);
}
\ No newline at end of file
......@@ -36,14 +36,15 @@ import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.ExperimentAccessionTrait;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.genesys2.server.model.genesys.Method;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
import org.springframework.jdbc.core.PreparedStatementSetter;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
......@@ -52,6 +53,7 @@ import org.springframework.transaction.annotation.Transactional;
public class TraitValueRepositoryImpl implements TraitValueRepository {
public static final Log LOG = LogFactory.getLog(TraitValueRepository.class);
private JdbcTemplate jdbcTemplate;
private NamedParameterJdbcTemplate namedJdbcTemplate;
@Autowired
private MethodRepository methodRepository;
......@@ -59,12 +61,16 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Autowired
private MetadataMethodRepository metadataMethodRepository;
@Autowired
private MetadataAccessionRepository metadataAccessionRepository;
@Autowired
private AccessionTraitRepository accessionTraitRepository;
@Autowired
public void setDataSource(final DataSource dataSource) {
this.jdbcTemplate = new JdbcTemplate(dataSource);
this.namedJdbcTemplate = new NamedParameterJdbcTemplate(dataSource);
}
// /*
......@@ -151,41 +157,41 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Override
public Map<Long, Map<Long, List<Object>>> getValues(Metadata metadata, List<Method> methods, List<Accession> accessions) {
Map<Long, Map<Long, List<Object>>> accessionMetadataValues = new HashMap<Long, Map<Long, List<Object>>>();
final Map<Long, Map<Long, List<Object>>> accessionMetadataValues = new HashMap<Long, Map<Long, List<Object>>>();
final Set<Long> accessionIds = new HashSet<Long>(accessions.size());
for (Accession accession : accessions) {
// create lists
accessionMetadataValues.put(accession.getId(), new HashMap<Long, List<Object>>());
accessionIds.add(accession.getId());
}
for (Method method : methods) {
LOG.debug("MethodID=" + method.getId());
Map<String, Object> paramMap = new HashMap<String, Object>();
paramMap.put("metadataId", metadata.getId());
paramMap.put("accessionIds", accessionIds);
// FIXME We're loading all entries for metadata, should limit only
// to the accessions in the list!
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select accessionId, `" + method.getFieldName() + "` as traitvalue from `"
+ method.getId() + "` where metadataId=?", new Object[] { metadata.getId() });
for (Method method : methods) {
final long methodId = method.getId();
LOG.debug("MethodID=" + methodId);
if (rows == null || rows.size() == 0) {
// Skip
continue;
}
this.namedJdbcTemplate.query("select accessionId, `" + method.getFieldName() + "` as traitvalue from `" + method.getId()
+ "` where metadataId = :metadataId and accessionId in ( :accessionIds )", paramMap, new RowCallbackHandler() {
for (final Map<String, Object> row : rows) {
long accessionId = (Long) row.get("accessionId");
Object value = row.get("traitvalue");
@Override
public void processRow(ResultSet rs) throws SQLException {
long accessionId = rs.getLong(1);
Object value = rs.getObject(2);
// TODO null values should not exist
if (value != null) {
Map<Long, List<Object>> accessionValues = accessionMetadataValues.get(accessionId);
if (accessionValues != null) {
List<Object> x = accessionValues.get(method.getId());
List<Object> x = accessionValues.get(methodId);
if (x == null)
accessionValues.put(method.getId(), x = new ArrayList<Object>(3));
accessionValues.put(methodId, x = new ArrayList<Object>(3));
x.add(value);
}
}
}
});
}
if (LOG.isDebugEnabled())
......@@ -193,9 +199,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
return accessionMetadataValues;
}
@Override
@Transactional
public void delete(final Metadata metadata, final Accession accession, Method method) {
private void delete(final Metadata metadata, final Accession accession, Method method) {
LOG.info("Deleting trait values in methodId=" + method.getId() + " for metadataId=" + metadata.getId() + " and accnId=" + accession.getId());
this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE accessionId=? AND metadataId=?", method.getId()), new PreparedStatementSetter() {
@Override
......@@ -206,9 +211,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
});
}
@Override
@Transactional
public List<Object> insert(Metadata metadata, Accession accession, final Method method, final List<Object> accessionValues) {
private List<Object> insert(Metadata metadata, Accession accession, final Method method, final List<Object> accessionValues) {
final long accessionId = accession.getId();
final long metadataId = metadata.getId();
......@@ -223,6 +227,11 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
}
}
if (values.size() == 0) {
// Nothing to insert
return values;
}
this.jdbcTemplate.batchUpdate(
String.format("INSERT INTO `%s` (accessionId, metadataId, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()),
new BatchPreparedStatementSetter() {
......@@ -253,86 +262,59 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Override
public List<Object> upsert(Metadata metadata, Accession accession, Method method, List<Object> list) {
delete(metadata, accession, method);
return insert(metadata, accession, method, list);
}
@Override
public void updateAccessionTraits(Metadata metadata, Accession accession, final Set<Long> newMethodIds) {
// Clone it!
Set<Long> methodIds = new HashSet<Long>(newMethodIds);
List<Object> x = insert(metadata, accession, method, list);
// need to check all existing accessionTraits for
// metadata+accession (recorded methods) if there's still data for the
// accession+method, delete if not
List<AccessionTrait> e = accessionTraitRepository.findByMetadataIdAndAccession(metadata.getId(), accession);
for (AccessionTrait at : e) {
if (0 == countTraitValues(at.getMethodId(), metadata.getId(), accession.getId())) {
// and not freshly inserted
if (!methodIds.contains(at.getMethodId())) {
// Remove it
LOG.info("Deleting AccessionTrait " + at.getId());
accessionTraitRepository.delete(at);
}
}
// remove it from list
methodIds.remove(at.getMethodId());
// Need to update MetadataMethod
if (0 == countTraitAccessions(method.getId(), metadata.getId())) {
this.metadataMethodRepository.deleteByMetadataAndMethodId(metadata, method.getId());
} else {
ensureMetadataMethod(metadata, method);
}
// need to upsert metadataMethods for metadata+current methods
// note: List only contains records we have not found, insert
if (methodIds.size() > 0) {
List<AccessionTrait> forInsert = new ArrayList<AccessionTrait>(methodIds.size());
for (long methodId : methodIds) {
AccessionTrait at = new AccessionTrait();
at.setAccession(accession);
at.setMetadataId(metadata.getId());
at.setMethodId(methodId);
forInsert.add(at);
// Need to update AccessionTrait
if (0 == countTraitValues(method.getId(), metadata.getId(), accession.getId())) {
this.accessionTraitRepository.deleteByAccessionAndMetadataIdAndMethodId(accession, metadata.getId(), method.getId());
} else {
ensureAccessionTrait(metadata, method, accession);
}
accessionTraitRepository.save(forInsert);
// Need to update metadataaccession
long mda = this.accessionTraitRepository.countByMetadataAndAccessionId(metadata.getId(), accession);
if (0 == mda) {
this.metadataAccessionRepository.deleteByMetadataAndAccessionId(metadata, accession.getId());
} else {
ensureMetadataAccession(metadata, accession);
}
// need to upsert accessionTraits for metadata+accession+current
// methods
updateMetadataMethods(metadata, methodIds);
return x;
}
@Override
public void updateMetadataMethods(Metadata metadata, final Set<Long> newMethodIds) {
// Clone it!
Set<Long> methodIds = new HashSet<Long>(newMethodIds);
// need to check all existing metadataMethods for metadata
// (recorded methods) if they still have data for the method, delete if
// not
List<MetadataMethod> e = metadataMethodRepository.findByMetadata(metadata);
for (MetadataMethod mm : e) {
if (0 == countTraitValues(mm.getMethodId(), metadata.getId())) {
// and not freshly inserted
if (!methodIds.contains(mm.getMethodId())) {
// Remove it
LOG.info("Deleting MetadataMethod " + mm.getId());
metadataMethodRepository.delete(mm);
}
private void ensureAccessionTrait(final Metadata metadata, final Method method, final Accession accession) {
this.jdbcTemplate.update("INSERT IGNORE INTO accessiontrait (metadataId, methodId, accessionId) values (?, ?, ?)", new PreparedStatementSetter() {
public void setValues(PreparedStatement ps) throws SQLException {
ps.setLong(1, metadata.getId());
ps.setLong(2, method.getId());
ps.setLong(3, accession.getId());
}
// remove it from list
methodIds.remove(mm.getMethodId());
});
}
// need to upsert metadataMethods for metadata+current methods
// note: List only contains records we have not found, insert
if (methodIds.size() > 0) {
List<MetadataMethod> forInsert = new ArrayList<MetadataMethod>(methodIds.size());
for (long methodId : methodIds) {
MetadataMethod mm = new MetadataMethod();
mm.setMetadata(metadata);
mm.setMethodId(methodId);
forInsert.add(mm);
private void ensureMetadataMethod(final Metadata metadata, final Method method) {
this.jdbcTemplate.update("INSERT IGNORE INTO metadatamethod (metadataId, methodId) values (?, ?)", new PreparedStatementSetter() {
public void setValues(PreparedStatement ps) throws SQLException {
ps.setLong(1, metadata.getId());
ps.setLong(2, method.getId());
}
});
}
metadataMethodRepository.save(forInsert);
private void ensureMetadataAccession(final Metadata metadata, final Accession accession) {
this.jdbcTemplate.update("INSERT IGNORE INTO metadataaccession (metadataId, accessionId) values (?, ?)", new PreparedStatementSetter() {
public void setValues(PreparedStatement ps) throws SQLException {
ps.setLong(1, metadata.getId());
ps.setLong(2, accession.getId());
}
});
}
/**
......@@ -342,7 +324,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
* @param metadataId
* @return
*/
private int countTraitValues(long methodId, long metadataId) {
private int countTraitAccessions(long methodId, long metadataId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE metadataId=?", methodId), Integer.class, metadataId);
}
......
......@@ -82,7 +82,6 @@ import org.genesys2.server.service.TraitService;
import org.genesys2.spring.SecurityContextUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.repository.query.Param;
import org.springframework.jdbc.core.RowCallbackHandler;
......@@ -644,14 +643,6 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
if (withValues.size() > 0)
methodsWithValues.add(method.getId());
}
// Soon
LOG.info("Updating accessionTraits records");
traitValueRepository.updateAccessionTraits(metadata, accession, methodsWithValues);
// Called by the one above
// // Delayed
// traitValueRepository.updateMetadataMethods(metadata,
// methodValues.keySet());
}
@Override
......@@ -850,21 +841,9 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
coreEntry.setTime(System.currentTimeMillis());
zos.putNextEntry(coreEntry);
// Accessions
csv = new CSVWriter(new BufferedWriter(new OutputStreamWriter(zos)), ',', '"', '\\', "\n");
csv.writeNext(new String[] { "coreId", "instCode", "acceNumb", "genus" });
// Write accession information
Page<Accession> datasetAccessions = null;
int page = 0;
do {
datasetAccessions = listMetadataAccessions(metadata.getId(), new PageRequest(page++, 50));
for (Accession accn : datasetAccessions.getContent()) {
csv.writeNext(new String[] { String.valueOf(accn.getId()), accn.getInstituteCode(), accn.getAccessionName(), accn.getGenus() });
}
} while (datasetAccessions.hasNextPage());
writeDatasetCore(metadata, zos);