Commit 1eafa259 authored by Matija Obreza's avatar Matija Obreza
Browse files

REST upsert data in datasets

parent 6b3e3b47
......@@ -38,4 +38,6 @@ public interface AccessionTraitRepository extends JpaRepository<AccessionTrait,
@Query("select distinct at.accession from AccessionTrait at where at.metadataId = ?1")
Page<Accession> listMetadataAccessions(long metadataId, Pageable pageable);
List<AccessionTrait> findByMetadataIdAndAccession(Long id, Accession accession);
}
......@@ -18,23 +18,55 @@ package org.genesys2.server.persistence.domain;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.genesys2.server.model.genesys.Method;
public interface TraitValueRepository {
// int count(Method method);
//
// List<ExperimentTrait> getValues(Accession accession, Method method);
// int count(Method method);
//
// List<ExperimentTrait> getValues(Accession accession, Method method);
Map<Long, List<ExperimentTrait>> getValues(List<AccessionTrait> accessionTraits);
Map<Long, Map<Long, List<Object>>> getValues(Metadata metadata, List<Method> methods, List<Accession> accessions);
void insert(Metadata metadata, Accession accession, Method method, List<Object> accessionValues);
/**
* @return list of non-null trait values
*/
List<Object> insert(Metadata metadata, Accession accession, Method method, List<Object> accessionValues);
void delete(Metadata metadata, Accession accession, Method method);
/**
* @return list of non-null trait values
*/
List<Object> upsert(Metadata metadata, Accession accession, Method method, List<Object> list);
/**
* Rescan and fix all {@link AccessionTrait} data for various methods
* recorded for the accession in all metadatas
*
* @param metadata
* @param accession
* @param methodIds
* newly inserted methodIds
*/
void updateAccessionTraits(Metadata metadata, Accession accession, Set<Long> methodIds);
/**
* Rescan and fix all {@link MetadataMethod} data
*
* @param metadata
* @param methodIds
* newly inserted methodIds
*/
void updateMetadataMethods(Metadata metadata, Set<Long> methodIds);
}
\ No newline at end of file
......@@ -20,8 +20,10 @@ import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.sql.DataSource;
......@@ -31,10 +33,12 @@ import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.ExperimentTrait;
import org.genesys2.server.model.genesys.Metadata;
import org.genesys2.server.model.genesys.MetadataMethod;
import org.genesys2.server.model.genesys.Method;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementSetter;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
......@@ -45,10 +49,13 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
private JdbcTemplate jdbcTemplate;
@Autowired
private AccessionTraitRepository accessionTraitRepository;
private MethodRepository methodRepository;
@Autowired
private MethodRepository methodRepository;
private MetadataMethodRepository metadataMethodRepository;
@Autowired
private AccessionTraitRepository accessionTraitRepository;
@Autowired
public void setDataSource(final DataSource dataSource) {
......@@ -162,23 +169,166 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Override
@Transactional
public void insert(Metadata metadata, Accession accession, Method method, final List<Object> accessionValues) {
public void delete(final Metadata metadata, final Accession accession, Method method) {
LOG.info("Deleting trait values in methodId=" + method.getId() + " for metadataId=" + metadata.getId() + " and accnId=" + accession.getId());
this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE Alis_Id=? AND Meta_id=?", method.getId()), new PreparedStatementSetter() {
@Override
public void setValues(PreparedStatement pst) throws SQLException {
pst.setLong(1, accession.getId());
pst.setLong(2, metadata.getId());
}
});
}
@Override
@Transactional
public List<Object> insert(Metadata metadata, Accession accession, final Method method, final List<Object> accessionValues) {
final long accessionId = accession.getId();
final long metadataId = metadata.getId();
// Make sure NULL values are skipped
final List<Object> values = new ArrayList<Object>(accessionValues.size());
for (Object v : accessionValues) {
if (v != null) {
LOG.debug("Adding value " + v);
values.add(v);
} else {
LOG.debug("Skipping value null");
}
}
this.jdbcTemplate.batchUpdate(String.format("INSERT INTO `%s` (Alis_Id, Meta_id, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()),
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement pst, int idx) throws SQLException {
pst.setLong(1, accessionId);
pst.setLong(2, metadataId);
pst.setObject(3, accessionValues.get(idx));
pst.setObject(3, values.get(idx));
LOG.info("Inserting trait value in methodId=" + method.getId() + " for metadataId=" + metadataId + " and accnId=" + accessionId
+ " val=" + values.get(idx));
}
@Override
public int getBatchSize() {
return accessionValues.size();
return values.size();
}
});
return values;
}
/**
* Basic upsert deletes all existing data and inserts new records (IDs will
* go crazy)
*
* @return
*/
@Override
public List<Object> upsert(Metadata metadata, Accession accession, Method method, List<Object> list) {
delete(metadata, accession, method);
return insert(metadata, accession, method, list);
}
@Override
public void updateAccessionTraits(Metadata metadata, Accession accession, final Set<Long> newMethodIds) {
// Clone it!
Set<Long> methodIds = new HashSet<Long>(newMethodIds);
// need to check all existing accessionTraits for
// metadata+accession (recorded methods) if there's still data for the
// accession+method, delete if not
List<AccessionTrait> e = accessionTraitRepository.findByMetadataIdAndAccession(metadata.getId(), accession);
for (AccessionTrait at : e) {
if (0 == countTraitValues(at.getMethodId(), metadata.getId(), accession.getId())) {
// and not freshly inserted
if (!methodIds.contains(at.getMethodId())) {
// Remove it
LOG.info("Deleting AccessionTrait " + at.getId());
accessionTraitRepository.delete(at);
}
}
// remove it from list
methodIds.remove(at.getMethodId());
}
// need to upsert metadataMethods for metadata+current methods
// note: List only contains records we have not found, insert
if (methodIds.size() > 0) {
List<AccessionTrait> forInsert = new ArrayList<AccessionTrait>(methodIds.size());
for (long methodId : methodIds) {
AccessionTrait at = new AccessionTrait();
at.setAccession(accession);
at.setMetadataId(metadata.getId());
at.setMethodId(methodId);
forInsert.add(at);
}
accessionTraitRepository.save(forInsert);
}
// need to upsert accessionTraits for metadata+accession+current
// methods
updateMetadataMethods(metadata, methodIds);
}
@Override
public void updateMetadataMethods(Metadata metadata, final Set<Long> newMethodIds) {
// Clone it!
Set<Long> methodIds = new HashSet<Long>(newMethodIds);
// need to check all existing metadataMethods for metadata
// (recorded methods) if they still have data for the method, delete if
// not
List<MetadataMethod> e = metadataMethodRepository.findByMetadata(metadata);
for (MetadataMethod mm : e) {
if (0 == countTraitValues(mm.getMethodId(), metadata.getId())) {
// and not freshly inserted
if (!methodIds.contains(mm.getMethodId())) {
// Remove it
LOG.info("Deleting MetadataMethod " + mm.getId());
metadataMethodRepository.delete(mm);
}
}
// remove it from list
methodIds.remove(mm.getMethodId());
}
// need to upsert metadataMethods for metadata+current methods
// note: List only contains records we have not found, insert
if (methodIds.size() > 0) {
List<MetadataMethod> forInsert = new ArrayList<MetadataMethod>(methodIds.size());
for (long methodId : methodIds) {
MetadataMethod mm = new MetadataMethod();
mm.setMetadata(metadata);
mm.setMethodId(methodId);
forInsert.add(mm);
}
metadataMethodRepository.save(forInsert);
}
}
/**
* Returns the number of accession trait-values of a method in a dataset
*
* @param method
* @param metadataId
* @return
*/
private int countTraitValues(long methodId, long metadataId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=?", methodId), Integer.class, metadataId);
}
/**
* Returns the number of per-accession trait-values of a method in a dataset
*
* @param methodId
* @param metadataId
* @param accessionId
* @return
*/
private int countTraitValues(long methodId, long metadataId, long accessionId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=? AND ALIS_Id=?", methodId), Integer.class, metadataId,
accessionId);
}
}
......@@ -18,6 +18,7 @@ package org.genesys2.server.service.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -26,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionTrait;
import org.genesys2.server.model.genesys.AllAccnames;
import org.genesys2.server.model.genesys.AllAcqBreeding;
import org.genesys2.server.model.genesys.AllAcqCollect;
......@@ -223,12 +225,14 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
public List<Metadata> listMetadata(Accession accession) {
return metadataRepository.findByIds(accessionTraitRepository.listMetadataIds(accession));
List<Long> x = accessionTraitRepository.listMetadataIds(accession);
return x.size() == 0 ? null : metadataRepository.findByIds(x);
}
@Override
public List<Method> listMethods(Accession accession) {
return methodRepository.findByIds(accessionTraitRepository.listMethodIds(accession));
List<Long> x = accessionTraitRepository.listMethodIds(accession);
return x.size() == 0 ? null : methodRepository.findByIds(x);
}
@Override
......@@ -243,7 +247,8 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
public List<Method> listMethods(Metadata metadata) {
return metadataMethodRepository.listMetadataMethods(metadata);
List<Long> x = metadataMethodRepository.listMetadataMethods(metadata);
return x.size() == 0 ? null : methodRepository.findByIds(x);
}
@Override
......@@ -254,12 +259,13 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
public Map<Long, List<ExperimentTrait>> getAccessionTraitValues(Accession accession) {
LOG.debug("Getting trait values for accession: " + accession);
return traitValueRepository.getValues(accessionTraitRepository.findByAccession(accession));
List<AccessionTrait> x = accessionTraitRepository.findByAccession(accession);
return x.size() == 0 ? null : traitValueRepository.getValues(x);
}
@Override
public Map<Long, Map<Long, List<Object>>> getMetadataTraitValues(Metadata metadata, List<Accession> accessions) {
List<Method> methods = metadataMethodRepository.listMetadataMethods(metadata);
List<Method> methods = methodRepository.findByIds(metadataMethodRepository.listMetadataMethods(metadata));
return traitValueRepository.getValues(metadata, methods, accessions);
}
......@@ -343,7 +349,7 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
public List<Metadata> listMetadataByMethod(Method method) {
return metadataMethodRepository.listMetadataByMethod(method);
return metadataMethodRepository.listMetadataByMethodId(method.getId());
}
@Override
......@@ -485,22 +491,67 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#metadata, 'WRITE')")
@Transactional(readOnly = false)
public void upsertAccessionData(Metadata metadata, Accession accession, Map<Long, List<Object>> methodValues) {
// TODO Load all existing records for metadata+accession
// // Load all existing records for metadata+accession
// List<AccessionTrait> existingEntries =
// accessionTraitRepository.findByMetadataAndAccession(metadata,
// accessionTraitRepository.findByMetadataIdAndAccession(metadata.getId(),
// accession);
// Remove entries missing from map
//
// // Remove entries missing from map
// Collection<AccessionTrait> forRemoval = new
// ArrayList<AccessionTrait>(methodValues.size());
// Collection<AccessionTrait> forAddition = new
// ArrayList<AccessionTrait>(methodValues.size());
//
// for (AccessionTrait accessionTrait : existingEntries) {
// LOG.debug("Existing trait " + accessionTrait);
// // TODO Remove
// if (methodValues.containsKey(accessionTrait.getMethodId())) {
// // keep it
// } else {
// LOG.info("New data does not have methodId=" +
// accessionTrait.getMethodId());
// forRemoval.add(accessionTrait);
// }
// }
// // Add new entries from map
// for (final long methodId : methodValues.keySet()) {
// if (null == CollectionUtils.find(existingEntries, new Predicate() {
// @Override
// public boolean evaluate(Object object) {
// return ((AccessionTrait) object).getMethodId() == methodId;
// }
// })) {
// // Create new AccessionTrait
// AccessionTrait at = new AccessionTrait();
// at.setAccession(accession);
// at.setMetadataId(metadata.getId());
// at.setMethodId(methodId);
// forAddition.add(at);
// }
// TODO Add new entries from map
// }
// accessionTraitRepository.save(forAddition);
// accessionTraitRepository.delete(forRemoval);
LOG.info("Upserting accession data " + accession);
Set<Long> methodsWithValues = new HashSet<Long>();
// just stick them to database
// stick them to database
for (long methodId : methodValues.keySet()) {
Method method = methodRepository.findOne(methodId);
traitValueRepository.insert(metadata, accession, method, methodValues.get(methodId));
// traitValueRepository.delete(metadata, accession, method);
// traitValueRepository.insert(metadata, accession, method,
// methodValues.get(methodId));
LOG.info("Upserting accession data for method " + method.getId() + " accn=" + accession);
List<Object> withValues = traitValueRepository.upsert(metadata, accession, method, methodValues.get(methodId));
if (withValues.size() > 0)
methodsWithValues.add(method.getId());
}
// Soon
LOG.info("Updating accessionTraits records");
traitValueRepository.updateAccessionTraits(metadata, accession, methodsWithValues);
// Called by the one above
// // Delayed
// traitValueRepository.updateMetadataMethods(metadata,
// methodValues.keySet());
}
}
......@@ -159,7 +159,7 @@ public class DatasetController extends RestController {
System.err.println(data);
ObjectMapper objectMapper = new ObjectMapper();
JsonNode json = null;
try {
json = objectMapper.readTree(data);
System.err.println(json);
......@@ -242,7 +242,7 @@ public class DatasetController extends RestController {
throw new UnsupportedOperationException("Handling coded descriptors is supported");
} else if (method.getFieldType() == 0) {
// String
return v.asText();
return v.isNull() ? null : v.asText();
} else if (method.getFieldType() == 1) {
// Double
return v.isNumber() ? v.doubleValue() : null;
......@@ -258,7 +258,7 @@ public class DatasetController extends RestController {
public String instCode;
public String acceNumb;
public String genus;
private Map<Long, List<Object>> methodValues;
private Map<Long, List<Object>> methodValues = new HashMap<Long, List<Object>>();
@Override
public String getHoldingInstitute() {
......@@ -266,13 +266,10 @@ public class DatasetController extends RestController {
}
public void addMethodValue(long methodId, Object value) {
if (value == null) {
// skip nulls
return;
}
if (methodValues == null) {
methodValues = new HashMap<Long, List<Object>>();
}
// if (value == null) {
// // skip nulls
// return;
// }
List<Object> values = methodValues.get(methodId);
if (values == null) {
methodValues.put(methodId, values = new ArrayList<Object>(5));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment