Commit 7c8c22a5 authored by Matija Obreza's avatar Matija Obreza
Browse files

Use new schema for method tables, recalculate metadatamethods and

accessiontraits from existing data.
parent ac9e512e
...@@ -34,4 +34,6 @@ public interface GenesysLowlevelRepository { ...@@ -34,4 +34,6 @@ public interface GenesysLowlevelRepository {
void updateFaoInstituteCountries(); void updateFaoInstituteCountries();
void ensureMethodTable(Method method); void ensureMethodTable(Method method);
void refreshMetadataMethods();
} }
...@@ -19,6 +19,7 @@ package org.genesys2.server.persistence.domain; ...@@ -19,6 +19,7 @@ package org.genesys2.server.persistence.domain;
import java.sql.DatabaseMetaData; import java.sql.DatabaseMetaData;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List;
import javax.sql.DataSource; import javax.sql.DataSource;
...@@ -26,6 +27,7 @@ import org.apache.commons.logging.Log; ...@@ -26,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Method; import org.genesys2.server.model.genesys.Method;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
...@@ -56,13 +58,14 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository ...@@ -56,13 +58,14 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
.update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X;"); .update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X;");
LOG.info("Institute accession counts updated: " + count); LOG.info("Institute accession counts updated: " + count);
} }
@Override @Override
@Transactional(readOnly = false) @Transactional(readOnly = false)
public void updateInstituteAccessionCount(long instituteId) { public void updateInstituteAccessionCount(long instituteId) {
LOG.info("Updating institute accession count for instituteId=" + instituteId); LOG.info("Updating institute accession count for instituteId=" + instituteId);
int count = jdbcTemplate int count = jdbcTemplate
.update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession where instituteId= ? group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X where FI.id = ?;", instituteId, instituteId); .update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession where instituteId= ? group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X where FI.id = ?;",
instituteId, instituteId);
LOG.info("Institute accession counts updated: " + count); LOG.info("Institute accession counts updated: " + count);
} }
...@@ -107,7 +110,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository ...@@ -107,7 +110,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
// Common table elements // Common table elements
createTableSql.append("CREATE TABLE `").append(method.getId()) createTableSql.append("CREATE TABLE `").append(method.getId())
.append("` (`id` BIGINT NOT NULL AUTO_INCREMENT, `ALIS_Id` BIGINT NOT NULL, `Meta_id` INT NOT NULL, "); .append("` (`id` BIGINT NOT NULL AUTO_INCREMENT, `accessionId` BIGINT NOT NULL, `metadataId` INT NOT NULL, ");
// Value column // Value column
createTableSql.append("`").append(method.getFieldName()).append("` "); createTableSql.append("`").append(method.getFieldName()).append("` ");
...@@ -115,12 +118,21 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository ...@@ -115,12 +118,21 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
createTableSql.append(" NOT NULL"); createTableSql.append(" NOT NULL");
// PK // PK
createTableSql.append(", PRIMARY KEY (`id`), KEY `ALIS_Id` (`ALIS_Id`)"); createTableSql.append(", PRIMARY KEY (`id`)");
// FK metadata
// createTableSql.append(", INDEX `genesys_Metadata_").append(method.getId()).append("` (`Meta_Id` ASC)");
createTableSql.append(", CONSTRAINT `genesys_Metadata_").append(method.getId())
.append("` FOREIGN KEY (`methodId`) REFERENCES `metadata` (`Meta_id` ) ON DELETE NO ACTION ON UPDATE NO ACTION");
// FK accession
// createTableSql.append(", INDEX `genesys_Accession_idx` (`ALIS_Id` ASC)");
createTableSql.append(", CONSTRAINT `genesys_Accession_").append(method.getId())
.append("` FOREIGN KEY (`accessionId`) REFERENCES `accession` (`id` ) ON DELETE NO ACTION ON UPDATE NO ACTION");
// This is optional, but let's keep it // This is optional, but let's keep it
createTableSql.append(", KEY `").append(method.getFieldName()).append("` (`").append(method.getFieldName()).append("`)"); // createTableSql.append(", KEY `").append(method.getFieldName()).append("` (`").append(method.getFieldName()).append("`)");
createTableSql.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;"); createTableSql.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
LOG.warn("Executing: " + createTableSql.toString()); LOG.warn("Creating method table: " + createTableSql.toString());
jdbcTemplate.execute(createTableSql.toString()); jdbcTemplate.execute(createTableSql.toString());
} }
LOG.info("Done"); LOG.info("Done");
...@@ -139,4 +151,51 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository ...@@ -139,4 +151,51 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
} }
throw new RuntimeException("Unsupported data type " + method.getFieldType()); throw new RuntimeException("Unsupported data type " + method.getFieldType());
} }
@Override
@Transactional(readOnly = false)
public void refreshMetadataMethods() {
jdbcTemplate.update("delete from metadatamethod");
jdbcTemplate.update("delete from accessiontrait");
List<Long> methodIds = jdbcTemplate.queryForList("select id from methods", Long.class);
for (long methodId : methodIds) {
LOG.info("Updating metadatamethod for methodId=" + methodId);
// // Delete stuff without Metadata
// int deleted = jdbcTemplate.update("delete from `" + methodId +
// "` where Meta_Id in (select X.Meta_Id from (select distinct Meta_Id from `"
// + methodId +
// "`) X left outer join metadata M on M.Meta_Id=X.Meta_Id where M.Meta_Id is null)");
// if (deleted > 0) {
// LOG.warn("Deleted " + deleted +
// " observations with missing metadata from method=" + methodId);
// }
// Insert reference to metadatamethod
try {
int inserted = jdbcTemplate.update("insert into metadatamethod (metadataId, methodId) (select distinct metadataId, " + methodId + " from `"
+ methodId + "`)");
if (inserted == 0) {
LOG.warn("Method " + methodId + " has no entries?");
} else {
LOG.warn("Inserted " + inserted + " method references for metadata for methodId=" + methodId);
}
} catch (BadSqlGrammarException e) {
LOG.warn(e);
}
try {
int inserted2 = jdbcTemplate.update("insert into accessiontrait (metadataId, accessionId, methodId) (select distinct metadataId, accessionId, "
+ methodId + " from `" + methodId + "`)");
if (inserted2 == 0) {
LOG.warn("Method " + methodId + " has no entries?");
} else {
LOG.warn("Inserted " + inserted2 + " accession references methodId=" + methodId);
}
} catch (BadSqlGrammarException e) {
LOG.warn(e);
}
}
}
} }
...@@ -101,8 +101,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -101,8 +101,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
return this.jdbcTemplate.query(new PreparedStatementCreator() { return this.jdbcTemplate.query(new PreparedStatementCreator() {
@Override @Override
public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
PreparedStatement pst = conn.prepareStatement("select `ALIS_Id` as accessionId, `" + method.getFieldName() + "` as traitvalue from `" PreparedStatement pst = conn.prepareStatement("select accessionId, `" + method.getFieldName() + "` as traitvalue from `" + method.getId()
+ method.getId() + "` where `Meta_Id`=?"); + "` where metadataId=?");
pst.setLong(1, metadataId); pst.setLong(1, metadataId);
return pst; return pst;
} }
...@@ -126,8 +126,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -126,8 +126,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
for (AccessionTrait at : accessionTraits) { for (AccessionTrait at : accessionTraits) {
Method method = methodRepository.findOne(at.getMethodId()); Method method = methodRepository.findOne(at.getMethodId());
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select Meta_id as metadataId, `" + method.getFieldName() final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select metadataId, `" + method.getFieldName() + "` as traitvalue from `"
+ "` as traitvalue from `" + method.getId() + "` where `ALIS_Id`=?", new Object[] { at.getAccession().getId() }); + method.getId() + "` where accessionId=?", new Object[] { at.getAccession().getId() });
if (rows == null || rows.size() == 0) { if (rows == null || rows.size() == 0) {
// Skip // Skip
...@@ -137,14 +137,15 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -137,14 +137,15 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
final List<ExperimentTrait> ret = new ArrayList<ExperimentTrait>(); final List<ExperimentTrait> ret = new ArrayList<ExperimentTrait>();
for (final Map<String, Object> row : rows) { for (final Map<String, Object> row : rows) {
final ExperimentTrait et = new ExperimentTrait((Integer) row.get("metadataId"), row.get("traitvalue")); final ExperimentTrait et = new ExperimentTrait((Long) row.get("metadataId"), row.get("traitvalue"));
ret.add(et); ret.add(et);
} }
methodValues.put(method.getId(), ret); methodValues.put(method.getId(), ret);
} }
LOG.warn("Done! " + methodValues); if (LOG.isDebugEnabled())
LOG.debug("Done! " + methodValues);
return methodValues; return methodValues;
} }
...@@ -162,8 +163,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -162,8 +163,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
// FIXME We're loading all entries for metadata, should limit only // FIXME We're loading all entries for metadata, should limit only
// to the accessions in the list! // to the accessions in the list!
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select `ALIS_Id` as accessionId, `" + method.getFieldName() final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select accessionId, `" + method.getFieldName() + "` as traitvalue from `"
+ "` as traitvalue from `" + method.getId() + "` where `Meta_Id`=?", new Object[] { metadata.getId() }); + method.getId() + "` where metadataId=?", new Object[] { metadata.getId() });
if (rows == null || rows.size() == 0) { if (rows == null || rows.size() == 0) {
// Skip // Skip
...@@ -187,7 +188,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -187,7 +188,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
} }
} }
LOG.warn("Done! "); if (LOG.isDebugEnabled())
LOG.debug("Done! ");
return accessionMetadataValues; return accessionMetadataValues;
} }
...@@ -195,7 +197,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -195,7 +197,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Transactional @Transactional
public void delete(final Metadata metadata, final Accession accession, Method method) { public void delete(final Metadata metadata, final Accession accession, Method method) {
LOG.info("Deleting trait values in methodId=" + method.getId() + " for metadataId=" + metadata.getId() + " and accnId=" + accession.getId()); LOG.info("Deleting trait values in methodId=" + method.getId() + " for metadataId=" + metadata.getId() + " and accnId=" + accession.getId());
this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE Alis_Id=? AND Meta_id=?", method.getId()), new PreparedStatementSetter() { this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE accessionId=? AND metadataId=?", method.getId()), new PreparedStatementSetter() {
@Override @Override
public void setValues(PreparedStatement pst) throws SQLException { public void setValues(PreparedStatement pst) throws SQLException {
pst.setLong(1, accession.getId()); pst.setLong(1, accession.getId());
...@@ -221,7 +223,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -221,7 +223,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
} }
} }
this.jdbcTemplate.batchUpdate(String.format("INSERT INTO `%s` (Alis_Id, Meta_id, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()), this.jdbcTemplate.batchUpdate(
String.format("INSERT INTO `%s` (accessionId, metadataId, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()),
new BatchPreparedStatementSetter() { new BatchPreparedStatementSetter() {
@Override @Override
public void setValues(PreparedStatement pst, int idx) throws SQLException { public void setValues(PreparedStatement pst, int idx) throws SQLException {
...@@ -340,7 +343,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -340,7 +343,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
* @return * @return
*/ */
private int countTraitValues(long methodId, long metadataId) { private int countTraitValues(long methodId, long metadataId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=?", methodId), Integer.class, metadataId); return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE metadataId=?", methodId), Integer.class, metadataId);
} }
/** /**
...@@ -352,7 +355,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository { ...@@ -352,7 +355,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
* @return * @return
*/ */
private int countTraitValues(long methodId, long metadataId, long accessionId) { private int countTraitValues(long methodId, long metadataId, long accessionId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=? AND ALIS_Id=?", methodId), Integer.class, metadataId, return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE metadataId=? AND accessionId=?", methodId), Integer.class,
accessionId); metadataId, accessionId);
} }
} }
...@@ -129,4 +129,6 @@ public interface GenesysService { ...@@ -129,4 +129,6 @@ public interface GenesysService {
void saveGeo(List<AccessionGeo> all); void saveGeo(List<AccessionGeo> all);
void refreshMetadataMethods();
} }
...@@ -721,4 +721,11 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset ...@@ -721,4 +721,11 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
zos.finish(); zos.finish();
} }
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void refreshMetadataMethods() {
genesysLowlevelRepository.refreshMetadataMethods();
}
} }
...@@ -114,6 +114,12 @@ public class AdminController { ...@@ -114,6 +114,12 @@ public class AdminController {
return "redirect:/admin/"; return "redirect:/admin/";
} }
@RequestMapping(method = RequestMethod.POST, value = "/refreshMetadataMethods")
public String refreshMetadataMethods() {
genesysService.refreshMetadataMethods();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateInstituteCountryRefs") @RequestMapping(method = RequestMethod.POST, value = "/updateInstituteCountryRefs")
public String updateInstituteCountryRefs() { public String updateInstituteCountryRefs() {
instituteService.updateCountryRefs(); instituteService.updateCountryRefs();
......
...@@ -49,6 +49,11 @@ ...@@ -49,6 +49,11 @@
<input type="submit" class="btn btn-default" value="Update accession institute info" /> <input type="submit" class="btn btn-default" value="Update accession institute info" />
</form> </form>
<h3>C&E</h3>
<form method="post" action="<c:url value="/admin/refreshMetadataMethods" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Recalculate metadata methods" />
</form>
<h3>Content</h3> <h3>Content</h3>
<form method="post" action="<c:url value="/admin/sanitize" />"> <form method="post" action="<c:url value="/admin/sanitize" />">
<input type="submit" class="btn btn-default" value="Sanitize HTML content" /> <input type="submit" class="btn btn-default" value="Sanitize HTML content" />
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment